diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e71fce135c..145ace7876 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -206,7 +206,6 @@ Appwrite's current structure is a combination of both [Monolithic](https://en.wi │ ├── Network │ ├── OpenSSL │ ├── Promises -│ ├── Resque │ ├── Specification │ ├── Task │ ├── Template @@ -251,7 +250,6 @@ Appwrite stack is a combination of a variety of open-source technologies and too - Imagemagick - for manipulating and managing image media files. - Webp - for better compression of images on supporting clients. - SMTP - for sending email messages and alerts. -- Resque - for managing data queues and scheduled tasks over a Redis server. ## Package Managers diff --git a/app/cli.php b/app/cli.php index c8ebd089b1..5b5ae47bf6 100644 --- a/app/cli.php +++ b/app/cli.php @@ -3,6 +3,8 @@ require_once __DIR__ . '/init.php'; require_once __DIR__ . '/controllers/general.php'; +use Appwrite\Event\Delete; +use Appwrite\Event\Certificate; use Appwrite\Event\Func; use Appwrite\Platform\Appwrite; use Utopia\CLI\CLI; @@ -17,6 +19,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Logger\Log; use Utopia\Pools\Group; +use Utopia\Queue\Connection; use Utopia\Registry\Registry; Authorization::disable(); @@ -86,7 +89,7 @@ CLI::setResource('dbForConsole', function ($pools, $cache) { CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) { $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools - $getProjectDB = function (Document $project) use ($pools, $dbForConsole, $cache, &$databases) { + return function (Document $project) use ($pools, $dbForConsole, $cache, &$databases) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForConsole; } @@ -112,8 +115,6 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, return $database; }; - - return $getProjectDB; }, ['pools', 'dbForConsole', 'cache']); CLI::setResource('influxdb', function (Registry $register) { @@ -140,10 +141,18 @@ CLI::setResource('influxdb', function (Registry $register) { return $database; }, ['register']); -CLI::setResource('queueForFunctions', function (Group $pools) { - return new Func($pools->get('queue')->pop()->getResource()); +CLI::setResource('queue', function (Group $pools) { + return $pools->get('queue')->pop()->getResource(); }, ['pools']); - +CLI::setResource('queueForFunctions', function (Connection $queue) { + return new Func($queue); +}, ['queue']); +CLI::setResource('queueForDeletes', function (Connection $queue) { + return new Delete($queue); +}, ['queue']); +CLI::setResource('queueForCertificates', function (Connection $queue) { + return new Certificate($queue); +}, ['queue']); CLI::setResource('logError', function (Registry $register) { return function (Throwable $error, string $namespace, string $action) use ($register) { $logger = $register->get('logger'); diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index c6f88c5c61..2bc7759620 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -76,8 +76,9 @@ App::post('/v1/account') ->inject('user') ->inject('project') ->inject('dbForProject') - ->inject('events') - ->action(function (string $userId, string $email, string $password, string $name, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $userId, string $email, string $password, string $name, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents) { + $email = \strtolower($email); if ('console' === $project->getId()) { $whitelistEmails = $project->getAttribute('authWhitelistEmails'); @@ -156,7 +157,7 @@ App::post('/v1/account') Authorization::setRole(Role::user($user->getId())->toString()); Authorization::setRole(Role::users()->toString()); - $events->setParam('userId', $user->getId()); + $queueForEvents->setParam('userId', $user->getId()); $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -193,8 +194,8 @@ App::post('/v1/account/sessions/email') ->inject('project') ->inject('locale') ->inject('geodb') - ->inject('events') - ->action(function (string $email, string $password, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $events) { + ->inject('queueForEvents') + ->action(function (string $email, string $password, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $queueForEvents) { $email = \strtolower($email); $protocol = $request->getProtocol(); @@ -276,7 +277,7 @@ App::post('/v1/account/sessions/email') ->setAttribute('expire', $expire) ; - $events + $queueForEvents ->setParam('userId', $user->getId()) ->setParam('sessionId', $session->getId()) ; @@ -440,8 +441,8 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') ->inject('user') ->inject('dbForProject') ->inject('geodb') - ->inject('events') - ->action(function (string $provider, string $code, string $state, string $error, string $error_description, Request $request, Response $response, Document $project, Document $user, Database $dbForProject, Reader $geodb, Event $events) use ($oauthDefaultSuccess) { + ->inject('queueForEvents') + ->action(function (string $provider, string $code, string $state, string $error, string $error_description, Request $request, Response $response, Document $project, Document $user, Database $dbForProject, Reader $geodb, Event $queueForEvents) use ($oauthDefaultSuccess) { $protocol = $request->getProtocol(); $callback = $protocol . '://' . $request->getHostname() . '/v1/account/sessions/oauth2/callback/' . $provider . '/' . $project->getId(); @@ -758,7 +759,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') $session->setAttribute('expire', $expire); - $events + $queueForEvents ->setParam('userId', $user->getId()) ->setParam('sessionId', $session->getId()) ->setPayload($response->output($session, Response::MODEL_SESSION)) @@ -897,9 +898,9 @@ App::post('/v1/account/sessions/magic-url') ->inject('project') ->inject('dbForProject') ->inject('locale') - ->inject('events') - ->inject('mails') - ->action(function (string $userId, string $email, string $url, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Locale $locale, Event $events, Mail $mails) { + ->inject('queueForEvents') + ->inject('queueForMails') + ->action(function (string $userId, string $email, string $url, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Locale $locale, Event $queueForEvents, Mail $queueForMails) { if (empty(App::getEnv('_APP_SMTP_HOST'))) { throw new Exception(Exception::GENERAL_SMTP_DISABLED, 'SMTP disabled'); @@ -1020,7 +1021,7 @@ App::post('/v1/account/sessions/magic-url') $replyTo = $smtp['replyTo']; } - $mails + $queueForMails ->setSmtpHost($smtp['host'] ?? '') ->setSmtpPort($smtp['port'] ?? '') ->setSmtpUsername($smtp['username'] ?? '') @@ -1042,7 +1043,7 @@ App::post('/v1/account/sessions/magic-url') $subject = $customTemplate['subject'] ?? $subject; } - $mails + $queueForMails ->setSmtpReplyTo($replyTo) ->setSmtpSenderEmail($senderEmail) ->setSmtpSenderName($senderName); @@ -1066,14 +1067,14 @@ App::post('/v1/account/sessions/magic-url') 'redirect' => $url ]; - $mails + $queueForMails ->setSubject($subject) ->setBody($body) ->setVariables($emailVariables) ->setRecipient($email) ->trigger(); - $events->setPayload( + $queueForEvents->setPayload( $response->output( $token->setAttribute('secret', $loginSecret), Response::MODEL_TOKEN @@ -1117,8 +1118,8 @@ App::put('/v1/account/sessions/magic-url') ->inject('project') ->inject('locale') ->inject('geodb') - ->inject('events') - ->action(function (string $userId, string $secret, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $events) { + ->inject('queueForEvents') + ->action(function (string $userId, string $secret, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $queueForEvents) { /** @var Utopia\Database\Document $user */ @@ -1186,7 +1187,7 @@ App::put('/v1/account/sessions/magic-url') throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed saving user to DB'); } - $events + $queueForEvents ->setParam('userId', $user->getId()) ->setParam('sessionId', $session->getId()); @@ -1235,10 +1236,10 @@ App::post('/v1/account/sessions/phone') ->inject('user') ->inject('project') ->inject('dbForProject') - ->inject('events') - ->inject('messaging') + ->inject('queueForEvents') + ->inject('queueForMessaging') ->inject('locale') - ->action(function (string $userId, string $phone, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $events, EventPhone $messaging, Locale $locale) { + ->action(function (string $userId, string $phone, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents, EventPhone $queueForMessaging, Locale $locale) { if (empty(App::getEnv('_APP_SMS_PROVIDER'))) { throw new Exception(Exception::GENERAL_PHONE_DISABLED, 'Phone provider not configured'); @@ -1326,12 +1327,12 @@ App::post('/v1/account/sessions/phone') $message = $message->setParam('{{token}}', $secret); $message = $message->render(); - $messaging + $queueForMessaging ->setRecipient($phone) ->setMessage($message) ->trigger(); - $events->setPayload( + $queueForEvents->setPayload( $response->output( $token->setAttribute('secret', $secret), Response::MODEL_TOKEN @@ -1370,8 +1371,8 @@ App::put('/v1/account/sessions/phone') ->inject('project') ->inject('locale') ->inject('geodb') - ->inject('events') - ->action(function (string $userId, string $secret, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $events) { + ->inject('queueForEvents') + ->action(function (string $userId, string $secret, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $queueForEvents) { $userFromRequest = Authorization::skip(fn() => $dbForProject->getDocument('users', $userId)); @@ -1435,7 +1436,7 @@ App::put('/v1/account/sessions/phone') throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed saving user to DB'); } - $events + $queueForEvents ->setParam('userId', $user->getId()) ->setParam('sessionId', $session->getId()) ; @@ -1490,8 +1491,8 @@ App::post('/v1/account/sessions/anonymous') ->inject('project') ->inject('dbForProject') ->inject('geodb') - ->inject('events') - ->action(function (Request $request, Response $response, Locale $locale, Document $user, Document $project, Database $dbForProject, Reader $geodb, Event $events) { + ->inject('queueForEvents') + ->action(function (Request $request, Response $response, Locale $locale, Document $user, Document $project, Database $dbForProject, Reader $geodb, Event $queueForEvents) { $protocol = $request->getProtocol(); @@ -1574,7 +1575,7 @@ App::post('/v1/account/sessions/anonymous') $dbForProject->deleteCachedDocument('users', $user->getId()); - $events + $queueForEvents ->setParam('userId', $user->getId()) ->setParam('sessionId', $session->getId()) ; @@ -1858,14 +1859,14 @@ App::patch('/v1/account/name') ->inject('response') ->inject('user') ->inject('dbForProject') - ->inject('events') - ->action(function (string $name, ?\DateTime $requestTimestamp, Response $response, Document $user, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $name, ?\DateTime $requestTimestamp, Response $response, Document $user, Database $dbForProject, Event $queueForEvents) { $user->setAttribute('name', $name); $user = $dbForProject->withRequestTimestamp($requestTimestamp, fn () => $dbForProject->updateDocument('users', $user->getId(), $user)); - $events->setParam('userId', $user->getId()); + $queueForEvents->setParam('userId', $user->getId()); $response->dynamic($user, Response::MODEL_ACCOUNT); }); @@ -1895,8 +1896,8 @@ App::patch('/v1/account/password') ->inject('user') ->inject('project') ->inject('dbForProject') - ->inject('events') - ->action(function (string $password, string $oldPassword, ?\DateTime $requestTimestamp, Response $response, Document $user, Document $project, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $password, string $oldPassword, ?\DateTime $requestTimestamp, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents) { // Check old password only if its an existing user. if (!empty($user->getAttribute('passwordUpdate')) && !Auth::passwordVerify($oldPassword, $user->getAttribute('password'), $user->getAttribute('hash'), $user->getAttribute('hashOptions'))) { // Double check user password @@ -1932,7 +1933,7 @@ App::patch('/v1/account/password') $user = $dbForProject->withRequestTimestamp($requestTimestamp, fn () => $dbForProject->updateDocument('users', $user->getId(), $user)); - $events->setParam('userId', $user->getId()); + $queueForEvents->setParam('userId', $user->getId()); $response->dynamic($user, Response::MODEL_ACCOUNT); }); @@ -1960,8 +1961,8 @@ App::patch('/v1/account/email') ->inject('response') ->inject('user') ->inject('dbForProject') - ->inject('events') - ->action(function (string $email, string $password, ?\DateTime $requestTimestamp, Response $response, Document $user, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $email, string $password, ?\DateTime $requestTimestamp, Response $response, Document $user, Database $dbForProject, Event $queueForEvents) { // passwordUpdate will be empty if the user has never set a password $passwordUpdate = $user->getAttribute('passwordUpdate'); @@ -2002,7 +2003,7 @@ App::patch('/v1/account/email') throw new Exception(Exception::USER_EMAIL_ALREADY_EXISTS); } - $events->setParam('userId', $user->getId()); + $queueForEvents->setParam('userId', $user->getId()); $response->dynamic($user, Response::MODEL_ACCOUNT); }); @@ -2030,8 +2031,8 @@ App::patch('/v1/account/phone') ->inject('response') ->inject('user') ->inject('dbForProject') - ->inject('events') - ->action(function (string $phone, string $password, ?\DateTime $requestTimestamp, Response $response, Document $user, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $phone, string $password, ?\DateTime $requestTimestamp, Response $response, Document $user, Database $dbForProject, Event $queueForEvents) { // passwordUpdate will be empty if the user has never set a password $passwordUpdate = $user->getAttribute('passwordUpdate'); @@ -2061,7 +2062,7 @@ App::patch('/v1/account/phone') throw new Exception(Exception::USER_PHONE_ALREADY_EXISTS); } - $events->setParam('userId', $user->getId()); + $queueForEvents->setParam('userId', $user->getId()); $response->dynamic($user, Response::MODEL_ACCOUNT); }); @@ -2088,14 +2089,14 @@ App::patch('/v1/account/prefs') ->inject('response') ->inject('user') ->inject('dbForProject') - ->inject('events') - ->action(function (array $prefs, ?\DateTime $requestTimestamp, Response $response, Document $user, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (array $prefs, ?\DateTime $requestTimestamp, Response $response, Document $user, Database $dbForProject, Event $queueForEvents) { $user->setAttribute('prefs', $prefs); $user = $dbForProject->withRequestTimestamp($requestTimestamp, fn () => $dbForProject->updateDocument('users', $user->getId(), $user)); - $events->setParam('userId', $user->getId()); + $queueForEvents->setParam('userId', $user->getId()); $response->dynamic($user, Response::MODEL_ACCOUNT); }); @@ -2120,14 +2121,14 @@ App::patch('/v1/account/status') ->inject('response') ->inject('user') ->inject('dbForProject') - ->inject('events') - ->action(function (?\DateTime $requestTimestamp, Request $request, Response $response, Document $user, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (?\DateTime $requestTimestamp, Request $request, Response $response, Document $user, Database $dbForProject, Event $queueForEvents) { $user->setAttribute('status', false); $user = $dbForProject->withRequestTimestamp($requestTimestamp, fn () => $dbForProject->updateDocument('users', $user->getId(), $user)); - $events + $queueForEvents ->setParam('userId', $user->getId()) ->setPayload($response->output($user, Response::MODEL_ACCOUNT)); @@ -2166,9 +2167,9 @@ App::delete('/v1/account/sessions/:sessionId') ->inject('user') ->inject('dbForProject') ->inject('locale') - ->inject('events') + ->inject('queueForEvents') ->inject('project') - ->action(function (?string $sessionId, ?\DateTime $requestTimestamp, Request $request, Response $response, Document $user, Database $dbForProject, Locale $locale, Event $events, Document $project) { + ->action(function (?string $sessionId, ?\DateTime $requestTimestamp, Request $request, Response $response, Document $user, Database $dbForProject, Locale $locale, Event $queueForEvents, Document $project) { $protocol = $request->getProtocol(); $authDuration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG; @@ -2208,7 +2209,7 @@ App::delete('/v1/account/sessions/:sessionId') $dbForProject->deleteCachedDocument('users', $user->getId()); - $events + $queueForEvents ->setParam('userId', $user->getId()) ->setParam('sessionId', $session->getId()) ->setPayload($response->output($session, Response::MODEL_SESSION)) @@ -2244,8 +2245,8 @@ App::patch('/v1/account/sessions/:sessionId') ->inject('dbForProject') ->inject('project') ->inject('locale') - ->inject('events') - ->action(function (?string $sessionId, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Event $events) { + ->inject('queueForEvents') + ->action(function (?string $sessionId, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Event $queueForEvents) { $authDuration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG; $sessionId = ($sessionId === 'current') ? Auth::sessionVerify($user->getAttribute('sessions'), Auth::$secret, $authDuration) @@ -2293,7 +2294,7 @@ App::patch('/v1/account/sessions/:sessionId') $session->setAttribute('expire', DateTime::formatTz(DateTime::addSeconds(new \DateTime($session->getCreatedAt()), $authDuration))); - $events + $queueForEvents ->setParam('userId', $user->getId()) ->setParam('sessionId', $session->getId()) ->setPayload($response->output($session, Response::MODEL_SESSION)) @@ -2326,8 +2327,8 @@ App::delete('/v1/account/sessions') ->inject('user') ->inject('dbForProject') ->inject('locale') - ->inject('events') - ->action(function (Request $request, Response $response, Document $user, Database $dbForProject, Locale $locale, Event $events) { + ->inject('queueForEvents') + ->action(function (Request $request, Response $response, Document $user, Database $dbForProject, Locale $locale, Event $queueForEvents) { $protocol = $request->getProtocol(); $sessions = $user->getAttribute('sessions', []); @@ -2354,13 +2355,13 @@ App::delete('/v1/account/sessions') ->addCookie(Auth::$cookieName, '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')); // Use current session for events. - $events->setPayload($response->output($session, Response::MODEL_SESSION)); + $queueForEvents->setPayload($response->output($session, Response::MODEL_SESSION)); } } $dbForProject->deleteCachedDocument('users', $user->getId()); - $events + $queueForEvents ->setParam('userId', $user->getId()) ->setParam('sessionId', $session->getId()); @@ -2393,9 +2394,9 @@ App::post('/v1/account/recovery') ->inject('dbForProject') ->inject('project') ->inject('locale') - ->inject('mails') - ->inject('events') - ->action(function (string $email, string $url, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Mail $mails, Event $events) { + ->inject('queueForMails') + ->inject('queueForEvents') + ->action(function (string $email, string $url, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Mail $queueForMails, Event $queueForEvents) { if (empty(App::getEnv('_APP_SMTP_HOST'))) { throw new Exception(Exception::GENERAL_SMTP_DISABLED, 'SMTP Disabled'); @@ -2477,7 +2478,7 @@ App::post('/v1/account/recovery') $replyTo = $smtp['replyTo']; } - $mails + $queueForMails ->setSmtpHost($smtp['host'] ?? '') ->setSmtpPort($smtp['port'] ?? '') ->setSmtpUsername($smtp['username'] ?? '') @@ -2499,7 +2500,7 @@ App::post('/v1/account/recovery') $subject = $customTemplate['subject'] ?? $subject; } - $mails + $queueForMails ->setSmtpReplyTo($replyTo) ->setSmtpSenderEmail($senderEmail) ->setSmtpSenderName($senderName); @@ -2524,7 +2525,7 @@ App::post('/v1/account/recovery') ]; - $mails + $queueForMails ->setRecipient($profile->getAttribute('email', '')) ->setName($profile->getAttribute('name')) ->setBody($body) @@ -2532,7 +2533,7 @@ App::post('/v1/account/recovery') ->setSubject($subject) ->trigger(); - $events + $queueForEvents ->setParam('userId', $profile->getId()) ->setParam('tokenId', $recovery->getId()) ->setUser($profile) @@ -2576,8 +2577,8 @@ App::put('/v1/account/recovery') ->inject('user') ->inject('dbForProject') ->inject('project') - ->inject('events') - ->action(function (string $userId, string $secret, string $password, string $passwordAgain, Response $response, Document $user, Database $dbForProject, Document $project, Event $events) { + ->inject('queueForEvents') + ->action(function (string $userId, string $secret, string $password, string $passwordAgain, Response $response, Document $user, Database $dbForProject, Document $project, Event $queueForEvents) { if ($password !== $passwordAgain) { throw new Exception(Exception::USER_PASSWORD_MISMATCH); } @@ -2630,7 +2631,7 @@ App::put('/v1/account/recovery') $dbForProject->deleteDocument('tokens', $recovery); $dbForProject->deleteCachedDocument('users', $profile->getId()); - $events + $queueForEvents ->setParam('userId', $profile->getId()) ->setParam('tokenId', $recoveryDocument->getId()) ; @@ -2662,9 +2663,9 @@ App::post('/v1/account/verification') ->inject('user') ->inject('dbForProject') ->inject('locale') - ->inject('events') - ->inject('mails') - ->action(function (string $url, Request $request, Response $response, Document $project, Document $user, Database $dbForProject, Locale $locale, Event $events, Mail $mails) { + ->inject('queueForEvents') + ->inject('queueForMails') + ->action(function (string $url, Request $request, Response $response, Document $project, Document $user, Database $dbForProject, Locale $locale, Event $queueForEvents, Mail $queueForMails) { if (empty(App::getEnv('_APP_SMTP_HOST'))) { throw new Exception(Exception::GENERAL_SMTP_DISABLED, 'SMTP Disabled'); @@ -2729,7 +2730,7 @@ App::post('/v1/account/verification') $replyTo = $smtp['replyTo']; } - $mails + $queueForMails ->setSmtpHost($smtp['host'] ?? '') ->setSmtpPort($smtp['port'] ?? '') ->setSmtpUsername($smtp['username'] ?? '') @@ -2751,7 +2752,7 @@ App::post('/v1/account/verification') $subject = $customTemplate['subject'] ?? $subject; } - $mails + $queueForMails ->setSmtpReplyTo($replyTo) ->setSmtpSenderEmail($senderEmail) ->setSmtpSenderName($senderName); @@ -2775,7 +2776,7 @@ App::post('/v1/account/verification') 'redirect' => $url ]; - $mails + $queueForMails ->setSubject($subject) ->setBody($body) ->setVariables($emailVariables) @@ -2783,7 +2784,7 @@ App::post('/v1/account/verification') ->setName($user->getAttribute('name') ?? '') ->trigger(); - $events + $queueForEvents ->setParam('userId', $user->getId()) ->setParam('tokenId', $verification->getId()) ->setPayload($response->output( @@ -2821,8 +2822,8 @@ App::put('/v1/account/verification') ->inject('response') ->inject('user') ->inject('dbForProject') - ->inject('events') - ->action(function (string $userId, string $secret, Response $response, Document $user, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $userId, string $secret, Response $response, Document $user, Database $dbForProject, Event $queueForEvents) { $profile = Authorization::skip(fn() => $dbForProject->getDocument('users', $userId)); @@ -2852,7 +2853,7 @@ App::put('/v1/account/verification') $dbForProject->deleteDocument('tokens', $verification); $dbForProject->deleteCachedDocument('users', $profile->getId()); - $events + $queueForEvents ->setParam('userId', $userId) ->setParam('tokenId', $verificationDocument->getId()) ; @@ -2881,11 +2882,11 @@ App::post('/v1/account/verification/phone') ->inject('response') ->inject('user') ->inject('dbForProject') - ->inject('events') - ->inject('messaging') + ->inject('queueForEvents') + ->inject('queueForMessaging') ->inject('project') ->inject('locale') - ->action(function (Request $request, Response $response, Document $user, Database $dbForProject, Event $events, EventPhone $messaging, Document $project, Locale $locale) { + ->action(function (Request $request, Response $response, Document $user, Database $dbForProject, Event $queueForEvents, EventPhone $queueForMessaging, Document $project, Locale $locale) { if (empty(App::getEnv('_APP_SMS_PROVIDER'))) { throw new Exception(Exception::GENERAL_PHONE_DISABLED); @@ -2933,13 +2934,13 @@ App::post('/v1/account/verification/phone') $message = $message->setParam('{{token}}', $secret); $message = $message->render(); - $messaging + $queueForMessaging ->setRecipient($user->getAttribute('phone')) ->setMessage($message) ->trigger() ; - $events + $queueForEvents ->setParam('userId', $user->getId()) ->setParam('tokenId', $verification->getId()) ->setPayload($response->output( @@ -2978,8 +2979,8 @@ App::put('/v1/account/verification/phone') ->inject('response') ->inject('user') ->inject('dbForProject') - ->inject('events') - ->action(function (string $userId, string $secret, Response $response, Document $user, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $userId, string $secret, Response $response, Document $user, Database $dbForProject, Event $queueForEvents) { $profile = Authorization::skip(fn() => $dbForProject->getDocument('users', $userId)); @@ -3007,7 +3008,7 @@ App::put('/v1/account/verification/phone') $dbForProject->deleteDocument('tokens', $verification); $dbForProject->deleteCachedDocument('users', $profile->getId()); - $events + $queueForEvents ->setParam('userId', $user->getId()) ->setParam('tokenId', $verificationDocument->getId()) ; diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index d41b3f27b0..d132273b5d 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -17,11 +17,11 @@ use MaxMind\Db\Reader; use Utopia\App; use Utopia\Audit\Audit; use Utopia\Config\Config; -use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception\Authorization as AuthorizationException; +use Utopia\Database\Exception\Conflict; use Utopia\Database\Exception\Duplicate as DuplicateException; use Utopia\Database\Exception\Limit as LimitException; use Utopia\Database\Exception\Restricted as RestrictedException; @@ -37,8 +37,6 @@ use Utopia\Database\Validator\Index as IndexValidator; use Utopia\Database\Validator\Key; use Utopia\Database\Validator\Permissions; use Utopia\Database\Validator\Queries; -use Utopia\Database\Validator\Queries\Document as DocumentQueriesValidator; -use Utopia\Database\Validator\Queries\Documents; use Utopia\Database\Validator\Query\Limit; use Utopia\Database\Validator\Query\Offset; use Utopia\Database\Validator\Structure; @@ -57,13 +55,26 @@ use Utopia\Validator\URL; use Utopia\Validator\WhiteList; /** - * Create attribute of varying type - * + * * Create attribute of varying type * + * @param string $databaseId + * @param string $collectionId + * @param Document $attribute + * @param Response $response + * @param Database $dbForProject + * @param EventDatabase $queueForDatabase + * @param Event $queueForEvents * @return Document Newly created attribute document + * @throws AuthorizationException + * @throws Exception + * @throws LimitException + * @throws RestrictedException + * @throws StructureException + * @throws \Utopia\Database\Exception + * @throws Conflict * @throws Exception */ -function createAttribute(string $databaseId, string $collectionId, Document $attribute, Response $response, Database $dbForProject, EventDatabase $database, Event $events): Document +function createAttribute(string $databaseId, string $collectionId, Document $attribute, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents): Document { $key = $attribute->getAttribute('key'); $type = $attribute->getAttribute('type', ''); @@ -193,13 +204,13 @@ function createAttribute(string $databaseId, string $collectionId, Document $att $dbForProject->deleteCachedCollection('database_' . $db->getInternalId() . '_collection_' . $relatedCollection->getInternalId()); } - $database + $queueForDatabase ->setType(DATABASE_TYPE_CREATE_ATTRIBUTE) ->setDatabase($db) ->setCollection($collection) ->setDocument($attribute); - $events + $queueForEvents ->setContext('collection', $collection) ->setContext('database', $db) ->setParam('databaseId', $databaseId) @@ -216,7 +227,7 @@ function updateAttribute( string $collectionId, string $key, Database $dbForProject, - Event $events, + Event $queueForEvents, string $type, string $filter = null, string|bool|int|float $default = null, @@ -360,7 +371,7 @@ function updateAttribute( $attribute = $dbForProject->updateDocument('attributes', $db->getInternalId() . '_' . $collection->getInternalId() . '_' . $key, $attribute); $dbForProject->deleteCachedDocument('database_' . $db->getInternalId(), $collection->getId()); - $events + $queueForEvents ->setContext('collection', $collection) ->setContext('database', $db) ->setParam('databaseId', $databaseId) @@ -390,8 +401,8 @@ App::post('/v1/databases') ->param('enabled', true, new Boolean(), 'Is the database enabled? When set to \'disabled\', users cannot access the database but Server SDKs with an API key can still read and write to the database. No data is lost when this is toggled.', true) ->inject('response') ->inject('dbForProject') - ->inject('events') - ->action(function (string $databaseId, string $name, bool $enabled, Response $response, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $databaseId, string $name, bool $enabled, Response $response, Database $dbForProject, Event $queueForEvents) { $databaseId = $databaseId == 'unique()' ? ID::unique() : $databaseId; @@ -440,7 +451,7 @@ App::post('/v1/databases') throw new Exception(Exception::DATABASE_ALREADY_EXISTS); } - $events->setParam('databaseId', $database->getId()); + $queueForEvents->setParam('databaseId', $database->getId()); $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -627,8 +638,8 @@ App::put('/v1/databases/:databaseId') ->param('enabled', true, new Boolean(), 'Is database enabled? When set to \'disabled\', users cannot access the database but Server SDKs with an API key can still read and write to the database. No data is lost when this is toggled.', true) ->inject('response') ->inject('dbForProject') - ->inject('events') - ->action(function (string $databaseId, string $name, bool $enabled, Response $response, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $databaseId, string $name, bool $enabled, Response $response, Database $dbForProject, Event $queueForEvents) { $database = $dbForProject->getDocument('databases', $databaseId); @@ -647,7 +658,7 @@ App::put('/v1/databases/:databaseId') throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, 'Bad structure. ' . $exception->getMessage()); } - $events->setParam('databaseId', $database->getId()); + $queueForEvents->setParam('databaseId', $database->getId()); $response->dynamic($database, Response::MODEL_DATABASE); }); @@ -669,9 +680,9 @@ App::delete('/v1/databases/:databaseId') ->param('databaseId', '', new UID(), 'Database ID.') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->inject('deletes') - ->action(function (string $databaseId, Response $response, Database $dbForProject, Event $events, Delete $deletes) { + ->inject('queueForDatabase') + ->inject('queueForEvents') + ->action(function (string $databaseId, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { $database = $dbForProject->getDocument('databases', $databaseId); @@ -686,11 +697,11 @@ App::delete('/v1/databases/:databaseId') $dbForProject->deleteCachedDocument('databases', $database->getId()); $dbForProject->deleteCachedCollection('databases_' . $database->getInternalId()); - $deletes - ->setType(DELETE_TYPE_DOCUMENT) - ->setDocument($database); + $queueForDatabase + ->setType(DATABASE_TYPE_DELETE_DATABASE) + ->setDatabase($database); - $events + $queueForEvents ->setParam('databaseId', $database->getId()) ->setPayload($response->output($database, Response::MODEL_DATABASE)); @@ -722,8 +733,8 @@ App::post('/v1/databases/:databaseId/collections') ->inject('response') ->inject('dbForProject') ->inject('mode') - ->inject('events') - ->action(function (string $databaseId, string $collectionId, string $name, ?array $permissions, bool $documentSecurity, bool $enabled, Response $response, Database $dbForProject, string $mode, Event $events) { + ->inject('queueForEvents') + ->action(function (string $databaseId, string $collectionId, string $name, ?array $permissions, bool $documentSecurity, bool $enabled, Response $response, Database $dbForProject, string $mode, Event $queueForEvents) { $database = Authorization::skip(fn() => $dbForProject->getDocument('databases', $databaseId)); @@ -756,7 +767,7 @@ App::post('/v1/databases/:databaseId/collections') throw new Exception(Exception::COLLECTION_LIMIT_EXCEEDED); } - $events + $queueForEvents ->setContext('database', $database) ->setParam('databaseId', $databaseId) ->setParam('collectionId', $collection->getId()); @@ -983,8 +994,8 @@ App::put('/v1/databases/:databaseId/collections/:collectionId') ->inject('response') ->inject('dbForProject') ->inject('mode') - ->inject('events') - ->action(function (string $databaseId, string $collectionId, string $name, ?array $permissions, bool $documentSecurity, bool $enabled, Response $response, Database $dbForProject, string $mode, Event $events) { + ->inject('queueForEvents') + ->action(function (string $databaseId, string $collectionId, string $name, ?array $permissions, bool $documentSecurity, bool $enabled, Response $response, Database $dbForProject, string $mode, Event $queueForEvents) { $database = Authorization::skip(fn() => $dbForProject->getDocument('databases', $databaseId)); @@ -1019,7 +1030,7 @@ App::put('/v1/databases/:databaseId/collections/:collectionId') throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, 'Bad structure. ' . $exception->getMessage()); } - $events + $queueForEvents ->setContext('database', $database) ->setParam('databaseId', $databaseId) ->setParam('collectionId', $collection->getId()); @@ -1047,10 +1058,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId') ->param('collectionId', '', new UID(), 'Collection ID.') ->inject('response') ->inject('dbForProject') + ->inject('queueForDatabase') + ->inject('queueForEvents') ->inject('mode') - ->inject('events') - ->inject('deletes') - ->action(function (string $databaseId, string $collectionId, Response $response, Database $dbForProject, string $mode, Event $events, Delete $deletes) { + ->action(function (string $databaseId, string $collectionId, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents, string $mode) { $database = Authorization::skip(fn() => $dbForProject->getDocument('databases', $databaseId)); @@ -1070,11 +1081,12 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId') $dbForProject->deleteCachedCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); - $deletes - ->setType(DELETE_TYPE_DOCUMENT) - ->setDocument($collection); + $queueForDatabase + ->setType(DATABASE_TYPE_DELETE_COLLECTION) + ->setDatabase($database) + ->setCollection($collection); - $events + $queueForEvents ->setContext('database', $database) ->setParam('databaseId', $databaseId) ->setParam('collectionId', $collection->getId()) @@ -1110,9 +1122,9 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/string ->param('encrypt', false, new Boolean(), 'Toggle encryption for the attribute. Encryption enhances security by not storing any plain text values in the database. However, encrypted attributes cannot be queried.', true) ->inject('response') ->inject('dbForProject') - ->inject('database') - ->inject('events') - ->action(function (string $databaseId, string $collectionId, string $key, ?int $size, ?bool $required, ?string $default, bool $array, bool $encrypt, Response $response, Database $dbForProject, EventDatabase $database, Event $events) { + ->inject('queueForDatabase') + ->inject('queueForEvents') + ->action(function (string $databaseId, string $collectionId, string $key, ?int $size, ?bool $required, ?string $default, bool $array, bool $encrypt, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { // Ensure attribute default is within required size $validator = new Text($size, 0); @@ -1134,7 +1146,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/string 'default' => $default, 'array' => $array, 'filters' => $filters, - ]), $response, $dbForProject, $database, $events); + ]), $response, $dbForProject, $queueForDatabase, $queueForEvents); $response ->setStatusCode(Response::STATUS_CODE_ACCEPTED) @@ -1166,9 +1178,9 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/email' ->param('array', false, new Boolean(), 'Is attribute an array?', true) ->inject('response') ->inject('dbForProject') - ->inject('database') - ->inject('events') - ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?string $default, bool $array, Response $response, Database $dbForProject, EventDatabase $database, Event $events) { + ->inject('queueForDatabase') + ->inject('queueForEvents') + ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?string $default, bool $array, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { $attribute = createAttribute($databaseId, $collectionId, new Document([ 'key' => $key, @@ -1178,7 +1190,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/email' 'default' => $default, 'array' => $array, 'format' => APP_DATABASE_ATTRIBUTE_EMAIL, - ]), $response, $dbForProject, $database, $events); + ]), $response, $dbForProject, $queueForDatabase, $queueForEvents); $response ->setStatusCode(Response::STATUS_CODE_ACCEPTED) @@ -1211,10 +1223,9 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum') ->param('array', false, new Boolean(), 'Is attribute an array?', true) ->inject('response') ->inject('dbForProject') - ->inject('database') - ->inject('events') - ->action(function (string $databaseId, string $collectionId, string $key, array $elements, ?bool $required, ?string $default, bool $array, Response $response, Database $dbForProject, EventDatabase $database, Event $events) { - + ->inject('queueForDatabase') + ->inject('queueForEvents') + ->action(function (string $databaseId, string $collectionId, string $key, array $elements, ?bool $required, ?string $default, bool $array, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { // use length of longest string as attribute size $size = 0; foreach ($elements as $element) { @@ -1238,7 +1249,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum') 'array' => $array, 'format' => APP_DATABASE_ATTRIBUTE_ENUM, 'formatOptions' => ['elements' => $elements], - ]), $response, $dbForProject, $database, $events); + ]), $response, $dbForProject, $queueForDatabase, $queueForEvents); $response ->setStatusCode(Response::STATUS_CODE_ACCEPTED) @@ -1270,9 +1281,9 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/ip') ->param('array', false, new Boolean(), 'Is attribute an array?', true) ->inject('response') ->inject('dbForProject') - ->inject('database') - ->inject('events') - ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?string $default, bool $array, Response $response, Database $dbForProject, EventDatabase $database, Event $events) { + ->inject('queueForDatabase') + ->inject('queueForEvents') + ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?string $default, bool $array, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { $attribute = createAttribute($databaseId, $collectionId, new Document([ 'key' => $key, @@ -1282,7 +1293,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/ip') 'default' => $default, 'array' => $array, 'format' => APP_DATABASE_ATTRIBUTE_IP, - ]), $response, $dbForProject, $database, $events); + ]), $response, $dbForProject, $queueForDatabase, $queueForEvents); $response ->setStatusCode(Response::STATUS_CODE_ACCEPTED) @@ -1314,9 +1325,9 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/url') ->param('array', false, new Boolean(), 'Is attribute an array?', true) ->inject('response') ->inject('dbForProject') - ->inject('database') - ->inject('events') - ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?string $default, bool $array, Response $response, Database $dbForProject, EventDatabase $database, Event $events) { + ->inject('queueForDatabase') + ->inject('queueForEvents') + ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?string $default, bool $array, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { $attribute = createAttribute($databaseId, $collectionId, new Document([ 'key' => $key, @@ -1326,7 +1337,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/url') 'default' => $default, 'array' => $array, 'format' => APP_DATABASE_ATTRIBUTE_URL, - ]), $response, $dbForProject, $database, $events); + ]), $response, $dbForProject, $queueForDatabase, $queueForEvents); $response ->setStatusCode(Response::STATUS_CODE_ACCEPTED) @@ -1360,9 +1371,9 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/intege ->param('array', false, new Boolean(), 'Is attribute an array?', true) ->inject('response') ->inject('dbForProject') - ->inject('database') - ->inject('events') - ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?int $min, ?int $max, ?int $default, bool $array, Response $response, Database $dbForProject, EventDatabase $database, Event $events) { + ->inject('queueForDatabase') + ->inject('queueForEvents') + ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?int $min, ?int $max, ?int $default, bool $array, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { // Ensure attribute default is within range $min = (is_null($min)) ? PHP_INT_MIN : \intval($min); @@ -1392,7 +1403,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/intege 'min' => $min, 'max' => $max, ], - ]), $response, $dbForProject, $database, $events); + ]), $response, $dbForProject, $queueForDatabase, $queueForEvents); $formatOptions = $attribute->getAttribute('formatOptions', []); @@ -1433,9 +1444,9 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/float' ->param('array', false, new Boolean(), 'Is attribute an array?', true) ->inject('response') ->inject('dbForProject') - ->inject('database') - ->inject('events') - ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?float $min, ?float $max, ?float $default, bool $array, Response $response, Database $dbForProject, EventDatabase $database, Event $events) { + ->inject('queueForDatabase') + ->inject('queueForEvents') + ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?float $min, ?float $max, ?float $default, bool $array, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { // Ensure attribute default is within range $min = (is_null($min)) ? -PHP_FLOAT_MAX : \floatval($min); @@ -1468,7 +1479,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/float' 'min' => $min, 'max' => $max, ], - ]), $response, $dbForProject, $database, $events); + ]), $response, $dbForProject, $queueForDatabase, $queueForEvents); $formatOptions = $attribute->getAttribute('formatOptions', []); @@ -1507,9 +1518,9 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/boolea ->param('array', false, new Boolean(), 'Is attribute an array?', true) ->inject('response') ->inject('dbForProject') - ->inject('database') - ->inject('events') - ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?bool $default, bool $array, Response $response, Database $dbForProject, EventDatabase $database, Event $events) { + ->inject('queueForDatabase') + ->inject('queueForEvents') + ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?bool $default, bool $array, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { $attribute = createAttribute($databaseId, $collectionId, new Document([ 'key' => $key, @@ -1518,7 +1529,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/boolea 'required' => $required, 'default' => $default, 'array' => $array, - ]), $response, $dbForProject, $database, $events); + ]), $response, $dbForProject, $queueForDatabase, $queueForEvents); $response ->setStatusCode(Response::STATUS_CODE_ACCEPTED) @@ -1550,9 +1561,9 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/dateti ->param('array', false, new Boolean(), 'Is attribute an array?', true) ->inject('response') ->inject('dbForProject') - ->inject('database') - ->inject('events') - ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?string $default, bool $array, Response $response, Database $dbForProject, EventDatabase $database, Event $events) { + ->inject('queueForDatabase') + ->inject('queueForEvents') + ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?string $default, bool $array, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { $filters[] = 'datetime'; @@ -1564,7 +1575,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/dateti 'default' => $default, 'array' => $array, 'filters' => $filters, - ]), $response, $dbForProject, $database, $events); + ]), $response, $dbForProject, $queueForDatabase, $queueForEvents); $response ->setStatusCode(Response::STATUS_CODE_ACCEPTED) @@ -1598,8 +1609,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/relati ->param('onDelete', Database::RELATION_MUTATE_RESTRICT, new WhiteList([Database::RELATION_MUTATE_CASCADE, Database::RELATION_MUTATE_RESTRICT, Database::RELATION_MUTATE_SET_NULL], true), 'Constraints option', true) ->inject('response') ->inject('dbForProject') - ->inject('database') - ->inject('events') + ->inject('queueForDatabase') + ->inject('queueForEvents') ->action(function ( string $databaseId, string $collectionId, @@ -1611,8 +1622,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/relati string $onDelete, Response $response, Database $dbForProject, - EventDatabase $database, - Event $events + EventDatabase $queueForDatabase, + Event $queueForEvents ) { $key ??= $relatedCollectionId; $twoWayKey ??= $collectionId; @@ -1638,8 +1649,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/relati ]), $response, $dbForProject, - $database, - $events + $queueForDatabase, + $queueForEvents ); $options = $attribute->getAttribute('options', []); @@ -1827,15 +1838,15 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/strin ->param('default', null, new Nullable(new Text(0, 0)), 'Default value for attribute when not provided. Cannot be set when attribute is required.') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?string $default, Response $response, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?string $default, Response $response, Database $dbForProject, Event $queueForEvents) { $attribute = updateAttribute( databaseId: $databaseId, collectionId: $collectionId, key: $key, dbForProject: $dbForProject, - events: $events, + queueForEvents: $queueForEvents, type: Database::VAR_STRING, default: $default, required: $required @@ -1868,14 +1879,14 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/email ->param('default', null, new Nullable(new Email()), 'Default value for attribute when not provided. Cannot be set when attribute is required.') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?string $default, Response $response, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?string $default, Response $response, Database $dbForProject, Event $queueForEvents) { $attribute = updateAttribute( databaseId: $databaseId, collectionId: $collectionId, key: $key, dbForProject: $dbForProject, - events: $events, + queueForEvents: $queueForEvents, type: Database::VAR_STRING, filter: APP_DATABASE_ATTRIBUTE_EMAIL, default: $default, @@ -1910,14 +1921,14 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/enum/ ->param('default', null, new Nullable(new Text(0)), 'Default value for attribute when not provided. Cannot be set when attribute is required.') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->action(function (string $databaseId, string $collectionId, string $key, ?array $elements, ?bool $required, ?string $default, Response $response, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $databaseId, string $collectionId, string $key, ?array $elements, ?bool $required, ?string $default, Response $response, Database $dbForProject, Event $queueForEvents) { $attribute = updateAttribute( databaseId: $databaseId, collectionId: $collectionId, key: $key, dbForProject: $dbForProject, - events: $events, + queueForEvents: $queueForEvents, type: Database::VAR_STRING, filter: APP_DATABASE_ATTRIBUTE_ENUM, default: $default, @@ -1952,14 +1963,14 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/ip/:k ->param('default', null, new Nullable(new IP()), 'Default value for attribute when not provided. Cannot be set when attribute is required.') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?string $default, Response $response, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?string $default, Response $response, Database $dbForProject, Event $queueForEvents) { $attribute = updateAttribute( databaseId: $databaseId, collectionId: $collectionId, key: $key, dbForProject: $dbForProject, - events: $events, + queueForEvents: $queueForEvents, type: Database::VAR_STRING, filter: APP_DATABASE_ATTRIBUTE_IP, default: $default, @@ -1993,14 +2004,14 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/url/: ->param('default', null, new Nullable(new URL()), 'Default value for attribute when not provided. Cannot be set when attribute is required.') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?string $default, Response $response, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?string $default, Response $response, Database $dbForProject, Event $queueForEvents) { $attribute = updateAttribute( databaseId: $databaseId, collectionId: $collectionId, key: $key, dbForProject: $dbForProject, - events: $events, + queueForEvents: $queueForEvents, type: Database::VAR_STRING, filter: APP_DATABASE_ATTRIBUTE_URL, default: $default, @@ -2036,14 +2047,14 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/integ ->param('default', null, new Nullable(new Integer()), 'Default value for attribute when not provided. Cannot be set when attribute is required.') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?int $min, ?int $max, ?int $default, Response $response, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?int $min, ?int $max, ?int $default, Response $response, Database $dbForProject, Event $queueForEvents) { $attribute = updateAttribute( databaseId: $databaseId, collectionId: $collectionId, key: $key, dbForProject: $dbForProject, - events: $events, + queueForEvents: $queueForEvents, type: Database::VAR_INTEGER, default: $default, required: $required, @@ -2087,14 +2098,14 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/float ->param('default', null, new Nullable(new FloatValidator()), 'Default value for attribute when not provided. Cannot be set when attribute is required.') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?float $min, ?float $max, ?float $default, Response $response, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?float $min, ?float $max, ?float $default, Response $response, Database $dbForProject, Event $queueForEvents) { $attribute = updateAttribute( databaseId: $databaseId, collectionId: $collectionId, key: $key, dbForProject: $dbForProject, - events: $events, + queueForEvents: $queueForEvents, type: Database::VAR_FLOAT, default: $default, required: $required, @@ -2136,14 +2147,14 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/boole ->param('default', null, new Nullable(new Boolean()), 'Default value for attribute when not provided. Cannot be set when attribute is required.') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?bool $default, Response $response, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?bool $default, Response $response, Database $dbForProject, Event $queueForEvents) { $attribute = updateAttribute( databaseId: $databaseId, collectionId: $collectionId, key: $key, dbForProject: $dbForProject, - events: $events, + queueForEvents: $queueForEvents, type: Database::VAR_BOOLEAN, default: $default, required: $required @@ -2176,14 +2187,14 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/datet ->param('default', null, new Nullable(new DatetimeValidator()), 'Default value for attribute when not provided. Cannot be set when attribute is required.') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?string $default, Response $response, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?string $default, Response $response, Database $dbForProject, Event $queueForEvents) { $attribute = updateAttribute( databaseId: $databaseId, collectionId: $collectionId, key: $key, dbForProject: $dbForProject, - events: $events, + queueForEvents: $queueForEvents, type: Database::VAR_DATETIME, default: $default, required: $required @@ -2215,7 +2226,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/:key/ ->param('onDelete', null, new WhiteList([Database::RELATION_MUTATE_CASCADE, Database::RELATION_MUTATE_RESTRICT, Database::RELATION_MUTATE_SET_NULL], true), 'Constraints option', true) ->inject('response') ->inject('dbForProject') - ->inject('events') + ->inject('queueForEvents') ->action(function ( string $databaseId, string $collectionId, @@ -2223,14 +2234,14 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/:key/ ?string $onDelete, Response $response, Database $dbForProject, - Event $events + Event $queueForEvents ) { $attribute = updateAttribute( $databaseId, $collectionId, $key, $dbForProject, - $events, + $queueForEvents, type: Database::VAR_RELATIONSHIP, required: false, options: [ @@ -2254,7 +2265,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key ->desc('Delete attribute') ->groups(['api', 'database', 'schema']) ->label('scope', 'collections.write') - ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].delete') + ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update') ->label('audits.event', 'attribute.delete') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') ->label('usage.metric', 'collections.{scope}.requests.update') @@ -2270,9 +2281,9 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key ->param('key', '', new Key(), 'Attribute Key.') ->inject('response') ->inject('dbForProject') - ->inject('database') - ->inject('events') - ->action(function (string $databaseId, string $collectionId, string $key, Response $response, Database $dbForProject, EventDatabase $database, Event $events) { + ->inject('queueForDatabase') + ->inject('queueForEvents') + ->action(function (string $databaseId, string $collectionId, string $key, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { $db = Authorization::skip(fn() => $dbForProject->getDocument('databases', $databaseId)); @@ -2323,7 +2334,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key } } - $database + $queueForDatabase ->setType(DATABASE_TYPE_DELETE_ATTRIBUTE) ->setCollection($collection) ->setDatabase($db) @@ -2349,7 +2360,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key default => Response::MODEL_ATTRIBUTE, }; - $events + $queueForEvents ->setParam('databaseId', $databaseId) ->setParam('collectionId', $collection->getId()) ->setParam('attributeId', $attribute->getId()) @@ -2385,9 +2396,9 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') ->param('orders', [], new ArrayList(new WhiteList(['ASC', 'DESC'], false, Database::VAR_STRING), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of index orders. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' orders are allowed.', true) ->inject('response') ->inject('dbForProject') - ->inject('database') - ->inject('events') - ->action(function (string $databaseId, string $collectionId, string $key, string $type, array $attributes, array $orders, Response $response, Database $dbForProject, EventDatabase $database, Event $events) { + ->inject('queueForDatabase') + ->inject('queueForEvents') + ->action(function (string $databaseId, string $collectionId, string $key, string $type, array $attributes, array $orders, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { $db = Authorization::skip(fn() => $dbForProject->getDocument('databases', $databaseId)); @@ -2502,13 +2513,13 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') $dbForProject->deleteCachedDocument('database_' . $db->getInternalId(), $collectionId); - $database + $queueForDatabase ->setType(DATABASE_TYPE_CREATE_INDEX) ->setDatabase($db) ->setCollection($collection) ->setDocument($index); - $events + $queueForEvents ->setParam('databaseId', $databaseId) ->setParam('collectionId', $collection->getId()) ->setParam('indexId', $index->getId()) @@ -2631,7 +2642,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/indexes/:key') ->desc('Delete index') ->groups(['api', 'database']) ->label('scope', 'collections.write') - ->label('event', 'databases.[databaseId].collections.[collectionId].indexes.[indexId].delete') + ->label('event', 'databases.[databaseId].collections.[collectionId].indexes.[indexId].update') ->label('audits.event', 'index.delete') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') ->label('usage.metric', 'collections.{scope}.requests.update') @@ -2647,9 +2658,9 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/indexes/:key') ->param('key', '', new Key(), 'Index Key.') ->inject('response') ->inject('dbForProject') - ->inject('database') - ->inject('events') - ->action(function (string $databaseId, string $collectionId, string $key, Response $response, Database $dbForProject, EventDatabase $database, Event $events) { + ->inject('queueForDatabase') + ->inject('queueForEvents') + ->action(function (string $databaseId, string $collectionId, string $key, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { $db = Authorization::skip(fn() => $dbForProject->getDocument('databases', $databaseId)); @@ -2675,13 +2686,13 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/indexes/:key') $dbForProject->deleteCachedDocument('database_' . $db->getInternalId(), $collectionId); - $database + $queueForDatabase ->setType(DATABASE_TYPE_DELETE_INDEX) ->setDatabase($db) ->setCollection($collection) ->setDocument($index); - $events + $queueForEvents ->setParam('databaseId', $databaseId) ->setParam('collectionId', $collection->getId()) ->setParam('indexId', $index->getId()) @@ -2722,9 +2733,9 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') ->inject('response') ->inject('dbForProject') ->inject('user') - ->inject('events') + ->inject('queueForEvents') ->inject('mode') - ->action(function (string $databaseId, string $documentId, string $collectionId, string|array $data, ?array $permissions, Response $response, Database $dbForProject, Document $user, Event $events, string $mode) { + ->action(function (string $databaseId, string $documentId, string $collectionId, string|array $data, ?array $permissions, Response $response, Database $dbForProject, Document $user, Event $queueForEvents, string $mode) { $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array @@ -2920,12 +2931,13 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $processDocument($collection, $document); - $events + $queueForEvents ->setParam('databaseId', $databaseId) ->setParam('collectionId', $collection->getId()) ->setParam('documentId', $document->getId()) ->setContext('collection', $collection) - ->setContext('database', $database); + ->setContext('database', $database) + ; $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -3285,9 +3297,9 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum ->inject('requestTimestamp') ->inject('response') ->inject('dbForProject') - ->inject('events') + ->inject('queueForEvents') ->inject('mode') - ->action(function (string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $events, string $mode) { + ->action(function (string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, string $mode) { $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array @@ -3474,12 +3486,13 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $processDocument($collection, $document); - $events + $queueForEvents ->setParam('databaseId', $databaseId) ->setParam('collectionId', $collection->getId()) ->setParam('documentId', $document->getId()) ->setContext('collection', $collection) - ->setContext('database', $database); + ->setContext('database', $database) + ; $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -3511,10 +3524,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->inject('requestTimestamp') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->inject('deletes') + ->inject('queueForDeletes') + ->inject('queueForEvents') ->inject('mode') - ->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $events, Delete $deletes, string $mode) { + ->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Delete $queueForDeletes, Event $queueForEvents, string $mode) { $database = Authorization::skip(fn() => $dbForProject->getDocument('databases', $databaseId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); @@ -3585,11 +3598,11 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu $processDocument($collection, $document); - $deletes + $queueForDeletes ->setType(DELETE_TYPE_AUDIT) ->setDocument($document); - $events + $queueForEvents ->setParam('databaseId', $databaseId) ->setParam('collectionId', $collection->getId()) ->setParam('documentId', $document->getId()) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 4c41c8bd34..4a96308492 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -6,7 +6,6 @@ use Appwrite\Event\Build; use Appwrite\Event\Delete; use Appwrite\Event\Event; use Appwrite\Event\Func; -use Appwrite\Event\Usage; use Appwrite\Event\Validator\FunctionEvent; use Appwrite\Utopia\Response\Model\Rule; use Appwrite\Extend\Exception; @@ -50,7 +49,7 @@ use Utopia\VCS\Adapter\Git\GitHub; include_once __DIR__ . '/../shared/api.php'; -$redeployVcs = function (Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, Document $template, GitHub $github) { +$redeployVcs = function (Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, Build $queueForBuilds, Document $template, GitHub $github) { $deploymentId = ID::unique(); $entrypoint = $function->getAttribute('entrypoint', ''); $providerInstallationId = $installation->getAttribute('providerInstallationId', ''); @@ -109,8 +108,7 @@ $redeployVcs = function (Request $request, Document $function, Document $project 'activate' => true, ])); - $buildEvent = new Build(); - $buildEvent + $queueForBuilds ->setType(BUILD_TYPE_DEPLOYMENT) ->setResource($function) ->setDeployment($deployment) @@ -158,10 +156,11 @@ App::post('/v1/functions') ->inject('dbForProject') ->inject('project') ->inject('user') - ->inject('events') + ->inject('queueForEvents') + ->inject('queueForBuilds') ->inject('dbForConsole') ->inject('gitHub') - ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateBranch, Request $request, Response $response, Database $dbForProject, Document $project, Document $user, Event $eventsInstance, Database $dbForConsole, GitHub $github) use ($redeployVcs) { + ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateBranch, Request $request, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Build $queueForBuilds, Database $dbForConsole, GitHub $github) use ($redeployVcs) { $functionId = ($functionId == 'unique()') ? ID::unique() : $functionId; // build from template @@ -261,7 +260,7 @@ App::post('/v1/functions') // Redeploy vcs logic if (!empty($providerRepositoryId)) { - $redeployVcs($request, $function, $project, $installation, $dbForProject, $template, $github); + $redeployVcs($request, $function, $project, $installation, $dbForProject, $queueForBuilds, $template, $github); } $functionsDomain = App::getEnv('_APP_DOMAIN_FUNCTIONS', ''); @@ -286,7 +285,12 @@ App::post('/v1/functions') /** Trigger Webhook */ $ruleModel = new Rule(); - $ruleCreate = new Event(Event::WEBHOOK_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME); + $ruleCreate = + $queueForEvents + ->setClass(Event::WEBHOOK_CLASS_NAME) + ->setQueue(Event::WEBHOOK_QUEUE_NAME) + ; + $ruleCreate ->setProject($project) ->setEvent('rules.[ruleId].create') @@ -326,7 +330,7 @@ App::post('/v1/functions') ); } - $eventsInstance->setParam('functionId', $function->getId()); + $queueForEvents->setParam('functionId', $function->getId()); $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -530,7 +534,7 @@ App::get('/v1/functions/:functionId/usage') 'range' => $range, 'executionsTotal' => $stats["executions.$functionId.compute.total"] ?? [], 'executionsFailure' => $stats["executions.$functionId.compute.failure"] ?? [], - 'executionsSuccesse' => $stats["executions.$functionId.compute.success"] ?? [], + 'executionsSuccess' => $stats["executions.$functionId.compute.success"] ?? [], 'executionsTime' => $stats["executions.$functionId.compute.time"] ?? [], 'buildsTotal' => $stats["builds.$functionId.compute.total"] ?? [], 'buildsFailure' => $stats["builds.$functionId.compute.failure"] ?? [], @@ -679,10 +683,11 @@ App::put('/v1/functions/:functionId') ->inject('response') ->inject('dbForProject') ->inject('project') - ->inject('events') + ->inject('queueForEvents') + ->inject('queueForBuilds') ->inject('dbForConsole') ->inject('gitHub') - ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, Request $request, Response $response, Database $dbForProject, Document $project, Event $eventsInstance, Database $dbForConsole, GitHub $github) use ($redeployVcs) { + ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, Request $request, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Build $queueForBuilds, Database $dbForConsole, GitHub $github) use ($redeployVcs) { // TODO: If only branch changes, re-deploy $function = $dbForProject->getDocument('functions', $functionId); @@ -807,7 +812,7 @@ App::put('/v1/functions/:functionId') // Redeploy logic if (!$isConnected && !empty($providerRepositoryId)) { - $redeployVcs($request, $function, $project, $installation, $dbForProject, new Document(), $github); + $redeployVcs($request, $function, $project, $installation, $dbForProject, $queueForBuilds, new Document(), $github); } // Inform scheduler if function is still active @@ -818,7 +823,7 @@ App::put('/v1/functions/:functionId') ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); - $eventsInstance->setParam('functionId', $function->getId()); + $queueForEvents->setParam('functionId', $function->getId()); $response->dynamic($function, Response::MODEL_FUNCTION); }); @@ -928,9 +933,9 @@ App::patch('/v1/functions/:functionId/deployments/:deploymentId') ->param('deploymentId', '', new UID(), 'Deployment ID.') ->inject('response') ->inject('dbForProject') - ->inject('events') + ->inject('queueForEvents') ->inject('dbForConsole') - ->action(function (string $functionId, string $deploymentId, Response $response, Database $dbForProject, Event $events, Database $dbForConsole) { + ->action(function (string $functionId, string $deploymentId, Response $response, Database $dbForProject, Event $queueForEvents, Database $dbForConsole) { $function = $dbForProject->getDocument('functions', $functionId); $deployment = $dbForProject->getDocument('deployments', $deploymentId); @@ -965,7 +970,7 @@ App::patch('/v1/functions/:functionId/deployments/:deploymentId') ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); - $events + $queueForEvents ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()); @@ -988,10 +993,10 @@ App::delete('/v1/functions/:functionId') ->param('functionId', '', new UID(), 'Function ID.') ->inject('response') ->inject('dbForProject') - ->inject('deletes') - ->inject('events') + ->inject('queueForDeletes') + ->inject('queueForEvents') ->inject('dbForConsole') - ->action(function (string $functionId, Response $response, Database $dbForProject, Delete $deletes, Event $events, Database $dbForConsole) { + ->action(function (string $functionId, Response $response, Database $dbForProject, Delete $queueForDeletes, Event $queueForEvents, Database $dbForConsole) { $function = $dbForProject->getDocument('functions', $functionId); @@ -1010,11 +1015,11 @@ App::delete('/v1/functions/:functionId') ->setAttribute('active', false); Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); - $deletes + $queueForDeletes ->setType(DELETE_TYPE_DOCUMENT) ->setDocument($function); - $events->setParam('functionId', $function->getId()); + $queueForEvents->setParam('functionId', $function->getId()); $response->noContent(); }); @@ -1043,11 +1048,13 @@ App::post('/v1/functions/:functionId/deployments') ->inject('request') ->inject('response') ->inject('dbForProject') - ->inject('events') + ->inject('queueForEvents') ->inject('project') ->inject('deviceFunctions') ->inject('deviceLocal') - ->action(function (string $functionId, ?string $entrypoint, ?string $commands, mixed $code, mixed $activate, Request $request, Response $response, Database $dbForProject, Event $events, Document $project, Device $deviceFunctions, Device $deviceLocal) { + ->inject('queueForBuilds') + ->action(function (string $functionId, ?string $entrypoint, ?string $commands, mixed $code, bool $activate, Request $request, Response $response, Database $dbForProject, Event $queueForEvents, Document $project, Device $deviceFunctions, Device $deviceLocal, Build $queueForBuilds) { + $activate = filter_var($activate, FILTER_VALIDATE_BOOLEAN); $function = $dbForProject->getDocument('functions', $functionId); @@ -1191,8 +1198,7 @@ App::post('/v1/functions/:functionId/deployments') } // Start the build - $buildEvent = new Build(); - $buildEvent + $queueForBuilds ->setType(BUILD_TYPE_DEPLOYMENT) ->setResource($function) ->setDeployment($deployment) @@ -1229,7 +1235,7 @@ App::post('/v1/functions/:functionId/deployments') $metadata = null; - $events + $queueForEvents ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()); @@ -1367,10 +1373,10 @@ App::delete('/v1/functions/:functionId/deployments/:deploymentId') ->param('deploymentId', '', new UID(), 'Deployment ID.') ->inject('response') ->inject('dbForProject') - ->inject('deletes') - ->inject('events') + ->inject('queueForDeletes') + ->inject('queueForEvents') ->inject('deviceFunctions') - ->action(function (string $functionId, string $deploymentId, Response $response, Database $dbForProject, Delete $deletes, Event $events, Device $deviceFunctions) { + ->action(function (string $functionId, string $deploymentId, Response $response, Database $dbForProject, Delete $queueForDeletes, Event $queueForEvents, Device $deviceFunctions) { $function = $dbForProject->getDocument('functions', $functionId); if ($function->isEmpty()) { @@ -1403,11 +1409,11 @@ App::delete('/v1/functions/:functionId/deployments/:deploymentId') ]))); } - $events + $queueForEvents ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()); - $deletes + $queueForDeletes ->setType(DELETE_TYPE_DOCUMENT) ->setDocument($deployment); @@ -1434,8 +1440,9 @@ App::post('/v1/functions/:functionId/deployments/:deploymentId/builds/:buildId') ->inject('response') ->inject('dbForProject') ->inject('project') - ->inject('events') - ->action(function (string $functionId, string $deploymentId, string $buildId, Request $request, Response $response, Database $dbForProject, Document $project, Event $events) use ($redeployVcs) { + ->inject('queueForEvents') + ->inject('queueForBuilds') + ->action(function (string $functionId, string $deploymentId, string $buildId, Request $request, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Build $queueForBuilds) { $function = $dbForProject->getDocument('functions', $functionId); @@ -1467,16 +1474,14 @@ App::post('/v1/functions/:functionId/deployments/:deploymentId/builds/:buildId') 'search' => implode(' ', [$deploymentId, $function->getAttribute('entrypoint')]), ])); - $buildEvent = new Build(); - - $buildEvent + $queueForBuilds ->setType(BUILD_TYPE_DEPLOYMENT) ->setResource($function) ->setDeployment($deployment) ->setProject($project) ->trigger(); - $events + $queueForEvents ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()); @@ -1505,12 +1510,12 @@ App::post('/v1/functions/:functionId/executions') ->inject('project') ->inject('dbForProject') ->inject('user') - ->inject('events') + ->inject('queueForEvents') ->inject('usage') ->inject('mode') ->inject('queueForFunctions') ->inject('geodb') - ->action(function (string $functionId, string $body, bool $async, string $path, string $method, array $headers, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, Stats $usage, string $mode, Func $queueForFunctions, Reader $geodb) { + ->action(function (string $functionId, string $body, bool $async, string $path, string $method, array $headers, Response $response, Document $project, Database $dbForProject, Document $user, Event $queueForEvents, Stats $usage, string $mode, Func $queueForFunctions, Reader $geodb) { $function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId)); @@ -1626,7 +1631,7 @@ App::post('/v1/functions/:functionId/executions') 'search' => implode(' ', [$functionId, $executionId]), ]); - $events + $queueForEvents ->setParam('functionId', $function->getId()) ->setParam('executionId', $execution->getId()) ->setContext('function', $function); diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 2b41e2fef8..124166543c 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -14,6 +14,7 @@ use Utopia\Registry\Registry; use Utopia\Storage\Device; use Utopia\Storage\Device\Local; use Utopia\Storage\Storage; +use Utopia\Validator\Text; App::get('/v1/health') ->desc('Get HTTP') @@ -347,10 +348,11 @@ App::get('/v1/health/queue/webhooks') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->inject('queue') ->inject('response') - ->action(function (Response $response) { - - $response->dynamic(new Document([ 'size' => Resque::size(Event::WEBHOOK_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE); + ->action(function (Connection $queue, Response $response) { + $client = new Client(Event::WEBHOOK_QUEUE_NAME, $queue); + $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/logs') @@ -364,10 +366,11 @@ App::get('/v1/health/queue/logs') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->inject('queue') ->inject('response') - ->action(function (Response $response) { - - $response->dynamic(new Document([ 'size' => Resque::size(Event::AUDITS_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE); + ->action(function (Connection $queue, Response $response) { + $client = new Client(Event::AUDITS_QUEUE_NAME, $queue); + $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/certificates') @@ -381,10 +384,11 @@ App::get('/v1/health/queue/certificates') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->inject('queue') ->inject('response') - ->action(function (Response $response) { - - $response->dynamic(new Document([ 'size' => Resque::size(Event::CERTIFICATES_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE); + ->action(function (Connection $queue, Response $response) { + $client = new Client(Event::CERTIFICATES_QUEUE_NAME, $queue); + $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/builds') @@ -398,13 +402,14 @@ App::get('/v1/health/queue/builds') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->inject('queue') ->inject('response') - ->action(function (Response $response) { - - $response->dynamic(new Document([ 'size' => Resque::size(Event::BUILDS_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE); + ->action(function (Connection $queue, Response $response) { + $client = new Client(Event::BUILDS_QUEUE_NAME, $queue); + $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); -App::get('/v1/health/queue/databases') +App::get('/v1/health/queue/databases/:databaseId') ->desc('Get databases queue') ->groups(['api', 'health']) ->label('scope', 'health.read') @@ -415,10 +420,12 @@ App::get('/v1/health/queue/databases') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('databaseId', 'database_db_main', new Text(256), 'Database for which to check the queue size', true) + ->inject('queue') ->inject('response') - ->action(function (Response $response) { - - $response->dynamic(new Document([ 'size' => Resque::size(Event::DATABASE_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE); + ->action(function (string $databaseId, Connection $queue, Response $response) { + $client = new Client($databaseId, $queue); + $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/deletes') @@ -432,10 +439,11 @@ App::get('/v1/health/queue/deletes') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->inject('queue') ->inject('response') - ->action(function (Response $response) { - - $response->dynamic(new Document([ 'size' => Resque::size(Event::DELETE_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE); + ->action(function (Connection $queue, Response $response) { + $client = new Client(Event::DELETE_QUEUE_NAME, $queue); + $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/mails') @@ -449,10 +457,11 @@ App::get('/v1/health/queue/mails') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->inject('queue') ->inject('response') - ->action(function (Response $response) { - - $response->dynamic(new Document([ 'size' => Resque::size(Event::MAILS_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE); + ->action(function (Connection $queue, Response $response) { + $client = new Client(Event::MAILS_QUEUE_NAME, $queue); + $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/messaging') @@ -466,10 +475,11 @@ App::get('/v1/health/queue/messaging') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->inject('queue') ->inject('response') - ->action(function (Response $response) { - - $response->dynamic(new Document([ 'size' => Resque::size(Event::MESSAGING_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE); + ->action(function (Connection $queue, Response $response) { + $client = new Client(Event::MESSAGING_QUEUE_NAME, $queue); + $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/migrations') @@ -483,10 +493,11 @@ App::get('/v1/health/queue/migrations') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->inject('queue') ->inject('response') - ->action(function (Response $response) { - - $response->dynamic(new Document([ 'size' => Resque::size(Event::MIGRATIONS_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE); + ->action(function (Connection $queue, Response $response) { + $client = new Client(Event::MIGRATIONS_QUEUE_NAME, $queue); + $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/functions') diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 83888fff2c..d51c6adbf7 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -51,8 +51,9 @@ App::post('/v1/migrations/appwrite') ->inject('dbForProject') ->inject('project') ->inject('user') - ->inject('events') - ->action(function (array $resources, string $endpoint, string $projectId, string $apiKey, Response $response, Database $dbForProject, Document $project, Document $user, Event $events) { + ->inject('queueForEvents') + ->inject('queueForMigrations') + ->action(function (array $resources, string $endpoint, string $projectId, string $apiKey, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations) { $migration = $dbForProject->createDocument('migrations', new Document([ '$id' => ID::unique(), 'status' => 'pending', @@ -69,11 +70,10 @@ App::post('/v1/migrations/appwrite') 'errors' => [], ])); - $events->setParam('migrationId', $migration->getId()); + $queueForEvents->setParam('migrationId', $migration->getId()); // Trigger Transfer - $event = new Migration(); - $event + $queueForMigrations ->setMigration($migration) ->setProject($project) ->setUser($user) @@ -104,9 +104,10 @@ App::post('/v1/migrations/firebase/oauth') ->inject('dbForConsole') ->inject('project') ->inject('user') - ->inject('events') + ->inject('queueForEvents') + ->inject('queueForMigrations') ->inject('request') - ->action(function (array $resources, string $projectId, Response $response, Database $dbForProject, Database $dbForConsole, Document $project, Document $user, Event $events, Request $request) { + ->action(function (array $resources, string $projectId, Response $response, Database $dbForProject, Database $dbForConsole, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations, Request $request) { $firebase = new OAuth2Firebase( App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', ''), App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', ''), @@ -171,11 +172,10 @@ App::post('/v1/migrations/firebase/oauth') 'errors' => [] ])); - $events->setParam('migrationId', $migration->getId()); + $queueForEvents->setParam('migrationId', $migration->getId()); // Trigger Transfer - $event = new Migration(); - $event + $queueForMigrations ->setMigration($migration) ->setProject($project) ->setUser($user) @@ -205,8 +205,9 @@ App::post('/v1/migrations/firebase') ->inject('dbForProject') ->inject('project') ->inject('user') - ->inject('events') - ->action(function (array $resources, string $serviceAccount, Response $response, Database $dbForProject, Document $project, Document $user, Event $events) { + ->inject('queueForEvents') + ->inject('queueForMigrations') + ->action(function (array $resources, string $serviceAccount, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations) { $migration = $dbForProject->createDocument('migrations', new Document([ '$id' => ID::unique(), 'status' => 'pending', @@ -221,11 +222,10 @@ App::post('/v1/migrations/firebase') 'errors' => [], ])); - $events->setParam('migrationId', $migration->getId()); + $queueForEvents->setParam('migrationId', $migration->getId()); // Trigger Transfer - $event = new Migration(); - $event + $queueForMigrations ->setMigration($migration) ->setProject($project) ->setUser($user) @@ -260,8 +260,9 @@ App::post('/v1/migrations/supabase') ->inject('dbForProject') ->inject('project') ->inject('user') - ->inject('events') - ->action(function (array $resources, string $endpoint, string $apiKey, string $databaseHost, string $username, string $password, int $port, Response $response, Database $dbForProject, Document $project, Document $user, Event $events) { + ->inject('queueForEvents') + ->inject('queueForMigrations') + ->action(function (array $resources, string $endpoint, string $apiKey, string $databaseHost, string $username, string $password, int $port, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations) { $migration = $dbForProject->createDocument('migrations', new Document([ '$id' => ID::unique(), 'status' => 'pending', @@ -281,11 +282,10 @@ App::post('/v1/migrations/supabase') 'errors' => [], ])); - $events->setParam('migrationId', $migration->getId()); + $queueForEvents->setParam('migrationId', $migration->getId()); // Trigger Transfer - $event = new Migration(); - $event + $queueForMigrations ->setMigration($migration) ->setProject($project) ->setUser($user) @@ -321,8 +321,9 @@ App::post('/v1/migrations/nhost') ->inject('dbForProject') ->inject('project') ->inject('user') - ->inject('events') - ->action(function (array $resources, string $subdomain, string $region, string $adminSecret, string $database, string $username, string $password, int $port, Response $response, Database $dbForProject, Document $project, Document $user, Event $events) { + ->inject('queueForEvents') + ->inject('queueForMigrations') + ->action(function (array $resources, string $subdomain, string $region, string $adminSecret, string $database, string $username, string $password, int $port, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations) { $migration = $dbForProject->createDocument('migrations', new Document([ '$id' => ID::unique(), 'status' => 'pending', @@ -343,11 +344,10 @@ App::post('/v1/migrations/nhost') 'errors' => [], ])); - $events->setParam('migrationId', $migration->getId()); + $queueForEvents->setParam('migrationId', $migration->getId()); // Trigger Transfer - $event = new Migration(); - $event + $queueForMigrations ->setMigration($migration) ->setProject($project) ->setUser($user) @@ -931,8 +931,8 @@ App::patch('/v1/migrations/:migrationId') ->inject('dbForProject') ->inject('project') ->inject('user') - ->inject('events') - ->action(function (string $migrationId, Response $response, Database $dbForProject, Document $project, Document $user, Event $eventInstance) { + ->inject('queueForMigrations') + ->action(function (string $migrationId, Response $response, Database $dbForProject, Document $project, Document $user, Migration $queueForMigrations) { $migration = $dbForProject->getDocument('migrations', $migrationId); if ($migration->isEmpty()) { @@ -948,8 +948,7 @@ App::patch('/v1/migrations/:migrationId') ->setAttribute('dateUpdated', \time()); // Trigger Migration - $event = new Migration(); - $event + $queueForMigrations ->setMigration($migration) ->setProject($project) ->setUser($user) @@ -974,8 +973,8 @@ App::delete('/v1/migrations/:migrationId') ->param('migrationId', '', new UID(), 'Migration ID.') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->action(function (string $migrationId, Response $response, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $migrationId, Response $response, Database $dbForProject, Event $queueForEvents) { $migration = $dbForProject->getDocument('migrations', $migrationId); if ($migration->isEmpty()) { @@ -986,7 +985,7 @@ App::delete('/v1/migrations/:migrationId') throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove migration from DB'); } - $events->setParam('migrationId', $migration->getId()); + $queueForEvents->setParam('migrationId', $migration->getId()); $response->noContent(); }); diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 7f349cb2d2..16f2653178 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -864,15 +864,15 @@ App::delete('/v1/projects/:projectId') ->inject('response') ->inject('user') ->inject('dbForConsole') - ->inject('deletes') - ->action(function (string $projectId, Response $response, Document $user, Database $dbForConsole, Delete $deletes) { + ->inject('queueForDeletes') + ->action(function (string $projectId, Response $response, Document $user, Database $dbForConsole, Delete $queueForDeletes) { $project = $dbForConsole->getDocument('projects', $projectId); if ($project->isEmpty()) { throw new Exception(Exception::PROJECT_NOT_FOUND); } - $deletes + $queueForDeletes ->setType(DELETE_TYPE_DOCUMENT) ->setDocument($project); diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 7b7b1068d7..04c3f300e9 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -72,8 +72,8 @@ App::post('/v1/storage/buckets') ->param('antivirus', true, new Boolean(true), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS, 0) . ' AntiVirus scanning is skipped even if it\'s enabled', true) ->inject('response') ->inject('dbForProject') - ->inject('events') - ->action(function (string $bucketId, string $name, ?array $permissions, bool $fileSecurity, bool $enabled, int $maximumFileSize, array $allowedFileExtensions, string $compression, bool $encryption, bool $antivirus, Response $response, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $bucketId, string $name, ?array $permissions, bool $fileSecurity, bool $enabled, int $maximumFileSize, array $allowedFileExtensions, string $compression, bool $encryption, bool $antivirus, Response $response, Database $dbForProject, Event $queueForEvents) { $bucketId = $bucketId === 'unique()' ? ID::unique() : $bucketId; @@ -135,7 +135,7 @@ App::post('/v1/storage/buckets') throw new Exception(Exception::STORAGE_BUCKET_ALREADY_EXISTS); } - $events + $queueForEvents ->setParam('bucketId', $bucket->getId()) ; @@ -246,8 +246,8 @@ App::put('/v1/storage/buckets/:bucketId') ->param('antivirus', true, new Boolean(true), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS, 0) . ' AntiVirus scanning is skipped even if it\'s enabled', true) ->inject('response') ->inject('dbForProject') - ->inject('events') - ->action(function (string $bucketId, string $name, ?array $permissions, bool $fileSecurity, bool $enabled, ?int $maximumFileSize, array $allowedFileExtensions, string $compression, bool $encryption, bool $antivirus, Response $response, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $bucketId, string $name, ?array $permissions, bool $fileSecurity, bool $enabled, ?int $maximumFileSize, array $allowedFileExtensions, string $compression, bool $encryption, bool $antivirus, Response $response, Database $dbForProject, Event $queueForEvents) { $bucket = $dbForProject->getDocument('buckets', $bucketId); if ($bucket->isEmpty()) { @@ -280,7 +280,7 @@ App::put('/v1/storage/buckets/:bucketId') ->setAttribute('antivirus', $antivirus)); $dbForProject->updateCollection('bucket_' . $bucket->getInternalId(), $permissions, $fileSecurity); - $events + $queueForEvents ->setParam('bucketId', $bucket->getId()) ; @@ -304,9 +304,9 @@ App::delete('/v1/storage/buckets/:bucketId') ->param('bucketId', '', new UID(), 'Bucket unique ID.') ->inject('response') ->inject('dbForProject') - ->inject('deletes') - ->inject('events') - ->action(function (string $bucketId, Response $response, Database $dbForProject, Delete $deletes, Event $events) { + ->inject('queueForDeletes') + ->inject('queueForEvents') + ->action(function (string $bucketId, Response $response, Database $dbForProject, Delete $queueForDeletes, Event $queueForEvents) { $bucket = $dbForProject->getDocument('buckets', $bucketId); if ($bucket->isEmpty()) { @@ -317,11 +317,11 @@ App::delete('/v1/storage/buckets/:bucketId') throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove bucket from DB'); } - $deletes + $queueForDeletes ->setType(DELETE_TYPE_DOCUMENT) ->setDocument($bucket); - $events + $queueForEvents ->setParam('bucketId', $bucket->getId()) ->setPayload($response->output($bucket, Response::MODEL_BUCKET)) ; @@ -359,11 +359,12 @@ App::post('/v1/storage/buckets/:bucketId/files') ->inject('response') ->inject('dbForProject') ->inject('user') - ->inject('events') + ->inject('queueForEvents') ->inject('mode') ->inject('deviceFiles') ->inject('deviceLocal') - ->action(function (string $bucketId, string $fileId, mixed $file, ?array $permissions, Request $request, Response $response, Database $dbForProject, Document $user, Event $events, string $mode, Device $deviceFiles, Device $deviceLocal) { + ->action(function (string $bucketId, string $fileId, mixed $file, ?array $permissions, Request $request, Response $response, Database $dbForProject, Document $user, Event $queueForEvents, string $mode, Device $deviceFiles, Device $deviceLocal) { + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); @@ -669,7 +670,7 @@ App::post('/v1/storage/buckets/:bucketId/files') } } - $events + $queueForEvents ->setParam('bucketId', $bucket->getId()) ->setParam('fileId', $file->getId()) ->setContext('bucket', $bucket) @@ -1305,8 +1306,9 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId') ->inject('dbForProject') ->inject('user') ->inject('mode') - ->inject('events') - ->action(function (string $bucketId, string $fileId, ?string $name, ?array $permissions, Response $response, Database $dbForProject, Document $user, string $mode, Event $events) { + ->inject('queueForEvents') + ->action(function (string $bucketId, string $fileId, ?string $name, ?array $permissions, Response $response, Database $dbForProject, Document $user, string $mode, Event $queueForEvents) { + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); @@ -1378,7 +1380,7 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId') $file = Authorization::skip(fn() => $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file)); } - $events + $queueForEvents ->setParam('bucketId', $bucket->getId()) ->setParam('fileId', $file->getId()) ->setContext('bucket', $bucket) @@ -1409,11 +1411,11 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') ->param('fileId', '', new UID(), 'File ID.') ->inject('response') ->inject('dbForProject') - ->inject('events') + ->inject('queueForEvents') ->inject('mode') ->inject('deviceFiles') - ->inject('deletes') - ->action(function (string $bucketId, string $fileId, Response $response, Database $dbForProject, Event $events, string $mode, Device $deviceFiles, Delete $deletes) { + ->inject('queueForDeletes') + ->action(function (string $bucketId, string $fileId, Response $response, Database $dbForProject, Event $queueForEvents, string $mode, Device $deviceFiles, Delete $queueForDeletes) { $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); @@ -1453,7 +1455,7 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') } if ($deviceDeleted) { - $deletes + $queueForDeletes ->setType(DELETE_TYPE_CACHE_BY_RESOURCE) ->setResource('file/' . $fileId) ; @@ -1475,7 +1477,7 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to delete file from device'); } - $events + $queueForEvents ->setParam('bucketId', $bucket->getId()) ->setParam('fileId', $file->getId()) ->setContext('bucket', $bucket) diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index f18fc3208a..0a887908bb 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -60,8 +60,8 @@ App::post('/v1/teams') ->inject('response') ->inject('user') ->inject('dbForProject') - ->inject('events') - ->action(function (string $teamId, string $name, array $roles, Response $response, Document $user, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $teamId, string $name, array $roles, Response $response, Document $user, Database $dbForProject, Event $queueForEvents) { $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); $isAppUser = Auth::isAppUser(Authorization::getRoles()); @@ -117,10 +117,10 @@ App::post('/v1/teams') $dbForProject->deleteCachedDocument('users', $user->getId()); } - $events->setParam('teamId', $team->getId()); + $queueForEvents->setParam('teamId', $team->getId()); if (!empty($user->getId())) { - $events->setParam('userId', $user->getId()); + $queueForEvents->setParam('userId', $user->getId()); } $response @@ -256,8 +256,8 @@ App::put('/v1/teams/:teamId') ->inject('requestTimestamp') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->action(function (string $teamId, string $name, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $teamId, string $name, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents) { $team = $dbForProject->getDocument('teams', $teamId); @@ -273,7 +273,7 @@ App::put('/v1/teams/:teamId') return $dbForProject->updateDocument('teams', $team->getId(), $team); }); - $events->setParam('teamId', $team->getId()); + $queueForEvents->setParam('teamId', $team->getId()); $response->dynamic($team, Response::MODEL_TEAM); }); @@ -298,8 +298,8 @@ App::put('/v1/teams/:teamId/prefs') ->param('prefs', '', new Assoc(), 'Prefs key-value JSON object.') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->action(function (string $teamId, array $prefs, Response $response, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $teamId, array $prefs, Response $response, Database $dbForProject, Event $queueForEvents) { $team = $dbForProject->getDocument('teams', $teamId); @@ -309,7 +309,7 @@ App::put('/v1/teams/:teamId/prefs') $team = $dbForProject->updateDocument('teams', $team->getId(), $team->setAttribute('prefs', $prefs)); - $events->setParam('teamId', $team->getId()); + $queueForEvents->setParam('teamId', $team->getId()); $response->dynamic(new Document($prefs), Response::MODEL_PREFERENCES); }); @@ -330,9 +330,9 @@ App::delete('/v1/teams/:teamId') ->param('teamId', '', new UID(), 'Team ID.') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->inject('deletes') - ->action(function (string $teamId, Response $response, Database $dbForProject, Event $events, Delete $deletes) { + ->inject('queueForEvents') + ->inject('queueForDeletes') + ->action(function (string $teamId, Response $response, Database $dbForProject, Event $queueForEvents, Delete $queueForDeletes) { $team = $dbForProject->getDocument('teams', $teamId); @@ -344,11 +344,11 @@ App::delete('/v1/teams/:teamId') throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove team from DB'); } - $deletes + $queueForDeletes ->setType(DELETE_TYPE_DOCUMENT) ->setDocument($team); - $events + $queueForEvents ->setParam('teamId', $team->getId()) ->setPayload($response->output($team, Response::MODEL_TEAM)) ; @@ -385,10 +385,10 @@ App::post('/v1/teams/:teamId/memberships') ->inject('user') ->inject('dbForProject') ->inject('locale') - ->inject('mails') - ->inject('messaging') - ->inject('events') - ->action(function (string $teamId, string $email, string $userId, string $phone, array $roles, string $url, string $name, Response $response, Document $project, Document $user, Database $dbForProject, Locale $locale, Mail $mails, EventPhone $messaging, Event $events) { + ->inject('queueForMails') + ->inject('queueForMessaging') + ->inject('queueForEvents') + ->action(function (string $teamId, string $email, string $userId, string $phone, array $roles, string $url, string $name, Response $response, Document $project, Document $user, Database $dbForProject, Locale $locale, Mail $queueForMails, EventPhone $queueForMessaging, Event $queueForEvents) { $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -576,7 +576,7 @@ App::post('/v1/teams/:teamId/memberships') $replyTo = $smtp['replyTo']; } - $mails + $queueForMails ->setSmtpHost($smtp['host'] ?? '') ->setSmtpPort($smtp['port'] ?? '') ->setSmtpUsername($smtp['username'] ?? '') @@ -598,7 +598,7 @@ App::post('/v1/teams/:teamId/memberships') $subject = $customTemplate['subject'] ?? $subject; } - $mails + $queueForMails ->setSmtpReplyTo($replyTo) ->setSmtpSenderEmail($senderEmail) ->setSmtpSenderName($senderName); @@ -623,7 +623,7 @@ App::post('/v1/teams/:teamId/memberships') 'redirect' => $url ]; - $mails + $queueForMails ->setSubject($subject) ->setBody($body) ->setRecipient($invitee->getAttribute('email')) @@ -642,14 +642,14 @@ App::post('/v1/teams/:teamId/memberships') $message = $message->setParam('{{token}}', $url); $message = $message->render(); - $messaging + $queueForMessaging ->setRecipient($phone) ->setMessage($message) ->trigger(); } } - $events + $queueForEvents ->setParam('teamId', $team->getId()) ->setParam('membershipId', $membership->getId()) ; @@ -812,8 +812,8 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId') ->inject('response') ->inject('user') ->inject('dbForProject') - ->inject('events') - ->action(function (string $teamId, string $membershipId, array $roles, Request $request, Response $response, Document $user, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $teamId, string $membershipId, array $roles, Request $request, Response $response, Document $user, Database $dbForProject, Event $queueForEvents) { $team = $dbForProject->getDocument('teams', $teamId); if ($team->isEmpty()) { @@ -849,7 +849,7 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId') */ $dbForProject->deleteCachedDocument('users', $profile->getId()); - $events + $queueForEvents ->setParam('teamId', $team->getId()) ->setParam('membershipId', $membership->getId()); @@ -887,8 +887,8 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId/status') ->inject('dbForProject') ->inject('project') ->inject('geodb') - ->inject('events') - ->action(function (string $teamId, string $membershipId, string $userId, string $secret, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Reader $geodb, Event $events) { + ->inject('queueForEvents') + ->action(function (string $teamId, string $membershipId, string $userId, string $secret, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Reader $geodb, Event $queueForEvents) { $protocol = $request->getProtocol(); $membership = $dbForProject->getDocument('memberships', $membershipId); @@ -972,7 +972,7 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId/status') $team = Authorization::skip(fn() => $dbForProject->updateDocument('teams', $team->getId(), $team->setAttribute('total', $team->getAttribute('total', 0) + 1))); - $events + $queueForEvents ->setParam('teamId', $team->getId()) ->setParam('membershipId', $membership->getId()) ; @@ -1014,8 +1014,8 @@ App::delete('/v1/teams/:teamId/memberships/:membershipId') ->param('membershipId', '', new UID(), 'Membership ID.') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->action(function (string $teamId, string $membershipId, Response $response, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $teamId, string $membershipId, Response $response, Database $dbForProject, Event $queueForEvents) { $membership = $dbForProject->getDocument('memberships', $membershipId); @@ -1054,7 +1054,7 @@ App::delete('/v1/teams/:teamId/memberships/:membershipId') Authorization::skip(fn() => $dbForProject->updateDocument('teams', $team->getId(), $team)); } - $events + $queueForEvents ->setParam('teamId', $team->getId()) ->setParam('membershipId', $membership->getId()) ->setPayload($response->output($membership, Response::MODEL_MEMBERSHIP)) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 84e09b6978..fcaa2ff641 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -41,7 +41,7 @@ use Appwrite\Auth\Validator\PasswordDictionary; use Appwrite\Auth\Validator\PersonalData; /** TODO: Remove function when we move to using utopia/platform */ -function createUser(string $hash, mixed $hashOptions, string $userId, ?string $email, ?string $password, ?string $phone, string $name, Document $project, Database $dbForProject, Event $events): Document +function createUser(string $hash, mixed $hashOptions, string $userId, ?string $email, ?string $password, ?string $phone, string $name, Document $project, Database $dbForProject, Event $queueForEvents): Document { $hashOptionsObject = (\is_string($hashOptions)) ? \json_decode($hashOptions, true) : $hashOptions; // Cast to JSON array $passwordHistory = $project->getAttribute('auths', [])['passwordHistory'] ?? 0; @@ -102,7 +102,7 @@ function createUser(string $hash, mixed $hashOptions, string $userId, ?string $e throw new Exception(Exception::USER_ALREADY_EXISTS); } - $events->setParam('userId', $user->getId()); + $queueForEvents->setParam('userId', $user->getId()); return $user; } @@ -130,10 +130,9 @@ App::post('/v1/users') ->inject('response') ->inject('project') ->inject('dbForProject') - ->inject('events') - ->action(function (string $userId, ?string $email, ?string $phone, ?string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $events) { - - $user = createUser('plaintext', '{}', $userId, $email, $password, $phone, $name, $project, $dbForProject, $events); + ->inject('queueForEvents') + ->action(function (string $userId, ?string $email, ?string $phone, ?string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) { + $user = createUser('plaintext', '{}', $userId, $email, $password, $phone, $name, $project, $dbForProject, $queueForEvents); $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -162,9 +161,9 @@ App::post('/v1/users/bcrypt') ->inject('response') ->inject('project') ->inject('dbForProject') - ->inject('events') - ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $events) { - $user = createUser('bcrypt', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $events); + ->inject('queueForEvents') + ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) { + $user = createUser('bcrypt', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents); $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -193,9 +192,9 @@ App::post('/v1/users/md5') ->inject('response') ->inject('project') ->inject('dbForProject') - ->inject('events') - ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $events) { - $user = createUser('md5', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $events); + ->inject('queueForEvents') + ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) { + $user = createUser('md5', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents); $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -224,9 +223,9 @@ App::post('/v1/users/argon2') ->inject('response') ->inject('project') ->inject('dbForProject') - ->inject('events') - ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $events) { - $user = createUser('argon2', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $events); + ->inject('queueForEvents') + ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) { + $user = createUser('argon2', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents); $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -256,15 +255,15 @@ App::post('/v1/users/sha') ->inject('response') ->inject('project') ->inject('dbForProject') - ->inject('events') - ->action(function (string $userId, string $email, string $password, string $passwordVersion, string $name, Response $response, Document $project, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $userId, string $email, string $password, string $passwordVersion, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) { $options = '{}'; if (!empty($passwordVersion)) { $options = '{"version":"' . $passwordVersion . '"}'; } - $user = createUser('sha', $options, $userId, $email, $password, null, $name, $project, $dbForProject, $events); + $user = createUser('sha', $options, $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents); $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -293,9 +292,9 @@ App::post('/v1/users/phpass') ->inject('response') ->inject('project') ->inject('dbForProject') - ->inject('events') - ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $events) { - $user = createUser('phpass', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $events); + ->inject('queueForEvents') + ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) { + $user = createUser('phpass', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents); $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -329,8 +328,8 @@ App::post('/v1/users/scrypt') ->inject('response') ->inject('project') ->inject('dbForProject') - ->inject('events') - ->action(function (string $userId, string $email, string $password, string $passwordSalt, int $passwordCpu, int $passwordMemory, int $passwordParallel, int $passwordLength, string $name, Response $response, Document $project, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $userId, string $email, string $password, string $passwordSalt, int $passwordCpu, int $passwordMemory, int $passwordParallel, int $passwordLength, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) { $options = [ 'salt' => $passwordSalt, 'costCpu' => $passwordCpu, @@ -339,7 +338,7 @@ App::post('/v1/users/scrypt') 'length' => $passwordLength ]; - $user = createUser('scrypt', \json_encode($options), $userId, $email, $password, null, $name, $project, $dbForProject, $events); + $user = createUser('scrypt', \json_encode($options), $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents); $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -371,9 +370,9 @@ App::post('/v1/users/scrypt-modified') ->inject('response') ->inject('project') ->inject('dbForProject') - ->inject('events') - ->action(function (string $userId, string $email, string $password, string $passwordSalt, string $passwordSaltSeparator, string $passwordSignerKey, string $name, Response $response, Document $project, Database $dbForProject, Event $events) { - $user = createUser('scryptMod', '{"signerKey":"' . $passwordSignerKey . '","saltSeparator":"' . $passwordSaltSeparator . '","salt":"' . $passwordSalt . '"}', $userId, $email, $password, null, $name, $project, $dbForProject, $events); + ->inject('queueForEvents') + ->action(function (string $userId, string $email, string $password, string $passwordSalt, string $passwordSaltSeparator, string $passwordSignerKey, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) { + $user = createUser('scryptMod', '{"signerKey":"' . $passwordSignerKey . '","saltSeparator":"' . $passwordSaltSeparator . '","salt":"' . $passwordSalt . '"}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents); $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -716,8 +715,8 @@ App::patch('/v1/users/:userId/status') ->param('status', null, new Boolean(true), 'User Status. To activate the user pass `true` and to block the user pass `false`.') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->action(function (string $userId, bool $status, Response $response, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $userId, bool $status, Response $response, Database $dbForProject, Event $queueForEvents) { $user = $dbForProject->getDocument('users', $userId); @@ -727,7 +726,7 @@ App::patch('/v1/users/:userId/status') $user = $dbForProject->updateDocument('users', $user->getId(), $user->setAttribute('status', (bool) $status)); - $events + $queueForEvents ->setParam('userId', $user->getId()); $response->dynamic($user, Response::MODEL_USER); @@ -752,8 +751,8 @@ App::put('/v1/users/:userId/labels') ->param('labels', [], new ArrayList(new Text(36, allowList: [...Text::NUMBERS, ...Text::ALPHABET_UPPER, ...Text::ALPHABET_LOWER]), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of user labels. Replaces the previous labels. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' labels are allowed, each up to 36 alphanumeric characters long.') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->action(function (string $userId, array $labels, Response $response, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $userId, array $labels, Response $response, Database $dbForProject, Event $queueForEvents) { $user = $dbForProject->getDocument('users', $userId); @@ -765,7 +764,7 @@ App::put('/v1/users/:userId/labels') $user = $dbForProject->updateDocument('users', $user->getId(), $user); - $events + $queueForEvents ->setParam('userId', $user->getId()); $response->dynamic($user, Response::MODEL_USER); @@ -790,8 +789,8 @@ App::patch('/v1/users/:userId/verification/phone') ->param('phoneVerification', false, new Boolean(), 'User phone verification status.') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->action(function (string $userId, bool $phoneVerification, Response $response, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $userId, bool $phoneVerification, Response $response, Database $dbForProject, Event $queueForEvents) { $user = $dbForProject->getDocument('users', $userId); @@ -801,7 +800,7 @@ App::patch('/v1/users/:userId/verification/phone') $user = $dbForProject->updateDocument('users', $user->getId(), $user->setAttribute('phoneVerification', $phoneVerification)); - $events + $queueForEvents ->setParam('userId', $user->getId()); $response->dynamic($user, Response::MODEL_USER); @@ -827,8 +826,8 @@ App::patch('/v1/users/:userId/name') ->param('name', '', new Text(128), 'User name. Max length: 128 chars.') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->action(function (string $userId, string $name, Response $response, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $userId, string $name, Response $response, Database $dbForProject, Event $queueForEvents) { $user = $dbForProject->getDocument('users', $userId); @@ -840,7 +839,7 @@ App::patch('/v1/users/:userId/name') $user = $dbForProject->updateDocument('users', $user->getId(), $user); - $events->setParam('userId', $user->getId()); + $queueForEvents->setParam('userId', $user->getId()); $response->dynamic($user, Response::MODEL_USER); }); @@ -866,8 +865,8 @@ App::patch('/v1/users/:userId/password') ->inject('response') ->inject('project') ->inject('dbForProject') - ->inject('events') - ->action(function (string $userId, string $password, Response $response, Document $project, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $userId, string $password, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) { $user = $dbForProject->getDocument('users', $userId); @@ -905,7 +904,7 @@ App::patch('/v1/users/:userId/password') $user = $dbForProject->updateDocument('users', $user->getId(), $user); - $events->setParam('userId', $user->getId()); + $queueForEvents->setParam('userId', $user->getId()); $response->dynamic($user, Response::MODEL_USER); }); @@ -930,8 +929,8 @@ App::patch('/v1/users/:userId/email') ->param('email', '', new Email(), 'User email.') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->action(function (string $userId, string $email, Response $response, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $userId, string $email, Response $response, Database $dbForProject, Event $queueForEvents) { $user = $dbForProject->getDocument('users', $userId); @@ -962,7 +961,7 @@ App::patch('/v1/users/:userId/email') throw new Exception(Exception::USER_EMAIL_ALREADY_EXISTS); } - $events->setParam('userId', $user->getId()); + $queueForEvents->setParam('userId', $user->getId()); $response->dynamic($user, Response::MODEL_USER); }); @@ -986,8 +985,8 @@ App::patch('/v1/users/:userId/phone') ->param('number', '', new Phone(), 'User phone number.') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->action(function (string $userId, string $number, Response $response, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $userId, string $number, Response $response, Database $dbForProject, Event $queueForEvents) { $user = $dbForProject->getDocument('users', $userId); @@ -1006,7 +1005,7 @@ App::patch('/v1/users/:userId/phone') throw new Exception(Exception::USER_PHONE_ALREADY_EXISTS); } - $events->setParam('userId', $user->getId()); + $queueForEvents->setParam('userId', $user->getId()); $response->dynamic($user, Response::MODEL_USER); }); @@ -1031,8 +1030,8 @@ App::patch('/v1/users/:userId/verification') ->param('emailVerification', false, new Boolean(), 'User email verification status.') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->action(function (string $userId, bool $emailVerification, Response $response, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $userId, bool $emailVerification, Response $response, Database $dbForProject, Event $queueForEvents) { $user = $dbForProject->getDocument('users', $userId); @@ -1042,7 +1041,7 @@ App::patch('/v1/users/:userId/verification') $user = $dbForProject->updateDocument('users', $user->getId(), $user->setAttribute('emailVerification', $emailVerification)); - $events->setParam('userId', $user->getId()); + $queueForEvents->setParam('userId', $user->getId()); $response->dynamic($user, Response::MODEL_USER); }); @@ -1064,8 +1063,8 @@ App::patch('/v1/users/:userId/prefs') ->param('prefs', '', new Assoc(), 'Prefs key-value JSON object.') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->action(function (string $userId, array $prefs, Response $response, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $userId, array $prefs, Response $response, Database $dbForProject, Event $queueForEvents) { $user = $dbForProject->getDocument('users', $userId); @@ -1075,7 +1074,7 @@ App::patch('/v1/users/:userId/prefs') $user = $dbForProject->updateDocument('users', $user->getId(), $user->setAttribute('prefs', $prefs)); - $events + $queueForEvents ->setParam('userId', $user->getId()); $response->dynamic(new Document($prefs), Response::MODEL_PREFERENCES); @@ -1099,8 +1098,8 @@ App::delete('/v1/users/:userId/sessions/:sessionId') ->param('sessionId', '', new UID(), 'Session ID.') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->action(function (string $userId, string $sessionId, Response $response, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $userId, string $sessionId, Response $response, Database $dbForProject, Event $queueForEvents) { $user = $dbForProject->getDocument('users', $userId); @@ -1117,7 +1116,7 @@ App::delete('/v1/users/:userId/sessions/:sessionId') $dbForProject->deleteDocument('sessions', $session->getId()); $dbForProject->deleteCachedDocument('users', $user->getId()); - $events + $queueForEvents ->setParam('userId', $user->getId()) ->setParam('sessionId', $sessionId) ->setPayload($response->output($session, Response::MODEL_SESSION)); @@ -1142,8 +1141,8 @@ App::delete('/v1/users/:userId/sessions') ->param('userId', '', new UID(), 'User ID.') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->action(function (string $userId, Response $response, Database $dbForProject, Event $events) { + ->inject('queueForEvents') + ->action(function (string $userId, Response $response, Database $dbForProject, Event $queueForEvents) { $user = $dbForProject->getDocument('users', $userId); @@ -1161,7 +1160,7 @@ App::delete('/v1/users/:userId/sessions') $dbForProject->deleteCachedDocument('users', $user->getId()); - $events + $queueForEvents ->setParam('userId', $user->getId()) ->setPayload($response->output($user, Response::MODEL_USER)); @@ -1185,9 +1184,9 @@ App::delete('/v1/users/:userId') ->param('userId', '', new UID(), 'User ID.') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->inject('deletes') - ->action(function (string $userId, Response $response, Database $dbForProject, Event $events, Delete $deletes) { + ->inject('queueForEvents') + ->inject('queueForDeletes') + ->action(function (string $userId, Response $response, Database $dbForProject, Event $queueForEvents, Delete $queueForDeletes) { $user = $dbForProject->getDocument('users', $userId); @@ -1200,11 +1199,11 @@ App::delete('/v1/users/:userId') $dbForProject->deleteDocument('users', $userId); - $deletes + $queueForDeletes ->setType(DELETE_TYPE_DOCUMENT) ->setDocument($clone); - $events + $queueForEvents ->setParam('userId', $user->getId()) ->setPayload($response->output($clone, Response::MODEL_USER)); @@ -1228,9 +1227,7 @@ App::delete('/v1/users/identities/:identityId') ->param('identityId', '', new UID(), 'Identity ID.') ->inject('response') ->inject('dbForProject') - ->inject('events') - ->inject('deletes') - ->action(function (string $identityId, Response $response, Database $dbForProject, Event $events, Delete $deletes) { + ->action(function (string $identityId, Response $response, Database $dbForProject) { $identity = $dbForProject->getDocument('identities', $identityId); diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php index b0050c61d4..55602136fc 100644 --- a/app/controllers/api/vcs.php +++ b/app/controllers/api/vcs.php @@ -39,7 +39,7 @@ use Utopia\Validator\Boolean; use function Swoole\Coroutine\batch; -$createGitDeployments = function (GitHub $github, string $providerInstallationId, array $repositories, string $providerBranch, string $providerBranchUrl, string $providerRepositoryName, string $providerRepositoryUrl, string $providerRepositoryOwner, string $providerCommitHash, string $providerCommitAuthor, string $providerCommitAuthorUrl, string $providerCommitMessage, string $providerCommitUrl, string $providerPullRequestId, bool $external, Database $dbForConsole, callable $getProjectDB, Request $request) { +$createGitDeployments = function (GitHub $github, string $providerInstallationId, array $repositories, string $providerBranch, string $providerBranchUrl, string $providerRepositoryName, string $providerRepositoryUrl, string $providerRepositoryOwner, string $providerCommitHash, string $providerCommitAuthor, string $providerCommitAuthorUrl, string $providerCommitMessage, string $providerCommitUrl, string $providerPullRequestId, bool $external, Database $dbForConsole, Build $queueForBuilds, callable $getProjectDB, Request $request) { foreach ($repositories as $resource) { $resourceType = $resource->getAttribute('resourceType'); @@ -213,8 +213,7 @@ $createGitDeployments = function (GitHub $github, string $providerInstallationId $github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, 'pending', $message, $providerTargetUrl, $name); } - $buildEvent = new Build(); - $buildEvent + $queueForBuilds ->setType(BUILD_TYPE_DEPLOYMENT) ->setResource($function) ->setDeployment($deployment) @@ -792,8 +791,9 @@ App::post('/v1/vcs/github/events') ->inject('response') ->inject('dbForConsole') ->inject('getProjectDB') + ->inject('queueForBuilds') ->action( - function (GitHub $github, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB) use ($createGitDeployments) { + function (GitHub $github, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Build $queueForBuilds) use ($createGitDeployments) { $payload = $request->getRawPayload(); $signatureRemote = $request->getHeader('x-hub-signature-256', ''); $signatureLocal = App::getEnv('_APP_VCS_GITHUB_WEBHOOK_SECRET', ''); @@ -834,7 +834,7 @@ App::post('/v1/vcs/github/events') // create new deployment only on push and not when branch is created if (!$providerBranchCreated) { - $createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, '', false, $dbForConsole, $getProjectDB, $request); + $createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, '', false, $dbForConsole, $queueForBuilds, $getProjectDB, $request); } } elseif ($event == $github::EVENT_INSTALLATION) { if ($parsedPayload["action"] == "deleted") { @@ -890,7 +890,7 @@ App::post('/v1/vcs/github/events') Query::orderDesc('$createdAt') ]); - $createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $external, $dbForConsole, $getProjectDB, $request); + $createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $external, $dbForConsole, $queueForBuilds, $getProjectDB, $request); } elseif ($parsedPayload["action"] == "closed") { // Allowed external contributions cleanup @@ -1054,7 +1054,8 @@ App::patch('/v1/vcs/github/installations/:installationId/repositories/:repositor ->inject('project') ->inject('dbForConsole') ->inject('getProjectDB') - ->action(function (string $installationId, string $repositoryId, string $providerPullRequestId, GitHub $github, Request $request, Response $response, Document $project, Database $dbForConsole, callable $getProjectDB) use ($createGitDeployments) { + ->inject('queueForBuilds') + ->action(function (string $installationId, string $repositoryId, string $providerPullRequestId, GitHub $github, Request $request, Response $response, Document $project, Database $dbForConsole, callable $getProjectDB, Build $queueForBuilds) use ($createGitDeployments) { $installation = $dbForConsole->getDocument('installations', $installationId); if ($installation->isEmpty()) { @@ -1095,7 +1096,7 @@ App::patch('/v1/vcs/github/installations/:installationId/repositories/:repositor $providerBranch = \explode(':', $pullRequestResponse['head']['label'])[1] ?? ''; $providerCommitHash = $pullRequestResponse['head']['sha'] ?? ''; - $createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerCommitHash, $providerPullRequestId, true, $dbForConsole, $getProjectDB, $request); + $createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerCommitHash, $providerPullRequestId, true, $dbForConsole, $queueForBuilds, $getProjectDB, $request); $response->noContent(); }); diff --git a/app/controllers/general.php b/app/controllers/general.php index bc21da50dd..44fc6ed540 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -208,7 +208,8 @@ App::init() ->inject('localeCodes') ->inject('clients') ->inject('servers') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForConsole, Document $user, Locale $locale, array $localeCodes, array $clients, array $servers) { + ->inject('queueForCertificates') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForConsole, Document $user, Locale $locale, array $localeCodes, array $clients, array $servers, Certificate $queueForCertificates) { /* * Appwrite Router */ @@ -299,7 +300,7 @@ App::init() Console::info('Issuing a TLS certificate for the main domain (' . $domain->get() . ') in a few seconds...'); - (new Certificate()) + $queueForCertificates ->setDomain($domainDocument) ->setSkipRenewCheck(true) ->trigger(); diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index c2102057fa..388851faef 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -96,15 +96,15 @@ App::init() ->inject('response') ->inject('project') ->inject('user') - ->inject('events') - ->inject('audits') - ->inject('deletes') - ->inject('database') + ->inject('queueForEvents') + ->inject('queueForAudits') + ->inject('queueForDeletes') + ->inject('queueForDatabase') ->inject('dbForProject') ->inject('mode') - ->inject('mails') + ->inject('queueForMails') ->inject('usage') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Delete $deletes, EventDatabase $database, Database $dbForProject, string $mode, Mail $mails, Stats $usage) use ($databaseListener) { + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Database $dbForProject, string $mode, Mail $queueForMails, Stats $usage) use ($databaseListener) { $route = $utopia->getRoute(); @@ -173,12 +173,12 @@ App::init() /* * Background Jobs */ - $events + $queueForEvents ->setEvent($route->getLabel('event', '')) ->setProject($project) ->setUser($user); - $audits + $queueForAudits ->setMode($mode) ->setUserAgent($request->getUserAgent('')) ->setIP($request->getIP()) @@ -194,8 +194,8 @@ App::init() ->setParam('project.{scope}.network.inbound', 0) ->setParam('project.{scope}.network.outbound', 0); - $deletes->setProject($project); - $database->setProject($project); + $queueForDeletes->setProject($project); + $queueForDatabase->setProject($project); $dbForProject->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, Document $document) => $databaseListener($event, $document, $usage)); $dbForProject->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, Document $document) => $databaseListener($event, $document, $usage)); @@ -360,35 +360,35 @@ App::shutdown() ->inject('response') ->inject('project') ->inject('user') - ->inject('events') - ->inject('audits') + ->inject('queueForEvents') + ->inject('queueForAudits') ->inject('usage') - ->inject('deletes') - ->inject('database') + ->inject('queueForDeletes') + ->inject('queueForDatabase') ->inject('dbForProject') ->inject('queueForFunctions') ->inject('mode') ->inject('dbForConsole') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Stats $usage, Delete $deletes, EventDatabase $database, Database $dbForProject, Func $queueForFunctions, string $mode, Database $dbForConsole) use ($parseLabel) { + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Stats $usage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Database $dbForProject, Func $queueForFunctions, string $mode, Database $dbForConsole) use ($parseLabel) { $responsePayload = $response->getPayload(); - if (!empty($events->getEvent())) { - if (empty($events->getPayload())) { - $events->setPayload($responsePayload); + if (!empty($queueForEvents->getEvent())) { + if (empty($queueForEvents->getPayload())) { + $queueForEvents->setPayload($responsePayload); } /** * Trigger functions. */ $queueForFunctions - ->from($events) + ->from($queueForEvents) ->trigger(); /** * Trigger webhooks. */ - $events + $queueForEvents ->setClass(Event::WEBHOOK_CLASS_NAME) ->setQueue(Event::WEBHOOK_QUEUE_NAME) ->trigger(); @@ -397,12 +397,12 @@ App::shutdown() * Trigger realtime. */ if ($project->getId() !== 'console') { - $allEvents = Event::generateEvents($events->getEvent(), $events->getParams()); - $payload = new Document($events->getPayload()); + $allEvents = Event::generateEvents($queueForEvents->getEvent(), $queueForEvents->getParams()); + $payload = new Document($queueForEvents->getPayload()); - $db = $events->getContext('database'); - $collection = $events->getContext('collection'); - $bucket = $events->getContext('bucket'); + $db = $queueForEvents->getContext('database'); + $collection = $queueForEvents->getContext('collection'); + $bucket = $queueForEvents->getContext('bucket'); $target = Realtime::fromPayload( // Pass first, most verbose event pattern @@ -416,13 +416,13 @@ App::shutdown() Realtime::send( projectId: $target['projectId'] ?? $project->getId(), - payload: $events->getPayload(), + payload: $queueForEvents->getPayload(), events: $allEvents, channels: $target['channels'], roles: $target['roles'], options: [ 'permissionsChanged' => $target['permissionsChanged'], - 'userId' => $events->getParam('userId') + 'userId' => $queueForEvents->getParam('userId') ] ); } @@ -438,36 +438,36 @@ App::shutdown() if (!empty($pattern)) { $resource = $parseLabel($pattern, $responsePayload, $requestParams, $user); if (!empty($resource) && $resource !== $pattern) { - $audits->setResource($resource); + $queueForAudits->setResource($resource); } } if (!$user->isEmpty()) { - $audits->setUser($user); + $queueForAudits->setUser($user); } - if (!empty($audits->getResource()) && !empty($audits->getUser()->getId())) { + if (!empty($queueForAudits->getResource()) && !empty($queueForAudits->getUser()->getId())) { /** * audits.payload is switched to default true * in order to auto audit payload for all endpoints */ $pattern = $route->getLabel('audits.payload', true); if (!empty($pattern)) { - $audits->setPayload($responsePayload); + $queueForAudits->setPayload($responsePayload); } - foreach ($events->getParams() as $key => $value) { - $audits->setParam($key, $value); + foreach ($queueForEvents->getParams() as $key => $value) { + $queueForAudits->setParam($key, $value); } - $audits->trigger(); + $queueForAudits->trigger(); } - if (!empty($deletes->getType())) { - $deletes->trigger(); + if (!empty($queueForDeletes->getType())) { + $queueForDeletes->trigger(); } - if (!empty($database->getType())) { - $database->trigger(); + if (!empty($queueForDatabase->getType())) { + $queueForDatabase->trigger(); } /** diff --git a/app/init.php b/app/init.php index c32c63f1f2..8aec6122fa 100644 --- a/app/init.php +++ b/app/init.php @@ -18,7 +18,7 @@ ini_set('display_startup_errors', 1); ini_set('default_socket_timeout', -1); error_reporting(E_ALL); -use Appwrite\Event\Usage; +use Appwrite\Event\Migration; use Appwrite\Extend\Exception; use Appwrite\Auth\Auth; use Appwrite\Event\Audit; @@ -69,7 +69,8 @@ use Utopia\Pools\Group; use Utopia\Pools\Pool; use Ahc\Jwt\JWT; use Ahc\Jwt\JWTException; -use Appwrite\Auth\OAuth2\Github; +use Appwrite\Event\Build; +use Appwrite\Event\Certificate; use Appwrite\Event\Func; use MaxMind\Db\Reader; use PHPMailer\PHPMailer\PHPMailer; @@ -145,6 +146,8 @@ const DATABASE_TYPE_CREATE_ATTRIBUTE = 'createAttribute'; const DATABASE_TYPE_CREATE_INDEX = 'createIndex'; const DATABASE_TYPE_DELETE_ATTRIBUTE = 'deleteAttribute'; const DATABASE_TYPE_DELETE_INDEX = 'deleteIndex'; +const DATABASE_TYPE_DELETE_COLLECTION = 'deleteCollection'; +const DATABASE_TYPE_DELETE_DATABASE = 'deleteDatabase'; // Build Worker Types const BUILD_TYPE_DEPLOYMENT = 'deployment'; const BUILD_TYPE_RETRY = 'retry'; @@ -259,14 +262,6 @@ Config::load('storage-mimes', __DIR__ . '/config/storage/mimes.php'); Config::load('storage-inputs', __DIR__ . '/config/storage/inputs.php'); Config::load('storage-outputs', __DIR__ . '/config/storage/outputs.php'); -$user = App::getEnv('_APP_REDIS_USER', ''); -$pass = App::getEnv('_APP_REDIS_PASS', ''); -if (!empty($user) || !empty($pass)) { - Resque::setBackend('redis://' . $user . ':' . $pass . '@' . App::getEnv('_APP_REDIS_HOST', '') . ':' . App::getEnv('_APP_REDIS_PORT', '')); -} else { - Resque::setBackend(App::getEnv('_APP_REDIS_HOST', '') . ':' . App::getEnv('_APP_REDIS_PORT', '')); -} - /** * New DB Filters */ @@ -887,18 +882,39 @@ App::setResource('localeCodes', function () { }); // Queues -App::setResource('events', fn() => new Event('', '')); -App::setResource('audits', fn() => new Audit()); -App::setResource('mails', fn() => new Mail()); -App::setResource('deletes', fn() => new Delete()); -App::setResource('database', fn() => new EventDatabase()); -App::setResource('messaging', fn() => new Phone()); App::setResource('queue', function (Group $pools) { return $pools->get('queue')->pop()->getResource(); }, ['pools']); +App::setResource('queueForMessaging', function (Connection $queue) { + return new Phone($queue); +}, ['queue']); +App::setResource('queueForMails', function (Connection $queue) { + return new Mail($queue); +}, ['queue']); +App::setResource('queueForBuilds', function (Connection $queue) { + return new Build($queue); +}, ['queue']); +App::setResource('queueForDatabase', function (Connection $queue) { + return new EventDatabase($queue); +}, ['queue']); +App::setResource('queueForDeletes', function (Connection $queue) { + return new Delete($queue); +}, ['queue']); +App::setResource('queueForEvents', function (Connection $queue) { + return new Event($queue); +}, ['queue']); +App::setResource('queueForAudits', function (Connection $queue) { + return new Audit($queue); +}, ['queue']); App::setResource('queueForFunctions', function (Connection $queue) { return new Func($queue); }, ['queue']); +App::setResource('queueForCertificates', function (Connection $queue) { + return new Certificate($queue); +}, ['queue']); +App::setResource('queueForMigrations', function (Connection $queue) { + return new Migration($queue); +}, ['queue']); App::setResource('usage', function ($register) { return new Stats($register->get('statsd')); }, ['register']); diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 158d5fcddb..bc64effeec 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -188,7 +188,6 @@ services: - traefik.http.routers.appwrite_realtime_wss.rule=PathPrefix(`/v1/realtime`) - traefik.http.routers.appwrite_realtime_wss.service=appwrite_realtime - traefik.http.routers.appwrite_realtime_wss.tls=true - - traefik.http.routers.appwrite_realtime_wss.tls.certresolver=dns networks: - appwrite depends_on: diff --git a/app/worker.php b/app/worker.php index 51e42969f4..8aa52ab931 100644 --- a/app/worker.php +++ b/app/worker.php @@ -2,32 +2,42 @@ require_once __DIR__ . '/init.php'; +use Appwrite\Event\Event; +use Appwrite\Event\Audit; +use Appwrite\Event\Build; +use Appwrite\Event\Certificate; +use Appwrite\Event\Database as EventDatabase; +use Appwrite\Event\Delete; use Appwrite\Event\Func; -use Appwrite\Event\Usage; +use Appwrite\Event\Mail; +use Appwrite\Event\Messaging; +use Appwrite\Event\Migration; +use Appwrite\Event\Phone; +use Appwrite\Platform\Appwrite; use Appwrite\Usage\Stats; use Swoole\Runtime; use Utopia\App; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; -use Utopia\CLI\CLI; use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; -use Utopia\Queue\Adapter\Swoole; +use Utopia\Platform\Service; use Utopia\Queue\Message; use Utopia\Queue\Server; use Utopia\Registry\Registry; use Utopia\Logger\Log; use Utopia\Logger\Logger; use Utopia\Pools\Group; +use Utopia\Queue\Connection; +Authorization::disable(); Runtime::enableCoroutine(SWOOLE_HOOK_ALL); -global $register; -Server::setResource('register', fn() => $register); +Server::setResource('register', fn () => $register); Server::setResource('dbForConsole', function (Cache $cache, Registry $register) { $pools = $register->get('pools'); @@ -63,6 +73,37 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, return $adapter; }, ['cache', 'register', 'message', 'dbForConsole']); +Server::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) { + $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools + + return function (Document $project) use ($pools, $dbForConsole, $cache, &$databases): Database { + if ($project->isEmpty() || $project->getId() === 'console') { + return $dbForConsole; + } + + $databaseName = $project->getAttribute('database'); + + if (isset($databases[$databaseName])) { + $database = $databases[$databaseName]; + $database->setNamespace('_' . $project->getInternalId()); + return $database; + } + + $dbAdapter = $pools + ->get($databaseName) + ->pop() + ->getResource(); + + $database = new Database($dbAdapter, $cache); + + $databases[$databaseName] = $database; + + $database->setNamespace('_' . $project->getInternalId()); + + return $database; + }; +}, ['pools', 'dbForConsole', 'cache']); + Server::setResource('cache', function (Registry $register) { $pools = $register->get('pools'); $list = Config::getParam('pools-cache', []); @@ -78,50 +119,121 @@ Server::setResource('cache', function (Registry $register) { return new Cache(new Sharding($adapters)); }, ['register']); - -Server::setResource('queueForFunctions', function (Registry $register) { - $pools = $register->get('pools'); - return new Func( - $pools - ->get('queue') - ->pop() - ->getResource() - ); -}, ['register']); - Server::setResource('log', fn() => new Log()); - -Server::setResource('logger', function ($register) { +Server::setResource('usage', function ($register) { + return new Stats($register->get('statsd')); +}, ['register']); +Server::setResource('queue', function (Group $pools) { + return $pools->get('queue')->pop()->getResource(); +}, ['pools']); +Server::setResource('queueForDatabase', function (Connection $queue) { + return new EventDatabase($queue); +}, ['queue']); +Server::setResource('queueForMessaging', function (Connection $queue) { + return new Phone($queue); +}, ['queue']); +Server::setResource('queueForMails', function (Connection $queue) { + return new Mail($queue); +}, ['queue']); +Server::setResource('queueForBuilds', function (Connection $queue) { + return new Build($queue); +}, ['queue']); +Server::setResource('queueForDeletes', function (Connection $queue) { + return new Delete($queue); +}, ['queue']); +Server::setResource('queueForEvents', function (Connection $queue) { + return new Event($queue); +}, ['queue']); +Server::setResource('queueForAudits', function (Connection $queue) { + return new Audit($queue); +}, ['queue']); +Server::setResource('queueForFunctions', function (Connection $queue) { + return new Func($queue); +}, ['queue']); +Server::setResource('queueForCertificates', function (Connection $queue) { + return new Certificate($queue); +}, ['queue']); +Server::setResource('queueForMigrations', function (Connection $queue) { + return new Migration($queue); +}, ['queue']); +Server::setResource('logger', function (Registry $register) { return $register->get('logger'); }, ['register']); - -Server::setResource('statsd', function ($register) { - return $register->get('statsd'); -}, ['register']); - -Server::setResource('pools', function ($register) { +Server::setResource('pools', function (Registry $register) { return $register->get('pools'); }, ['register']); +Server::setResource('getFunctionsDevice', function () { + return function (string $projectId) { + return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $projectId); + }; +}); +Server::setResource('getFilesDevice', function () { + return function (string $projectId) { + return getDevice(APP_STORAGE_UPLOADS . '/app-' . $projectId); + }; +}); +Server::setResource('getBuildsDevice', function () { + return function (string $projectId) { + return getDevice(APP_STORAGE_BUILDS . '/app-' . $projectId); + }; +}); +Server::setResource('getCacheDevice', function () { + return function (string $projectId) { + return getDevice(APP_STORAGE_CACHE . '/app-' . $projectId); + }; +}); $pools = $register->get('pools'); -$connection = $pools->get('queue')->pop()->getResource(); -$workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); +$platform = new Appwrite(); +$args = $_SERVER['argv']; -if (empty(App::getEnv('QUEUE'))) { - throw new Exception('Please configure "QUEUE" environment variable.'); +if (!isset($args[1])) { + Console::error('Missing worker name'); + Console::exit(1); } -$adapter = new Swoole($connection, $workerNumber, App::getEnv('QUEUE')); -$server = new Server($adapter); +\array_shift($args); +$workerName = $args[0]; +$workerIndex = $args[1] ?? ''; -$server +if (!empty($workerIndex)) { + $workerName .= '_' . $workerIndex; +} + +try { + /** + * Any worker can be configured with the following env vars: + * - _APP_WORKERS_NUM The total number of worker processes + * - _APP_WORKER_PER_CORE The number of worker processes per core (ignored if _APP_WORKERS_NUM is set) + * - _APP_QUEUE_NAME The name of the queue to read for database events + */ + if ($workerName === 'databases') { + $queueName = App::getEnv('_APP_QUEUE_NAME', 'database_db_main'); + } else { + $queueName = App::getEnv('_APP_QUEUE_NAME', 'v1-' . strtolower($workerName)); + } + + $platform->init(Service::TYPE_WORKER, [ + 'workersNum' => App::getEnv('_APP_WORKERS_NUM', 1), + 'connection' => $pools->get('queue')->pop()->getResource(), + 'workerName' => strtolower($workerName) ?? null, + 'queueName' => $queueName + ]); +} catch (\Exception $e) { + Console::error($e->getMessage() . ', File: ' . $e->getFile() . ', Line: ' . $e->getLine()); +} + + +$worker = $platform->getWorker(); + +$worker ->shutdown() ->inject('pools') ->action(function (Group $pools) { $pools->reclaim(); }); -$server +$worker ->error() ->inject('error') ->inject('logger') @@ -160,3 +272,10 @@ $server Console::error('[Error] File: ' . $error->getFile()); Console::error('[Error] Line: ' . $error->getLine()); }); + + $worker->workerStart() + ->action(function () use ($workerName) { + Console::info("Worker $workerName started"); + }); + + $worker->start(); diff --git a/app/workers/audits.php b/app/workers/audits.php deleted file mode 100644 index 8369ec74ec..0000000000 --- a/app/workers/audits.php +++ /dev/null @@ -1,62 +0,0 @@ -args['event']; - $payload = $this->args['payload']; - $mode = $this->args['mode']; - $resource = $this->args['resource']; - $userAgent = $this->args['userAgent']; - $ip = $this->args['ip']; - - $user = new Document($this->args['user']); - $project = new Document($this->args['project']); - - $userName = $user->getAttribute('name', ''); - $userEmail = $user->getAttribute('email', ''); - - $dbForProject = $this->getProjectDB($project); - $audit = new Audit($dbForProject); - $audit->log( - userId: $user->getInternalId(), - // Pass first, most verbose event pattern - event: $event, - resource: $resource, - userAgent: $userAgent, - ip: $ip, - location: '', - data: [ - 'userId' => $user->getId(), - 'userName' => $userName, - 'userEmail' => $userEmail, - 'mode' => $mode, - 'data' => $payload, - ] - ); - } - - public function shutdown(): void - { - } -} diff --git a/app/workers/messaging.php b/app/workers/messaging.php deleted file mode 100644 index 5732c8c00b..0000000000 --- a/app/workers/messaging.php +++ /dev/null @@ -1,78 +0,0 @@ -getUser(); - $secret = $dsn->getPassword(); - - $this->sms = match ($dsn->getHost()) { - 'mock' => new Mock($user, $secret), // used for tests - 'twilio' => new Twilio($user, $secret), - 'text-magic' => new TextMagic($user, $secret), - 'telesign' => new Telesign($user, $secret), - 'msg91' => new Msg91($user, $secret), - 'vonage' => new Vonage($user, $secret), - default => null - }; - - $this->from = App::getEnv('_APP_SMS_FROM'); - } - - public function run(): void - { - if (empty(App::getEnv('_APP_SMS_PROVIDER'))) { - Console::info('Skipped sms processing. No Phone provider has been set.'); - return; - } - - if (empty($this->from)) { - Console::info('Skipped sms processing. No phone number has been set.'); - return; - } - - $message = new SMS( - to: [$this->args['recipient']], - content: $this->args['message'], - from: $this->from, - ); - - try { - $this->sms->send($message); - } catch (\Exception $error) { - throw new Exception('Error sending message: ' . $error->getMessage(), 500); - } - } - - public function shutdown(): void - { - } -} diff --git a/bin/worker-audits b/bin/worker-audits index 7dd25c75ca..3df65d65e8 100644 --- a/bin/worker-audits +++ b/bin/worker-audits @@ -1,10 +1,3 @@ #!/bin/sh -if [ -z "$_APP_REDIS_USER" ] && [ -z "$_APP_REDIS_PASS" ] -then - REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -else - REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -fi - -INTERVAL=1 QUEUE='v1-audits' APP_INCLUDE='/usr/src/code/app/workers/audits.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +php /usr/src/code/app/worker.php audits $@ \ No newline at end of file diff --git a/bin/worker-builds b/bin/worker-builds index 2ba26ef4d1..3400111cb5 100644 --- a/bin/worker-builds +++ b/bin/worker-builds @@ -1,10 +1,3 @@ #!/bin/sh -if [ -z "$_APP_REDIS_USER" ] && [ -z "$_APP_REDIS_PASS" ] -then - REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -else - REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -fi - -INTERVAL=0.1 QUEUE='v1-builds' APP_INCLUDE='/usr/src/code/app/workers/builds.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +php /usr/src/code/app/worker.php builds $@ \ No newline at end of file diff --git a/bin/worker-certificates b/bin/worker-certificates index 679885fa46..901688c4c8 100755 --- a/bin/worker-certificates +++ b/bin/worker-certificates @@ -1,10 +1,3 @@ #!/bin/sh -if [ -z "$_APP_REDIS_USER" ] && [ -z "$_APP_REDIS_PASS" ] -then - REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -else - REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -fi - -INTERVAL=1 QUEUE='v1-certificates' APP_INCLUDE='/usr/src/code/app/workers/certificates.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +php /usr/src/code/app/worker.php certificates $@ \ No newline at end of file diff --git a/bin/worker-databases b/bin/worker-databases index bbec58268f..61e09aa9f1 100644 --- a/bin/worker-databases +++ b/bin/worker-databases @@ -1,10 +1,3 @@ #!/bin/sh -if [ -z "$_APP_REDIS_USER" ] && [ -z "$_APP_REDIS_PASS" ] -then - REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -else - REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -fi - -INTERVAL=0.1 QUEUE='v1-database' APP_INCLUDE='/usr/src/code/app/workers/databases.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +php /usr/src/code/app/worker.php databases $@ diff --git a/bin/worker-deletes b/bin/worker-deletes index 02c2311fa7..7c9793e6cb 100644 --- a/bin/worker-deletes +++ b/bin/worker-deletes @@ -1,10 +1,3 @@ #!/bin/sh -if [ -z "$_APP_REDIS_USER" ] && [ -z "$_APP_REDIS_PASS" ] -then - REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -else - REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -fi - -INTERVAL=1 QUEUE='v1-deletes' APP_INCLUDE='/usr/src/code/app/workers/deletes.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +php /usr/src/code/app/worker.php deletes $@ \ No newline at end of file diff --git a/bin/worker-functions b/bin/worker-functions index 687f9fd0cd..4757b1b72a 100644 --- a/bin/worker-functions +++ b/bin/worker-functions @@ -1,3 +1,3 @@ #!/bin/sh -QUEUE=v1-functions php /usr/src/code/app/workers/functions.php $@ \ No newline at end of file +php /usr/src/code/app/worker.php functions $@ \ No newline at end of file diff --git a/bin/worker-mails b/bin/worker-mails index 87fa64cf7c..fee8a96da7 100644 --- a/bin/worker-mails +++ b/bin/worker-mails @@ -1,10 +1,3 @@ #!/bin/sh -if [ -z "$_APP_REDIS_USER" ] && [ -z "$_APP_REDIS_PASS" ] -then - REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -else - REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -fi - -INTERVAL=1 QUEUE='v1-mails' APP_INCLUDE='/usr/src/code/app/workers/mails.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +php /usr/src/code/app/worker.php mails $@ \ No newline at end of file diff --git a/bin/worker-messaging b/bin/worker-messaging index 51057bc5a5..e6edf80f06 100644 --- a/bin/worker-messaging +++ b/bin/worker-messaging @@ -1,10 +1,3 @@ #!/bin/sh -if [ -z "$_APP_REDIS_USER" ] && [ -z "$_APP_REDIS_PASS" ] -then - REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -else - REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -fi - -INTERVAL=1 QUEUE='v1-messaging' APP_INCLUDE='/usr/src/code/app/workers/messaging.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +php /usr/src/code/app/worker.php messaging $@ \ No newline at end of file diff --git a/bin/worker-migrations b/bin/worker-migrations index 54e57001b0..32d4aef468 100644 --- a/bin/worker-migrations +++ b/bin/worker-migrations @@ -1,10 +1,3 @@ #!/bin/sh -if [ -z "$_APP_REDIS_USER" ] && [ -z "$_APP_REDIS_PASS" ] -then - REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -else - REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -fi - -INTERVAL=0.1 QUEUE='v1-migrations' APP_INCLUDE='/usr/src/code/app/workers/migrations.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +php /usr/src/code/app/worker.php migrations $@ \ No newline at end of file diff --git a/bin/worker-webhooks b/bin/worker-webhooks index e34c79658a..93f8027a81 100644 --- a/bin/worker-webhooks +++ b/bin/worker-webhooks @@ -1,10 +1,3 @@ #!/bin/sh -if [ -z "$_APP_REDIS_USER" ] && [ -z "$_APP_REDIS_PASS" ] -then - REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -else - REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -fi - -INTERVAL=0.1 QUEUE='v1-webhooks' APP_INCLUDE='/usr/src/code/app/workers/webhooks.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +php /usr/src/code/app/worker.php webhooks $@ \ No newline at end of file diff --git a/composer.json b/composer.json index 0d415677ce..0ffbfe6200 100644 --- a/composer.json +++ b/composer.json @@ -59,7 +59,7 @@ "utopia-php/messaging": "0.1.*", "utopia-php/migration": "0.3.*", "utopia-php/orchestration": "0.9.*", - "utopia-php/platform": "0.4.*", + "utopia-php/platform": "0.5.*", "utopia-php/pools": "0.4.*", "utopia-php/preloader": "0.2.*", "utopia-php/queue": "0.5.*", @@ -68,7 +68,6 @@ "utopia-php/swoole": "0.5.*", "utopia-php/vcs": "0.5.*", "utopia-php/websocket": "0.1.*", - "resque/php-resque": "1.3.6", "matomo/device-detector": "6.1.*", "dragonmantank/cron-expression": "3.3.2", "influxdb/influxdb-php": "1.15.2", diff --git a/composer.lock b/composer.lock index e9a0319834..e45e4b3d86 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "13a3bdc7c1dec5756bf58ec73a49753d", + "content-hash": "3a0624bf1df70e602233efa5916aa6ce", "packages": [ { "name": "adhocore/jwt", @@ -339,53 +339,6 @@ ], "time": "2022-07-05T22:32:14+00:00" }, - { - "name": "colinmollenhour/credis", - "version": "v1.15.0", - "source": { - "type": "git", - "url": "https://github.com/colinmollenhour/credis.git", - "reference": "28810439de1d9597b7ba11794ed9479fb6f3de7c" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/colinmollenhour/credis/zipball/28810439de1d9597b7ba11794ed9479fb6f3de7c", - "reference": "28810439de1d9597b7ba11794ed9479fb6f3de7c", - "shasum": "" - }, - "require": { - "php": ">=5.6.0" - }, - "suggest": { - "ext-redis": "Improved performance for communicating with redis" - }, - "type": "library", - "autoload": { - "classmap": [ - "Client.php", - "Cluster.php", - "Sentinel.php", - "Module.php" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Colin Mollenhour", - "email": "colin@mollenhour.com" - } - ], - "description": "Credis is a lightweight interface to the Redis key-value store which wraps the phpredis library when available for better performance.", - "homepage": "https://github.com/colinmollenhour/credis", - "support": { - "issues": "https://github.com/colinmollenhour/credis/issues", - "source": "https://github.com/colinmollenhour/credis/tree/v1.15.0" - }, - "time": "2023-04-18T15:34:23+00:00" - }, { "name": "dragonmantank/cron-expression", "version": "v3.3.2", @@ -1476,56 +1429,6 @@ }, "time": "2023-04-04T09:54:51+00:00" }, - { - "name": "psr/log", - "version": "1.1.4", - "source": { - "type": "git", - "url": "https://github.com/php-fig/log.git", - "reference": "d49695b909c3b7628b6289db5479a1c204601f11" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/php-fig/log/zipball/d49695b909c3b7628b6289db5479a1c204601f11", - "reference": "d49695b909c3b7628b6289db5479a1c204601f11", - "shasum": "" - }, - "require": { - "php": ">=5.3.0" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.1.x-dev" - } - }, - "autoload": { - "psr-4": { - "Psr\\Log\\": "Psr/Log/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "PHP-FIG", - "homepage": "https://www.php-fig.org/" - } - ], - "description": "Common interface for logging libraries", - "homepage": "https://github.com/php-fig/log", - "keywords": [ - "log", - "psr", - "psr-3" - ], - "support": { - "source": "https://github.com/php-fig/log/tree/1.1.4" - }, - "time": "2021-05-03T11:20:27+00:00" - }, { "name": "ralouphie/getallheaders", "version": "3.0.3", @@ -1570,89 +1473,6 @@ }, "time": "2019-03-08T08:55:37+00:00" }, - { - "name": "resque/php-resque", - "version": "v1.3.6", - "source": { - "type": "git", - "url": "https://github.com/resque/php-resque.git", - "reference": "fe41c04763699b1318d97ed14cc78583e9380161" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/resque/php-resque/zipball/fe41c04763699b1318d97ed14cc78583e9380161", - "reference": "fe41c04763699b1318d97ed14cc78583e9380161", - "shasum": "" - }, - "require": { - "colinmollenhour/credis": "~1.7", - "php": ">=5.6.0", - "psr/log": "~1.0" - }, - "require-dev": { - "phpunit/phpunit": "^5.7" - }, - "suggest": { - "ext-pcntl": "REQUIRED for forking processes on platforms that support it (so anything but Windows).", - "ext-proctitle": "Allows php-resque to rename the title of UNIX processes to show the status of a worker.", - "ext-redis": "Native PHP extension for Redis connectivity. Credis will automatically utilize when available." - }, - "bin": [ - "bin/resque", - "bin/resque-scheduler" - ], - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.0-dev" - } - }, - "autoload": { - "psr-0": { - "Resque": "lib", - "ResqueScheduler": "lib" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Dan Hunsaker", - "email": "danhunsaker+resque@gmail.com", - "role": "Maintainer" - }, - { - "name": "Rajib Ahmed", - "homepage": "https://github.com/rajibahmed", - "role": "Maintainer" - }, - { - "name": "Steve Klabnik", - "email": "steve@steveklabnik.com", - "role": "Maintainer" - }, - { - "name": "Chris Boulton", - "email": "chris@bigcommerce.com", - "role": "Creator" - } - ], - "description": "Redis backed library for creating background jobs and processing them later. Based on resque for Ruby.", - "homepage": "http://www.github.com/resque/php-resque/", - "keywords": [ - "background", - "job", - "redis", - "resque" - ], - "support": { - "issues": "https://github.com/resque/php-resque/issues", - "source": "https://github.com/resque/php-resque/tree/v1.3.6" - }, - "time": "2020-04-16T16:39:50+00:00" - }, { "name": "slickdeals/statsd", "version": "3.1.0", @@ -2463,22 +2283,23 @@ }, { "name": "utopia-php/logger", - "version": "0.3.1", + "version": "0.3.2", "source": { "type": "git", "url": "https://github.com/utopia-php/logger.git", - "reference": "de623f1ec1c672c795d113dd25c5bf212f7ef4fc" + "reference": "9151b7d16eab18d4c37c34643041cc0f33ca4a6c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/logger/zipball/de623f1ec1c672c795d113dd25c5bf212f7ef4fc", - "reference": "de623f1ec1c672c795d113dd25c5bf212f7ef4fc", + "url": "https://api.github.com/repos/utopia-php/logger/zipball/9151b7d16eab18d4c37c34643041cc0f33ca4a6c", + "reference": "9151b7d16eab18d4c37c34643041cc0f33ca4a6c", "shasum": "" }, "require": { "php": ">=8.0" }, "require-dev": { + "laravel/pint": "1.2.*", "phpstan/phpstan": "1.9.x-dev", "phpunit/phpunit": "^9.3", "vimeo/psalm": "4.0.1" @@ -2510,9 +2331,9 @@ ], "support": { "issues": "https://github.com/utopia-php/logger/issues", - "source": "https://github.com/utopia-php/logger/tree/0.3.1" + "source": "https://github.com/utopia-php/logger/tree/0.3.2" }, - "time": "2023-02-10T15:52:50+00:00" + "time": "2023-10-16T08:16:19+00:00" }, { "name": "utopia-php/messaging", @@ -2722,16 +2543,16 @@ }, { "name": "utopia-php/platform", - "version": "0.4.2", + "version": "0.5.0", "source": { "type": "git", "url": "https://github.com/utopia-php/platform.git", - "reference": "6e3d6db9ee8f99e36c2df331b58de2e6e3e37eb9" + "reference": "229a7b1fa1f39afd1532f7a515326a6afc222a26" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/platform/zipball/6e3d6db9ee8f99e36c2df331b58de2e6e3e37eb9", - "reference": "6e3d6db9ee8f99e36c2df331b58de2e6e3e37eb9", + "url": "https://api.github.com/repos/utopia-php/platform/zipball/229a7b1fa1f39afd1532f7a515326a6afc222a26", + "reference": "229a7b1fa1f39afd1532f7a515326a6afc222a26", "shasum": "" }, "require": { @@ -2765,9 +2586,9 @@ ], "support": { "issues": "https://github.com/utopia-php/platform/issues", - "source": "https://github.com/utopia-php/platform/tree/0.4.2" + "source": "https://github.com/utopia-php/platform/tree/0.5.0" }, - "time": "2023-08-30T16:28:31+00:00" + "time": "2023-10-16T20:28:49+00:00" }, { "name": "utopia-php/pools", @@ -4604,6 +4425,56 @@ ], "time": "2022-04-01T12:37:26+00:00" }, + { + "name": "psr/log", + "version": "3.0.0", + "source": { + "type": "git", + "url": "https://github.com/php-fig/log.git", + "reference": "fe5ea303b0887d5caefd3d431c3e61ad47037001" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/log/zipball/fe5ea303b0887d5caefd3d431c3e61ad47037001", + "reference": "fe5ea303b0887d5caefd3d431c3e61ad47037001", + "shasum": "" + }, + "require": { + "php": ">=8.0.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Log\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "https://www.php-fig.org/" + } + ], + "description": "Common interface for logging libraries", + "homepage": "https://github.com/php-fig/log", + "keywords": [ + "log", + "psr", + "psr-3" + ], + "support": { + "source": "https://github.com/php-fig/log/tree/3.0.0" + }, + "time": "2021-07-14T16:46:02+00:00" + }, { "name": "sebastian/cli-parser", "version": "1.0.1", @@ -6019,5 +5890,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.6.0" } diff --git a/docker-compose.yml b/docker-compose.yml index e6755c04c9..71d7365d14 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -211,7 +211,6 @@ services: - traefik.http.routers.appwrite_realtime_wss.rule=PathPrefix(`/v1/realtime`) - traefik.http.routers.appwrite_realtime_wss.service=appwrite_realtime - traefik.http.routers.appwrite_realtime_wss.tls=true - - traefik.http.routers.appwrite_realtime_wss.tls.certresolver=dns networks: - appwrite volumes: @@ -379,6 +378,8 @@ services: - _APP_DB_PASS - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG + - _APP_WORKERS_NUM + - _APP_QUEUE_NAME appwrite-worker-builds: entrypoint: worker-builds @@ -871,7 +872,6 @@ services: # MailCatcher - An SMTP server. Catches all system emails and displays them in a nice UI. # RequestCatcher - An HTTP server. Catches all system https calls and displays them using a simple HTTP API. Used to debug & tests webhooks and HTTP tasks # RedisCommander - A nice UI for exploring Redis data - # Resque - A nice UI for exploring Redis pub/sub, view the different queues workloads, pending and failed tasks # Chronograf - A nice UI for exploring InfluxDB data # Webgrind - A nice UI for exploring and debugging code-level stuff @@ -913,19 +913,6 @@ services: # ports: # - "8081:8081" - # resque: - # image: appwrite/resque-web:1.1.0 - # restart: unless-stopped - # networks: - # - appwrite - # ports: - # - "5678:5678" - # environment: - # - RESQUE_WEB_HOST=redis - # - RESQUE_WEB_PORT=6379 - # - RESQUE_WEB_HTTP_BASIC_AUTH_USER=user - # - RESQUE_WEB_HTTP_BASIC_AUTH_PASSWORD=password - # chronograf: # image: chronograf:1.6 # container_name: appwrite-chronograf diff --git a/phpunit.xml b/phpunit.xml index f83f9f0fae..975b54962a 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -19,9 +19,11 @@ ./tests/e2e/Client.php ./tests/e2e/General ./tests/e2e/Scopes - ./tests/e2e/Services/Account - ./tests/e2e/Services/Console + ./tests/e2e/Services/Teams ./tests/e2e/Services/Realtime + ./tests/e2e/Services/Account + ./tests/e2e/Services/Users + ./tests/e2e/Services/Console ./tests/e2e/Services/Avatars ./tests/e2e/Services/Databases ./tests/e2e/Services/GraphQL @@ -29,8 +31,6 @@ ./tests/e2e/Services/Locale ./tests/e2e/Services/Projects ./tests/e2e/Services/Storage - ./tests/e2e/Services/Teams - ./tests/e2e/Services/Users ./tests/e2e/Services/Webhooks ./tests/e2e/Services/Functions/FunctionsBase.php ./tests/e2e/Services/Functions/FunctionsCustomServerTest.php diff --git a/src/Appwrite/Event/Audit.php b/src/Appwrite/Event/Audit.php index 254f7c294a..4b02849970 100644 --- a/src/Appwrite/Event/Audit.php +++ b/src/Appwrite/Event/Audit.php @@ -2,7 +2,8 @@ namespace Appwrite\Event; -use Resque; +use Utopia\Queue\Client; +use Utopia\Queue\Connection; class Audit extends Event { @@ -11,9 +12,13 @@ class Audit extends Event protected string $userAgent = ''; protected string $ip = ''; - public function __construct() + public function __construct(protected Connection $connection) { - parent::__construct(Event::AUDITS_QUEUE_NAME, Event::AUDITS_CLASS_NAME); + parent::__construct($connection); + + $this + ->setQueue(Event::AUDITS_QUEUE_NAME) + ->setClass(Event::AUDITS_CLASS_NAME); } /** @@ -116,7 +121,9 @@ class Audit extends Event */ public function trigger(): string|bool { - return Resque::enqueue($this->queue, $this->class, [ + $client = new Client($this->queue, $this->connection); + + return $client->enqueue([ 'project' => $this->project, 'user' => $this->user, 'payload' => $this->payload, diff --git a/src/Appwrite/Event/Build.php b/src/Appwrite/Event/Build.php index 428b6aa832..496db87d64 100644 --- a/src/Appwrite/Event/Build.php +++ b/src/Appwrite/Event/Build.php @@ -2,8 +2,9 @@ namespace Appwrite\Event; -use Resque; use Utopia\Database\Document; +use Utopia\Queue\Client; +use Utopia\Queue\Connection; class Build extends Event { @@ -12,9 +13,13 @@ class Build extends Event protected ?Document $deployment = null; protected ?Document $template = null; - public function __construct() + public function __construct(protected Connection $connection) { - parent::__construct(Event::BUILDS_QUEUE_NAME, Event::BUILDS_CLASS_NAME); + parent::__construct($connection); + + $this + ->setQueue(Event::BUILDS_QUEUE_NAME) + ->setClass(Event::BUILDS_CLASS_NAME); } /** @@ -107,7 +112,9 @@ class Build extends Event */ public function trigger(): string|bool { - return Resque::enqueue($this->queue, $this->class, [ + $client = new Client($this->queue, $this->connection); + + return $client->enqueue([ 'project' => $this->project, 'resource' => $this->resource, 'deployment' => $this->deployment, diff --git a/src/Appwrite/Event/Certificate.php b/src/Appwrite/Event/Certificate.php index d3d9091804..85058c96fe 100644 --- a/src/Appwrite/Event/Certificate.php +++ b/src/Appwrite/Event/Certificate.php @@ -2,17 +2,22 @@ namespace Appwrite\Event; -use Resque; use Utopia\Database\Document; +use Utopia\Queue\Client; +use Utopia\Queue\Connection; class Certificate extends Event { protected bool $skipRenewCheck = false; protected ?Document $domain = null; - public function __construct() + public function __construct(protected Connection $connection) { - parent::__construct(Event::CERTIFICATES_QUEUE_NAME, Event::CERTIFICATES_CLASS_NAME); + parent::__construct($connection); + + $this + ->setQueue(Event::CERTIFICATES_QUEUE_NAME) + ->setClass(Event::CERTIFICATES_CLASS_NAME); } /** @@ -69,7 +74,9 @@ class Certificate extends Event */ public function trigger(): string|bool { - return Resque::enqueue($this->queue, $this->class, [ + $client = new Client($this->queue, $this->connection); + + return $client->enqueue([ 'project' => $this->project, 'domain' => $this->domain, 'skipRenewCheck' => $this->skipRenewCheck diff --git a/src/Appwrite/Event/Database.php b/src/Appwrite/Event/Database.php index 1822f06c71..442cbe4bbc 100644 --- a/src/Appwrite/Event/Database.php +++ b/src/Appwrite/Event/Database.php @@ -2,8 +2,9 @@ namespace Appwrite\Event; -use Resque; use Utopia\Database\Document; +use Utopia\Queue\Client; +use Utopia\Queue\Connection; class Database extends Event { @@ -12,9 +13,11 @@ class Database extends Event protected ?Document $collection = null; protected ?Document $document = null; - public function __construct() + public function __construct(protected Connection $connection) { - parent::__construct(Event::DATABASE_QUEUE_NAME, Event::DATABASE_CLASS_NAME); + parent::__construct($connection); + + $this->setClass(Event::DATABASE_CLASS_NAME); } /** @@ -104,7 +107,11 @@ class Database extends Event */ public function trigger(): string|bool { - return Resque::enqueue($this->queue, $this->class, [ + $this->setQueue($this->getProject()->getAttribute('database')); + + $client = new Client($this->queue, $this->connection); + + return $client->enqueue([ 'project' => $this->project, 'user' => $this->user, 'type' => $this->type, diff --git a/src/Appwrite/Event/Delete.php b/src/Appwrite/Event/Delete.php index 8e593b1020..57300feb72 100644 --- a/src/Appwrite/Event/Delete.php +++ b/src/Appwrite/Event/Delete.php @@ -2,8 +2,9 @@ namespace Appwrite\Event; -use Resque; use Utopia\Database\Document; +use Utopia\Queue\Client; +use Utopia\Queue\Connection; class Delete extends Event { @@ -13,9 +14,14 @@ class Delete extends Event protected ?string $datetime = null; protected ?string $hourlyUsageRetentionDatetime = null; - public function __construct() + + public function __construct(protected Connection $connection) { - parent::__construct(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME); + parent::__construct($connection); + + $this + ->setQueue(Event::DELETE_QUEUE_NAME) + ->setClass(Event::DELETE_CLASS_NAME); } /** @@ -120,7 +126,9 @@ class Delete extends Event */ public function trigger(): string|bool { - return Resque::enqueue($this->queue, $this->class, [ + $client = new Client($this->queue, $this->connection); + + return $client->enqueue([ 'project' => $this->project, 'type' => $this->type, 'document' => $this->document, diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index 23dde49637..46b430d122 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -3,8 +3,9 @@ namespace Appwrite\Event; use InvalidArgumentException; -use Resque; use Utopia\Database\Document; +use Utopia\Queue\Client; +use Utopia\Queue\Connection; class Event { @@ -52,14 +53,11 @@ class Event protected bool $paused = false; /** - * @param string $queue - * @param string $class + * @param Connection $connection * @return void */ - public function __construct(string $queue, string $class) + public function __construct(protected Connection $connection) { - $this->queue = $queue; - $this->class = $class; } /** @@ -271,7 +269,9 @@ class Event return false; } - return Resque::enqueue($this->queue, $this->class, [ + $client = new Client($this->queue, $this->connection); + + return $client->enqueue([ 'project' => $this->project, 'user' => $this->user, 'payload' => $this->payload, diff --git a/src/Appwrite/Event/Func.php b/src/Appwrite/Event/Func.php index b8c81cdc2d..11c9e980ed 100644 --- a/src/Appwrite/Event/Func.php +++ b/src/Appwrite/Event/Func.php @@ -19,7 +19,11 @@ class Func extends Event public function __construct(protected Connection $connection) { - parent::__construct(Event::FUNCTIONS_QUEUE_NAME, Event::FUNCTIONS_CLASS_NAME); + parent::__construct($connection); + + $this + ->setQueue(Event::FUNCTIONS_QUEUE_NAME) + ->setClass(Event::FUNCTIONS_CLASS_NAME); } /** diff --git a/src/Appwrite/Event/Mail.php b/src/Appwrite/Event/Mail.php index e1c015fb2b..c2de8023b0 100644 --- a/src/Appwrite/Event/Mail.php +++ b/src/Appwrite/Event/Mail.php @@ -2,8 +2,8 @@ namespace Appwrite\Event; -use Resque; -use Utopia\Database\Document; +use Utopia\Queue\Client; +use Utopia\Queue\Connection; class Mail extends Event { @@ -14,9 +14,13 @@ class Mail extends Event protected array $smtp = []; protected array $variables = []; - public function __construct() + public function __construct(protected Connection $connection) { - parent::__construct(Event::MAILS_QUEUE_NAME, Event::MAILS_CLASS_NAME); + parent::__construct($connection); + + $this + ->setQueue(Event::MAILS_QUEUE_NAME) + ->setClass(Event::MAILS_CLASS_NAME); } /** @@ -317,7 +321,9 @@ class Mail extends Event */ public function trigger(): string|bool { - return Resque::enqueue($this->queue, $this->class, [ + $client = new Client($this->queue, $this->connection); + + return $client->enqueue([ 'recipient' => $this->recipient, 'name' => $this->name, 'subject' => $this->subject, diff --git a/src/Appwrite/Event/Migration.php b/src/Appwrite/Event/Migration.php index 4d53f16796..478291829b 100644 --- a/src/Appwrite/Event/Migration.php +++ b/src/Appwrite/Event/Migration.php @@ -2,19 +2,22 @@ namespace Appwrite\Event; -use DateTime; -use Resque; -use ResqueScheduler; use Utopia\Database\Document; +use Utopia\Queue\Client; +use Utopia\Queue\Connection; class Migration extends Event { protected string $type = ''; protected ?Document $migration = null; - public function __construct() + public function __construct(protected Connection $connection) { - parent::__construct(Event::MIGRATIONS_QUEUE_NAME, Event::MIGRATIONS_CLASS_NAME); + parent::__construct($connection); + + $this + ->setQueue(Event::MIGRATIONS_QUEUE_NAME) + ->setClass(Event::MIGRATIONS_CLASS_NAME); } /** @@ -72,24 +75,10 @@ class Migration extends Event */ public function trigger(): string|bool { - return Resque::enqueue($this->queue, $this->class, [ - 'project' => $this->project, - 'user' => $this->user, - 'migration' => $this->migration - ]); - } - /** - * Schedules the migration event and schedules it in the migrations worker queue. - * - * @param \DateTime|int $at - * @return void - * @throws \Resque_Exception - * @throws \ResqueScheduler_InvalidTimestampException - */ - public function schedule(DateTime|int $at): void - { - ResqueScheduler::enqueueAt($at, $this->queue, $this->class, [ + $client = new Client($this->queue, $this->connection); + + return $client->enqueue([ 'project' => $this->project, 'user' => $this->user, 'migration' => $this->migration diff --git a/src/Appwrite/Event/Phone.php b/src/Appwrite/Event/Phone.php index 8baa5120c9..45f193a540 100644 --- a/src/Appwrite/Event/Phone.php +++ b/src/Appwrite/Event/Phone.php @@ -2,16 +2,21 @@ namespace Appwrite\Event; -use Resque; +use Utopia\Queue\Client; +use Utopia\Queue\Connection; class Phone extends Event { protected string $recipient = ''; protected string $message = ''; - public function __construct() + public function __construct(protected Connection $connection) { - parent::__construct(Event::MESSAGING_QUEUE_NAME, Event::MESSAGING_CLASS_NAME); + parent::__construct($connection); + + $this + ->setQueue(Event::MESSAGING_QUEUE_NAME) + ->setClass(Event::MESSAGING_CLASS_NAME); } /** @@ -68,7 +73,9 @@ class Phone extends Event */ public function trigger(): string|bool { - return Resque::enqueue($this->queue, $this->class, [ + $client = new Client($this->queue, $this->connection); + + return $client->enqueue([ 'project' => $this->project, 'user' => $this->user, 'payload' => $this->payload, diff --git a/src/Appwrite/Event/Usage.php b/src/Appwrite/Event/Usage.php index b302b88808..398c3319f2 100644 --- a/src/Appwrite/Event/Usage.php +++ b/src/Appwrite/Event/Usage.php @@ -13,7 +13,11 @@ class Usage extends Event public function __construct(protected Connection $connection) { - parent::__construct(Event::USAGE_QUEUE_NAME, Event::USAGE_CLASS_NAME); + parent::__construct($connection); + + $this + ->setQueue(Event::USAGE_QUEUE_NAME) + ->setClass(Event::USAGE_CLASS_NAME); } /** diff --git a/src/Appwrite/Platform/Appwrite.php b/src/Appwrite/Platform/Appwrite.php index 9ef6f38c36..05224799f3 100644 --- a/src/Appwrite/Platform/Appwrite.php +++ b/src/Appwrite/Platform/Appwrite.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform; use Appwrite\Platform\Services\Tasks; +use Appwrite\Platform\Services\Workers; use Utopia\Platform\Platform; class Appwrite extends Platform @@ -10,5 +11,6 @@ class Appwrite extends Platform public function __construct() { $this->addService('tasks', new Tasks()); + $this->addService('workers', new Workers()); } } diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index bc8d1bbc72..00779084d4 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -8,20 +8,15 @@ use Appwrite\Platform\Tasks\Install; use Appwrite\Platform\Tasks\Maintenance; use Appwrite\Platform\Tasks\Migrate; use Appwrite\Platform\Tasks\Schedule; -use Appwrite\Platform\Tasks\PatchCreateMissingSchedules; use Appwrite\Platform\Tasks\SDKs; use Appwrite\Platform\Tasks\Specs; use Appwrite\Platform\Tasks\SSL; use Appwrite\Platform\Tasks\Hamster; -use Appwrite\Platform\Tasks\PatchDeleteScheduleUpdatedAtAttribute; -use Appwrite\Platform\Tasks\ClearCardCache; use Appwrite\Platform\Tasks\Usage; use Appwrite\Platform\Tasks\Vars; use Appwrite\Platform\Tasks\Version; use Appwrite\Platform\Tasks\VolumeSync; -use Appwrite\Platform\Tasks\CalcUsersStats; use Appwrite\Platform\Tasks\CalcTierStats; -use Appwrite\Platform\Tasks\PatchDeleteProjectCollections; use Appwrite\Platform\Tasks\Upgrade; class Tasks extends Service @@ -39,17 +34,12 @@ class Tasks extends Service ->addAction(Install::getName(), new Install()) ->addAction(Upgrade::getName(), new Upgrade()) ->addAction(Maintenance::getName(), new Maintenance()) - ->addAction(PatchCreateMissingSchedules::getName(), new PatchCreateMissingSchedules()) - ->addAction(ClearCardCache::getName(), new ClearCardCache()) - ->addAction(PatchDeleteScheduleUpdatedAtAttribute::getName(), new PatchDeleteScheduleUpdatedAtAttribute()) ->addAction(Schedule::getName(), new Schedule()) ->addAction(Migrate::getName(), new Migrate()) ->addAction(SDKs::getName(), new SDKs()) ->addAction(VolumeSync::getName(), new VolumeSync()) ->addAction(Specs::getName(), new Specs()) - ->addAction(CalcUsersStats::getName(), new CalcUsersStats()) ->addAction(CalcTierStats::getName(), new CalcTierStats()) - ->addAction(PatchDeleteProjectCollections::getName(), new PatchDeleteProjectCollections()) ; } } diff --git a/src/Appwrite/Platform/Services/Workers.php b/src/Appwrite/Platform/Services/Workers.php new file mode 100644 index 0000000000..07fc25434e --- /dev/null +++ b/src/Appwrite/Platform/Services/Workers.php @@ -0,0 +1,36 @@ +type = self::TYPE_WORKER; + $this + ->addAction(Audits::getName(), new Audits()) + ->addAction(Webhooks::getName(), new Webhooks()) + ->addAction(Mails::getName(), new Mails()) + ->addAction(Messaging::getName(), new Messaging()) + ->addAction(Certificates::getName(), new Certificates()) + ->addAction(Databases::getName(), new Databases()) + ->addAction(Functions::getName(), new Functions()) + ->addAction(Builds::getName(), new Builds()) + ->addAction(Deletes::getName(), new Deletes()) + ->addAction(Migrations::getName(), new Migrations()) + + ; + } +} diff --git a/src/Appwrite/Platform/Tasks/CalcTierStats.php b/src/Appwrite/Platform/Tasks/CalcTierStats.php index 1678c6c621..2a2bc20af9 100644 --- a/src/Appwrite/Platform/Tasks/CalcTierStats.php +++ b/src/Appwrite/Platform/Tasks/CalcTierStats.php @@ -155,12 +155,12 @@ class CalcTierStats extends Action } /** Get Usage stats */ - $range = '90d'; + $range = '30d'; $periods = [ - '90d' => [ + '30d' => [ 'period' => '1d', - 'limit' => 90, - ], + 'limit' => 30, + ] ]; $tmp = []; diff --git a/src/Appwrite/Platform/Tasks/CalcUsersStats.php b/src/Appwrite/Platform/Tasks/CalcUsersStats.php deleted file mode 100644 index 6310fe17b4..0000000000 --- a/src/Appwrite/Platform/Tasks/CalcUsersStats.php +++ /dev/null @@ -1,176 +0,0 @@ -desc('Get stats for projects') - ->inject('pools') - ->inject('cache') - ->inject('dbForConsole') - ->inject('register') - ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { - $this->action($pools, $cache, $dbForConsole, $register); - }); - } - - public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void - { - //docker compose exec -t appwrite calc-users-stats - - Console::title('Cloud Users calculation V1'); - Console::success(APP_NAME . ' cloud Users calculation has started'); - - /* Initialise new Utopia app */ - $app = new App('UTC'); - $console = $app->getResource('console'); - - /** CSV stuff */ - $this->date = date('Y-m-d'); - $this->path = "{$this->directory}/users_stats_{$this->date}.csv"; - $csv = Writer::createFromPath($this->path, 'w'); - $csv->insertOne($this->columns); - - /** Database connections */ - $totalProjects = $dbForConsole->count('projects'); - Console::success("Found a total of: {$totalProjects} projects"); - - $projects = [$console]; - $count = 0; - $limit = 30; - $sum = 30; - $offset = 0; - while (!empty($projects)) { - foreach ($projects as $project) { - - /** - * Skip user projects with id 'console' - */ - if ($project->getId() === 'console') { - continue; - } - - Console::info("Getting stats for {$project->getId()}"); - - try { - $db = $project->getAttribute('database'); - $adapter = $pools - ->get($db) - ->pop() - ->getResource(); - - $dbForProject = new Database($adapter, $cache); - $dbForProject->setDefaultDatabase('appwrite'); - $dbForProject->setNamespace('_' . $project->getInternalId()); - - /** Get Project ID */ - $stats['Project ID'] = $project->getId(); - - /** Get Project Name */ - $stats['Project Name'] = $project->getAttribute('name'); - - - /** Get Team Name and Id */ - $teamId = $project->getAttribute('teamId', null); - $teamName = null; - if ($teamId) { - $team = $dbForConsole->getDocument('teams', $teamId); - $teamName = $team->getAttribute('name'); - } - - $stats['Team ID'] = $teamId; - $stats['Team name'] = $teamName; - - /** Get Total Users */ - $stats['users'] = $dbForProject->count('users', []); - - $csv->insertOne(array_values($stats)); - } catch (\Throwable $th) { - Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); - } finally { - $pools - ->get($db) - ->reclaim(); - } - } - - $sum = \count($projects); - - $projects = $dbForConsole->find('projects', [ - Query::limit($limit), - Query::offset($offset), - ]); - - $offset = $offset + $limit; - $count = $count + $sum; - } - Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...'); - $pools - ->get('console') - ->reclaim(); - - /** @var PHPMailer $mail */ - $mail = $register->get('smtp'); - - $mail->clearAddresses(); - $mail->clearAllRecipients(); - $mail->clearReplyTos(); - $mail->clearAttachments(); - $mail->clearBCCs(); - $mail->clearCCs(); - - try { - /** Addresses */ - $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster'); - $recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', '')); - - foreach ($recipients as $recipient) { - $mail->addAddress($recipient); - } - - /** Attachments */ - $mail->addAttachment($this->path); - - /** Content */ - $mail->Subject = "Cloud Report for {$this->date}"; - $mail->Body = "Please find the daily cloud report atttached"; - $mail->send(); - Console::success('Email has been sent!'); - } catch (Exception $e) { - Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}"); - } - } -} diff --git a/src/Appwrite/Platform/Tasks/ClearCardCache.php b/src/Appwrite/Platform/Tasks/ClearCardCache.php deleted file mode 100644 index d3153b995c..0000000000 --- a/src/Appwrite/Platform/Tasks/ClearCardCache.php +++ /dev/null @@ -1,62 +0,0 @@ -desc('Deletes card cache for specific user') - ->param('userId', '', new UID(), 'User UID.', false) - ->inject('dbForConsole') - ->callback(fn (string $userId, Database $dbForConsole) => $this->action($userId, $dbForConsole)); - } - - public function action(string $userId, Database $dbForConsole): void - { - Authorization::disable(); - Authorization::setDefaultStatus(false); - - Console::title('ClearCardCache V1'); - Console::success(APP_NAME . ' ClearCardCache v1 has started'); - $resources = ['card/' . $userId, 'card-back/' . $userId, 'card-og/' . $userId]; - - $caches = Authorization::skip(fn () => $dbForConsole->find('cache', [ - Query::equal('resource', $resources), - Query::limit(100) - ])); - - $count = \count($caches); - Console::info("Going to delete {$count} cache records in 10 seconds..."); - \sleep(10); - - foreach ($caches as $cache) { - $key = $cache->getId(); - - $cacheFolder = new Cache( - new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-console') - ); - - $cacheFolder->purge($key); - - Authorization::skip(fn () => $dbForConsole->deleteDocument('cache', $cache->getId())); - } - - Console::success(APP_NAME . ' ClearCardCache v1 has finished'); - } -} diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index eb98b1bb3b..82a62ffed1 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -22,129 +22,131 @@ class Maintenance extends Action public function __construct() { $this - ->desc('Schedules maintenance tasks and publishes them to resque') + ->desc('Schedules maintenance tasks and publishes them to our queues') ->inject('dbForConsole') - ->callback(fn (Database $dbForConsole) => $this->action($dbForConsole)); + ->inject('queueForCertificates') + ->inject('queueForDeletes') + ->callback(fn (Database $dbForConsole, Certificate $queueForCertificates, Delete $queueForDeletes) => $this->action($dbForConsole, $queueForCertificates, $queueForDeletes)); } - public function action(Database $dbForConsole): void + public function action(Database $dbForConsole, Certificate $queueForCertificates, Delete $queueForDeletes): void { Console::title('Maintenance V1'); Console::success(APP_NAME . ' maintenance process v1 has started'); - function notifyDeleteExecutionLogs(int $interval) - { - (new Delete()) - ->setType(DELETE_TYPE_EXECUTIONS) - ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval)) - ->trigger(); - } - - function notifyDeleteAbuseLogs(int $interval) - { - (new Delete()) - ->setType(DELETE_TYPE_ABUSE) - ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval)) - ->trigger(); - } - - function notifyDeleteAuditLogs(int $interval) - { - (new Delete()) - ->setType(DELETE_TYPE_AUDIT) - ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval)) - ->trigger(); - } - - function notifyDeleteUsageStats(int $usageStatsRetentionHourly) - { - (new Delete()) - ->setType(DELETE_TYPE_USAGE) - ->setUsageRetentionHourlyDateTime(DateTime::addSeconds(new \DateTime(), -1 * $usageStatsRetentionHourly)) - ->trigger(); - } - - function notifyDeleteConnections() - { - (new Delete()) - ->setType(DELETE_TYPE_REALTIME) - ->setDatetime(DateTime::addSeconds(new \DateTime(), -60)) - ->trigger(); - } - - function notifyDeleteExpiredSessions() - { - (new Delete()) - ->setType(DELETE_TYPE_SESSIONS) - ->trigger(); - } - - function renewCertificates($dbForConsole) - { - $time = DateTime::now(); - - $certificates = $dbForConsole->find('certificates', [ - Query::lessThan('attempts', 5), // Maximum 5 attempts - Query::lessThanEqual('renewDate', $time), // includes 60 days cooldown (we have 30 days to renew) - Query::limit(200), // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains) - ]); - - - if (\count($certificates) > 0) { - Console::info("[{$time}] Found " . \count($certificates) . " certificates for renewal, scheduling jobs."); - - $event = new Certificate(); - foreach ($certificates as $certificate) { - $event - ->setDomain(new Document([ - 'domain' => $certificate->getAttribute('domain') - ])) - ->trigger(); - } - } else { - Console::info("[{$time}] No certificates for renewal."); - } - } - - function notifyDeleteCache($interval) - { - (new Delete()) - ->setType(DELETE_TYPE_CACHE_BY_TIMESTAMP) - ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval)) - ->trigger(); - } - - function notifyDeleteSchedules($interval) - { - (new Delete()) - ->setType(DELETE_TYPE_SCHEDULES) - ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval)) - ->trigger(); - } - // # of days in seconds (1 day = 86400s) $interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400'); $executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600'); $auditLogRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', '1209600'); $abuseLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', '86400'); $usageStatsRetentionHourly = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_HOURLY', '8640000'); //100 days - $cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days $schedulesDeletionRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_SCHEDULES', '86400'); // 1 Day - Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForConsole) { + Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForConsole, $queueForDeletes, $queueForCertificates) { $time = DateTime::now(); Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds"); - notifyDeleteExecutionLogs($executionLogsRetention); - notifyDeleteAbuseLogs($abuseLogsRetention); - notifyDeleteAuditLogs($auditLogRetention); - notifyDeleteUsageStats($usageStatsRetentionHourly); - notifyDeleteConnections(); - notifyDeleteExpiredSessions(); - renewCertificates($dbForConsole); - notifyDeleteCache($cacheRetention); - notifyDeleteSchedules($schedulesDeletionRetention); + $this->notifyDeleteExecutionLogs($executionLogsRetention, $queueForDeletes); + $this->notifyDeleteAbuseLogs($abuseLogsRetention, $queueForDeletes); + $this->notifyDeleteAuditLogs($auditLogRetention, $queueForDeletes); + $this->notifyDeleteUsageStats($usageStatsRetentionHourly, $queueForDeletes); + $this->notifyDeleteConnections($queueForDeletes); + $this->notifyDeleteExpiredSessions($queueForDeletes); + $this->renewCertificates($dbForConsole, $queueForCertificates); + $this->notifyDeleteCache($cacheRetention, $queueForDeletes); + $this->notifyDeleteSchedules($schedulesDeletionRetention, $queueForDeletes); }, $interval); } + + private function notifyDeleteExecutionLogs(int $interval, Delete $queueForDeletes): void + { + ($queueForDeletes) + ->setType(DELETE_TYPE_EXECUTIONS) + ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval)) + ->trigger(); + } + + private function notifyDeleteAbuseLogs(int $interval, Delete $queueForDeletes): void + { + ($queueForDeletes) + ->setType(DELETE_TYPE_ABUSE) + ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval)) + ->trigger(); + } + + private function notifyDeleteAuditLogs(int $interval, Delete $queueForDeletes): void + { + ($queueForDeletes) + ->setType(DELETE_TYPE_AUDIT) + ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval)) + ->trigger(); + } + + private function notifyDeleteUsageStats(int $usageStatsRetentionHourly, Delete $queueForDeletes): void + { + ($queueForDeletes) + ->setType(DELETE_TYPE_USAGE) + ->setUsageRetentionHourlyDateTime(DateTime::addSeconds(new \DateTime(), -1 * $usageStatsRetentionHourly)) + ->trigger(); + } + + private function notifyDeleteConnections(Delete $queueForDeletes): void + { + ($queueForDeletes) + ->setType(DELETE_TYPE_REALTIME) + ->setDatetime(DateTime::addSeconds(new \DateTime(), -60)) + ->trigger(); + } + + private function notifyDeleteExpiredSessions(Delete $queueForDeletes): void + { + ($queueForDeletes) + ->setType(DELETE_TYPE_SESSIONS) + ->trigger(); + } + + private function renewCertificates(Database $dbForConsole, Certificate $queueForCertificate): void + { + $time = DateTime::now(); + + $certificates = $dbForConsole->find('certificates', [ + Query::lessThan('attempts', 5), // Maximum 5 attempts + Query::lessThanEqual('renewDate', $time), // includes 60 days cooldown (we have 30 days to renew) + Query::limit(200), // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains) + ]); + + + if (\count($certificates) > 0) { + Console::info("[{$time}] Found " . \count($certificates) . " certificates for renewal, scheduling jobs."); + + foreach ($certificates as $certificate) { + $queueForCertificate + ->setDomain(new Document([ + 'domain' => $certificate->getAttribute('domain') + ])) + ->trigger(); + } + } else { + Console::info("[{$time}] No certificates for renewal."); + } + } + + private function notifyDeleteCache($interval, Delete $queueForDeletes): void + { + + ($queueForDeletes) + ->setType(DELETE_TYPE_CACHE_BY_TIMESTAMP) + ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval)) + ->trigger(); + } + + private function notifyDeleteSchedules($interval, Delete $queueForDeletes): void + { + + ($queueForDeletes) + ->setType(DELETE_TYPE_SCHEDULES) + ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval)) + ->trigger(); + } } diff --git a/src/Appwrite/Platform/Tasks/PatchCreateMissingSchedules.php b/src/Appwrite/Platform/Tasks/PatchCreateMissingSchedules.php deleted file mode 100644 index 74ef644498..0000000000 --- a/src/Appwrite/Platform/Tasks/PatchCreateMissingSchedules.php +++ /dev/null @@ -1,97 +0,0 @@ -desc('Ensure every function has a schedule') - ->inject('dbForConsole') - ->inject('getProjectDB') - ->callback(fn (Database $dbForConsole, callable $getProjectDB) => $this->action($dbForConsole, $getProjectDB)); - } - - /** - * Iterate over every function on every project to make sure there is a schedule. If not, recreate the schedule. - */ - public function action(Database $dbForConsole, callable $getProjectDB): void - { - Authorization::disable(); - Authorization::setDefaultStatus(false); - - Console::title('PatchCreateMissingSchedules V1'); - Console::success(APP_NAME . ' PatchCreateMissingSchedules v1 has started'); - - $limit = 100; - $projectCursor = null; - while (true) { - $projectsQueries = [Query::limit($limit)]; - if ($projectCursor !== null) { - $projectsQueries[] = Query::cursorAfter($projectCursor); - } - $projects = $dbForConsole->find('projects', $projectsQueries); - - if (count($projects) === 0) { - break; - } - - foreach ($projects as $project) { - Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getId() . ")"); - $dbForProject = $getProjectDB($project); - $functionCursor = null; - - while (true) { - $functionsQueries = [Query::limit($limit)]; - if ($functionCursor !== null) { - $functionsQueries[] = Query::cursorAfter($functionCursor); - } - $functions = $dbForProject->find('functions', $functionsQueries); - if (count($functions) === 0) { - break; - } - - foreach ($functions as $function) { - $scheduleId = $function->getAttribute('scheduleId'); - $schedule = $dbForConsole->getDocument('schedules', $scheduleId); - - if ($schedule->isEmpty()) { - $functionId = $function->getId(); - $schedule = $dbForConsole->createDocument('schedules', new Document([ - '$id' => ID::custom($scheduleId), - 'region' => $project->getAttribute('region', 'default'), - 'resourceType' => 'function', - 'resourceId' => $functionId, - 'resourceUpdatedAt' => DateTime::now(), - 'projectId' => $project->getId(), - 'schedule' => $function->getAttribute('schedule'), - 'active' => !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')), - ])); - - Console::success('Recreated schedule for function ' . $functionId); - } - } - - $functionCursor = $functions[array_key_last($functions)]; - } - } - - $projectCursor = $projects[array_key_last($projects)]; - } - } -} diff --git a/src/Appwrite/Platform/Tasks/PatchDeleteProjectCollections.php b/src/Appwrite/Platform/Tasks/PatchDeleteProjectCollections.php deleted file mode 100644 index a909e68595..0000000000 --- a/src/Appwrite/Platform/Tasks/PatchDeleteProjectCollections.php +++ /dev/null @@ -1,129 +0,0 @@ -desc('Delete unnecessary project collections') - ->param('offset', 0, new Numeric(), 'Resume deletion from param pos', true) - ->inject('pools') - ->inject('cache') - ->inject('dbForConsole') - ->callback(function (int $offset, Group $pools, Cache $cache, Database $dbForConsole) { - $this->action($offset, $pools, $cache, $dbForConsole); - }); - } - - public function action(int $offset, Group $pools, Cache $cache, Database $dbForConsole): void - { - //docker compose exec -t appwrite patch-delete-project-collections - - Console::title('Delete project collections V1'); - Console::success(APP_NAME . ' delete project collections has started'); - - /* Initialise new Utopia app */ - $app = new App('UTC'); - $console = $app->getResource('console'); - - /** Database connections */ - $totalProjects = $dbForConsole->count('projects'); - Console::success("Found a total of: {$totalProjects} projects"); - - $projects = [$console]; - $count = 0; - $limit = 50; - $sum = 50; - $offset = $offset; - while (!empty($projects)) { - foreach ($projects as $project) { - - /** - * Skip user projects with id 'console' - */ - if ($project->getId() === 'console') { - continue; - } - - Console::info("Deleting collections for {$project->getId()}"); - - try { - $db = $project->getAttribute('database'); - $adapter = $pools - ->get($db) - ->pop() - ->getResource(); - - $dbForProject = new Database($adapter, $cache); - $dbForProject->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $dbForProject->setNamespace('_' . $project->getInternalId()); - - foreach ($this->names as $name) { - if (empty($name)) { - continue; - } - if ($dbForProject->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), $name)) { - if ($dbForProject->deleteCollection($name)) { - Console::log('Deleted ' . $name); - } else { - Console::error('Failed to delete ' . $name); - } - } - } - } catch (\Throwable $th) { - Console::error('Failed on project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); - } finally { - $pools - ->get($db) - ->reclaim(); - } - } - - $sum = \count($projects); - - $projects = $dbForConsole->find('projects', [ - Query::limit($limit), - Query::offset($offset), - ]); - - if (!empty($projects)) { - Console::log('Querying..... offset=' . $offset . ' , limit=' . $limit . ', count=' . $count); - } - - $offset = $offset + $limit; - $count = $count + $sum; - } - Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...'); - $pools - ->get('console') - ->reclaim(); - } -} diff --git a/src/Appwrite/Platform/Tasks/PatchDeleteScheduleUpdatedAtAttribute.php b/src/Appwrite/Platform/Tasks/PatchDeleteScheduleUpdatedAtAttribute.php deleted file mode 100644 index 95a7c4ffe1..0000000000 --- a/src/Appwrite/Platform/Tasks/PatchDeleteScheduleUpdatedAtAttribute.php +++ /dev/null @@ -1,74 +0,0 @@ -desc('Ensure function collections do not have scheduleUpdatedAt attribute') - ->inject('pools') - ->inject('dbForConsole') - ->inject('getProjectDB') - ->callback(fn (Group $pools, Database $dbForConsole, callable $getProjectDB) => $this->action($pools, $dbForConsole, $getProjectDB)); - } - - /** - * Iterate over every function on every project to make sure there is a schedule. If not, recreate the schedule. - */ - public function action(Group $pools, Database $dbForConsole, callable $getProjectDB): void - { - Authorization::disable(); - Authorization::setDefaultStatus(false); - - Console::title('PatchDeleteScheduleUpdatedAtAttribute V1'); - Console::success(APP_NAME . ' PatchDeleteScheduleUpdatedAtAttribute v1 has started'); - - $limit = 100; - $projectCursor = null; - while (true) { - $projectsQueries = [Query::limit($limit)]; - if ($projectCursor !== null) { - $projectsQueries[] = Query::cursorAfter($projectCursor); - } - $projects = $dbForConsole->find('projects', $projectsQueries); - - if (count($projects) === 0) { - break; - } - - foreach ($projects as $project) { - Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getId() . ")"); - $dbForProject = $getProjectDB($project); - - try { - /** - * Delete 'scheduleUpdatedAt' attribute - */ - $dbForProject->deleteAttribute('functions', 'scheduleUpdatedAt'); - $dbForProject->deleteCachedCollection('functions'); - Console::success("'scheduleUpdatedAt' deleted."); - } catch (\Throwable $th) { - Console::warning("'scheduleUpdatedAt' errored: {$th->getMessage()}"); - } - - $pools->reclaim(); - } - - $projectCursor = $projects[array_key_last($projects)]; - } - } -} diff --git a/src/Appwrite/Platform/Tasks/SSL.php b/src/Appwrite/Platform/Tasks/SSL.php index 43026b0753..6dbf4dcd70 100644 --- a/src/Appwrite/Platform/Tasks/SSL.php +++ b/src/Appwrite/Platform/Tasks/SSL.php @@ -21,14 +21,15 @@ class SSL extends Action $this ->desc('Validate server certificates') ->param('domain', App::getEnv('_APP_DOMAIN', ''), new Hostname(), 'Domain to generate certificate for. If empty, main domain will be used.', true) - ->callback(fn ($domain) => $this->action($domain)); + ->inject('queueForCertificates') + ->callback(fn (string $domain, Certificate $queueForCertificates) => $this->action($domain, $queueForCertificates)); } - public function action(string $domain): void + public function action(string $domain, Certificate $queueForCertificates): void { Console::success('Scheduling a job to issue a TLS certificate for domain: ' . $domain); - (new Certificate()) + $queueForCertificates ->setDomain(new Document([ 'domain' => $domain ])) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php new file mode 100644 index 0000000000..53c16d8eb3 --- /dev/null +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -0,0 +1,82 @@ +desc('Audits worker') + ->inject('message') + ->inject('dbForProject') + ->callback(fn ($message, $dbForProject) => $this->action($message, $dbForProject)); + } + + + /** + * @param Message $message + * @param Database $dbForProject + * @return void + * @throws Throwable + * @throws \Utopia\Database\Exception + * @throws Authorization + * @throws Structure + */ + public function action(Message $message, Database $dbForProject): void + { + + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + $event = $payload['event'] ?? ''; + $auditPayload = $payload['payload'] ?? ''; + $mode = $payload['mode'] ?? ''; + $resource = $payload['resource'] ?? ''; + $userAgent = $payload['userAgent'] ?? ''; + $ip = $payload['ip'] ?? ''; + $user = new Document($payload['user'] ?? []); + + $userName = $user->getAttribute('name', ''); + $userEmail = $user->getAttribute('email', ''); + + $audit = new Audit($dbForProject); + $audit->log( + userId: $user->getInternalId(), + // Pass first, most verbose event pattern + event: $event, + resource: $resource, + userAgent: $userAgent, + ip: $ip, + location: '', + data: [ + 'userId' => $user->getId(), + 'userName' => $userName, + 'userEmail' => $userEmail, + 'mode' => $mode, + 'data' => $auditPayload, + ] + ); + } +} diff --git a/app/workers/builds.php b/src/Appwrite/Platform/Workers/Builds.php similarity index 77% rename from app/workers/builds.php rename to src/Appwrite/Platform/Workers/Builds.php index f3eb1fb972..f1ef0df602 100644 --- a/app/workers/builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -1,81 +1,118 @@ executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); + $this + ->desc('Builds worker') + ->inject('message') + ->inject('dbForConsole') + ->inject('queueForEvents') + ->inject('queueForFunctions') + ->inject('usage') + ->inject('cache') + ->inject('dbForProject') + ->inject('getFunctionsDevice') + ->callback(fn($message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Stats $usage, Cache $cache, Database $dbForProject, callable $getFunctionsDevice) => $this->action($message, $dbForConsole, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $getFunctionsDevice)); } - public function run(): void + /** + * @param Message $message + * @param Database $dbForConsole + * @param Event $queueForEvents + * @param Func $queueForFunctions + * @param Stats $usage + * @param Cache $cache + * @param Database $dbForProject + * @param callable $getFunctionsDevice + * @return void + * @throws \Utopia\Database\Exception + */ + public function action(Message $message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Stats $usage, Cache $cache, Database $dbForProject, callable $getFunctionsDevice): void { - $type = $this->args['type'] ?? ''; - $project = new Document($this->args['project'] ?? []); - $resource = new Document($this->args['resource'] ?? []); - $deployment = new Document($this->args['deployment'] ?? []); - $template = new Document($this->args['template'] ?? []); + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + $type = $payload['type'] ?? ''; + $project = new Document($payload['project'] ?? []); + $resource = new Document($payload['resource'] ?? []); + $deployment = new Document($payload['deployment'] ?? []); + $template = new Document($payload['template'] ?? []); switch ($type) { case BUILD_TYPE_DEPLOYMENT: case BUILD_TYPE_RETRY: Console::info('Creating build for deployment: ' . $deployment->getId()); - $github = new GitHub($this->getCache()); - $this->buildDeployment($github, $project, $resource, $deployment, $template); + $github = new GitHub($cache); + $this->buildDeployment($getFunctionsDevice, $queueForFunctions, $queueForEvents, $usage, $dbForConsole, $dbForProject, $github, $project, $resource, $deployment, $template); break; default: throw new \Exception('Invalid build type'); - break; } } /** - * @throws \Utopia\Database\Exception\Authorization - * @throws \Utopia\Database\Exception\Structure - * @throws Throwable + * @param callable $getFunctionsDevice + * @param Func $queueForFunctions + * @param Event $queueForEvents + * @param Stats $usage + * @param Database $dbForConsole + * @param Database $dbForProject + * @param GitHub $github + * @param Document $project + * @param Document $function + * @param Document $deployment + * @param Document $template + * @return void + * @throws \Utopia\Database\Exception + * @throws Exception */ - protected function buildDeployment(GitHub $github, Document $project, Document $function, Document $deployment, Document $template) + protected function buildDeployment(callable $getFunctionsDevice, Func $queueForFunctions, Event $queueForEvents, Stats $usage, Database $dbForConsole, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template): void { - global $register; - - $dbForProject = $this->getProjectDB($project); - $dbForConsole = $this->getConsoleDB(); + $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); $function = $dbForProject->getDocument('functions', $function->getId()); if ($function->isEmpty()) { @@ -94,7 +131,7 @@ class BuildsV1 extends Worker $version = $function->getAttribute('version', 'v2'); $runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []); $key = $function->getAttribute('runtime'); - $runtime = isset($runtimes[$key]) ? $runtimes[$key] : null; + $runtime = $runtimes[$key] ?? null; if (\is_null($runtime)) { throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); } @@ -107,12 +144,9 @@ class BuildsV1 extends Worker $startTime = DateTime::now(); $durationStart = \microtime(true); - $buildId = $deployment->getAttribute('buildId', ''); - $isNewBuild = empty($buildId); - - $deviceFunctions = $this->getFunctionsDevice($project->getId()); + $deviceFunctions = $getFunctionsDevice($project->getId()); if ($isNewBuild) { $buildId = ID::unique(); @@ -188,7 +222,7 @@ class BuildsV1 extends Worker $templateOwnerName = $template->getAttribute('ownerName', ''); $templateBranch = $template->getAttribute('branch', ''); - $templateRootDirectory = $template->getAttribute('rootDirectory', ''); + $templateRootDirectory = $template->getAttribute('rootDirectory', ''); $templateRootDirectory = \rtrim($templateRootDirectory, '/'); $templateRootDirectory = \ltrim($templateRootDirectory, '.'); $templateRootDirectory = \ltrim($templateRootDirectory, '/'); @@ -211,7 +245,7 @@ class BuildsV1 extends Worker Console::execute('cp -rfn ' . $tmpTemplateDirectory . '/' . $templateRootDirectory . '/* ' . $tmpDirectory . '/' . $rootDirectory, '', $stdout, $stderr); // Commit and push - $exit = Console::execute('git config --global user.email "team@appwrite.io" && git config --global user.name "Appwrite" && cd ' . $tmpDirectory . ' && git add . && git commit -m "Create \'' . \escapeshellcmd($function->getAttribute('name', '')) . '\' function" && git push origin ' . \escapeshellcmd($branchName), '', $stdout, $stderr); + $exit = Console::execute('git config --global user.email "team@appwrite.io" && git config --global user.name "Appwrite" && cd ' . $tmpDirectory . ' && git add . && git commit -m "Create \'' . \escapeshellcmd($function->getAttribute('name', '')) . '\' function" && git push origin ' . \escapeshellcmd($branchName), '', $stdout, $stderr); if ($exit !== 0) { throw new \Exception('Unable to push code repository: ' . $stderr); @@ -229,7 +263,7 @@ class BuildsV1 extends Worker $deployment->setAttribute('providerCommitHash', $providerCommitHash ?? ''); $deployment->setAttribute('providerCommitAuthorUrl', $authorUrl); $deployment->setAttribute('providerCommitAuthor', 'Appwrite'); - $deployment->setAttribute('providerCommitMessage', "Create '" . $function->getAttribute('name', '') . "' function"); + $deployment->setAttribute('providerCommitMessage', "Create '" . $function->getAttribute('name', '') . "' function"); $deployment->setAttribute('providerCommitUrl', "https://github.com/$cloneOwner/$cloneRepository/commit/$providerCommitHash"); $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); @@ -237,7 +271,7 @@ class BuildsV1 extends Worker * Send realtime Event */ $target = Realtime::fromPayload( - // Pass first, most verbose event pattern + // Pass first, most verbose event pattern event: $allEvents[0], payload: $build, project: $project @@ -267,7 +301,7 @@ class BuildsV1 extends Worker Console::execute('tar --exclude code.tar.gz -czf ' . $tmpPathFile . ' -C /tmp/builds/' . \escapeshellcmd($buildId) . '/code' . (empty($rootDirectory) ? '' : '/' . $rootDirectory) . ' .', '', $stdout, $stderr); - $deviceFunctions = $this->getFunctionsDevice($project->getId()); + $deviceFunctions = $getFunctionsDevice($project->getId()); $path = $deviceFunctions->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); $result = $localDevice->transfer($tmpPathFile, $path, $deviceFunctions); @@ -295,30 +329,27 @@ class BuildsV1 extends Worker /** Trigger Webhook */ $deploymentModel = new Deployment(); + $deploymentUpdate = + $queueForEvents + ->setQueue(Event::WEBHOOK_QUEUE_NAME) + ->setClass(Event::WEBHOOK_CLASS_NAME) + ->setProject($project) + ->setEvent('functions.[functionId].deployments.[deploymentId].update') + ->setParam('functionId', $function->getId()) + ->setParam('deploymentId', $deployment->getId()) + ->setPayload($deployment->getArrayCopy(array_keys($deploymentModel->getRules()))) + ; - $deploymentUpdate = new Event(Event::WEBHOOK_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME); - $deploymentUpdate - ->setProject($project) - ->setEvent('functions.[functionId].deployments.[deploymentId].update') - ->setParam('functionId', $function->getId()) - ->setParam('deploymentId', $deployment->getId()) - ->setPayload($deployment->getArrayCopy(array_keys($deploymentModel->getRules()))) - ->trigger(); + $deploymentUpdate->trigger(); /** Trigger Functions */ - $pools = $register->get('pools'); - $connection = $pools->get('queue')->pop(); - - $functions = new Func($connection->getResource()); - $functions + $queueForFunctions ->from($deploymentUpdate) ->trigger(); - $connection->reclaim(); - /** Trigger Realtime */ $target = Realtime::fromPayload( - // Pass first, most verbose event pattern + // Pass first, most verbose event pattern event: $allEvents[0], payload: $build, project: $project @@ -358,36 +389,33 @@ class BuildsV1 extends Worker $command = \str_replace('"', '\\"', $command); $response = null; - $err = null; - // TODO: Remove run() wrapper when switching to new utopia queue. That should be done on Swoole adapter in the libary - Co\run(function () use ($project, $deployment, &$response, $source, $function, $runtime, $vars, $command, &$build, $dbForProject, $allEvents, &$err) { - Co::join([ - Co\go(function () use (&$response, $project, $deployment, $source, $function, $runtime, $vars, $command, &$err) { - try { - $version = $function->getAttribute('version', 'v2'); - $command = $version === 'v2' ? 'tar -zxf /tmp/code.tar.gz -C /usr/code && cd /usr/local/src/ && ./build.sh' : 'tar -zxf /tmp/code.tar.gz -C /mnt/code && helpers/build.sh "' . $command . '"'; + Co::join([ + Co\go(function () use ($executor, &$response, $project, $deployment, $source, $function, $runtime, $vars, $command, &$err) { + try { + $version = $function->getAttribute('version', 'v2'); + $command = $version === 'v2' ? 'tar -zxf /tmp/code.tar.gz -C /usr/code && cd /usr/local/src/ && ./build.sh' : 'tar -zxf /tmp/code.tar.gz -C /mnt/code && helpers/build.sh "' . $command . '"'; - $response = $this->executor->createRuntime( - deploymentId: $deployment->getId(), - projectId: $project->getId(), - source: $source, - image: $runtime['image'], - version: $version, - remove: true, - entrypoint: $deployment->getAttribute('entrypoint'), - destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}", - variables: $vars, - command: $command - ); - } catch (Exception $error) { - $err = $error; - } - }), - Co\go(function () use ($project, $deployment, &$response, &$build, $dbForProject, $allEvents, &$err) { + $response = $executor->createRuntime( + deploymentId: $deployment->getId(), + projectId: $project->getId(), + source: $source, + image: $runtime['image'], + version: $version, + remove: true, + entrypoint: $deployment->getAttribute('entrypoint'), + destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}", + variables: $vars, + command: $command + ); + } catch (Exception $error) { + $err = $error; + } + }), + Co\go(function () use ($executor, $project, $deployment, &$response, &$build, $dbForProject, $allEvents, &$err) { try { - $this->executor->getLogs( + $executor->getLogs( deploymentId: $deployment->getId(), projectId: $project->getId(), callback: function ($logs) use (&$response, &$build, $dbForProject, $allEvents, $project) { @@ -405,7 +433,7 @@ class BuildsV1 extends Worker * Send realtime Event */ $target = Realtime::fromPayload( - // Pass first, most verbose event pattern + // Pass first, most verbose event pattern event: $allEvents[0], payload: $build, project: $project @@ -427,7 +455,6 @@ class BuildsV1 extends Worker } }), ]); - }); if ($err) { throw $err; @@ -460,14 +487,14 @@ class BuildsV1 extends Worker } /** Update function schedule */ - $dbForConsole = $this->getConsoleDB(); + // Inform scheduler if function is still active $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); $schedule ->setAttribute('resourceUpdatedAt', DateTime::now()) ->setAttribute('schedule', $function->getAttribute('schedule')) ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); - Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); + Authorization::skip(fn() => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); } catch (\Throwable $th) { $endTime = DateTime::now(); $durationEnd = \microtime(true); @@ -489,7 +516,7 @@ class BuildsV1 extends Worker * Send realtime Event */ $target = Realtime::fromPayload( - // Pass first, most verbose event pattern + // Pass first, most verbose event pattern event: $allEvents[0], payload: $build, project: $project @@ -504,8 +531,6 @@ class BuildsV1 extends Worker /** Update usage stats */ if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { - $statsd = $register->get('statsd'); - $usage = new Stats($statsd); $usage ->setParam('projectInternalId', $project->getInternalId()) ->setParam('projectId', $project->getId()) @@ -520,6 +545,24 @@ class BuildsV1 extends Worker } } + /** + * @param string $status + * @param GitHub $github + * @param string $providerCommitHash + * @param string $owner + * @param string $repositoryName + * @param Document $project + * @param Document $function + * @param string $deploymentId + * @param Database $dbForProject + * @param Database $dbForConsole + * @return void + * @throws Structure + * @throws \Utopia\Database\Exception + * @throws Authorization + * @throws Conflict + * @throws Restricted + */ protected function runGitAction(string $status, GitHub $github, string $providerCommitHash, string $owner, string $repositoryName, Document $project, Document $function, string $deploymentId, Database $dbForProject, Database $dbForConsole): void { if ($function->getAttribute('providerSilentMode', false) === true) { @@ -589,8 +632,4 @@ class BuildsV1 extends Worker } } } - - public function shutdown(): void - { - } } diff --git a/app/workers/certificates.php b/src/Appwrite/Platform/Workers/Certificates.php similarity index 78% rename from app/workers/certificates.php rename to src/Appwrite/Platform/Workers/Certificates.php index 4a3b9bccb5..8fb5094a8f 100644 --- a/app/workers/certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -1,46 +1,90 @@ desc('Certificates worker') + ->inject('message') + ->inject('dbForConsole') + ->inject('queueForMails') + ->inject('queueForEvents') + ->inject('queueForFunctions') + ->callback(fn(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions) => $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions)); } - public function init(): void + /** + * @param Message $message + * @param Database $dbForConsole + * @param Mail $queueForMails + * @param Event $queueForEvents + * @param Func $queueForFunctions + * @return void + * @throws Throwable + * @throws \Utopia\Database\Exception + */ + public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions): void { + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + $document = new Document($payload['domain'] ?? []); + $domain = new Domain($document->getAttribute('domain', '')); + $skipRenewCheck = $payload['skipRenewCheck'] ?? false; + + $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $skipRenewCheck); } - public function run(): void + /** + * @param Domain $domain + * @param Database $dbForConsole + * @param Mail $queueForMails + * @param Event $queueForEvents + * @param Func $queueForFunctions + * @param bool $skipRenewCheck + * @return void + * @throws Throwable + * @throws \Utopia\Database\Exception + */ + private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, bool $skipRenewCheck = false): void { /** * 1. Read arguments and validate domain @@ -49,7 +93,7 @@ class CertificatesV1 extends Worker * 4. Validate security email. Cannot be empty, required by LetsEncrypt * 5. Validate renew date with certificate file, unless requested to skip by parameter * 6. Issue a certificate using certbot CLI - * 7. Update 'logs' attribute on certificate document with Certbot message + * 7. Update 'log' attribute on certificate document with Certbot message * 8. Create storage folder for certificate, if not ready already * 9. Move certificates from Certbot location to our Storage * 10. Create/Update our Storage with new Traefik config with new certificate paths @@ -59,7 +103,7 @@ class CertificatesV1 extends Worker * If at any point unexpected error occurs, program stops without applying changes to document, and error is thrown into worker * * If code stops with expected error: - * 1. 'logs' attribute on document is updated with error message + * 1. 'log' attribute on document is updated with error message * 2. 'attempts' amount is increased * 3. Console log is shown * 4. Email is sent to security email @@ -72,14 +116,8 @@ class CertificatesV1 extends Worker * Note: Renewals are checked and scheduled from maintenence worker */ - $this->dbForConsole = $this->getConsoleDB(); - - $skipCheck = $this->args['skipRenewCheck'] ?? false; // If true, we won't double-check expiry from cert file - $document = new Document($this->args['domain'] ?? []); - $domain = new Domain($document->getAttribute('domain', '')); - // Get current certificate - $certificate = $this->dbForConsole->findOne('certificates', [Query::equal('domain', [$domain->get()])]); + $certificate = $dbForConsole->findOne('certificates', [Query::equal('domain', [$domain->get()])]); // If we don't have certificate for domain yet, let's create new document. At the end we save it if (!$certificate) { @@ -97,14 +135,14 @@ class CertificatesV1 extends Worker } // Validate domain and DNS records. Skip if job is forced - if (!$skipCheck) { + if (!$skipRenewCheck) { $mainDomain = $this->getMainDomain(); $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain; $this->validateDomain($domain, $isMainDomain); } // If certificate exists already, double-check expiry date. Skip if job is forced - if (!$skipCheck && !$this->isRenewRequired($domain->get())) { + if (!$skipRenewCheck && !$this->isRenewRequired($domain->get())) { throw new Exception('Renew isn\'t required.'); } @@ -118,6 +156,7 @@ class CertificatesV1 extends Worker $logs = 'Certificate successfully generated.'; $certificate->setAttribute('logs', \mb_strcut($logs, 0, 1000000));// Limit to 1MB + // Give certificates to Traefik $this->applyCertificateFiles($folder, $domain->get(), $letsEncryptData); @@ -125,7 +164,6 @@ class CertificatesV1 extends Worker $certificate->setAttribute('renewDate', $this->getRenewDate($domain->get())); $certificate->setAttribute('attempts', 0); $certificate->setAttribute('issueDate', DateTime::now()); - $success = true; } catch (Throwable $e) { $logs = $e->getMessage(); @@ -141,45 +179,46 @@ class CertificatesV1 extends Worker $certificate->setAttribute('renewDate', DateTime::now()); // Send email to security email - $this->notifyError($domain->get(), $e->getMessage(), $attempts); + $this->notifyError($domain->get(), $e->getMessage(), $attempts, $queueForMails); } finally { // All actions result in new updatedAt date $certificate->setAttribute('updated', DateTime::now()); // Save all changes we made to certificate document into database - $this->saveCertificateDocument($domain->get(), $certificate, $success); + $this->saveCertificateDocument($domain->get(), $certificate, $success, $dbForConsole, $queueForEvents, $queueForFunctions); } } - public function shutdown(): void - { - } - /** * Save certificate data into database. * * @param string $domain Domain name that certificate is for * @param Document $certificate Certificate document that we need to save - * @param bool $success Was certificate generation successful? - * + * @param bool $success + * @param Database $dbForConsole Database connection for console + * @param Event $queueForEvents + * @param Func $queueForFunctions * @return void + * @throws \Utopia\Database\Exception + * @throws Authorization + * @throws Conflict + * @throws Structure */ - private function saveCertificateDocument(string $domain, Document $certificate, bool $success): void + private function saveCertificateDocument(string $domain, Document $certificate, bool $success, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions): void { // Check if update or insert required - $certificateDocument = $this->dbForConsole->findOne('certificates', [Query::equal('domain', [$domain])]); + $certificateDocument = $dbForConsole->findOne('certificates', [Query::equal('domain', [$domain])]); if (!empty($certificateDocument) && !$certificateDocument->isEmpty()) { // Merge new data with current data $certificate = new Document(\array_merge($certificateDocument->getArrayCopy(), $certificate->getArrayCopy())); - - $certificate = $this->dbForConsole->updateDocument('certificates', $certificate->getId(), $certificate); + $certificate = $dbForConsole->updateDocument('certificates', $certificate->getId(), $certificate); } else { $certificate->removeAttribute('$internalId'); - $certificate = $this->dbForConsole->createDocument('certificates', $certificate); + $certificate = $dbForConsole->createDocument('certificates', $certificate); } $certificateId = $certificate->getId(); - $this->updateDomainDocuments($certificateId, $domain, $success); + $this->updateDomainDocuments($certificateId, $domain, $success, $dbForConsole, $queueForEvents, $queueForFunctions); } /** @@ -206,6 +245,7 @@ class CertificatesV1 extends Worker * @param bool $isMainDomain In case of master domain, we look for different DNS configurations * * @return void + * @throws Exception */ private function validateDomain(Domain $domain, bool $isMainDomain): void { @@ -219,7 +259,6 @@ class CertificatesV1 extends Worker if (!$isMainDomain) { // TODO: Would be awesome to also support A/AAAA records here. Maybe dry run? - // Validate if domain target is properly configured $target = new Domain(App::getEnv('_APP_DOMAIN_TARGET', '')); @@ -242,8 +281,8 @@ class CertificatesV1 extends Worker * Reads expiry date of certificate from file and decides if renewal is required or not. * * @param string $domain Domain for which we check certificate file - * * @return bool True, if certificate needs to be renewed + * @throws Exception */ private function isRenewRequired(string $domain): bool { @@ -273,8 +312,8 @@ class CertificatesV1 extends Worker * * @param string $folder Folder into which certificates should be generated * @param string $domain Domain to generate certificate for - * * @return array Named array with keys 'stdout' and 'stderr', both string + * @throws Exception */ private function issueCertificate(string $folder, string $domain, string $email): array { @@ -303,8 +342,8 @@ class CertificatesV1 extends Worker * Read new renew date from certificate file generated by Let's Encrypt * * @param string $domain Domain which certificate was generated for - * * @return string + * @throws \Utopia\Database\Exception */ private function getRenewDate(string $domain): string { @@ -321,11 +360,12 @@ class CertificatesV1 extends Worker * @param string $domain Domain which certificate was generated for * @param string $folder Folder in which certificates were generated * @param array $letsEncryptData Let's Encrypt logs to use for additional info when throwing error - * * @return void + * @throws Exception */ private function applyCertificateFiles(string $folder, string $domain, array $letsEncryptData): void { + // Prepare folder in storage for domain $path = APP_STORAGE_CERTIFICATES . '/' . $domain; if (!\is_readable($path)) { @@ -370,10 +410,11 @@ class CertificatesV1 extends Worker * @param string $domain Domain that caused the error * @param string $errorMessage Verbose error message * @param int $attempt How many times it failed already - * + * @param Mail $queueForMails * @return void + * @throws Exception */ - private function notifyError(string $domain, string $errorMessage, int $attempt): void + private function notifyError(string $domain, string $errorMessage, int $attempt, Mail $queueForMails): void { // Log error into console Console::warning('Cannot renew domain (' . $domain . ') on attempt no. ' . $attempt . ' certificate: ' . $errorMessage); @@ -387,12 +428,11 @@ class CertificatesV1 extends Worker $body = Template::fromFile(__DIR__ . '/../config/locale/templates/email-base.tpl'); - $subject = \sprintf($locale->getText("emails.certificate.subject"), $domain); - $body->setParam('{{domain}}', $domain); - $body->setParam('{{error}}', $errorMessage); - $body->setParam('{{attempt}}', $attempt); - + $subject = \sprintf($locale->getText("emails.certificate.subject"), $domain); $body + ->setParam('{{domain}}', $domain) + ->setParam('{{error}}', $errorMessage) + ->setParam('{{attempt}}', $attempt) ->setParam('{{subject}}', $subject) ->setParam('{{hello}}', $locale->getText("emails.certificate.hello")) ->setParam('{{body}}', $locale->getText("emails.certificate.body")) @@ -406,10 +446,9 @@ class CertificatesV1 extends Worker ->setParam('{{bg-content}}', '#ffffff') ->setParam('{{text-content}}', '#000000'); - $body = $body->render(); - $mail = new Mail(); - $mail + $queueForMails ->setRecipient(App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS')) + ->setBody($body->render()) ->setName('Appwrite Administrator') ->trigger(); } @@ -427,16 +466,17 @@ class CertificatesV1 extends Worker * * @return void */ - private function updateDomainDocuments(string $certificateId, string $domain, bool $success): void + private function updateDomainDocuments(string $certificateId, string $domain, bool $success, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions): void { - $rule = $this->dbForConsole->findOne('rules', [ + + $rule = $dbForConsole->findOne('rules', [ Query::equal('domain', [$domain]), ]); if ($rule !== false && !$rule->isEmpty()) { $rule->setAttribute('certificateId', $certificateId); $rule->setAttribute('status', $success ? 'verified' : 'unverified'); - $this->dbForConsole->updateDocument('rules', $rule->getId(), $rule); + $dbForConsole->updateDocument('rules', $rule->getId(), $rule); $projectId = $rule->getAttribute('projectId'); @@ -445,22 +485,24 @@ class CertificatesV1 extends Worker return; } - $project = $this->dbForConsole->getDocument('projects', $projectId); + $project = $dbForConsole->getDocument('projects', $projectId); /** Trigger Webhook */ $ruleModel = new Rule(); - $ruleUpdate = new Event(Event::WEBHOOK_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME); - $ruleUpdate + $queueForEvents ->setProject($project) ->setEvent('rules.[ruleId].update') ->setParam('ruleId', $rule->getId()) ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))) ->trigger(); + /** Trigger Functions */ - $ruleUpdate - ->setClass(Event::FUNCTIONS_CLASS_NAME) - ->setQueue(Event::FUNCTIONS_QUEUE_NAME) + $queueForFunctions + ->setProject($project) + ->setEvent('rules.[ruleId].update') + ->setParam('ruleId', $rule->getId()) + ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))) ->trigger(); /** Trigger realtime event */ @@ -468,7 +510,7 @@ class CertificatesV1 extends Worker 'ruleId' => $rule->getId(), ]); $target = Realtime::fromPayload( - // Pass first, most verbose event pattern + // Pass first, most verbose event pattern event: $allEvents[0], payload: $rule, project: $project diff --git a/app/workers/databases.php b/src/Appwrite/Platform/Workers/Databases.php similarity index 58% rename from app/workers/databases.php rename to src/Appwrite/Platform/Workers/Databases.php index 764f668f33..e0ec75e1d4 100644 --- a/app/workers/databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -1,63 +1,78 @@ args['type']; - $project = new Document($this->args['project']); - $collection = new Document($this->args['collection'] ?? []); - $document = new Document($this->args['document'] ?? []); - $database = new Document($this->args['database'] ?? []); - - if ($collection->isEmpty()) { - throw new DatabaseException('Missing collection'); - } - - if ($document->isEmpty()) { - throw new DatabaseException('Missing document'); - } - - switch (strval($type)) { - case DATABASE_TYPE_CREATE_ATTRIBUTE: - $this->createAttribute($database, $collection, $document, $project); - break; - case DATABASE_TYPE_DELETE_ATTRIBUTE: - $this->deleteAttribute($database, $collection, $document, $project); - break; - case DATABASE_TYPE_CREATE_INDEX: - $this->createIndex($database, $collection, $document, $project); - break; - case DATABASE_TYPE_DELETE_INDEX: - $this->deleteIndex($database, $collection, $document, $project); - break; - - default: - Console::error('No database operation for type: ' . $type); - break; - } + $this + ->desc('Databases worker') + ->inject('message') + ->inject('dbForConsole') + ->inject('dbForProject') + ->callback(fn($message, $dbForConsole, $dbForProject) => $this->action($message, $dbForConsole, $dbForProject)); } - public function shutdown(): void + /** + * @param Message $message + * @param Database $dbForConsole + * @param Database $dbForProject + * @return void + * @throws \Exception + */ + public function action(Message $message, Database $dbForConsole, Database $dbForProject): void { + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + throw new \Exception('Missing payload'); + } + + $type = $payload['type']; + $project = new Document($payload['project']); + $collection = new Document($payload['collection'] ?? []); + $document = new Document($payload['document'] ?? []); + $database = new Document($payload['database'] ?? []); + + if ($database->isEmpty()) { + throw new Exception('Missing database'); + } + + match (strval($type)) { + DATABASE_TYPE_DELETE_DATABASE => $this->deleteDatabase($database, $project, $dbForProject), + DATABASE_TYPE_DELETE_COLLECTION => $this->deleteCollection($database, $collection, $project, $dbForProject), + DATABASE_TYPE_CREATE_ATTRIBUTE => $this->createAttribute($database, $collection, $document, $project, $dbForConsole, $dbForProject), + DATABASE_TYPE_DELETE_ATTRIBUTE => $this->deleteAttribute($database, $collection, $document, $project, $dbForConsole, $dbForProject), + DATABASE_TYPE_CREATE_INDEX => $this->createIndex($database, $collection, $document, $project, $dbForConsole, $dbForProject), + DATABASE_TYPE_DELETE_INDEX => $this->deleteIndex($database, $collection, $document, $project, $dbForConsole, $dbForProject), + default => Console::error('No database operation for type: ' . $type), + }; } /** @@ -65,12 +80,23 @@ class DatabaseV1 extends Worker * @param Document $collection * @param Document $attribute * @param Document $project + * @param Database $dbForConsole + * @param Database $dbForProject + * @return void + * @throws Authorization + * @throws Conflict + * @throws \Exception */ - protected function createAttribute(Document $database, Document $collection, Document $attribute, Document $project): void + private function createAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForConsole, Database $dbForProject): void { + if ($collection->isEmpty()) { + throw new Exception('Missing collection'); + } + if ($attribute->isEmpty()) { + throw new Exception('Missing attribute'); + } + $projectId = $project->getId(); - $dbForConsole = $this->getConsoleDB(); - $dbForProject = $this->getProjectDB($project); $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].attributes.[attributeId].update', [ 'databaseId' => $database->getId(), @@ -78,6 +104,7 @@ class DatabaseV1 extends Worker 'attributeId' => $attribute->getId() ]); /** + * TODO @christyjacob4 verify if this is still the case * Fetch attribute from the database, since with Resque float values are loosing informations. */ $attribute = $dbForProject->getDocument('attributes', $attribute->getId()); @@ -96,6 +123,7 @@ class DatabaseV1 extends Worker $options = $attribute->getAttribute('options', []); $project = $dbForConsole->getDocument('projects', $projectId); + try { switch ($type) { case Database::VAR_RELATIONSHIP: @@ -125,7 +153,7 @@ class DatabaseV1 extends Worker break; default: if (!$dbForProject->createAttribute('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key, $type, $size, $required, $default, $signed, $array, $format, $formatOptions, $filters)) { - throw new Exception('Failed to create Attribute'); + throw new \Exception('Failed to create Attribute'); } } @@ -154,25 +182,7 @@ class DatabaseV1 extends Worker ); } } finally { - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $events[0], - payload: $attribute, - project: $project, - ); - - Realtime::send( - projectId: 'console', - payload: $attribute->getArrayCopy(), - events: $events, - channels: $target['channels'], - roles: $target['roles'], - options: [ - 'projectId' => $projectId, - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId() - ] - ); + $this->trigger($database, $collection, $attribute, $project, $projectId, $events); } if ($type === Database::VAR_RELATIONSHIP && $options['twoWay']) { @@ -187,13 +197,23 @@ class DatabaseV1 extends Worker * @param Document $collection * @param Document $attribute * @param Document $project - * @throws Throwable - */ - protected function deleteAttribute(Document $database, Document $collection, Document $attribute, Document $project): void + * @param Database $dbForConsole + * @param Database $dbForProject + * @return void + * @throws Authorization + * @throws Conflict + * @throws \Exception + **/ + private function deleteAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForConsole, Database $dbForProject): void { + if ($collection->isEmpty()) { + throw new Exception('Missing collection'); + } + if ($attribute->isEmpty()) { + throw new Exception('Missing attribute'); + } + $projectId = $project->getId(); - $dbForConsole = $this->getConsoleDB(); - $dbForProject = $this->getProjectDB($project); $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].attributes.[attributeId].delete', [ 'databaseId' => $database->getId(), @@ -262,25 +282,7 @@ class DatabaseV1 extends Worker ); } } finally { - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $events[0], - payload: $attribute, - project: $project - ); - - Realtime::send( - projectId: 'console', - payload: $attribute->getArrayCopy(), - events: $events, - channels: $target['channels'], - roles: $target['roles'], - options: [ - 'projectId' => $projectId, - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId() - ] - ); + $this->trigger($database, $collection, $attribute, $project, $projectId, $events); } // The underlying database removes/rebuilds indexes when attribute is removed @@ -326,7 +328,7 @@ class DatabaseV1 extends Worker } if ($exists) { // Delete the duplicate if created, else update in db - $this->deleteIndex($database, $collection, $index, $project); + $this->deleteIndex($database, $collection, $index, $project, $dbForConsole, $dbForProject); } else { $dbForProject->updateDocument('indexes', $index->getId(), $index); } @@ -348,13 +350,24 @@ class DatabaseV1 extends Worker * @param Document $collection * @param Document $index * @param Document $project - * @throws \Exception + * @param Database $dbForConsole + * @param Database $dbForProject + * @return void + * @throws Authorization + * @throws Conflict + * @throws Structure + * @throws DatabaseException */ - protected function createIndex(Document $database, Document $collection, Document $index, Document $project): void + private function createIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForConsole, Database $dbForProject): void { + if ($collection->isEmpty()) { + throw new Exception('Missing collection'); + } + if ($index->isEmpty()) { + throw new Exception('Missing index'); + } + $projectId = $project->getId(); - $dbForConsole = $this->getConsoleDB(); - $dbForProject = $this->getProjectDB($project); $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].update', [ 'databaseId' => $database->getId(), @@ -386,25 +399,7 @@ class DatabaseV1 extends Worker $index->setAttribute('status', 'failed') ); } finally { - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $events[0], - payload: $index, - project: $project - ); - - Realtime::send( - projectId: 'console', - payload: $index->getArrayCopy(), - events: $events, - channels: $target['channels'], - roles: $target['roles'], - options: [ - 'projectId' => $projectId, - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId() - ] - ); + $this->trigger($database, $collection, $index, $project, $projectId, $events); } $dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $collectionId); @@ -415,12 +410,24 @@ class DatabaseV1 extends Worker * @param Document $collection * @param Document $index * @param Document $project + * @param Database $dbForConsole + * @param Database $dbForProject + * @return void + * @throws Authorization + * @throws Conflict + * @throws Structure + * @throws DatabaseException */ - protected function deleteIndex(Document $database, Document $collection, Document $index, Document $project): void + private function deleteIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForConsole, Database $dbForProject): void { + if ($collection->isEmpty()) { + throw new Exception('Missing collection'); + } + if ($index->isEmpty()) { + throw new Exception('Missing index'); + } + $projectId = $project->getId(); - $dbForConsole = $this->getConsoleDB(); - $dbForProject = $this->getProjectDB($project); $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].delete', [ 'databaseId' => $database->getId(), @@ -436,6 +443,7 @@ class DatabaseV1 extends Worker throw new DatabaseException('Failed to delete index'); } $dbForProject->deleteDocument('indexes', $index->getId()); + $index->setAttribute('status', 'deleted'); } catch (\Exception $e) { Console::error($e->getMessage()); @@ -448,27 +456,167 @@ class DatabaseV1 extends Worker $index->setAttribute('status', 'stuck') ); } finally { - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $events[0], - payload: $index, - project: $project - ); - - Realtime::send( - projectId: 'console', - payload: $index->getArrayCopy(), - events: $events, - channels: $target['channels'], - roles: $target['roles'], - options: [ - 'projectId' => $projectId, - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId() - ] - ); + $this->trigger($database, $collection, $index, $project, $projectId, $events); } $dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $collection->getId()); } + + /** + * @param Document $database + * @param Document $project + * @param $dbForProject + * @return void + * @throws Exception + */ + protected function deleteDatabase(Document $database, Document $project, $dbForProject): void + { + $this->deleteByGroup('database_' . $database->getInternalId(), [], $dbForProject, function ($collection) use ($database, $project, $dbForProject) { + $this->deleteCollection($database, $collection, $project, $dbForProject); + }); + + $dbForProject->deleteCollection('database_' . $database->getInternalId()); + + $this->deleteAuditLogsByResource('database/' . $database->getId(), $project, $dbForProject); + } + + /** + * @param Document $database + * @param Document $collection + * @param Document $project + * @param Database $dbForProject + * @return void + * @throws Authorization + * @throws Conflict + * @throws DatabaseException + * @throws Restricted + * @throws Structure + */ + protected function deleteCollection(Document $database, Document $collection, Document $project, Database $dbForProject): void + { + if ($collection->isEmpty()) { + throw new Exception('Missing collection'); + } + + $collectionId = $collection->getId(); + $collectionInternalId = $collection->getInternalId(); + $databaseId = $database->getId(); + $databaseInternalId = $database->getInternalId(); + + $relationships = \array_filter( + $collection->getAttribute('attributes'), + fn ($attribute) => $attribute['type'] === Database::VAR_RELATIONSHIP + ); + + foreach ($relationships as $relationship) { + if (!$relationship['twoWay']) { + continue; + } + $relatedCollection = $dbForProject->getDocument('database_' . $databaseInternalId, $relationship['relatedCollection']); + $dbForProject->deleteDocument('attributes', $databaseInternalId . '_' . $relatedCollection->getInternalId() . '_' . $relationship['twoWayKey']); + $dbForProject->deleteCachedDocument('database_' . $databaseInternalId, $relatedCollection->getId()); + $dbForProject->deleteCachedCollection('database_' . $databaseInternalId . '_collection_' . $relatedCollection->getInternalId()); + } + + $dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $collection->getInternalId()); + + $this->deleteByGroup('attributes', [ + Query::equal('databaseInternalId', [$databaseInternalId]), + Query::equal('collectionInternalId', [$collectionInternalId]) + ], $dbForProject); + + $this->deleteByGroup('indexes', [ + Query::equal('databaseInternalId', [$databaseInternalId]), + Query::equal('collectionInternalId', [$collectionInternalId]) + ], $dbForProject); + + $this->deleteAuditLogsByResource('database/' . $databaseId . '/collection/' . $collectionId, $project, $dbForProject); + } + + /** + * @param string $resource + * @param Document $project + * @param Database $dbForProject + * @return void + * @throws Exception + */ + protected function deleteAuditLogsByResource(string $resource, Document $project, Database $dbForProject): void + { + $this->deleteByGroup(Audit::COLLECTION, [ + Query::equal('resource', [$resource]) + ], $dbForProject); + } + + /** + * @param string $collection collectionID + * @param array $queries + * @param Database $database + * @param callable|null $callback + * @return void + * @throws Exception + */ + protected function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null): void + { + $count = 0; + $chunk = 0; + $limit = 50; + $sum = $limit; + + $executionStart = \microtime(true); + + while ($sum === $limit) { + $chunk++; + + $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries)); + + $sum = count($results); + + Console::info('Deleting chunk #' . $chunk . '. Found ' . $sum . ' documents'); + + foreach ($results as $document) { + if ($database->deleteDocument($document->getCollection(), $document->getId())) { + Console::success('Deleted document "' . $document->getId() . '" successfully'); + + if (\is_callable($callback)) { + $callback($document); + } + } else { + Console::error('Failed to delete document: ' . $document->getId()); + } + $count++; + } + } + + $executionEnd = \microtime(true); + + Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); + } + + protected function trigger( + Document $database, + Document $collection, + Document $attribute, + Document $project, + string $projectId, + array $events + ): void { + $target = Realtime::fromPayload( + // Pass first, most verbose event pattern + event: $events[0], + payload: $attribute, + project: $project, + ); + Realtime::send( + projectId: 'console', + payload: $attribute->getArrayCopy(), + events: $events, + channels: $target['channels'], + roles: $target['roles'], + options: [ + 'projectId' => $projectId, + 'databaseId' => $database->getId(), + 'collectionId' => $collection->getId() + ] + ); + } } diff --git a/app/workers/deletes.php b/src/Appwrite/Platform/Workers/Deletes.php similarity index 60% rename from app/workers/deletes.php rename to src/Appwrite/Platform/Workers/Deletes.php index 14f05a06a6..2130499257 100644 --- a/app/workers/deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -1,82 +1,111 @@ desc('Deletes worker') + ->inject('message') + ->inject('dbForConsole') + ->inject('getProjectDB') + ->inject('getFilesDevice') + ->inject('getFunctionsDevice') + ->inject('getBuildsDevice') + ->inject('getCacheDevice') + ->callback(fn($message, $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice) => $this->action($message, $dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice)); } - public function run(): void + /** + * @throws Exception + * @throws Throwable + */ + public function action(Message $message, Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice): void { - $project = new Document($this->args['project'] ?? []); - $type = $this->args['type'] ?? ''; + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + $type = $payload['type'] ?? ''; + $datetime = $payload['datetime'] ?? null; + $hourlyUsageRetentionDatetime = $payload['hourlyUsageRetentionDatetime'] ?? null; + $resource = $payload['resource'] ?? null; + $document = new Document($payload['document'] ?? []); + $project = new Document($payload['project'] ?? []); + switch (strval($type)) { case DELETE_TYPE_DOCUMENT: - $document = new Document($this->args['document'] ?? []); - switch ($document->getCollection()) { case DELETE_TYPE_DATABASES: - $this->deleteDatabase($document, $project); + $this->deleteDatabase($getProjectDB, $document, $project); break; case DELETE_TYPE_COLLECTIONS: - $this->deleteCollection($document, $project); + $this->deleteCollection($getProjectDB, $document, $project); break; case DELETE_TYPE_PROJECTS: - $this->deleteProject($document); + $this->deleteProject($dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $document); break; case DELETE_TYPE_FUNCTIONS: - $this->deleteFunction($document, $project); + $this->deleteFunction($dbForConsole, $getProjectDB, $getFunctionsDevice, $getBuildsDevice, $document, $project); break; case DELETE_TYPE_DEPLOYMENTS: - $this->deleteDeployment($document, $project); + $this->deleteDeployment($getProjectDB, $getFunctionsDevice, $getBuildsDevice, $document, $project); break; case DELETE_TYPE_USERS: - $this->deleteUser($document, $project); + $this->deleteUser($getProjectDB, $document, $project); break; case DELETE_TYPE_TEAMS: - $this->deleteMemberships($document, $project); + $this->deleteMemberships($getProjectDB, $document, $project); if ($project->getId() === 'console') { - $this->deleteProjectsByTeam($document); + $this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $document); } break; case DELETE_TYPE_BUCKETS: - $this->deleteBucket($document, $project); + $this->deleteBucket($getProjectDB, $getFilesDevice, $document, $project); break; case DELETE_TYPE_INSTALLATIONS: - $this->deleteInstallation($document, $project); + $this->deleteInstallation($dbForConsole, $getProjectDB, $document, $project); break; case DELETE_TYPE_RULES: - $this->deleteRule($document); + $this->deleteRule($dbForConsole, $document); break; default: if (\str_starts_with($document->getCollection(), 'database_')) { - $this->deleteCollection($document, $project); + $this->deleteCollection($getProjectDB, $document, $project); break; } Console::error('No lazy delete operation available for document of type: ' . $document->getCollection()); @@ -85,47 +114,40 @@ class DeletesV1 extends Worker break; case DELETE_TYPE_EXECUTIONS: - $this->deleteExecutionLogs($this->args['datetime']); + $this->deleteExecutionLogs($dbForConsole, $getProjectDB, $datetime); break; case DELETE_TYPE_AUDIT: - $datetime = $this->args['datetime'] ?? null; if (!empty($datetime)) { - $this->deleteAuditLogs($datetime); + $this->deleteAuditLogs($dbForConsole, $getProjectDB, $datetime); } - $document = new Document($this->args['document'] ?? []); - if (!$document->isEmpty()) { - $this->deleteAuditLogsByResource('document/' . $document->getId(), $project); + $this->deleteAuditLogsByResource($getProjectDB, 'document/' . $document->getId(), $project); } - break; - case DELETE_TYPE_ABUSE: - $this->deleteAbuseLogs($this->args['datetime']); + $this->deleteAbuseLogs($dbForConsole, $getProjectDB, $datetime); break; case DELETE_TYPE_REALTIME: - $this->deleteRealtimeUsage($this->args['datetime']); + $this->deleteRealtimeUsage($dbForConsole, $datetime); break; case DELETE_TYPE_SESSIONS: - $this->deleteExpiredSessions(); + $this->deleteExpiredSessions($dbForConsole, $getProjectDB); break; - case DELETE_TYPE_USAGE: - $this->deleteUsageStats($this->args['hourlyUsageRetentionDatetime']); + $this->deleteUsageStats($dbForConsole, $getProjectDB, $hourlyUsageRetentionDatetime); break; - case DELETE_TYPE_CACHE_BY_RESOURCE: - $this->deleteCacheByResource($project, $this->args['resource']); + $this->deleteCacheByResource($project, $getProjectDB, $resource); break; case DELETE_TYPE_CACHE_BY_TIMESTAMP: - $this->deleteCacheByDate($this->args['datetime']); + $this->deleteCacheByDate($project, $getProjectDB, $datetime); break; case DELETE_TYPE_SCHEDULES: - $this->deleteSchedules($this->args['datetime']); + $this->deleteSchedules($dbForConsole, $getProjectDB, $datetime); break; default: Console::error('No delete operation for type: ' . $type); @@ -133,14 +155,15 @@ class DeletesV1 extends Worker } } - public function shutdown(): void - { - } - /** - * @throws Exception + * @param Database $dbForConsole + * @param callable $getProjectDB + * @param string $datetime + * @return void + * @throws Authorization + * @throws Throwable */ - protected function deleteSchedules(string $datetime): void + private function deleteSchedules(Database $dbForConsole, callable $getProjectDB, string $datetime): void { $this->listByGroup( 'schedules', @@ -150,21 +173,20 @@ class DeletesV1 extends Worker Query::lessThanEqual('resourceUpdatedAt', $datetime), Query::equal('active', [false]), ], - $this->getConsoleDB(), - function (Document $document) { - $project = $this->getConsoleDB()->getDocument('projects', $document->getAttribute('projectId')); + $dbForConsole, + function (Document $document) use ($dbForConsole, $getProjectDB) { + $project = $dbForConsole->getDocument('projects', $document->getAttribute('projectId')); if ($project->isEmpty()) { - $this->getConsoleDB()->deleteDocument('schedules', $document->getId()); - Console::success('Deleted schedule for deleted project ' . $document->getAttribute('projectId')); + Console::warning('Unable to delete schedule for function ' . $document->getAttribute('resourceId')); return; } - $function = $this->getProjectDB($project)->getDocument('functions', $document->getAttribute('resourceId')); + $function = $getProjectDB($project)->getDocument('functions', $document->getAttribute('resourceId')); if ($function->isEmpty()) { - $this->getConsoleDB()->deleteDocument('schedules', $document->getId()); - Console::success('Deleted schedule for function ' . $document->getAttribute('resourceId')); + $dbForConsole->deleteDocument('schedules', $document->getId()); + Console::success('Deleting schedule for function ' . $document->getAttribute('resourceId')); } } ); @@ -172,13 +194,15 @@ class DeletesV1 extends Worker /** * @param Document $project + * @param callable $getProjectDB * @param string $resource - * @throws Exception + * @return void + * @throws Authorization */ - protected function deleteCacheByResource(Document $project, string $resource): void + private function deleteCacheByResource(Document $project, callable $getProjectDB, string $resource): void { $projectId = $project->getId(); - $dbForProject = $this->getProjectDB($project); + $dbForProject = $getProjectDB($project); $document = $dbForProject->findOne('cache', [Query::equal('resource', [$resource])]); if ($document) { @@ -203,72 +227,77 @@ class DeletesV1 extends Worker } /** + * Document $project + * @param Document $project + * @param callable $getProjectDB * @param string $datetime + * @return void * @throws Exception */ - protected function deleteCacheByDate(string $datetime): void + private function deleteCacheByDate(Document $project, callable $getProjectDB, string $datetime): void { - $this->deleteForProjectIds(function (Document $project) use ($datetime) { - $projectId = $project->getId(); - $dbForProject = $this->getProjectDB($project); - $cache = new Cache( - new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) - ); + $projectId = $project->getId(); + $dbForProject = $getProjectDB($project); - $query = [ - Query::lessThan('accessedAt', $datetime), - ]; + $cache = new Cache( + new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) + ); - $this->deleteByGroup( - 'cache', - $query, - $dbForProject, - function (Document $document) use ($cache, $projectId) { - $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId(); + $query = [ + Query::lessThan('accessedAt', $datetime), + ]; - if ($cache->purge($document->getId())) { - Console::success('Deleting cache file: ' . $path); - } else { - Console::error('Failed to delete cache file: ' . $path); - } + $this->deleteByGroup( + 'cache', + $query, + $dbForProject, + function (Document $document) use ($cache, $projectId) { + $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId(); + + if ($cache->purge($document->getId())) { + Console::success('Deleting cache file: ' . $path); + } else { + Console::error('Failed to delete cache file: ' . $path); } - ); - }); + } + ); } - /** - * @param Document $document database document + * @param callable $getProjectDB + * @param Document $document * @param Document $project + * @return void + * @throws Exception */ - protected function deleteDatabase(Document $document, Document $project): void + private function deleteDatabase(callable $getProjectDB, Document $document, Document $project): void { $databaseId = $document->getId(); - $projectId = $project->getId(); + $dbForProject = $getProjectDB($project); - $dbForProject = $this->getProjectDB($project); - - $this->deleteByGroup('database_' . $document->getInternalId(), [], $dbForProject, function ($document) use ($project) { - $this->deleteCollection($document, $project); + $this->deleteByGroup('database_' . $document->getInternalId(), [], $dbForProject, function ($document) use ($getProjectDB, $project) { + $this->deleteCollection($getProjectDB, $document, $project); }); $dbForProject->deleteCollection('database_' . $document->getInternalId()); - - $this->deleteAuditLogsByResource('database/' . $databaseId, $project); + $this->deleteAuditLogsByResource($getProjectDB, 'database/' . $databaseId, $project); } /** + * @param callable $getProjectDB * @param Document $document teams document * @param Document $project + * @return void + * @throws Exception */ - protected function deleteCollection(Document $document, Document $project): void + private function deleteCollection(callable $getProjectDB, Document $document, Document $project): void { $collectionId = $document->getId(); $collectionInternalId = $document->getInternalId(); $databaseId = $document->getAttribute('databaseId'); $databaseInternalId = $document->getAttribute('databaseInternalId'); - $dbForProject = $this->getProjectDB($project); + $dbForProject = $getProjectDB($project); $relationships = \array_filter( $document->getAttribute('attributes'), @@ -297,17 +326,20 @@ class DeletesV1 extends Worker Query::equal('collectionInternalId', [$collectionInternalId]) ], $dbForProject); - $this->deleteAuditLogsByResource('database/' . $databaseId . '/collection/' . $collectionId, $project); + $this->deleteAuditLogsByResource($getProjectDB, 'database/' . $databaseId . '/collection/' . $collectionId, $project); } /** + * @param Database $dbForConsole + * @param callable $getProjectDB * @param string $hourlyUsageRetentionDatetime + * @return void * @throws Exception */ - protected function deleteUsageStats(string $hourlyUsageRetentionDatetime) + private function deleteUsageStats(Database $dbForConsole, callable $getProjectDB, string $hourlyUsageRetentionDatetime): void { - $this->deleteForProjectIds(function (Document $project) use ($hourlyUsageRetentionDatetime) { - $dbForProject = $this->getProjectDB($project); + $this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $hourlyUsageRetentionDatetime) { + $dbForProject = $getProjectDB($project); // Delete Usage stats $this->deleteByGroup('stats', [ Query::lessThan('time', $hourlyUsageRetentionDatetime), @@ -317,12 +349,15 @@ class DeletesV1 extends Worker } /** + * @param callable $getProjectDB * @param Document $document teams document * @param Document $project + * @return void + * @throws Exception */ - protected function deleteMemberships(Document $document, Document $project): void + private function deleteMemberships(callable $getProjectDB, Document $document, Document $project): void { - $dbForProject = $this->getProjectDB($project); + $dbForProject = $getProjectDB($project); $teamInternalId = $document->getInternalId(); // Delete Memberships @@ -340,37 +375,46 @@ class DeletesV1 extends Worker } /** - * @param \Utopia\Database\Document $document + * @param Database $dbForConsole + * @param Document $document * @return void - * @throws \Exception + * @throws Authorization + * @throws \Utopia\Database\Exception + * @throws Conflict + * @throws Restricted + * @throws Structure */ - protected function deleteProjectsByTeam(Document $document): void + private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, Document $document): void { - $dbForConsole = $this->getConsoleDB(); - $projects = $dbForConsole->find('projects', [ Query::equal('teamInternalId', [$document->getInternalId()]) ]); - foreach ($projects as $project) { - $this->deleteProject($project); + $this->deleteProject($dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $project); $dbForConsole->deleteDocument('projects', $project->getId()); } } /** - * @param Document $document project document + * @param Database $dbForConsole + * @param callable $getProjectDB + * @param callable $getFilesDevice + * @param callable $getFunctionsDevice + * @param callable $getBuildsDevice + * @param callable $getCacheDevice + * @param Document $document + * @return void * @throws Exception + * @throws Authorization + * @throws \Utopia\Database\Exception */ - protected function deleteProject(Document $document): void + private function deleteProject(Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, Document $document): void { $projectId = $document->getId(); $projectInternalId = $document->getInternalId(); - $dbForConsole = $this->getConsoleDB(); - // Delete project tables - $dbForProject = $this->getProjectDB($document); + $dbForProject = $getProjectDB($document); while (true) { $collections = $dbForProject->listCollections(); @@ -392,8 +436,8 @@ class DeletesV1 extends Worker // Delete project and function rules $this->deleteByGroup('rules', [ Query::equal('projectInternalId', [$projectInternalId]) - ], $dbForConsole, function (Document $document) { - $this->deleteRule($document); + ], $dbForConsole, function (Document $document) use ($dbForConsole) { + $this->deleteRule($dbForConsole, $document); }); // Delete Keys @@ -415,10 +459,10 @@ class DeletesV1 extends Worker } // Delete all storage directories - $uploads = $this->getFilesDevice($projectId); - $functions = $this->getFunctionsDevice($projectId); - $builds = $this->getBuildsDevice($projectId); - $cache = $this->getCacheDevice($projectId); + $uploads = $getFilesDevice($projectId); + $functions = $getFunctionsDevice($projectId); + $builds = $getBuildsDevice($projectId); + $cache = $getCacheDevice($projectId); $uploads->delete($uploads->getRoot(), true); $functions->delete($functions->getRoot(), true); @@ -427,15 +471,17 @@ class DeletesV1 extends Worker } /** + * @param callable $getProjectDB * @param Document $document user document * @param Document $project + * @return void + * @throws Exception */ - protected function deleteUser(Document $document, Document $project): void + private function deleteUser(callable $getProjectDB, Document $document, Document $project): void { $userId = $document->getId(); $userInternalId = $document->getInternalId(); - - $dbForProject = $this->getProjectDB($project); + $dbForProject = $getProjectDB($project); // Delete all sessions of this user from the sessions table and update the sessions field of the user record $this->deleteByGroup('sessions', [ @@ -474,13 +520,16 @@ class DeletesV1 extends Worker } /** + * @param database $dbForConsole + * @param callable $getProjectDB * @param string $datetime + * @return void * @throws Exception */ - protected function deleteExecutionLogs(string $datetime): void + private function deleteExecutionLogs(database $dbForConsole, callable $getProjectDB, string $datetime): void { - $this->deleteForProjectIds(function (Document $project) use ($datetime) { - $dbForProject = $this->getProjectDB($project); + $this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $datetime) { + $dbForProject = $getProjectDB($project); // Delete Executions $this->deleteByGroup('executions', [ Query::lessThan('$createdAt', $datetime) @@ -488,14 +537,18 @@ class DeletesV1 extends Worker }); } - protected function deleteExpiredSessions(): void + /** + * @param Database $dbForConsole + * @param callable $getProjectDB + * @return void + * @throws Exception|Throwable + */ + private function deleteExpiredSessions(Database $dbForConsole, callable $getProjectDB): void { - $consoleDB = $this->getConsoleDB(); - $this->deleteForProjectIds(function (Document $project) use ($consoleDB) { - $dbForProject = $this->getProjectDB($project); - - $project = $consoleDB->getDocument('projects', $project->getId()); + $this->deleteForProjectIds($dbForConsole, function (Document $project) use ($dbForConsole, $getProjectDB) { + $dbForProject = $getProjectDB($project); + $project = $dbForConsole->getDocument('projects', $project->getId()); $duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG; $expired = DateTime::addSeconds(new \DateTime(), -1 * $duration); @@ -507,12 +560,13 @@ class DeletesV1 extends Worker } /** + * @param Database $dbForConsole * @param string $datetime + * @return void * @throws Exception */ - protected function deleteRealtimeUsage(string $datetime): void + private function deleteRealtimeUsage(Database $dbForConsole, string $datetime): void { - $dbForConsole = $this->getConsoleDB(); // Delete Dead Realtime Logs $this->deleteByGroup('realtime', [ Query::lessThan('timestamp', $datetime) @@ -520,18 +574,21 @@ class DeletesV1 extends Worker } /** + * @param Database $dbForConsole + * @param callable $getProjectDB * @param string $datetime + * @return void * @throws Exception */ - protected function deleteAbuseLogs(string $datetime): void + private function deleteAbuseLogs(Database $dbForConsole, callable $getProjectDB, string $datetime): void { if (empty($datetime)) { throw new Exception('Failed to delete audit logs. No datetime provided'); } - $this->deleteForProjectIds(function (Document $project) use ($datetime) { + $this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $datetime) { $projectId = $project->getId(); - $dbForProject = $this->getProjectDB($project); + $dbForProject = $getProjectDB($project); $timeLimit = new TimeLimit("", 0, 1, $dbForProject); $abuse = new Abuse($timeLimit); $status = $abuse->cleanup($datetime); @@ -542,18 +599,21 @@ class DeletesV1 extends Worker } /** + * @param Database $dbForConsole + * @param callable $getProjectDB * @param string $datetime + * @return void * @throws Exception */ - protected function deleteAuditLogs(string $datetime): void + private function deleteAuditLogs(Database $dbForConsole, callable $getProjectDB, string $datetime): void { if (empty($datetime)) { throw new Exception('Failed to delete audit logs. No datetime provided'); } - $this->deleteForProjectIds(function (Document $project) use ($datetime) { + $this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $datetime) { $projectId = $project->getId(); - $dbForProject = $this->getProjectDB($project); + $dbForProject = $getProjectDB($project); $audit = new Audit($dbForProject); $status = $audit->cleanup($datetime); if (!$status) { @@ -563,12 +623,15 @@ class DeletesV1 extends Worker } /** + * @param callable $getProjectDB * @param string $resource * @param Document $project + * @return void + * @throws Exception */ - protected function deleteAuditLogsByResource(string $resource, Document $project): void + private function deleteAuditLogsByResource(callable $getProjectDB, string $resource, Document $project): void { - $dbForProject = $this->getProjectDB($project); + $dbForProject = $getProjectDB($project); $this->deleteByGroup(Audit::COLLECTION, [ Query::equal('resource', [$resource]) @@ -576,14 +639,18 @@ class DeletesV1 extends Worker } /** + * @param callable $getProjectDB + * @param callable $getFunctionsDevice + * @param callable $getBuildsDevice * @param Document $document function document * @param Document $project + * @return void + * @throws Exception */ - protected function deleteFunction(Document $document, Document $project): void + private function deleteFunction(Database $dbForConsole, callable $getProjectDB, callable $getFunctionsDevice, callable $getBuildsDevice, Document $document, Document $project): void { $projectId = $project->getId(); - $dbForProject = $this->getProjectDB($project); - $dbForConsole = $this->getConsoleDB(); + $dbForProject = $getProjectDB($project); $functionId = $document->getId(); $functionInternalId = $document->getInternalId(); @@ -595,8 +662,8 @@ class DeletesV1 extends Worker Query::equal('resourceType', ['function']), Query::equal('resourceInternalId', [$functionInternalId]), Query::equal('projectInternalId', [$project->getInternalId()]) - ], $dbForConsole, function (Document $document) use ($project) { - $this->deleteRule($document, $project); + ], $dbForConsole, function (Document $document) use ($project, $dbForConsole) { + $this->deleteRule($dbForConsole, $document); }); /** @@ -612,25 +679,25 @@ class DeletesV1 extends Worker * Delete Deployments */ Console::info("Deleting deployments for function " . $functionId); - $deviceFunctions = $this->getFunctionsDevice($projectId); + $functionsStorage = $getFunctionsDevice($projectId); $deploymentInternalIds = []; $this->deleteByGroup('deployments', [ Query::equal('resourceInternalId', [$functionInternalId]) - ], $dbForProject, function (Document $document) use ($deviceFunctions, &$deploymentInternalIds) { + ], $dbForProject, function (Document $document) use ($functionsStorage, &$deploymentInternalIds) { $deploymentInternalIds[] = $document->getInternalId(); - $this->deleteDeploymentFiles($deviceFunctions, $document); + $this->deleteDeploymentFiles($functionsStorage, $document); }); - /** + /** * Delete builds */ Console::info("Deleting builds for function " . $functionId); - $deviceBuilds = $this->getBuildsDevice($projectId); + $buildsStorage = $getBuildsDevice($projectId); foreach ($deploymentInternalIds as $deploymentInternalId) { $this->deleteByGroup('builds', [ Query::equal('deploymentInternalId', [$deploymentInternalId]) - ], $dbForProject, function (Document $document) use ($deviceBuilds) { - $this->deleteBuildFiles($deviceBuilds, $document); + ], $dbForProject, function (Document $document) use ($buildsStorage) { + $this->deleteBuildFiles($buildsStorage, $document); }); } @@ -646,10 +713,15 @@ class DeletesV1 extends Worker * Request executor to delete all deployment containers */ Console::info("Requesting executor to delete all deployment containers for function " . $functionId); - $this->deleteRuntimes($document, $project); + $this->deleteRuntimes($getProjectDB, $document, $project); } - protected function deleteDeploymentFiles(Device $device, Document $deployment) + /** + * @param Device $device + * @param Document $deployment + * @return void + */ + private function deleteDeploymentFiles(Device $device, Document $deployment): void { $deploymentId = $deployment->getId(); $deploymentPath = $deployment->getAttribute('path', ''); @@ -676,7 +748,12 @@ class DeletesV1 extends Worker } } - protected function deleteBuildFiles(Device $device, Document $build) + /** + * @param Device $device + * @param Document $build + * @return void + */ + private function deleteBuildFiles(Device $device, Document $build): void { $buildId = $build->getId(); $buildPath = $build->getAttribute('path', ''); @@ -702,51 +779,52 @@ class DeletesV1 extends Worker } /** - * @param Document $document deployment document + * @param callable $getProjectDB + * @param callable $getFunctionsDevice + * @param callable $getBuildsDevice + * @param Document $document * @param Document $project + * @return void + * @throws Exception */ - protected function deleteDeployment(Document $document, Document $project): void + private function deleteDeployment(callable $getProjectDB, callable $getFunctionsDevice, callable $getBuildsDevice, Document $document, Document $project): void { $projectId = $project->getId(); - $dbForProject = $this->getProjectDB($project); + $dbForProject = $getProjectDB($project); $deploymentId = $document->getId(); $deploymentInternalId = $document->getInternalId(); /** * Delete deployment files */ - $deviceFunctions = $this->getFunctionsDevice($projectId); - $this->deleteDeploymentFiles($deviceFunctions, $document); + $functionsStorage = $getFunctionsDevice($projectId); + $this->deleteDeploymentFiles($functionsStorage, $document); /** * Delete builds */ Console::info("Deleting builds for deployment " . $deploymentId); - $deviceBuilds = $this->getBuildsDevice($projectId); + $buildsStorage = $getBuildsDevice($projectId); $this->deleteByGroup('builds', [ Query::equal('deploymentInternalId', [$deploymentInternalId]) - ], $dbForProject, function (Document $document) use ($deviceBuilds) { - $this->deleteBuildFiles($deviceBuilds, $document); + ], $dbForProject, function (Document $document) use ($buildsStorage) { + $this->deleteBuildFiles($buildsStorage, $document); }); - /** * Request executor to delete all deployment containers */ Console::info("Requesting executor to delete deployment container for deployment " . $deploymentId); - $this->deleteRuntimes($document, $project); + $this->deleteRuntimes($getProjectDB, $document, $project); } - /** * @param Document $document to be deleted * @param Database $database to delete it from * @param callable|null $callback to perform after document is deleted - * - * @return bool - * @throws \Utopia\Database\Exception\Authorization + * @return void */ - protected function deleteById(Document $document, Database $database, callable $callback = null): bool + private function deleteById(Document $document, Database $database, callable $callback = null): void { if ($database->deleteDocument($document->getCollection(), $document->getId())) { Console::success('Deleted document "' . $document->getId() . '" successfully'); @@ -754,31 +832,27 @@ class DeletesV1 extends Worker if (is_callable($callback)) { $callback($document); } - - return true; } else { Console::error('Failed to delete document: ' . $document->getId()); - return false; } } /** + * @param Database $dbForConsole * @param callable $callback * @throws Exception */ - protected function deleteForProjectIds(callable $callback): void + private function deleteForProjectIds(database $dbForConsole, callable $callback): void { // TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document $count = 0; $chunk = 0; $limit = 50; - $projects = []; $sum = $limit; - $executionStart = \microtime(true); while ($sum === $limit) { - $projects = $this->getConsoleDB()->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]); + $projects = $dbForConsole->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]); $chunk++; @@ -801,35 +875,31 @@ class DeletesV1 extends Worker * @param array $queries * @param Database $database * @param callable|null $callback + * @return void * @throws Exception */ - protected function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null): void + private function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null): void { $count = 0; $chunk = 0; $limit = 50; - $results = []; $sum = $limit; $executionStart = \microtime(true); - try { - while ($sum === $limit) { - $chunk++; + while ($sum === $limit) { + $chunk++; - $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries)); + $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries)); - $sum = count($results); + $sum = count($results); - Console::info('Deleting chunk #' . $chunk . '. Found ' . $sum . ' documents in collection ' . $database->getNamespace() . '_' . $collection); + Console::info('Deleting chunk #' . $chunk . '. Found ' . $sum . ' documents'); - foreach ($results as $document) { - $this->deleteById($document, $database, $callback); - $count++; - } + foreach ($results as $document) { + $this->deleteById($document, $database, $callback); + $count++; } - } catch (\Exception $e) { - Console::error($e->getMessage()); } $executionEnd = \microtime(true); @@ -841,35 +911,26 @@ class DeletesV1 extends Worker * @param string $collection collectionID * @param Query[] $queries * @param Database $database - * @param callable $callback + * @param callable|null $callback + * @return void + * @throws Exception */ - protected function listByGroup(string $collection, array $queries, Database $database, callable $callback = null): void + private function listByGroup(string $collection, array $queries, Database $database, callable $callback = null): void { $count = 0; $chunk = 0; $limit = 50; - $results = []; $sum = $limit; - $cursor = null; $executionStart = \microtime(true); while ($sum === $limit) { $chunk++; - $mergedQueries = \array_merge([Query::limit($limit)], $queries); - if ($cursor instanceof Document) { - $mergedQueries[] = Query::cursorAfter($cursor); - } - - $results = $database->find($collection, $mergedQueries); + $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries)); $sum = count($results); - if ($sum > 0) { - $cursor = $results[$sum - 1]; - } - foreach ($results as $document) { if (is_callable($callback)) { $callback($document); @@ -885,12 +946,12 @@ class DeletesV1 extends Worker } /** + * @param Database $dbForConsole * @param Document $document rule document - * @param Document $project project document + * @return void */ - protected function deleteRule(Document $document): void + private function deleteRule(Database $dbForConsole, Document $document): void { - $consoleDB = $this->getConsoleDB(); $domain = $document->getAttribute('domain'); $directory = APP_STORAGE_CERTIFICATES . '/' . $domain; @@ -907,25 +968,40 @@ class DeletesV1 extends Worker // Delete certificate document, so Appwrite is aware of change if (isset($document['certificateId'])) { - $consoleDB->deleteDocument('certificates', $document['certificateId']); + $dbForConsole->deleteDocument('certificates', $document['certificateId']); } } - protected function deleteBucket(Document $document, Document $project) + /** + * @param callable $getProjectDB + * @param callable $getFilesDevice + * @param Document $document + * @param Document $project + * @return void + */ + private function deleteBucket(callable $getProjectDB, callable $getFilesDevice, Document $document, Document $project): void { $projectId = $project->getId(); - $dbForProject = $this->getProjectDB($project); + $dbForProject = $getProjectDB($project); + $dbForProject->deleteCollection('bucket_' . $document->getInternalId()); - $device = $this->getFilesDevice($projectId); + $device = $getFilesDevice($projectId); $device->deletePath($document->getId()); } - protected function deleteInstallation(Document $document, Document $project) + /** + * @param Database $dbForConsole + * @param callable $getProjectDB + * @param Document $document + * @param Document $project + * @return void + * @throws Exception + */ + private function deleteInstallation(Database $dbForConsole, callable $getProjectDB, Document $document, Document $project): void { - $dbForProject = $this->getProjectDB($project); - $dbForConsole = $this->getConsoleDB(); + $dbForProject = $getProjectDB($project); $this->listByGroup('functions', [ Query::equal('installationInternalId', [$document->getInternalId()]) @@ -945,18 +1021,25 @@ class DeletesV1 extends Worker }); } - protected function deleteRuntimes(?Document $function, Document $project) + /** + * @param callable $getProjectDB + * @param ?Document $function + * @param Document $project + * @return void + * @throws Exception + */ + private function deleteRuntimes(callable $getProjectDB, ?Document $function, Document $project): void { $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); - $deleteByFunction = function (Document $function) use ($project, $executor) { + $deleteByFunction = function (Document $function) use ($getProjectDB, $project, $executor) { $this->listByGroup( 'deployments', [ Query::equal('resourceInternalId', [$function->getInternalId()]), Query::equal('resourceType', ['functions']), ], - $this->getProjectDB($project), + $getProjectDB($project), function (Document $deployment) use ($project, $executor) { $deploymentId = $deployment->getId(); @@ -982,7 +1065,7 @@ class DeletesV1 extends Worker $this->listByGroup( 'functions', [], - $this->getProjectDB($project), + $getProjectDB($project), function (Document $function) use ($deleteByFunction) { $deleteByFunction($function); } diff --git a/app/workers/functions.php b/src/Appwrite/Platform/Workers/Functions.php similarity index 77% rename from app/workers/functions.php rename to src/Appwrite/Platform/Workers/Functions.php index 20eda492bd..6b7b5efe03 100644 --- a/app/workers/functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -1,59 +1,250 @@ desc('Functions worker') + ->inject('message') + ->inject('dbForProject') + ->inject('queueForFunctions') + ->inject('queueForEvents') + ->inject('usage') + ->inject('log') + ->callback(fn(Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Stats $usage, Log $log) => $this->action($message, $dbForProject, $queueForFunctions, $queueForEvents, $usage, $log)); + } + + /** + * @param Message $message + * @param Database $dbForProject + * @param Func $queueForFunctions + * @param Event $queueForEvents + * @param Stats $usage + * @param Log $log + * @return void + * @throws Authorization + * @throws Structure + * @throws \Utopia\Database\Exception + * @throws Conflict + */ + public function action(Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Stats $usage, Log $log): void + { + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + $type = $payload['type'] ?? ''; + $events = $payload['events'] ?? []; + $data = $payload['data'] ?? ''; + $eventData = $payload['payload'] ?? ''; + $project = new Document($payload['project'] ?? []); + $function = new Document($payload['function'] ?? []); + $user = new Document($payload['user'] ?? []); + $method = $payload['method'] ?? 'POST'; + $headers = $payload['headers'] ?? []; + $path = $payload['path'] ?? '/'; + + if ($project->getId() === 'console') { + return; + } + + if (!empty($events)) { + $limit = 30; + $sum = 30; + $offset = 0; + /** @var Document[] $functions */ + while ($sum >= $limit) { + $functions = $dbForProject->find('functions', [ + Query::limit($limit), + Query::offset($offset), + Query::orderAsc('name'), + ]); + + $sum = \count($functions); + $offset = $offset + $limit; + + Console::log('Fetched ' . $sum . ' functions...'); + + foreach ($functions as $function) { + if (!array_intersect($events, $function->getAttribute('events', []))) { + continue; + } + Console::success('Iterating function: ' . $function->getAttribute('name')); + + $this->execute( + log: $log, + dbForProject: $dbForProject, + queueForFunctions: $queueForFunctions, + usage: $usage, + queueForEvents: $queueForEvents, + project: $project, + function: $function, + trigger: 'event', + path: '/', + method: 'POST', + headers: [ + 'user-agent' => 'Appwrite/' . APP_VERSION_STABLE, + 'content-type' => 'application/json' + ], + data: null, + user: $user, + jwt: null, + event: $events[0], + eventData: \is_string($eventData) ? $eventData : \json_encode($eventData), + executionId: null, + ); + Console::success('Triggered function: ' . $events[0]); + } + } + return; + } + + /** + * Handle Schedule and HTTP execution. + */ + switch ($type) { + case 'http': + $jwt = $payload['jwt'] ?? ''; + $execution = new Document($payload['execution'] ?? []); + $user = new Document($payload['user'] ?? []); + $this->execute( + log: $log, + dbForProject: $dbForProject, + queueForFunctions: $queueForFunctions, + usage: $usage, + queueForEvents: $queueForEvents, + project: $project, + function: $function, + trigger: 'http', + path: $path, + method: $method, + headers: $headers, + data: $data, + user: $user, + jwt: $jwt, + event: null, + eventData: null, + executionId: $execution->getId() + ); + break; + case 'schedule': + $this->execute( + log: $log, + dbForProject: $dbForProject, + queueForFunctions: $queueForFunctions, + usage: $usage, + queueForEvents: $queueForEvents, + project: $project, + function: $function, + trigger: 'schedule', + path: $path, + method: $method, + headers: $headers, + data: null, + user: null, + jwt: null, + event: null, + eventData: null, + executionId: null, + ); + break; + } + } + + /** + * @param Log $log + * @param Database $dbForProject + * @param Func $queueForFunctions + * @param Stats $usage + * @param Event $queueForEvents + * @param Document $project + * @param Document $function + * @param string $trigger + * @param string $path + * @param string $method + * @param array $headers + * @param string|null $data + * @param Document|null $user + * @param string|null $jwt + * @param string|null $event + * @param string|null $eventData + * @param string|null $executionId + * @return void + * @throws Authorization + * @throws Structure + * @throws \Utopia\Database\Exception + * @throws Conflict + */ + private function execute( Log $log, - Func $queueForFunctions, Database $dbForProject, - Client $statsd, + Func $queueForFunctions, + stats $usage, + Event $queueForEvents, Document $project, Document $function, string $trigger, - string $data = null, string $path, string $method, array $headers, + string $data = null, ?Document $user = null, string $jwt = null, string $event = null, string $eventData = null, string $executionId = null, - ) { - $user ??= new Document(); - $functionId = $function->getId(); - $deploymentId = $function->getAttribute('deployment', ''); + ): void { + $user ??= new Document(); + $functionId = $function->getId(); + $deploymentId = $function->getAttribute('deployment', ''); - $log->addTag('functionId', $functionId); - $log->addTag('projectId', $project->getId()); + $log->addTag('functionId', $functionId); + $log->addTag('projectId', $project->getId()); - /** Check if deployment exists */ - $deployment = $dbForProject->getDocument('deployments', $deploymentId); + /** Check if deployment exists */ + $deployment = $dbForProject->getDocument('deployments', $deploymentId); if ($deployment->getAttribute('resourceId') !== $functionId) { throw new Exception('Deployment not found. Create deployment before trying to execute a function'); @@ -63,8 +254,8 @@ Server::setResource('execute', function () { throw new Exception('Deployment not found. Create deployment before trying to execute a function'); } - /** Check if build has exists */ - $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); + /** Check if build has exists */ + $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); if ($build->isEmpty()) { throw new Exception('Build not found'); } @@ -73,7 +264,7 @@ Server::setResource('execute', function () { throw new Exception('Build not ready'); } - /** Check if runtime is supported */ + /** Check if runtime is supported */ $version = $function->getAttribute('version', 'v2'); $runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []); @@ -88,6 +279,7 @@ Server::setResource('execute', function () { $headers['x-appwrite-user-id'] = $user->getId() ?? ''; $headers['x-appwrite-user-jwt'] = $jwt ?? ''; + /** Create execution or update execution status */ /** Create execution or update execution status */ $execution = $dbForProject->getDocument('executions', $executionId ?? ''); if ($execution->isEmpty()) { @@ -220,7 +412,6 @@ Server::setResource('execute', function () { ->setAttribute('duration', $executionResponse['duration']); } catch (\Throwable $th) { $durationEnd = \microtime(true); - $execution ->setAttribute('duration', $durationEnd - $durationStart) ->setAttribute('status', 'failed') @@ -234,11 +425,11 @@ Server::setResource('execute', function () { if ($function->getAttribute('logging')) { $execution = $dbForProject->updateDocument('executions', $executionId, $execution); } - /** Trigger Webhook */ $executionModel = new Execution(); - $executionUpdate = new Event(Event::WEBHOOK_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME); - $executionUpdate + $queueForEvents + ->setQueue(Event::WEBHOOK_QUEUE_NAME) + ->setClass(Event::WEBHOOK_CLASS_NAME) ->setProject($project) ->setUser($user) ->setEvent('functions.[functionId].executions.[executionId].update') @@ -249,7 +440,7 @@ Server::setResource('execute', function () { /** Trigger Functions */ $queueForFunctions - ->from($executionUpdate) + ->from($queueForEvents) ->trigger(); /** Trigger realtime event */ @@ -258,7 +449,7 @@ Server::setResource('execute', function () { 'executionId' => $execution->getId() ]); $target = Realtime::fromPayload( - // Pass first, most verbose event pattern + // Pass first, most verbose event pattern event: $allEvents[0], payload: $execution ); @@ -277,9 +468,12 @@ Server::setResource('execute', function () { roles: $target['roles'] ); + if (!empty($error)) { + throw new Exception($error, $errorCode); + } + /** Update usage stats */ if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { - $usage = new Stats($statsd); $usage ->setParam('projectId', $project->getId()) ->setParam('projectInternalId', $project->getInternalId()) @@ -291,137 +485,5 @@ Server::setResource('execute', function () { ->setParam('networkResponseSize', 0) ->submit(); } - - if (!empty($error)) { - throw new Exception($error, $errorCode); - } - }; -}); - -$server->job() - ->inject('message') - ->inject('dbForProject') - ->inject('queueForFunctions') - ->inject('statsd') - ->inject('execute') - ->inject('log') - ->action(function (Message $message, Database $dbForProject, Func $queueForFunctions, Client $statsd, callable $execute, Log $log) { - $payload = $message->getPayload() ?? []; - - if (empty($payload)) { - throw new Exception('Missing payload'); - } - - $type = $payload['type'] ?? ''; - $events = $payload['events'] ?? []; - $data = $payload['body'] ?? ''; - $eventData = $payload['payload'] ?? ''; - $project = new Document($payload['project'] ?? []); - $function = new Document($payload['function'] ?? []); - $user = new Document($payload['user'] ?? []); - - if ($project->getId() === 'console') { - return; - } - - if (!empty($events)) { - $limit = 30; - $sum = 30; - $offset = 0; - $functions = []; - /** @var Document[] $functions */ - while ($sum >= $limit) { - $functions = $dbForProject->find('functions', [ - Query::limit($limit), - Query::offset($offset) - ]); - - $sum = \count($functions); - $offset = $offset + $limit; - - Console::log('Fetched ' . $sum . ' functions...'); - - foreach ($functions as $function) { - if (!array_intersect($events, $function->getAttribute('events', []))) { - continue; - } - Console::success('Iterating function: ' . $function->getAttribute('name')); - $execute( - log: $log, - statsd: $statsd, - dbForProject: $dbForProject, - project: $project, - function: $function, - queueForFunctions: $queueForFunctions, - trigger: 'event', - event: $events[0], - eventData: \is_string($eventData) ? $eventData : \json_encode($eventData), - user: $user, - data: null, - executionId: null, - jwt: null, - path: '/', - method: 'POST', - headers: [ - 'user-agent' => 'Appwrite/' . APP_VERSION_STABLE, - 'content-type' => 'application/json' - ], - ); - Console::success('Triggered function: ' . $events[0]); - } - } - return; - } - - /** - * Handle Schedule and HTTP execution. - */ - switch ($type) { - case 'http': - $jwt = $payload['jwt'] ?? ''; - $execution = new Document($payload['execution'] ?? []); - $user = new Document($payload['user'] ?? []); - $execute( - log: $log, - project: $project, - function: $function, - dbForProject: $dbForProject, - queueForFunctions: $queueForFunctions, - trigger: 'http', - executionId: $execution->getId(), - event: null, - eventData: null, - data: $data, - user: $user, - jwt: $jwt, - path: $payload['path'] ?? '', - method: $payload['method'] ?? 'POST', - headers: $payload['headers'] ?? [], - statsd: $statsd, - ); - break; - case 'schedule': - $execute( - log: $log, - project: $project, - function: $function, - dbForProject: $dbForProject, - queueForFunctions: $queueForFunctions, - trigger: 'schedule', - executionId: null, - event: null, - eventData: null, - data: null, - user: null, - jwt: null, - path: $payload['path'] ?? '/', - method: $payload['method'] ?? 'POST', - headers: $payload['headers'] ?? [], - statsd: $statsd, - ); - break; - } - }); - -$server->workerStart(); -$server->start(); + } +} diff --git a/app/workers/mails.php b/src/Appwrite/Platform/Workers/Mails.php similarity index 60% rename from app/workers/mails.php rename to src/Appwrite/Platform/Workers/Mails.php index 56676b4aed..b0cf28caeb 100644 --- a/app/workers/mails.php +++ b/src/Appwrite/Platform/Workers/Mails.php @@ -1,45 +1,63 @@ desc('Mails worker') + ->inject('message') + ->inject('register') + ->callback(fn($message, $register) => $this->action($message, $register)); } - public function run(): void + /** + * @param Message $message + * @param Registry $register + * @throws \PHPMailer\PHPMailer\Exception + * @return void + * @throws Exception + */ + public function action(Message $message, Registry $register): void { - global $register; - $smtp = $this->args['smtp']; + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + $smtp = $payload['smtp']; if (empty($smtp) && empty(App::getEnv('_APP_SMTP_HOST'))) { Console::info('Skipped mail processing. No SMTP configuration has been set.'); return; } - $recipient = $this->args['recipient']; - $subject = $this->args['subject']; - $body = $this->args['body']; - $variables = $this->args['variables']; - $name = $this->args['name']; - - $body = Template::fromFile(__DIR__ . '/../config/locale/templates/email-base.tpl'); + $recipient = $payload['recipient']; + $subject = $payload['subject']; + $variables = $payload['variables']; + $name = $payload['name']; + $body = Template::fromFile(__DIR__ . '/../../../../app/config/locale/templates/email-base.tpl'); foreach ($variables as $key => $value) { $body->setParam('{{' . $key . '}}', $value); @@ -70,6 +88,11 @@ class MailsV1 extends Worker } } + /** + * @param array $smtp + * @return PHPMailer + * @throws \PHPMailer\PHPMailer\Exception + */ protected function getMailer(array $smtp): PHPMailer { $mail = new PHPMailer(true); @@ -99,8 +122,4 @@ class MailsV1 extends Worker return $mail; } - - public function shutdown(): void - { - } } diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php new file mode 100644 index 0000000000..76b86e4f0c --- /dev/null +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -0,0 +1,107 @@ +provider = App::getEnv('_APP_SMS_PROVIDER', ''); + if (!empty($this->provider)) { + $this->dsn = new DSN($this->provider); + $this->user = $this->dsn->getUser(); + $this->secret = $this->dsn->getPassword(); + } + + $this + ->desc('Messaging worker') + ->inject('message') + ->callback(fn($message) => $this->action($message)); + } + + /** + * @param Message $message + * @return void + * @throws Exception + */ + public function action(Message $message): void + { + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + Console::error('Payload arg not found'); + return; + } + + if (empty($payload['recipient'])) { + Console::error('Recipient arg not found'); + return; + } + + if (empty($payload['message'])) { + Console::error('Message arg not found'); + return; + } + + $sms = match ($this->dsn->getHost()) { + 'mock' => new Mock($this->user, $this->secret), // used for tests + 'twilio' => new Twilio($this->user, $this->secret), + 'text-magic' => new TextMagic($this->user, $this->secret), + 'telesign' => new Telesign($this->user, $this->secret), + 'msg91' => new Msg91($this->user, $this->secret), + 'vonage' => new Vonage($this->user, $this->secret), + default => null + }; + + if (empty(App::getEnv('_APP_SMS_PROVIDER'))) { + Console::error('Skipped sms processing. No Phone provider has been set.'); + return; + } + + $from = App::getEnv('_APP_SMS_FROM'); + + if (empty($from)) { + Console::error('Skipped sms processing. No phone number has been set.'); + return; + } + + $message = new SMS( + to: [$payload['recipient']], + content: $payload['message'], + from: $from, + ); + + try { + $sms->send($message); + } catch (\Exception $error) { + throw new Exception('Error sending message: ' . $error->getMessage(), 500); + } + } +} diff --git a/app/workers/migrations.php b/src/Appwrite/Platform/Workers/Migrations.php similarity index 63% rename from app/workers/migrations.php rename to src/Appwrite/Platform/Workers/Migrations.php index 0f8194acf8..31b0df59a3 100644 --- a/app/workers/migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -1,13 +1,21 @@ desc('Migrations worker') + ->inject('message') + ->inject('dbForProject') + ->inject('dbForConsole') + ->callback(fn(Message $message, Database $dbForProject, Database $dbForConsole) => $this->action($message, $dbForProject, $dbForConsole)); } - public function run(): void + /** + * @param Message $message + * @param Database $dbForProject + * @param Database $dbForConsole + * @return void + * @throws Exception + */ + public function action(Message $message, Database $dbForProject, Database $dbForConsole): void { - $type = $this->args['type'] ?? ''; - $events = $this->args['events'] ?? []; - $project = new Document($this->args['project'] ?? []); - $user = new Document($this->args['user'] ?? []); - $payload = json_encode($this->args['payload'] ?? []); + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + $events = $payload['events'] ?? []; + $project = new Document($payload['project'] ?? []); + $migration = new Document($payload['migration'] ?? []); if ($project->getId() === 'console') { return; } + $this->dbForProject = $dbForProject; + $this->dbForConsole = $dbForConsole; + /** * Handle Event execution. */ @@ -60,57 +82,51 @@ class MigrationsV1 extends Worker return; } - $this->dbForProject = $this->getProjectDB($project); - - $this->processMigration(); + $this->processMigration($project, $migration); } /** - * Process Source - * + * @param string $source + * @param array $credentials * @return Source - * - * @throws \Exception + * @throws Exception */ protected function processSource(string $source, array $credentials): Source { - switch ($source) { - case Firebase::getName(): - return new Firebase( - json_decode($credentials['serviceAccount'], true), - ); - break; - case Supabase::getName(): - return new Supabase( - $credentials['endpoint'], - $credentials['apiKey'], - $credentials['databaseHost'], - 'postgres', - $credentials['username'], - $credentials['password'], - $credentials['port'], - ); - break; - case NHost::getName(): - return new NHost( - $credentials['subdomain'], - $credentials['region'], - $credentials['adminSecret'], - $credentials['database'], - $credentials['username'], - $credentials['password'], - $credentials['port'], - ); - break; - case Appwrite::getName(): - return new Appwrite($credentials['projectId'], str_starts_with($credentials['endpoint'], 'http://localhost/v1') ? 'http://appwrite/v1' : $credentials['endpoint'], $credentials['apiKey']); - break; - default: - throw new \Exception('Invalid source type'); - break; - } + return match ($source) { + Firebase::getName() => new Firebase( + json_decode($credentials['serviceAccount'], true), + ), + Supabase::getName() => new Supabase( + $credentials['endpoint'], + $credentials['apiKey'], + $credentials['databaseHost'], + 'postgres', + $credentials['username'], + $credentials['password'], + $credentials['port'], + ), + NHost::getName() => new NHost( + $credentials['subdomain'], + $credentials['region'], + $credentials['adminSecret'], + $credentials['database'], + $credentials['username'], + $credentials['password'], + $credentials['port'], + ), + Appwrite::getName() => new Appwrite($credentials['projectId'], str_starts_with($credentials['endpoint'], 'http://localhost/v1') ? 'http://appwrite/v1' : $credentials['endpoint'], $credentials['apiKey']), + default => throw new \Exception('Invalid source type'), + }; } + /** + * @throws Authorization + * @throws Structure + * @throws Conflict + * @throws \Utopia\Database\Exception + * @throws Exception + */ protected function updateMigrationDocument(Document $migration, Document $project): Document { /** Trigger Realtime */ @@ -143,16 +159,30 @@ class MigrationsV1 extends Worker return $this->dbForProject->updateDocument('migrations', $migration->getId(), $migration); } - protected function removeAPIKey(Document $apiKey) + /** + * @param Document $apiKey + * @return void + * @throws \Utopia\Database\Exception + * @throws Authorization + * @throws Conflict + * @throws Restricted + * @throws Structure + */ + protected function removeAPIKey(Document $apiKey): void { - $consoleDB = $this->getConsoleDB(); - - $consoleDB->deleteDocument('keys', $apiKey->getId()); + $this->dbForConsole->deleteDocument('keys', $apiKey->getId()); } + /** + * @param Document $project + * @return Document + * @throws Authorization + * @throws Structure + * @throws \Utopia\Database\Exception + * @throws Exception + */ protected function generateAPIKey(Document $project): Document { - $consoleDB = $this->getConsoleDB(); $generatedSecret = bin2hex(\random_bytes(128)); $key = new Document([ @@ -189,18 +219,23 @@ class MigrationsV1 extends Worker 'secret' => $generatedSecret, ]); - $consoleDB->createDocument('keys', $key); - $consoleDB->deleteCachedDocument('projects', $project->getId()); + $this->dbForConsole->createDocument('keys', $key); + $this->dbForConsole->deleteCachedDocument('projects', $project->getId()); return $key; } /** - * Process Migration - * + * @param Document $project + * @param Document $migration * @return void + * @throws Authorization + * @throws Conflict + * @throws Restricted + * @throws Structure + * @throws \Utopia\Database\Exception */ - protected function processMigration(): void + protected function processMigration(Document $project, Document $migration): void { /** * @var Document $migrationDocument @@ -208,11 +243,11 @@ class MigrationsV1 extends Worker */ $migrationDocument = null; $transfer = null; - $projectDocument = $this->getConsoleDB()->getDocument('projects', $this->args['project']['$id']); + $projectDocument = $this->dbForConsole->getDocument('projects', $project->getId()); $tempAPIKey = $this->generateAPIKey($projectDocument); try { - $migrationDocument = $this->dbForProject->getDocument('migrations', $this->args['migration']['$id']); + $migrationDocument = $this->dbForProject->getDocument('migrations', $migration->getId()); $migrationDocument->setAttribute('stage', 'processing'); $migrationDocument->setAttribute('status', 'processing'); $this->updateMigrationDocument($migrationDocument, $projectDocument); @@ -292,17 +327,4 @@ class MigrationsV1 extends Worker } } } - - /** - * Process Verification - * - * @return void - */ - protected function processVerification(): void - { - } - - public function shutdown(): void - { - } } diff --git a/app/workers/webhooks.php b/src/Appwrite/Platform/Workers/Webhooks.php similarity index 50% rename from app/workers/webhooks.php rename to src/Appwrite/Platform/Workers/Webhooks.php index 4048581c0c..dd7b92bf5e 100644 --- a/app/workers/webhooks.php +++ b/src/Appwrite/Platform/Workers/Webhooks.php @@ -1,38 +1,54 @@ desc('Webhooks worker') + ->inject('message') + ->callback(fn($message) => $this->action($message)); } - public function run(): void + /** + * @param Message $message + * @return void + * @throws Exception + */ + public function action(Message $message): void { - $events = $this->args['events']; - $payload = json_encode($this->args['payload']); - $project = new Document($this->args['project']); - $user = new Document($this->args['user'] ?? []); + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + $events = $payload['events']; + $webhookPayload = json_encode($payload['payload']); + $project = new Document($payload['project']); + $user = new Document($payload['user'] ?? []); foreach ($project->getAttribute('webhooks', []) as $webhook) { if (array_intersect($webhook->getAttribute('events', []), $events)) { - $this->execute($events, $payload, $webhook, $user, $project); + $this->execute($events, $webhookPayload, $webhook, $user, $project); } } @@ -41,8 +57,17 @@ class WebhooksV1 extends Worker } } - protected function execute(array $events, string $payload, Document $webhook, Document $user, Document $project): void + /** + * @param array $events + * @param string $payload + * @param Document $webhook + * @param Document $user + * @param Document $project + * @return void + */ + private function execute(array $events, string $payload, Document $webhook, Document $user, Document $project): void { + $url = \rawurldecode($webhook->getAttribute('url')); $signatureKey = $webhook->getAttribute('signatureKey'); $signature = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true)); @@ -63,14 +88,14 @@ class WebhooksV1 extends Worker $ch, CURLOPT_HTTPHEADER, [ - 'Content-Type: application/json', - 'Content-Length: ' . \strlen($payload), - 'X-' . APP_NAME . '-Webhook-Id: ' . $webhook->getId(), - 'X-' . APP_NAME . '-Webhook-Events: ' . implode(',', $events), - 'X-' . APP_NAME . '-Webhook-Name: ' . $webhook->getAttribute('name', ''), - 'X-' . APP_NAME . '-Webhook-User-Id: ' . $user->getId(), - 'X-' . APP_NAME . '-Webhook-Project-Id: ' . $project->getId(), - 'X-' . APP_NAME . '-Webhook-Signature: ' . $signature, + 'Content-Type: application/json', + 'Content-Length: ' . \strlen($payload), + 'X-' . APP_NAME . '-Webhook-Id: ' . $webhook->getId(), + 'X-' . APP_NAME . '-Webhook-Events: ' . implode(',', $events), + 'X-' . APP_NAME . '-Webhook-Name: ' . $webhook->getAttribute('name', ''), + 'X-' . APP_NAME . '-Webhook-User-Id: ' . $user->getId(), + 'X-' . APP_NAME . '-Webhook-Project-Id: ' . $project->getId(), + 'X-' . APP_NAME . '-Webhook-Signature: ' . $signature, ] ); @@ -90,9 +115,4 @@ class WebhooksV1 extends Worker \curl_close($ch); } - - public function shutdown(): void - { - $this->errors = []; - } } diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php deleted file mode 100644 index 7693f1e6ff..0000000000 --- a/src/Appwrite/Resque/Worker.php +++ /dev/null @@ -1,409 +0,0 @@ -init(); - } catch (\Throwable $error) { - foreach (self::$errorCallbacks as $errorCallback) { - $errorCallback($error, "init", $this->getName()); - } - - throw $error; - } - } - - /** - * A wrapper around 'run' function with non-worker-specific code - * - * @return void - * @throws \Exception|\Throwable - */ - public function perform(): void - { - try { - /** - * Disabling global authorization in workers. - */ - Authorization::disable(); - Authorization::setDefaultStatus(false); - $this->run(); - } catch (\Throwable $error) { - foreach (self::$errorCallbacks as $errorCallback) { - $errorCallback($error, "run", $this->getName(), $this->args); - } - - throw $error; - } - } - - /** - * A wrapper around 'shutdown' function with non-worker-specific code - * - * @return void - * @throws \Exception|\Throwable - */ - public function tearDown(): void - { - global $register; - - try { - $pools = $register->get('pools'); /** @var Group $pools */ - $pools->reclaim(); - - $this->shutdown(); - } catch (\Throwable $error) { - foreach (self::$errorCallbacks as $errorCallback) { - $errorCallback($error, "shutdown", $this->getName()); - } - - throw $error; - } - } - - - /** - * Register callback. Will be executed when error occurs. - * @param callable $callback - * @return void - */ - public static function error(callable $callback): void - { - self::$errorCallbacks[] = $callback; - } - - /** - * Get internal project database - * @param Document $project - * @return Database - * @throws Exception - */ - protected static $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools - - protected function getProjectDB(Document $project): Database - { - global $register; - - $pools = $register->get('pools'); /** @var Group $pools */ - - if ($project->isEmpty() || $project->getId() === 'console') { - return $this->getConsoleDB(); - } - - $databaseName = $project->getAttribute('database'); - - if (isset(self::$databases[$databaseName])) { - $database = self::$databases[$databaseName]; - $database->setNamespace('_' . $project->getInternalId()); - return $database; - } - - $dbAdapter = $pools - ->get($project->getAttribute('database')) - ->pop() - ->getResource() - ; - - $database = new Database($dbAdapter, $this->getCache()); - - self::$databases[$databaseName] = $database; - - $database->setNamespace('_' . $project->getInternalId()); - - return $database; - } - - /** - * Get console database - * @return Database - * @throws Exception - */ - protected function getConsoleDB(): Database - { - global $register; - - $pools = $register->get('pools'); /** @var Group $pools */ - - $databaseName = 'console'; - - if (isset(self::$databases[$databaseName])) { - $database = self::$databases[$databaseName]; - $database->setNamespace('_console'); - return $database; - } - - $dbAdapter = $pools - ->get('console') - ->pop() - ->getResource() - ; - - $database = new Database($dbAdapter, $this->getCache()); - - self::$databases[$databaseName] = $database; - - $database->setNamespace('_console'); - - return $database; - } - - - /** - * Get Cache - * @return Cache - */ - protected function getCache(): Cache - { - global $register; - - $pools = $register->get('pools'); /** @var Group $pools */ - - $list = Config::getParam('pools-cache', []); - $adapters = []; - - foreach ($list as $value) { - $adapters[] = $pools - ->get($value) - ->pop() - ->getResource() - ; - } - - return new Cache(new Sharding($adapters)); - } - - /** - * Get usage queue - * @return Usage - * @throws Exception - */ - protected function getUsageQueue(): Usage - { - global $register; - - $pools = $register->get('pools'); /** @var Group $pools */ - $queue = $pools - ->get('queue') - ->pop() - ->getResource(); - - return new Usage($queue); - } - - /** - * Get Functions Storage Device - * @param string $projectId of the project - * @return Device - */ - protected function getFunctionsDevice(string $projectId): Device - { - return $this->getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $projectId); - } - - /** - * Get Files Storage Device - * @param string $projectId of the project - * @return Device - */ - protected function getFilesDevice(string $projectId): Device - { - return $this->getDevice(APP_STORAGE_UPLOADS . '/app-' . $projectId); - } - - /** - * Get Builds Storage Device - * @param string $projectId of the project - * @return Device - */ - protected function getBuildsDevice(string $projectId): Device - { - return $this->getDevice(APP_STORAGE_BUILDS . '/app-' . $projectId); - } - - protected function getCacheDevice(string $projectId): Device - { - return $this->getDevice(APP_STORAGE_CACHE . '/app-' . $projectId); - } - - /** - * Get Device based on selected storage environment - * @param string $root path of the device - * @return Device - */ - public function getDevice(string $root): Device - { - $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); - - if (!empty($connection)) { - $acl = 'private'; - $device = Storage::DEVICE_LOCAL; - $accessKey = ''; - $accessSecret = ''; - $bucket = ''; - $region = ''; - - try { - $dsn = new DSN($connection); - $device = $dsn->getScheme(); - $accessKey = $dsn->getUser() ?? ''; - $accessSecret = $dsn->getPassword() ?? ''; - $bucket = $dsn->getPath() ?? ''; - $region = $dsn->getParam('region'); - } catch (\Exception $e) { - Console::warning($e->getMessage() . 'Invalid DSN. Defaulting to Local device.'); - } - - switch ($device) { - case Storage::DEVICE_S3: - return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case STORAGE::DEVICE_DO_SPACES: - return new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case Storage::DEVICE_BACKBLAZE: - return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case Storage::DEVICE_LINODE: - return new Linode($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case Storage::DEVICE_WASABI: - return new Wasabi($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case Storage::DEVICE_LOCAL: - default: - return new Local($root); - } - } else { - switch (strtolower(App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) ?? '')) { - case Storage::DEVICE_LOCAL: - default: - return new Local($root); - case Storage::DEVICE_S3: - $s3AccessKey = App::getEnv('_APP_STORAGE_S3_ACCESS_KEY', ''); - $s3SecretKey = App::getEnv('_APP_STORAGE_S3_SECRET', ''); - $s3Region = App::getEnv('_APP_STORAGE_S3_REGION', ''); - $s3Bucket = App::getEnv('_APP_STORAGE_S3_BUCKET', ''); - $s3Acl = 'private'; - return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); - case Storage::DEVICE_DO_SPACES: - $doSpacesAccessKey = App::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', ''); - $doSpacesSecretKey = App::getEnv('_APP_STORAGE_DO_SPACES_SECRET', ''); - $doSpacesRegion = App::getEnv('_APP_STORAGE_DO_SPACES_REGION', ''); - $doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', ''); - $doSpacesAcl = 'private'; - return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); - case Storage::DEVICE_BACKBLAZE: - $backblazeAccessKey = App::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', ''); - $backblazeSecretKey = App::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', ''); - $backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', ''); - $backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', ''); - $backblazeAcl = 'private'; - return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); - case Storage::DEVICE_LINODE: - $linodeAccessKey = App::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', ''); - $linodeSecretKey = App::getEnv('_APP_STORAGE_LINODE_SECRET', ''); - $linodeRegion = App::getEnv('_APP_STORAGE_LINODE_REGION', ''); - $linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', ''); - $linodeAcl = 'private'; - return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl); - case Storage::DEVICE_WASABI: - $wasabiAccessKey = App::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', ''); - $wasabiSecretKey = App::getEnv('_APP_STORAGE_WASABI_SECRET', ''); - $wasabiRegion = App::getEnv('_APP_STORAGE_WASABI_REGION', ''); - $wasabiBucket = App::getEnv('_APP_STORAGE_WASABI_BUCKET', ''); - $wasabiAcl = 'private'; - return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl); - } - } - } -} diff --git a/tests/e2e/Services/Account/AccountCustomClientTest.php b/tests/e2e/Services/Account/AccountCustomClientTest.php index 1441ab7f98..0e0634850a 100644 --- a/tests/e2e/Services/Account/AccountCustomClientTest.php +++ b/tests/e2e/Services/Account/AccountCustomClientTest.php @@ -1011,7 +1011,7 @@ class AccountCustomClientTest extends Scope $smsRequest = $this->getLastRequest(); return \array_merge($data, [ - 'token' => $smsRequest['data']['message'] + 'token' => $smsRequest['data']['secret'] ]); } diff --git a/tests/e2e/Services/Databases/DatabasesConsoleClientTest.php b/tests/e2e/Services/Databases/DatabasesConsoleClientTest.php index 04ac00c263..91a0d089ae 100644 --- a/tests/e2e/Services/Databases/DatabasesConsoleClientTest.php +++ b/tests/e2e/Services/Databases/DatabasesConsoleClientTest.php @@ -263,7 +263,7 @@ class DatabasesConsoleClientTest extends Scope 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ - 'limit' => 1 + 'queries' => ['limit(1)'] ]); $this->assertEquals(200, $logs['headers']['status-code']); @@ -275,7 +275,7 @@ class DatabasesConsoleClientTest extends Scope 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ - 'offset' => 1 + 'queries' => ['offset(1)'] ]); $this->assertEquals(200, $logs['headers']['status-code']); @@ -286,8 +286,7 @@ class DatabasesConsoleClientTest extends Scope 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ - 'offset' => 1, - 'limit' => 1 + 'queries' => ['offset(1)', 'limit(1)'] ]); $this->assertEquals(200, $logs['headers']['status-code']); diff --git a/tests/e2e/Services/Health/HealthCustomServerTest.php b/tests/e2e/Services/Health/HealthCustomServerTest.php index 4ac15bd95c..c923491942 100644 --- a/tests/e2e/Services/Health/HealthCustomServerTest.php +++ b/tests/e2e/Services/Health/HealthCustomServerTest.php @@ -214,7 +214,7 @@ class HealthCustomServerTest extends Scope /** * Test for SUCCESS */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/databases', array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/databases/database_db_main', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); diff --git a/tests/e2e/Services/Realtime/RealtimeConsoleClientTest.php b/tests/e2e/Services/Realtime/RealtimeConsoleClientTest.php index 9c91ac4a52..82068f1301 100644 --- a/tests/e2e/Services/Realtime/RealtimeConsoleClientTest.php +++ b/tests/e2e/Services/Realtime/RealtimeConsoleClientTest.php @@ -346,15 +346,32 @@ class RealtimeConsoleClientTest extends Scope /** * Test Delete Index */ - $attribute = $this->client->call(Client::METHOD_DELETE, '/databases/' . $databaseId . '/collections/' . $actorsId . '/indexes/key_name', array_merge([ + $indexKey = 'key_name'; + $attribute = $this->client->call(Client::METHOD_DELETE, '/databases/' . $databaseId . '/collections/' . $actorsId . '/indexes/' . $indexKey, array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders())); $this->assertEquals($attribute['headers']['status-code'], 204); - $indexKey = 'key_name'; $response = json_decode($client->receive(), true); + $this->assertArrayHasKey('type', $response); + $this->assertArrayHasKey('data', $response); + $this->assertEquals('event', $response['type']); + $this->assertNotEmpty($response['data']); + $this->assertArrayHasKey('timestamp', $response['data']); + $this->assertCount(1, $response['data']['channels']); + $this->assertContains('console', $response['data']['channels']); + $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.indexes.*.update", $response['data']['events']); + $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.indexes.*", $response['data']['events']); + $this->assertContains("databases.{$databaseId}.collections.{$actorsId}", $response['data']['events']); + $this->assertContains("databases.{$databaseId}.collections.*.indexes.*.update", $response['data']['events']); + $this->assertContains("databases.{$databaseId}.collections.*.indexes.*", $response['data']['events']); + $this->assertContains("databases.{$databaseId}.collections.*", $response['data']['events']); + $this->assertNotEmpty($response['data']['payload']); + + /** Delete index generates two events. One from the API and one from the database worker */ + $response = json_decode($client->receive(), true); $this->assertArrayHasKey('type', $response); $this->assertArrayHasKey('data', $response); $this->assertEquals('event', $response['type']); @@ -402,13 +419,30 @@ class RealtimeConsoleClientTest extends Scope /** * Test Delete Attribute */ - $attribute = $this->client->call(Client::METHOD_DELETE, '/databases/' . $databaseId . '/collections/' . $data['actorsId'] . '/attributes/name', array_merge([ + $attributeKey = 'name'; + $attribute = $this->client->call(Client::METHOD_DELETE, '/databases/' . $databaseId . '/collections/' . $data['actorsId'] . '/attributes/' . $attributeKey, array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders())); $this->assertEquals($attribute['headers']['status-code'], 204); - $attributeKey = 'name'; + $response = json_decode($client->receive(), true); + + $this->assertArrayHasKey('type', $response); + $this->assertArrayHasKey('data', $response); + $this->assertEquals('event', $response['type']); + $this->assertNotEmpty($response['data']); + $this->assertArrayHasKey('timestamp', $response['data']); + $this->assertCount(1, $response['data']['channels']); + $this->assertContains('console', $response['data']['channels']); + $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.attributes.*.update", $response['data']['events']); + $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.attributes.*", $response['data']['events']); + $this->assertContains("databases.{$databaseId}.collections.{$actorsId}", $response['data']['events']); + $this->assertContains("databases.{$databaseId}.collections.*.attributes.*.update", $response['data']['events']); + $this->assertContains("databases.{$databaseId}.collections.*.attributes.*", $response['data']['events']); + $this->assertContains("databases.{$databaseId}.collections.*", $response['data']['events']); + $this->assertNotEmpty($response['data']['payload']); + $response = json_decode($client->receive(), true); $this->assertArrayHasKey('type', $response); diff --git a/tests/e2e/Services/Webhooks/WebhooksBase.php b/tests/e2e/Services/Webhooks/WebhooksBase.php index 6e30e7abc1..e2eb29c74d 100644 --- a/tests/e2e/Services/Webhooks/WebhooksBase.php +++ b/tests/e2e/Services/Webhooks/WebhooksBase.php @@ -167,10 +167,10 @@ trait WebhooksBase $this->assertEquals($webhook['headers']['User-Agent'], 'Appwrite-Server vdev. Please report abuse at security@appwrite.io'); $this->assertStringContainsString('databases.' . $databaseId . '.collections.*', $webhook['headers']['X-Appwrite-Webhook-Events']); $this->assertStringContainsString('databases.' . $databaseId . '.collections.*.attributes.*', $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString('databases.' . $databaseId . '.collections.*.attributes.*.delete', $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString('databases.' . $databaseId . '.collections.*.attributes.*.update', $webhook['headers']['X-Appwrite-Webhook-Events']); $this->assertStringContainsString("databases.{$databaseId}.collections.{$actorsId}", $webhook['headers']['X-Appwrite-Webhook-Events']); $this->assertStringContainsString("databases.{$databaseId}.collections.{$actorsId}.attributes.*", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("databases.{$databaseId}.collections.{$actorsId}.attributes.*.delete", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("databases.{$databaseId}.collections.{$actorsId}.attributes.*.update", $webhook['headers']['X-Appwrite-Webhook-Events']); $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Signature'], $signatureExpected); $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Id'] ?? '', $this->getProject()['webhookId']); $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Project-Id'] ?? '', $this->getProject()['$id']); diff --git a/tests/e2e/Services/Webhooks/WebhooksCustomServerTest.php b/tests/e2e/Services/Webhooks/WebhooksCustomServerTest.php index 01aa00cd84..7f8031ce94 100644 --- a/tests/e2e/Services/Webhooks/WebhooksCustomServerTest.php +++ b/tests/e2e/Services/Webhooks/WebhooksCustomServerTest.php @@ -124,10 +124,10 @@ class WebhooksCustomServerTest extends Scope $this->assertEquals($webhook['headers']['User-Agent'], 'Appwrite-Server vdev. Please report abuse at security@appwrite.io'); $this->assertStringContainsString('databases.' . $databaseId . '.collections.*', $webhook['headers']['X-Appwrite-Webhook-Events']); $this->assertStringContainsString('databases.' . $databaseId . '.collections.*.indexes.*', $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString('databases.' . $databaseId . '.collections.*.indexes.*.delete', $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString('databases.' . $databaseId . '.collections.*.indexes.*.update', $webhook['headers']['X-Appwrite-Webhook-Events']); $this->assertStringContainsString("databases.{$databaseId}.collections.{$actorsId}", $webhook['headers']['X-Appwrite-Webhook-Events']); $this->assertStringContainsString("databases.{$databaseId}.collections.{$actorsId}.indexes.*", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("databases.{$databaseId}.collections.{$actorsId}.indexes.*.delete", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("databases.{$databaseId}.collections.{$actorsId}.indexes.*.update", $webhook['headers']['X-Appwrite-Webhook-Events']); $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Signature'], $signatureExpected); $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Id'] ?? '', $this->getProject()['webhookId']); $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Project-Id'] ?? '', $this->getProject()['$id']); diff --git a/tests/resources/docker/docker-compose.yml b/tests/resources/docker/docker-compose.yml index bdb9bf49a4..50635018db 100644 --- a/tests/resources/docker/docker-compose.yml +++ b/tests/resources/docker/docker-compose.yml @@ -376,20 +376,6 @@ services: # ports: # - "8081:8081" - # resque: - # image: registry.gitlab.com/appwrite/appwrite/resque-web:v1.0.2 - # restart: unless-stopped - # networks: - # - appwrite - # ports: - # - "5678:5678" - # environment: - # - RESQUE_WEB_HOST=redis - # - RESQUE_WEB_PORT=6379 - # - RESQUE_WEB_HTTP_BASIC_AUTH_USER=user - # - RESQUE_WEB_HTTP_BASIC_AUTH_PASSWORD=password - - # webgrind: # image: 'jokkedk/webgrind:latest' # volumes: diff --git a/tests/unit/Event/EventTest.php b/tests/unit/Event/EventTest.php index a328c8d599..a430a7fdc6 100644 --- a/tests/unit/Event/EventTest.php +++ b/tests/unit/Event/EventTest.php @@ -3,9 +3,15 @@ namespace Tests\Unit\Event; use Appwrite\Event\Event; +use Appwrite\URL\URL; use InvalidArgumentException; use PHPUnit\Framework\TestCase; use Utopia\App; +use Utopia\DSN\DSN; +use Utopia\Queue; +use Utopia\Queue\Client; + +require_once __DIR__ . '/../../../app/init.php'; class EventTest extends TestCase { @@ -14,56 +20,56 @@ class EventTest extends TestCase public function setUp(): void { - $redisHost = App::getEnv('_APP_REDIS_HOST', ''); - $redisPort = App::getEnv('_APP_REDIS_PORT', ''); - \Resque::setBackend($redisHost . ':' . $redisPort); + $fallbackForRedis = URL::unparse([ + 'scheme' => 'redis', + 'host' => App::getEnv('_APP_REDIS_HOST', 'redis'), + 'port' => App::getEnv('_APP_REDIS_PORT', '6379'), + 'user' => App::getEnv('_APP_REDIS_USER', ''), + 'pass' => App::getEnv('_APP_REDIS_PASS', ''), + ]); + $dsn = App::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis); + $dsn = explode('=', $dsn); + $dsn = $dsn[0] ?? ''; + $dsn = new DSN($dsn); + $connection = new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort()); $this->queue = 'v1-tests' . uniqid(); - $this->object = new Event($this->queue, 'TestsV1'); + $this->object = new Event($connection); + $this->object->setClass('TestsV1'); + $this->object->setQueue($this->queue); } public function testQueue(): void { $this->assertEquals($this->queue, $this->object->getQueue()); - $this->object->setQueue('demo'); - $this->assertEquals('demo', $this->object->getQueue()); - $this->object->setQueue($this->queue); } public function testClass(): void { $this->assertEquals('TestsV1', $this->object->getClass()); - $this->object->setClass('TestsV2'); - $this->assertEquals('TestsV2', $this->object->getClass()); - $this->object->setClass('TestsV1'); } public function testParams(): void { + $this->object ->setParam('eventKey1', 'eventValue1') ->setParam('eventKey2', 'eventValue2'); $this->object->trigger(); - $this->assertEquals('eventValue1', $this->object->getParam('eventKey1')); $this->assertEquals('eventValue2', $this->object->getParam('eventKey2')); $this->assertEquals(null, $this->object->getParam('eventKey3')); - $this->assertEquals(\Resque::size($this->queue), 1); - } - - public function testPause(): void - { - $this->object->setPaused(true); - $this->assertTrue($this->object->isPaused()); - $this->object->setPaused(false); - $this->assertNotTrue($this->object->isPaused()); + global $register; + $pools = $register->get('pools'); + $client = new Client($this->object->getQueue(), $pools->get('queue')->pop()->getResource()); + $this->assertEquals($client->getQueueSize(), 1); } public function testReset(): void