diff --git a/.env b/.env index 0b5ddf24c6..f89386b5cf 100644 --- a/.env +++ b/.env @@ -102,4 +102,5 @@ _APP_ASSISTANT_OPENAI_API_KEY= _APP_MESSAGE_SMS_TEST_DSN= _APP_MESSAGE_EMAIL_TEST_DSN= _APP_MESSAGE_PUSH_TEST_DSN= -_APP_WEBHOOK_MAX_FAILED_ATTEMPTS=10 \ No newline at end of file +_APP_WEBHOOK_MAX_FAILED_ATTEMPTS=10 +_APP_PROJECT_REGIONS=default \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index e259782156..bed812bea0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "app/console"] path = app/console url = https://github.com/appwrite/console - branch = 1.5.x + branch = chore-update-sdk diff --git a/app/config/collections.php b/app/config/collections.php index e8a24c910e..cdc5d03b8d 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -2113,6 +2113,13 @@ $commonCollections = [ 'lengths' => [], 'orders' => [], ], + [ + '$id' => ID::custom('_unique_target_topic'), + 'type' => Database::INDEX_UNIQUE, + 'attributes' => ['targetInternalId', 'topicInternalId'], + 'lengths' => [], + 'orders' => [], + ], [ '$id' => ID::custom('_fulltext_search'), 'type' => Database::INDEX_FULLTEXT, @@ -4362,17 +4369,6 @@ $consoleCollections = array_merge([ 'array' => false, 'filters' => [], ], - [ - '$id' => ID::custom('resourceCollection'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => true, - 'default' => null, - 'array' => false, - 'filters' => [], - ], [ '$id' => ID::custom('resourceInternalId'), 'type' => Database::VAR_STRING, diff --git a/app/config/errors.php b/app/config/errors.php index 3f12b5953a..9e41d43120 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -420,6 +420,11 @@ return [ 'description' => 'The value for x-appwrite-id header is invalid. Please check the value of the x-appwrite-id header is a valid id and not unique().', 'code' => 400, ], + Exception::STORAGE_FILE_NOT_PUBLIC => [ + 'name' => Exception::STORAGE_FILE_NOT_PUBLIC, + 'description' => 'The requested file is not publicly readable.', + 'code' => 403, + ], /** VCS */ Exception::INSTALLATION_NOT_FOUND => [ @@ -718,6 +723,11 @@ return [ 'description' => 'You can\'t delete default template. If you are trying to reset your template changes, you can ignore this error as it\'s already been reset.', 'code' => 401, ], + Exception::PROJECT_REGION_UNSUPPORTED => [ + 'name' => Exception::PROJECT_REGION_UNSUPPORTED, + 'description' => 'The requested region is either inactive or unsupported. Please check the value of the _APP_REGIONS environment variable.', + 'code' => 400, + ], Exception::WEBHOOK_NOT_FOUND => [ 'name' => Exception::WEBHOOK_NOT_FOUND, 'description' => 'Webhook with the requested ID could not be found.', diff --git a/app/config/regions.php b/app/config/regions.php index 0dc5fab1ed..b40667ab5e 100644 --- a/app/config/regions.php +++ b/app/config/regions.php @@ -2,8 +2,73 @@ return [ 'default' => [ - 'name' => 'Default', - 'default' => true, + '$id' => 'default', + 'name' => 'Frankfurt', 'disabled' => false, - ] + 'flag' => 'de', + 'default' => true, + ], + 'fra' => [ + '$id' => 'fra', + 'name' => 'Frankfurt', + 'disabled' => false, + 'flag' => 'de', + 'default' => true, + ], + 'nyc' => [ + '$id' => 'nyc', + 'name' => 'New York', + 'disabled' => true, + 'flag' => 'us', + 'default' => true, + ], + 'sfo' => [ + '$id' => 'sfo', + 'name' => 'San Francisco', + 'disabled' => true, + 'flag' => 'us', + 'default' => true, + ], + 'blr' => [ + '$id' => 'blr', + 'name' => 'Banglore', + 'disabled' => true, + 'flag' => 'in', + 'default' => true, + ], + 'lon' => [ + '$id' => 'lon', + 'name' => 'London', + 'disabled' => true, + 'flag' => 'gb', + 'default' => true, + ], + 'ams' => [ + '$id' => 'ams', + 'name' => 'Amsterdam', + 'disabled' => true, + 'flag' => 'nl', + 'default' => true, + ], + 'sgp' => [ + '$id' => 'sgp', + 'name' => 'Singapore', + 'disabled' => true, + 'flag' => 'sg', + 'default' => true, + ], + 'tor' => [ + '$id' => 'tor', + 'name' => 'Toronto', + 'disabled' => true, + 'flag' => 'ca', + 'default' => true, + ], + 'syd' => [ + '$id' => 'syd', + 'name' => 'Sydney', + 'disabled' => true, + 'flag' => 'au', + 'default' => true, + ], ]; diff --git a/app/config/runtimes.php b/app/config/runtimes.php index 2cd73c1b70..a55e0b3fb4 100644 --- a/app/config/runtimes.php +++ b/app/config/runtimes.php @@ -1,16 +1,9 @@ getAll(true, $allowList); - -return $runtimes; +return (new Runtimes('v3'))->getAll(); diff --git a/app/console b/app/console index 1d942975d1..44edd461c6 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit 1d942975d16397a252a58ab730fb57819d679213 +Subproject commit 44edd461c6036cb462047c1424b80f0903cdc15e diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index dd009864a7..11facf04a7 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -1773,10 +1773,10 @@ App::post('/v1/account/tokens/phone') ]); $queueForMessaging + ->setType(MESSAGE_SEND_TYPE_INTERNAL) ->setMessage($messageDoc) ->setRecipients([$phone]) - ->setProviderType(MESSAGE_TYPE_SMS) - ->trigger(); + ->setProviderType(MESSAGE_TYPE_SMS); $queueForEvents->setPayload( $response->output( @@ -3314,10 +3314,10 @@ App::post('/v1/account/verification/phone') ]); $queueForMessaging + ->setType(MESSAGE_SEND_TYPE_INTERNAL) ->setMessage($messageDoc) ->setRecipients([$user->getAttribute('phone')]) - ->setProviderType(MESSAGE_TYPE_SMS) - ->trigger(); + ->setProviderType(MESSAGE_TYPE_SMS); $queueForEvents ->setParam('userId', $user->getId()) @@ -3677,14 +3677,14 @@ App::post('/v1/account/mfa/challenge') } $queueForMessaging + ->setType(MESSAGE_SEND_TYPE_INTERNAL) ->setMessage(new Document([ '$id' => $challenge->getId(), 'data' => [ 'content' => $code, ], ])) - ->setRecipients([$user->getAttribute('phone')]) - ->trigger(); + ->setRecipients([$user->getAttribute('phone')]); break; case 'email': if (empty(App::getEnv('_APP_SMTP_HOST'))) { diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 21d5928267..8c4eb8b09a 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -121,9 +121,7 @@ $redeployVcs = function (Request $request, Document $function, Document $project ->setType(BUILD_TYPE_DEPLOYMENT) ->setResource($function) ->setDeployment($deployment) - ->setTemplate($template) - ->setProject($project) - ->trigger(); + ->setTemplate($template); }; App::post('/v1/functions') @@ -172,6 +170,12 @@ App::post('/v1/functions') ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateBranch, Request $request, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Build $queueForBuilds, Database $dbForConsole, GitHub $github) use ($redeployVcs) { $functionId = ($functionId == 'unique()') ? ID::unique() : $functionId; + $allowList = \array_filter(\explode(',', App::getEnv('_APP_FUNCTIONS_RUNTIMES', ''))); + + if (!empty($allowList) && !\in_array($runtime, $allowList)) { + throw new Exception(Exception::FUNCTION_RUNTIME_UNSUPPORTED, 'Runtime "' . $runtime . '" is not supported'); + } + // build from template $template = new Document([]); if ( @@ -229,7 +233,6 @@ App::post('/v1/functions') fn () => $dbForConsole->createDocument('schedules', new Document([ 'region' => App::getEnv('_APP_REGION', 'default'), // Todo replace with projects region 'resourceType' => 'function', - 'resourceCollection' => 'functions', 'resourceId' => $function->getId(), 'resourceInternalId' => $function->getInternalId(), 'resourceUpdatedAt' => DateTime::now(), @@ -841,8 +844,8 @@ App::get('/v1/functions/:functionId/deployments/:deploymentId/download') ->inject('response') ->inject('request') ->inject('dbForProject') - ->inject('deviceFunctions') - ->action(function (string $functionId, string $deploymentId, Response $response, Request $request, Database $dbForProject, Device $deviceFunctions) { + ->inject('deviceForFunctions') + ->action(function (string $functionId, string $deploymentId, Response $response, Request $request, Database $dbForProject, Device $deviceForFunctions) { $function = $dbForProject->getDocument('functions', $functionId); if ($function->isEmpty()) { @@ -859,7 +862,7 @@ App::get('/v1/functions/:functionId/deployments/:deploymentId/download') } $path = $deployment->getAttribute('path', ''); - if (!$deviceFunctions->exists($path)) { + if (!$deviceForFunctions->exists($path)) { throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); } @@ -869,7 +872,7 @@ App::get('/v1/functions/:functionId/deployments/:deploymentId/download') ->addHeader('X-Peak', \memory_get_peak_usage()) ->addHeader('Content-Disposition', 'attachment; filename="' . $deploymentId . '.tar.gz"'); - $size = $deviceFunctions->getFileSize($path); + $size = $deviceForFunctions->getFileSize($path); $rangeHeader = $request->getHeader('range'); if (!empty($rangeHeader)) { @@ -891,13 +894,13 @@ App::get('/v1/functions/:functionId/deployments/:deploymentId/download') ->addHeader('Content-Length', $end - $start + 1) ->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT); - $response->send($deviceFunctions->read($path, $start, ($end - $start + 1))); + $response->send($deviceForFunctions->read($path, $start, ($end - $start + 1))); } if ($size > APP_STORAGE_READ_BUFFER) { for ($i = 0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) { $response->chunk( - $deviceFunctions->read( + $deviceForFunctions->read( $path, ($i * MAX_OUTPUT_CHUNK_SIZE), min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE)) @@ -906,7 +909,7 @@ App::get('/v1/functions/:functionId/deployments/:deploymentId/download') ); } } else { - $response->send($deviceFunctions->read($path)); + $response->send($deviceForFunctions->read($path)); } }); @@ -1045,10 +1048,10 @@ App::post('/v1/functions/:functionId/deployments') ->inject('dbForProject') ->inject('queueForEvents') ->inject('project') - ->inject('deviceFunctions') - ->inject('deviceLocal') + ->inject('deviceForFunctions') + ->inject('deviceForLocal') ->inject('queueForBuilds') - ->action(function (string $functionId, ?string $entrypoint, ?string $commands, mixed $code, bool $activate, Request $request, Response $response, Database $dbForProject, Event $queueForEvents, Document $project, Device $deviceFunctions, Device $deviceLocal, Build $queueForBuilds) { + ->action(function (string $functionId, ?string $entrypoint, ?string $commands, mixed $code, bool $activate, Request $request, Response $response, Database $dbForProject, Event $queueForEvents, Document $project, Device $deviceForFunctions, Device $deviceForLocal, Build $queueForBuilds) { $activate = filter_var($activate, FILTER_VALIDATE_BOOLEAN); @@ -1129,11 +1132,11 @@ App::post('/v1/functions/:functionId/deployments') } // Save to storage - $fileSize ??= $deviceLocal->getFileSize($fileTmpName); - $path = $deviceFunctions->getPath($deploymentId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION)); + $fileSize ??= $deviceForLocal->getFileSize($fileTmpName); + $path = $deviceForFunctions->getPath($deploymentId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION)); $deployment = $dbForProject->getDocument('deployments', $deploymentId); - $metadata = ['content_type' => $deviceLocal->getFileMimeType($fileTmpName)]; + $metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)]; if (!$deployment->isEmpty()) { $chunks = $deployment->getAttribute('chunksTotal', 1); $metadata = $deployment->getAttribute('metadata', []); @@ -1142,7 +1145,7 @@ App::post('/v1/functions/:functionId/deployments') } } - $chunksUploaded = $deviceFunctions->upload($fileTmpName, $path, $chunk, $chunks, $metadata); + $chunksUploaded = $deviceForFunctions->upload($fileTmpName, $path, $chunk, $chunks, $metadata); if (empty($chunksUploaded)) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed moving file'); @@ -1165,7 +1168,7 @@ App::post('/v1/functions/:functionId/deployments') } } - $fileSize = $deviceFunctions->getFileSize($path); + $fileSize = $deviceForFunctions->getFileSize($path); if ($deployment->isEmpty()) { $deployment = $dbForProject->createDocument('deployments', new Document([ @@ -1196,9 +1199,7 @@ App::post('/v1/functions/:functionId/deployments') $queueForBuilds ->setType(BUILD_TYPE_DEPLOYMENT) ->setResource($function) - ->setDeployment($deployment) - ->setProject($project) - ->trigger(); + ->setDeployment($deployment); } else { if ($deployment->isEmpty()) { $deployment = $dbForProject->createDocument('deployments', new Document([ @@ -1376,8 +1377,8 @@ App::delete('/v1/functions/:functionId/deployments/:deploymentId') ->inject('dbForProject') ->inject('queueForDeletes') ->inject('queueForEvents') - ->inject('deviceFunctions') - ->action(function (string $functionId, string $deploymentId, Response $response, Database $dbForProject, Delete $queueForDeletes, Event $queueForEvents, Device $deviceFunctions) { + ->inject('deviceForFunctions') + ->action(function (string $functionId, string $deploymentId, Response $response, Database $dbForProject, Delete $queueForDeletes, Event $queueForEvents, Device $deviceForFunctions) { $function = $dbForProject->getDocument('functions', $functionId); if ($function->isEmpty()) { @@ -1398,7 +1399,7 @@ App::delete('/v1/functions/:functionId/deployments/:deploymentId') } if (!empty($deployment->getAttribute('path', ''))) { - if (!($deviceFunctions->delete($deployment->getAttribute('path', '')))) { + if (!($deviceForFunctions->delete($deployment->getAttribute('path', '')))) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove deployment from storage'); } } @@ -1478,9 +1479,7 @@ App::post('/v1/functions/:functionId/deployments/:deploymentId/builds/:buildId') $queueForBuilds ->setType(BUILD_TYPE_DEPLOYMENT) ->setResource($function) - ->setDeployment($deployment) - ->setProject($project) - ->trigger(); + ->setDeployment($deployment); $queueForEvents ->setParam('functionId', $function->getId()) diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 4e0b094926..3ea96c9744 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -792,8 +792,8 @@ App::get('/v1/health/stats') // Currently only used internally ->label('docs', false) ->inject('response') ->inject('register') - ->inject('deviceFiles') - ->action(function (Response $response, Registry $register, Device $deviceFiles) { + ->inject('deviceForFiles') + ->action(function (Response $response, Registry $register, Device $deviceForFiles) { $cache = $register->get('cache'); @@ -802,9 +802,9 @@ App::get('/v1/health/stats') // Currently only used internally $response ->json([ 'storage' => [ - 'used' => Storage::human($deviceFiles->getDirectorySize($deviceFiles->getRoot() . '/')), - 'partitionTotal' => Storage::human($deviceFiles->getPartitionTotalSpace()), - 'partitionFree' => Storage::human($deviceFiles->getPartitionFreeSpace()), + 'used' => Storage::human($deviceForFiles->getDirectorySize($deviceForFiles->getRoot() . '/')), + 'partitionTotal' => Storage::human($deviceForFiles->getPartitionTotalSpace()), + 'partitionFree' => Storage::human($deviceForFiles->getPartitionFreeSpace()), ], 'cache' => [ 'uptime' => $cacheStats['uptime_in_seconds'] ?? 0, diff --git a/app/controllers/api/messaging.php b/app/controllers/api/messaging.php index 8e6c73f3bc..abd52f8ced 100644 --- a/app/controllers/api/messaging.php +++ b/app/controllers/api/messaging.php @@ -10,6 +10,7 @@ use Appwrite\Messaging\Status as MessageStatus; use Appwrite\Network\Validator\Email; use Appwrite\Permission; use Appwrite\Role; +use Appwrite\Utopia\Database\Validator\CompoundUID; use Appwrite\Utopia\Database\Validator\CustomId; use Appwrite\Utopia\Database\Validator\Queries\Messages; use Appwrite\Utopia\Database\Validator\Queries\Providers; @@ -27,11 +28,13 @@ use Utopia\Database\Helpers\ID; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\Datetime as DatetimeValidator; +use Utopia\Database\Validator\Key; use Utopia\Database\Validator\Queries; use Utopia\Database\Validator\Query\Limit; use Utopia\Database\Validator\Query\Offset; use Utopia\Database\Validator\Roles; use Utopia\Database\Validator\UID; +use Utopia\Domains\Domain; use Utopia\Locale\Locale; use Utopia\Validator\ArrayList; use Utopia\Validator\Boolean; @@ -2573,6 +2576,7 @@ App::post('/v1/messaging/messages/email') ->param('targets', [], new ArrayList(new UID()), 'List of Targets IDs.', true) ->param('cc', [], new ArrayList(new UID()), 'Array of target IDs to be added as CC.', true) ->param('bcc', [], new ArrayList(new UID()), 'Array of target IDs to be added as BCC.', true) + ->param('attachments', [], new ArrayList(new CompoundUID()), 'Array of compound bucket IDs to file IDs to be attached to the email.', true) ->param('status', MessageStatus::DRAFT, new WhiteList([MessageStatus::DRAFT, MessageStatus::SCHEDULED, MessageStatus::PROCESSING]), 'Message Status. Value must be one of: ' . implode(', ', [MessageStatus::DRAFT, MessageStatus::SCHEDULED, MessageStatus::PROCESSING]) . '.', true) ->param('html', false, new Boolean(), 'Is content of type HTML', true) ->param('scheduledAt', null, new DatetimeValidator(requireDateInFuture: true), 'Scheduled delivery time for message in [ISO 8601](https://www.iso.org/iso-8601-date-and-time-format.html) format. DateTime value must be in future.', true) @@ -2582,7 +2586,7 @@ App::post('/v1/messaging/messages/email') ->inject('project') ->inject('queueForMessaging') ->inject('response') - ->action(function (string $messageId, string $subject, string $content, array $topics, array $users, array $targets, array $cc, array $bcc, string $status, bool $html, ?string $scheduledAt, Event $queueForEvents, Database $dbForProject, Database $dbForConsole, Document $project, Messaging $queueForMessaging, Response $response) { + ->action(function (string $messageId, string $subject, string $content, array $topics, array $users, array $targets, array $cc, array $bcc, array $attachments, string $status, bool $html, ?string $scheduledAt, Event $queueForEvents, Database $dbForProject, Database $dbForConsole, Document $project, Messaging $queueForMessaging, Response $response) { $messageId = $messageId == 'unique()' ? ID::unique() : $messageId; @@ -2615,6 +2619,29 @@ App::post('/v1/messaging/messages/email') } } + if (!empty($attachments)) { + foreach ($attachments as &$attachment) { + [$bucketId, $fileId] = CompoundUID::parse($attachment); + + $bucket = $dbForProject->getDocument('buckets', $bucketId); + + if ($bucket->isEmpty()) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); + + if ($file->isEmpty()) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); + } + + $attachment = [ + 'bucketId' => $bucketId, + 'fileId' => $fileId, + ]; + } + } + $message = $dbForProject->createDocument('messages', new Document([ '$id' => $messageId, 'providerType' => MESSAGE_TYPE_EMAIL, @@ -2628,6 +2655,7 @@ App::post('/v1/messaging/messages/email') 'html' => $html, 'cc' => $cc, 'bcc' => $bcc, + 'attachments' => $attachments, ], 'status' => $status, ])); @@ -2635,14 +2663,13 @@ App::post('/v1/messaging/messages/email') switch ($status) { case MessageStatus::PROCESSING: $queueForMessaging - ->setMessageId($message->getId()) - ->trigger(); + ->setType(MESSAGE_SEND_TYPE_EXTERNAL) + ->setMessageId($message->getId()); break; case MessageStatus::SCHEDULED: $schedule = $dbForConsole->createDocument('schedules', new Document([ 'region' => App::getEnv('_APP_REGION', 'default'), 'resourceType' => 'message', - 'resourceCollection' => 'messages', 'resourceId' => $message->getId(), 'resourceInternalId' => $message->getInternalId(), 'resourceUpdatedAt' => DateTime::now(), @@ -2744,14 +2771,13 @@ App::post('/v1/messaging/messages/sms') switch ($status) { case MessageStatus::PROCESSING: $queueForMessaging - ->setMessageId($message->getId()) - ->trigger(); + ->setType(MESSAGE_SEND_TYPE_EXTERNAL) + ->setMessageId($message->getId()); break; case MessageStatus::SCHEDULED: $schedule = $dbForConsole->createDocument('schedules', new Document([ 'region' => App::getEnv('_APP_REGION', 'default'), 'resourceType' => 'message', - 'resourceCollection' => 'messages', 'resourceId' => $message->getId(), 'resourceInternalId' => $message->getInternalId(), 'resourceUpdatedAt' => DateTime::now(), @@ -2802,6 +2828,7 @@ App::post('/v1/messaging/messages/push') ->param('targets', [], new ArrayList(new UID()), 'List of Targets IDs.', true) ->param('data', null, new JSON(), 'Additional Data for push notification.', true) ->param('action', '', new Text(256), 'Action for push notification.', true) + ->param('image', '', new CompoundUID(), 'Image for push notification. Must be a compound bucket ID to file ID of a jpeg, png, or bmp image in Appwrite Storage.', true) ->param('icon', '', new Text(256), 'Icon for push notification. Available only for Android and Web Platform.', true) ->param('sound', '', new Text(256), 'Sound for push notification. Available only for Android and IOS Platform.', true) ->param('color', '', new Text(256), 'Color for push notification. Available only for Android Platform.', true) @@ -2815,7 +2842,7 @@ App::post('/v1/messaging/messages/push') ->inject('project') ->inject('queueForMessaging') ->inject('response') - ->action(function (string $messageId, string $title, string $body, array $topics, array $users, array $targets, ?array $data, string $action, string $icon, string $sound, string $color, string $tag, string $badge, string $status, ?string $scheduledAt, Event $queueForEvents, Database $dbForProject, Database $dbForConsole, Document $project, Messaging $queueForMessaging, Response $response) { + ->action(function (string $messageId, string $title, string $body, array $topics, array $users, array $targets, ?array $data, string $action, string $image, string $icon, string $sound, string $color, string $tag, string $badge, string $status, ?string $scheduledAt, Event $queueForEvents, Database $dbForProject, Database $dbForConsole, Document $project, Messaging $queueForMessaging, Response $response) { $messageId = $messageId == 'unique()' ? ID::unique() : $messageId; @@ -2846,9 +2873,41 @@ App::post('/v1/messaging/messages/push') } } + if (!empty($image)) { + [$bucketId, $fileId] = CompoundUID::parse($image); + + $bucket = $dbForProject->getDocument('buckets', $bucketId); + if ($bucket->isEmpty()) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); + if ($file->isEmpty()) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + if (!\in_array(Permission::read(Role::any()), \array_merge($file->getRead(), $bucket->getRead()))) { + throw new Exception(Exception::STORAGE_FILE_NOT_PUBLIC); + } + + if (!\in_array($file->getAttribute('mimeType'), ['image/png', 'image/jpeg'])) { + throw new Exception(Exception::STORAGE_FILE_TYPE_UNSUPPORTED); + } + + $host = App::getEnv('_APP_DOMAIN', 'localhost'); + $domain = new Domain(\parse_url($host, PHP_URL_HOST)); + $protocol = App::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https'; + + if (!$domain->isKnown()) { + throw new Exception(Exception::STORAGE_FILE_NOT_PUBLIC); + } + + $image = "{$protocol}://{$host}/v1/storage/buckets/{$bucket->getId()}/files/{$file->getId()}/view?project={$project->getId()}"; + } + $pushData = []; - $keys = ['title', 'body', 'data', 'action', 'icon', 'sound', 'color', 'tag', 'badge']; + $keys = ['title', 'body', 'data', 'action', 'image', 'icon', 'sound', 'color', 'tag', 'badge']; foreach ($keys as $key) { if (!empty($$key)) { @@ -2870,14 +2929,13 @@ App::post('/v1/messaging/messages/push') switch ($status) { case MessageStatus::PROCESSING: $queueForMessaging - ->setMessageId($message->getId()) - ->trigger(); + ->setType(MESSAGE_SEND_TYPE_EXTERNAL) + ->setMessageId($message->getId()); break; case MessageStatus::SCHEDULED: $schedule = $dbForConsole->createDocument('schedules', new Document([ 'region' => App::getEnv('_APP_REGION', 'default'), 'resourceType' => 'message', - 'resourceCollection' => 'messages', 'resourceId' => $message->getId(), 'resourceInternalId' => $message->getInternalId(), 'resourceUpdatedAt' => DateTime::now(), @@ -3231,7 +3289,6 @@ App::patch('/v1/messaging/messages/email/:messageId') $schedule = $dbForConsole->createDocument('schedules', new Document([ 'region' => App::getEnv('_APP_REGION', 'default'), 'resourceType' => 'message', - 'resourceCollection' => 'messages', 'resourceId' => $message->getId(), 'resourceInternalId' => $message->getInternalId(), 'resourceUpdatedAt' => DateTime::now(), @@ -3263,8 +3320,8 @@ App::patch('/v1/messaging/messages/email/:messageId') if ($status === MessageStatus::PROCESSING) { $queueForMessaging - ->setMessageId($message->getId()) - ->trigger(); + ->setType(MESSAGE_SEND_TYPE_EXTERNAL) + ->setMessageId($message->getId()); } $queueForEvents @@ -3350,7 +3407,6 @@ App::patch('/v1/messaging/messages/sms/:messageId') $schedule = $dbForConsole->createDocument('schedules', new Document([ 'region' => App::getEnv('_APP_REGION', 'default'), 'resourceType' => 'message', - 'resourceCollection' => 'messages', 'resourceId' => $message->getId(), 'resourceInternalId' => $message->getInternalId(), 'resourceUpdatedAt' => DateTime::now(), @@ -3382,8 +3438,8 @@ App::patch('/v1/messaging/messages/sms/:messageId') if ($status === MessageStatus::PROCESSING) { $queueForMessaging - ->setMessageId($message->getId()) - ->trigger(); + ->setType(MESSAGE_SEND_TYPE_EXTERNAL) + ->setMessageId($message->getId()); } $queueForEvents @@ -3415,6 +3471,7 @@ App::patch('/v1/messaging/messages/push/:messageId') ->param('body', null, new Text(64230), 'Body for push notification.', true) ->param('data', null, new JSON(), 'Additional Data for push notification.', true) ->param('action', null, new Text(256), 'Action for push notification.', true) + ->param('image', null, new CompoundUID(), 'Image for push notification. Must be a compound bucket ID to file ID of a jpeg, png, or bmp image in Appwrite Storage.', true) ->param('icon', null, new Text(256), 'Icon for push notification. Available only for Android and Web platforms.', true) ->param('sound', null, new Text(256), 'Sound for push notification. Available only for Android and iOS platforms.', true) ->param('color', null, new Text(256), 'Color for push notification. Available only for Android platforms.', true) @@ -3428,7 +3485,7 @@ App::patch('/v1/messaging/messages/push/:messageId') ->inject('project') ->inject('queueForMessaging') ->inject('response') - ->action(function (string $messageId, ?array $topics, ?array $users, ?array $targets, ?string $title, ?string $body, ?array $data, ?string $action, ?string $icon, ?string $sound, ?string $color, ?string $tag, ?int $badge, ?string $status, ?string $scheduledAt, Event $queueForEvents, Database $dbForProject, Database $dbForConsole, Document $project, Messaging $queueForMessaging, Response $response) { + ->action(function (string $messageId, ?array $topics, ?array $users, ?array $targets, ?string $title, ?string $body, ?array $data, ?string $action, ?string $image, ?string $icon, ?string $sound, ?string $color, ?string $tag, ?int $badge, ?string $status, ?string $scheduledAt, Event $queueForEvents, Database $dbForProject, Database $dbForConsole, Document $project, Messaging $queueForMessaging, Response $response) { $message = $dbForProject->getDocument('messages', $messageId); if ($message->isEmpty()) { @@ -3498,6 +3555,38 @@ App::patch('/v1/messaging/messages/push/:messageId') $pushData['badge'] = $badge; } + if (!\is_null($image)) { + [$bucketId, $fileId] = CompoundUID::parse($image); + + $bucket = $dbForProject->getDocument('buckets', $bucketId); + if ($bucket->isEmpty()) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); + if ($file->isEmpty()) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + if (!\in_array(Permission::read(Role::any()), \array_merge($file->getRead(), $bucket->getRead()))) { + throw new Exception(Exception::STORAGE_FILE_NOT_PUBLIC); + } + + if (!\in_array($file->getAttribute('mimeType'), ['image/png', 'image/jpeg'])) { + throw new Exception(Exception::STORAGE_FILE_TYPE_UNSUPPORTED); + } + + $host = App::getEnv('_APP_DOMAIN', 'localhost'); + $domain = new Domain(\parse_url($host, PHP_URL_HOST)); + $protocol = App::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https'; + + if (!$domain->isKnown()) { + throw new Exception(Exception::STORAGE_FILE_NOT_PUBLIC); + } + + $pushData['image'] = "{$protocol}://{$host}/v1/storage/buckets/{$bucket->getId()}/files/{$file->getId()}/view?project={$project->getId()}"; + } + $message->setAttribute('data', $pushData); if (!\is_null($status)) { @@ -3509,7 +3598,6 @@ App::patch('/v1/messaging/messages/push/:messageId') $schedule = $dbForConsole->createDocument('schedules', new Document([ 'region' => App::getEnv('_APP_REGION', 'default'), 'resourceType' => 'message', - 'resourceCollection' => 'messages', 'resourceId' => $message->getId(), 'resourceInternalId' => $message->getInternalId(), 'resourceUpdatedAt' => DateTime::now(), @@ -3541,8 +3629,8 @@ App::patch('/v1/messaging/messages/push/:messageId') if ($status === MessageStatus::PROCESSING) { $queueForMessaging - ->setMessageId($message->getId()) - ->trigger(); + ->setType(MESSAGE_SEND_TYPE_EXTERNAL) + ->setMessageId($message->getId()); } $queueForEvents diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index bea893064f..73202a6e0a 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -78,13 +78,18 @@ App::post('/v1/projects') ->inject('pools') ->action(function (string $projectId, string $name, string $teamId, string $region, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, Cache $cache, Group $pools) { - $team = $dbForConsole->getDocument('teams', $teamId); if ($team->isEmpty()) { throw new Exception(Exception::TEAM_NOT_FOUND); } + $allowList = \array_filter(\explode(',', App::getEnv('_APP_PROJECT_REGIONS', ''))); + + if (!empty($allowList) && !\in_array($region, $allowList)) { + throw new Exception(Exception::PROJECT_REGION_UNSUPPORTED, 'Region "' . $region . '" is not supported'); + } + $auth = Config::getParam('auth', []); $auths = ['limit' => 0, 'maxSessions' => APP_LIMIT_USER_SESSIONS_DEFAULT, 'passwordHistory' => 0, 'passwordDictionary' => false, 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, 'personalDataCheck' => false]; foreach ($auth as $index => $method) { diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 7fe2580aec..4968fc381d 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -360,9 +360,9 @@ App::post('/v1/storage/buckets/:bucketId/files') ->inject('user') ->inject('queueForEvents') ->inject('mode') - ->inject('deviceFiles') - ->inject('deviceLocal') - ->action(function (string $bucketId, string $fileId, mixed $file, ?array $permissions, Request $request, Response $response, Database $dbForProject, Document $user, Event $queueForEvents, string $mode, Device $deviceFiles, Device $deviceLocal) { + ->inject('deviceForFiles') + ->inject('deviceForLocal') + ->action(function (string $bucketId, string $fileId, mixed $file, ?array $permissions, Request $request, Response $response, Database $dbForProject, Document $user, Event $queueForEvents, string $mode, Device $deviceForFiles, Device $deviceForLocal) { $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); @@ -493,13 +493,13 @@ App::post('/v1/storage/buckets/:bucketId/files') } // Save to storage - $fileSize ??= $deviceLocal->getFileSize($fileTmpName); - $path = $deviceFiles->getPath($fileId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION)); - $path = str_ireplace($deviceFiles->getRoot(), $deviceFiles->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $path); // Add bucket id to path after root + $fileSize ??= $deviceForLocal->getFileSize($fileTmpName); + $path = $deviceForFiles->getPath($fileId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION)); + $path = str_ireplace($deviceForFiles->getRoot(), $deviceForFiles->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $path); // Add bucket id to path after root $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); - $metadata = ['content_type' => $deviceLocal->getFileMimeType($fileTmpName)]; + $metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)]; if (!$file->isEmpty()) { $chunks = $file->getAttribute('chunksTotal', 1); $uploaded = $file->getAttribute('chunksUploaded', 0); @@ -514,32 +514,32 @@ App::post('/v1/storage/buckets/:bucketId/files') } } - $chunksUploaded = $deviceFiles->upload($fileTmpName, $path, $chunk, $chunks, $metadata); + $chunksUploaded = $deviceForFiles->upload($fileTmpName, $path, $chunk, $chunks, $metadata); if (empty($chunksUploaded)) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed uploading file'); } if ($chunksUploaded === $chunks) { - if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && $deviceFiles->getType() === Storage::DEVICE_LOCAL) { + if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && $deviceForFiles->getType() === Storage::DEVICE_LOCAL) { $antivirus = new Network( App::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'), (int) App::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310) ); if (!$antivirus->fileScan($path)) { - $deviceFiles->delete($path); + $deviceForFiles->delete($path); throw new Exception(Exception::STORAGE_INVALID_FILE); } } - $mimeType = $deviceFiles->getFileMimeType($path); // Get mime-type before compression and encryption - $fileHash = $deviceFiles->getFileHash($path); // Get file hash before compression and encryption + $mimeType = $deviceForFiles->getFileMimeType($path); // Get mime-type before compression and encryption + $fileHash = $deviceForFiles->getFileHash($path); // Get file hash before compression and encryption $data = ''; // Compression $algorithm = $bucket->getAttribute('compression', Compression::NONE); if ($fileSize <= APP_STORAGE_READ_BUFFER && $algorithm != Compression::NONE) { - $data = $deviceFiles->read($path); + $data = $deviceForFiles->read($path); switch ($algorithm) { case Compression::ZSTD: $compressor = new Zstd(); @@ -559,7 +559,7 @@ App::post('/v1/storage/buckets/:bucketId/files') if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) { if (empty($data)) { - $data = $deviceFiles->read($path); + $data = $deviceForFiles->read($path); } $key = App::getEnv('_APP_OPENSSL_KEY_V1'); $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM)); @@ -567,12 +567,12 @@ App::post('/v1/storage/buckets/:bucketId/files') } if (!empty($data)) { - if (!$deviceFiles->write($path, $data, $mimeType)) { + if (!$deviceForFiles->write($path, $data, $mimeType)) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to save file'); } } - $sizeActual = $deviceFiles->getFileSize($path); + $sizeActual = $deviceForFiles->getFileSize($path); $openSSLVersion = null; $openSSLCipher = null; @@ -872,9 +872,9 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') ->inject('project') ->inject('dbForProject') ->inject('mode') - ->inject('deviceFiles') - ->inject('deviceLocal') - ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, Request $request, Response $response, Document $project, Database $dbForProject, string $mode, Device $deviceFiles, Device $deviceLocal) { + ->inject('deviceForFiles') + ->inject('deviceForLocal') + ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, Request $request, Response $response, Document $project, Database $dbForProject, string $mode, Device $deviceForFiles, Device $deviceForLocal) { if (!\extension_loaded('imagick')) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing'); @@ -931,10 +931,10 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $cipher = null; $background = (empty($background)) ? 'eceff1' : $background; $type = \strtolower(\pathinfo($path, PATHINFO_EXTENSION)); - $deviceFiles = $deviceLocal; + $deviceForFiles = $deviceForLocal; } - if (!$deviceFiles->exists($path)) { + if (!$deviceForFiles->exists($path)) { throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); } @@ -950,7 +950,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $output = empty($type) ? (array_search($mime, $outputs) ?? 'jpg') : $type; } - $source = $deviceFiles->read($path); + $source = $deviceForFiles->read($path); if (!empty($cipher)) { // Decrypt $source = OpenSSL::decrypt( @@ -1033,8 +1033,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') ->inject('response') ->inject('dbForProject') ->inject('mode') - ->inject('deviceFiles') - ->action(function (string $bucketId, string $fileId, Request $request, Response $response, Database $dbForProject, string $mode, Device $deviceFiles) { + ->inject('deviceForFiles') + ->action(function (string $bucketId, string $fileId, Request $request, Response $response, Database $dbForProject, string $mode, Device $deviceForFiles) { $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); @@ -1064,7 +1064,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') $path = $file->getAttribute('path', ''); - if (!$deviceFiles->exists($path)) { + if (!$deviceForFiles->exists($path)) { throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); } @@ -1100,7 +1100,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') $source = ''; if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt - $source = $deviceFiles->read($path); + $source = $deviceForFiles->read($path); $source = OpenSSL::decrypt( $source, $file->getAttribute('openSSLCipher'), @@ -1114,14 +1114,14 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') switch ($file->getAttribute('algorithm', Compression::NONE)) { case Compression::ZSTD: if (empty($source)) { - $source = $deviceFiles->read($path); + $source = $deviceForFiles->read($path); } $compressor = new Zstd(); $source = $compressor->decompress($source); break; case Compression::GZIP: if (empty($source)) { - $source = $deviceFiles->read($path); + $source = $deviceForFiles->read($path); } $compressor = new GZIP(); $source = $compressor->decompress($source); @@ -1136,13 +1136,13 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') } if (!empty($rangeHeader)) { - $response->send($deviceFiles->read($path, $start, ($end - $start + 1))); + $response->send($deviceForFiles->read($path, $start, ($end - $start + 1))); } if ($size > APP_STORAGE_READ_BUFFER) { for ($i = 0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) { $response->chunk( - $deviceFiles->read( + $deviceForFiles->read( $path, ($i * MAX_OUTPUT_CHUNK_SIZE), min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE)) @@ -1151,7 +1151,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') ); } } else { - $response->send($deviceFiles->read($path)); + $response->send($deviceForFiles->read($path)); } }); @@ -1173,8 +1173,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') ->inject('request') ->inject('dbForProject') ->inject('mode') - ->inject('deviceFiles') - ->action(function (string $bucketId, string $fileId, Response $response, Request $request, Database $dbForProject, string $mode, Device $deviceFiles) { + ->inject('deviceForFiles') + ->action(function (string $bucketId, string $fileId, Response $response, Request $request, Database $dbForProject, string $mode, Device $deviceForFiles) { $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); @@ -1205,7 +1205,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') $path = $file->getAttribute('path', ''); - if (!$deviceFiles->exists($path)) { + if (!$deviceForFiles->exists($path)) { throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); } @@ -1249,7 +1249,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') $source = ''; if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt - $source = $deviceFiles->read($path); + $source = $deviceForFiles->read($path); $source = OpenSSL::decrypt( $source, $file->getAttribute('openSSLCipher'), @@ -1263,14 +1263,14 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') switch ($file->getAttribute('algorithm', Compression::NONE)) { case Compression::ZSTD: if (empty($source)) { - $source = $deviceFiles->read($path); + $source = $deviceForFiles->read($path); } $compressor = new Zstd(); $source = $compressor->decompress($source); break; case Compression::GZIP: if (empty($source)) { - $source = $deviceFiles->read($path); + $source = $deviceForFiles->read($path); } $compressor = new GZIP(); $source = $compressor->decompress($source); @@ -1286,15 +1286,15 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') } if (!empty($rangeHeader)) { - $response->send($deviceFiles->read($path, $start, ($end - $start + 1))); + $response->send($deviceForFiles->read($path, $start, ($end - $start + 1))); return; } - $size = $deviceFiles->getFileSize($path); + $size = $deviceForFiles->getFileSize($path); if ($size > APP_STORAGE_READ_BUFFER) { for ($i = 0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) { $response->chunk( - $deviceFiles->read( + $deviceForFiles->read( $path, ($i * MAX_OUTPUT_CHUNK_SIZE), min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE)) @@ -1303,7 +1303,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') ); } } else { - $response->send($deviceFiles->read($path)); + $response->send($deviceForFiles->read($path)); } }); @@ -1438,9 +1438,9 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') ->inject('dbForProject') ->inject('queueForEvents') ->inject('mode') - ->inject('deviceFiles') + ->inject('deviceForFiles') ->inject('queueForDeletes') - ->action(function (string $bucketId, string $fileId, Response $response, Database $dbForProject, Event $queueForEvents, string $mode, Device $deviceFiles, Delete $queueForDeletes) { + ->action(function (string $bucketId, string $fileId, Response $response, Database $dbForProject, Event $queueForEvents, string $mode, Device $deviceForFiles, Delete $queueForDeletes) { $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); @@ -1471,12 +1471,12 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') $deviceDeleted = false; if ($file->getAttribute('chunksTotal') !== $file->getAttribute('chunksUploaded')) { - $deviceDeleted = $deviceFiles->abort( + $deviceDeleted = $deviceForFiles->abort( $file->getAttribute('path'), ($file->getAttribute('metadata', [])['uploadId'] ?? '') ); } else { - $deviceDeleted = $deviceFiles->delete($file->getAttribute('path')); + $deviceDeleted = $deviceForFiles->delete($file->getAttribute('path')); } if ($deviceDeleted) { diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 775fc27bb7..d5db918cd7 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -658,10 +658,10 @@ App::post('/v1/teams/:teamId/memberships') ]); $queueForMessaging + ->setType(MESSAGE_SEND_TYPE_INTERNAL) ->setMessage($messageDoc) ->setRecipients([$phone]) - ->setProviderType('SMS') - ->trigger(); + ->setProviderType('SMS'); } } diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php index 2bc94885b1..09ec5d7690 100644 --- a/app/controllers/api/vcs.php +++ b/app/controllers/api/vcs.php @@ -238,9 +238,7 @@ $createGitDeployments = function (GitHub $github, string $providerInstallationId $queueForBuilds ->setType(BUILD_TYPE_DEPLOYMENT) ->setResource($function) - ->setDeployment($deployment) - ->setProject($project) - ->trigger(); + ->setDeployment($deployment); //TODO: Add event? } diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 810d778a21..141553e12d 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -2,6 +2,7 @@ use Appwrite\Auth\Auth; use Appwrite\Event\Audit; +use Appwrite\Event\Build; use Appwrite\Event\Database as EventDatabase; use Appwrite\Event\Delete; use Appwrite\Event\Event; @@ -305,10 +306,11 @@ App::init() ->inject('queueForAudits') ->inject('queueForDeletes') ->inject('queueForDatabase') + ->inject('queueForBuilds') ->inject('queueForUsage') ->inject('dbForProject') ->inject('mode') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Usage $queueForUsage, Database $dbForProject, string $mode) use ($databaseListener) { + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Usage $queueForUsage, Database $dbForProject, string $mode) use ($databaseListener) { $route = $utopia->getRoute(); @@ -382,9 +384,6 @@ App::init() ->setProject($project) ->setUser($user); - $queueForMessaging - ->setProject($project); - $queueForAudits ->setMode($mode) ->setUserAgent($request->getUserAgent('')) @@ -393,9 +392,10 @@ App::init() ->setProject($project) ->setUser($user); - $queueForDeletes->setProject($project); $queueForDatabase->setProject($project); + $queueForBuilds->setProject($project); + $queueForMessaging->setProject($project); $dbForProject ->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $databaseListener($event, $document, $project, $queueForUsage, $dbForProject)) @@ -513,11 +513,13 @@ App::shutdown() ->inject('queueForUsage') ->inject('queueForDeletes') ->inject('queueForDatabase') + ->inject('queueForBuilds') + ->inject('queueForMessaging') ->inject('dbForProject') ->inject('queueForFunctions') ->inject('mode') ->inject('dbForConsole') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Usage $queueForUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Database $dbForProject, Func $queueForFunctions, string $mode, Database $dbForConsole) use ($parseLabel) { + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Usage $queueForUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Database $dbForProject, Func $queueForFunctions, string $mode, Database $dbForConsole) use ($parseLabel) { $responsePayload = $response->getPayload(); @@ -618,6 +620,14 @@ App::shutdown() $queueForDatabase->trigger(); } + if (!empty($queueForBuilds->getType())) { + $queueForBuilds->trigger(); + } + + if (!empty($queueForMessaging->getType())) { + $queueForMessaging->trigger(); + } + /** * Cache label */ diff --git a/app/init.php b/app/init.php index db4fe4d669..17b4a4321d 100644 --- a/app/init.php +++ b/app/init.php @@ -112,8 +112,8 @@ const APP_LIMIT_LIST_DEFAULT = 25; // Default maximum number of items to return const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours const APP_USER_ACCCESS = 24 * 60 * 60; // 24 hours const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours -const APP_CACHE_BUSTER = 329; -const APP_VERSION_STABLE = '1.4.13'; +const APP_CACHE_BUSTER = 330; +const APP_VERSION_STABLE = '1.5.0'; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; const APP_DATABASE_ATTRIBUTE_IP = 'ip'; @@ -142,9 +142,11 @@ const APP_SOCIAL_DEV = 'https://dev.to/appwrite'; const APP_SOCIAL_STACKSHARE = 'https://stackshare.io/appwrite'; const APP_SOCIAL_YOUTUBE = 'https://www.youtube.com/c/appwrite?sub_confirmation=1'; const APP_HOSTNAME_INTERNAL = 'appwrite'; + // Database Reconnect const DATABASE_RECONNECT_SLEEP = 2; const DATABASE_RECONNECT_MAX_ATTEMPTS = 10; + // Database Worker Types const DATABASE_TYPE_CREATE_ATTRIBUTE = 'createAttribute'; const DATABASE_TYPE_CREATE_INDEX = 'createIndex'; @@ -152,9 +154,11 @@ const DATABASE_TYPE_DELETE_ATTRIBUTE = 'deleteAttribute'; const DATABASE_TYPE_DELETE_INDEX = 'deleteIndex'; const DATABASE_TYPE_DELETE_COLLECTION = 'deleteCollection'; const DATABASE_TYPE_DELETE_DATABASE = 'deleteDatabase'; + // Build Worker Types const BUILD_TYPE_DEPLOYMENT = 'deployment'; const BUILD_TYPE_RETRY = 'retry'; + // Deletion Types const DELETE_TYPE_DATABASES = 'databases'; const DELETE_TYPE_DOCUMENT = 'document'; @@ -180,6 +184,10 @@ const DELETE_TYPE_TOPIC = 'topic'; const DELETE_TYPE_TARGET = 'target'; const DELETE_TYPE_EXPIRED_TARGETS = 'invalid_targets'; const DELETE_TYPE_SESSION_TARGETS = 'session_targets'; + +// Message types +const MESSAGE_SEND_TYPE_INTERNAL = 'internal'; +const MESSAGE_SEND_TYPE_EXTERNAL = 'external'; // Mail Types const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_MAGIC_SESSION = 'magicSession'; @@ -1375,19 +1383,19 @@ App::setResource('cache', function (Group $pools) { return new Cache(new Sharding($adapters)); }, ['pools']); -App::setResource('deviceLocal', function () { +App::setResource('deviceForLocal', function () { return new Local(); }); -App::setResource('deviceFiles', function ($project) { +App::setResource('deviceForFiles', function ($project) { return getDevice(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); }, ['project']); -App::setResource('deviceFunctions', function ($project) { +App::setResource('deviceForFunctions', function ($project) { return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId()); }, ['project']); -App::setResource('deviceBuilds', function ($project) { +App::setResource('deviceForBuilds', function ($project) { return getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId()); }, ['project']); diff --git a/app/worker.php b/app/worker.php index 2080970acb..cd0601afad 100644 --- a/app/worker.php +++ b/app/worker.php @@ -34,6 +34,7 @@ use Utopia\Logger\Log; use Utopia\Logger\Logger; use Utopia\Pools\Group; use Utopia\Queue\Connection; +use Utopia\Storage\Device\Local; Authorization::disable(); Runtime::enableCoroutine(SWOOLE_HOOK_ALL); @@ -45,8 +46,7 @@ Server::setResource('dbForConsole', function (Cache $cache, Registry $register) $database = $pools ->get('console') ->pop() - ->getResource() - ; + ->getResource(); $adapter = new Database($database, $cache); $adapter->setNamespace('_console'); @@ -54,26 +54,6 @@ Server::setResource('dbForConsole', function (Cache $cache, Registry $register) return $adapter; }, ['cache', 'register']); -Server::setResource('dbForProject', function (Cache $cache, Registry $register, Message $message, Database $dbForConsole) { - $payload = $message->getPayload() ?? []; - $project = new Document($payload['project'] ?? []); - - if ($project->isEmpty() || $project->getId() === 'console') { - return $dbForConsole; - } - - $pools = $register->get('pools'); - $database = $pools - ->get($project->getAttribute('database')) - ->pop() - ->getResource() - ; - - $adapter = new Database($database, $cache); - $adapter->setNamespace('_' . $project->getInternalId()); - return $adapter; -}, ['cache', 'register', 'message', 'dbForConsole']); - Server::setResource('project', function (Message $message, Database $dbForConsole) { $payload = $message->getPayload() ?? []; $project = new Document($payload['project'] ?? []); @@ -81,10 +61,26 @@ Server::setResource('project', function (Message $message, Database $dbForConsol if ($project->getId() === 'console') { return $project; } + return $dbForConsole->getDocument('projects', $project->getId()); - ; }, ['message', 'dbForConsole']); +Server::setResource('dbForProject', function (Cache $cache, Registry $register, Message $message, Document $project, Database $dbForConsole) { + if ($project->isEmpty() || $project->getId() === 'console') { + return $dbForConsole; + } + + $pools = $register->get('pools'); + $database = $pools + ->get($project->getAttribute('database')) + ->pop() + ->getResource(); + + $adapter = new Database($database, $cache); + $adapter->setNamespace('_' . $project->getInternalId()); + return $adapter; +}, ['cache', 'register', 'message', 'project', 'dbForConsole']); + Server::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) { $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools @@ -143,72 +139,88 @@ Server::setResource('cache', function (Registry $register) { return new Cache(new Sharding($adapters)); }, ['register']); + Server::setResource('log', fn() => new Log()); + Server::setResource('queueForUsage', function (Connection $queue) { return new Usage($queue); }, ['queue']); + Server::setResource('queue', function (Group $pools) { return $pools->get('queue')->pop()->getResource(); }, ['pools']); + Server::setResource('queueForDatabase', function (Connection $queue) { return new EventDatabase($queue); }, ['queue']); + Server::setResource('queueForMessaging', function (Connection $queue) { return new Messaging($queue); }, ['queue']); + Server::setResource('queueForMails', function (Connection $queue) { return new Mail($queue); }, ['queue']); + Server::setResource('queueForBuilds', function (Connection $queue) { return new Build($queue); }, ['queue']); + Server::setResource('queueForDeletes', function (Connection $queue) { return new Delete($queue); }, ['queue']); + Server::setResource('queueForEvents', function (Connection $queue) { return new Event($queue); }, ['queue']); + Server::setResource('queueForAudits', function (Connection $queue) { return new Audit($queue); }, ['queue']); + Server::setResource('queueForFunctions', function (Connection $queue) { return new Func($queue); }, ['queue']); + Server::setResource('queueForCertificates', function (Connection $queue) { return new Certificate($queue); }, ['queue']); + Server::setResource('queueForMigrations', function (Connection $queue) { return new Migration($queue); }, ['queue']); + Server::setResource('queueForHamster', function (Connection $queue) { return new Hamster($queue); }, ['queue']); + Server::setResource('logger', function (Registry $register) { return $register->get('logger'); }, ['register']); + Server::setResource('pools', function (Registry $register) { return $register->get('pools'); }, ['register']); -Server::setResource('getFunctionsDevice', function () { - return function (string $projectId) { - return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $projectId); - }; -}); -Server::setResource('getFilesDevice', function () { - return function (string $projectId) { - return getDevice(APP_STORAGE_UPLOADS . '/app-' . $projectId); - }; -}); -Server::setResource('getBuildsDevice', function () { - return function (string $projectId) { - return getDevice(APP_STORAGE_BUILDS . '/app-' . $projectId); - }; -}); -Server::setResource('getCacheDevice', function () { - return function (string $projectId) { - return getDevice(APP_STORAGE_CACHE . '/app-' . $projectId); - }; -}); + +Server::setResource('deviceForFunctions', function (Document $project) { + return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId()); +}, ['project']); + +Server::setResource('deviceForFiles', function (Document $project) { + return getDevice(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); +}, ['project']); + +Server::setResource('deviceForBuilds', function (Document $project) { + return getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId()); +}, ['project']); + +Server::setResource('deviceForCache', function (Document $project) { + return getDevice(APP_STORAGE_CACHE . '/app-' . $project->getId()); +}, ['project']); + +Server::setResource('deviceForLocalFiles', function (Document $project) { + return new Local(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); +}, ['project']); $pools = $register->get('pools'); $platform = new Appwrite(); diff --git a/composer.json b/composer.json index 9bad82b3e6..90a7086c12 100644 --- a/composer.json +++ b/composer.json @@ -56,7 +56,7 @@ "utopia-php/image": "0.6.*", "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.3.*", - "utopia-php/messaging": "0.9.*", + "utopia-php/messaging": "0.10.*", "utopia-php/migration": "0.3.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.5.*", @@ -70,7 +70,7 @@ "utopia-php/websocket": "0.1.*", "matomo/device-detector": "6.1.*", "dragonmantank/cron-expression": "3.3.2", - "phpmailer/phpmailer": "6.8.0", + "phpmailer/phpmailer": "6.9.1", "chillerlan/php-qrcode": "4.3.4", "adhocore/jwt": "1.1.2", "spomky-labs/otphp": "^10.0", diff --git a/composer.lock b/composer.lock index 048a0955f7..a3de6fb74a 100644 --- a/composer.lock +++ b/composer.lock @@ -885,16 +885,16 @@ }, { "name": "phpmailer/phpmailer", - "version": "v6.8.0", + "version": "v6.9.1", "source": { "type": "git", "url": "https://github.com/PHPMailer/PHPMailer.git", - "reference": "df16b615e371d81fb79e506277faea67a1be18f1" + "reference": "039de174cd9c17a8389754d3b877a2ed22743e18" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/PHPMailer/PHPMailer/zipball/df16b615e371d81fb79e506277faea67a1be18f1", - "reference": "df16b615e371d81fb79e506277faea67a1be18f1", + "url": "https://api.github.com/repos/PHPMailer/PHPMailer/zipball/039de174cd9c17a8389754d3b877a2ed22743e18", + "reference": "039de174cd9c17a8389754d3b877a2ed22743e18", "shasum": "" }, "require": { @@ -904,16 +904,17 @@ "php": ">=5.5.0" }, "require-dev": { - "dealerdirect/phpcodesniffer-composer-installer": "^0.7.2", + "dealerdirect/phpcodesniffer-composer-installer": "^1.0", "doctrine/annotations": "^1.2.6 || ^1.13.3", "php-parallel-lint/php-console-highlighter": "^1.0.0", "php-parallel-lint/php-parallel-lint": "^1.3.2", "phpcompatibility/php-compatibility": "^9.3.5", "roave/security-advisories": "dev-latest", - "squizlabs/php_codesniffer": "^3.7.1", + "squizlabs/php_codesniffer": "^3.7.2", "yoast/phpunit-polyfills": "^1.0.4" }, "suggest": { + "decomplexity/SendOauth2": "Adapter for using XOAUTH2 authentication", "ext-mbstring": "Needed to send email in multibyte encoding charset or decode encoded addresses", "ext-openssl": "Needed for secure SMTP sending and DKIM signing", "greew/oauth2-azure-provider": "Needed for Microsoft Azure XOAUTH2 authentication", @@ -953,7 +954,7 @@ "description": "PHPMailer is a full-featured email creation and transfer class for PHP", "support": { "issues": "https://github.com/PHPMailer/PHPMailer/issues", - "source": "https://github.com/PHPMailer/PHPMailer/tree/v6.8.0" + "source": "https://github.com/PHPMailer/PHPMailer/tree/v6.9.1" }, "funding": [ { @@ -961,7 +962,7 @@ "type": "github" } ], - "time": "2023-03-06T14:43:22+00:00" + "time": "2023-11-25T22:23:28+00:00" }, { "name": "spomky-labs/otphp", @@ -1911,28 +1912,28 @@ }, { "name": "utopia-php/messaging", - "version": "0.9.1", + "version": "0.10.0", "source": { "type": "git", "url": "https://github.com/utopia-php/messaging.git", - "reference": "7beec07684e9e1dfcf4ab5b1ba731fa396dccbdf" + "reference": "71dce00ad43eb278a877cb2c329f7b8d677adfeb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/messaging/zipball/7beec07684e9e1dfcf4ab5b1ba731fa396dccbdf", - "reference": "7beec07684e9e1dfcf4ab5b1ba731fa396dccbdf", + "url": "https://api.github.com/repos/utopia-php/messaging/zipball/71dce00ad43eb278a877cb2c329f7b8d677adfeb", + "reference": "71dce00ad43eb278a877cb2c329f7b8d677adfeb", "shasum": "" }, "require": { "ext-curl": "*", "ext-openssl": "*", - "php": ">=8.0.0" + "php": ">=8.0.0", + "phpmailer/phpmailer": "6.9.1" }, "require-dev": { - "laravel/pint": "1.13.*", - "phpmailer/phpmailer": "6.8.*", - "phpstan/phpstan": "1.10.*", - "phpunit/phpunit": "9.6.10" + "laravel/pint": "1.13.11", + "phpstan/phpstan": "1.10.58", + "phpunit/phpunit": "10.5.10" }, "type": "library", "autoload": { @@ -1955,9 +1956,9 @@ ], "support": { "issues": "https://github.com/utopia-php/messaging/issues", - "source": "https://github.com/utopia-php/messaging/tree/0.9.1" + "source": "https://github.com/utopia-php/messaging/tree/0.10.0" }, - "time": "2024-02-15T03:44:44+00:00" + "time": "2024-02-20T07:30:15+00:00" }, { "name": "utopia-php/migration", @@ -3409,16 +3410,16 @@ }, { "name": "phpdocumentor/type-resolver", - "version": "1.8.0", + "version": "1.8.1", "source": { "type": "git", "url": "https://github.com/phpDocumentor/TypeResolver.git", - "reference": "fad452781b3d774e3337b0c0b245dd8e5a4455fc" + "reference": "bc3dc91a5e9b14aa06d1d9e90647c5c5a2cc5353" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/fad452781b3d774e3337b0c0b245dd8e5a4455fc", - "reference": "fad452781b3d774e3337b0c0b245dd8e5a4455fc", + "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/bc3dc91a5e9b14aa06d1d9e90647c5c5a2cc5353", + "reference": "bc3dc91a5e9b14aa06d1d9e90647c5c5a2cc5353", "shasum": "" }, "require": { @@ -3461,9 +3462,9 @@ "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names", "support": { "issues": "https://github.com/phpDocumentor/TypeResolver/issues", - "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.8.0" + "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.8.1" }, - "time": "2024-01-11T11:49:22+00:00" + "time": "2024-01-18T19:15:27+00:00" }, { "name": "phpspec/prophecy", @@ -5019,16 +5020,16 @@ }, { "name": "squizlabs/php_codesniffer", - "version": "3.8.1", + "version": "3.9.0", "source": { "type": "git", "url": "https://github.com/PHPCSStandards/PHP_CodeSniffer.git", - "reference": "14f5fff1e64118595db5408e946f3a22c75807f7" + "reference": "d63cee4890a8afaf86a22e51ad4d97c91dd4579b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/PHPCSStandards/PHP_CodeSniffer/zipball/14f5fff1e64118595db5408e946f3a22c75807f7", - "reference": "14f5fff1e64118595db5408e946f3a22c75807f7", + "url": "https://api.github.com/repos/PHPCSStandards/PHP_CodeSniffer/zipball/d63cee4890a8afaf86a22e51ad4d97c91dd4579b", + "reference": "d63cee4890a8afaf86a22e51ad4d97c91dd4579b", "shasum": "" }, "require": { @@ -5095,7 +5096,7 @@ "type": "open_collective" } ], - "time": "2024-01-11T20:47:48+00:00" + "time": "2024-02-16T15:06:51+00:00" }, { "name": "swoole/ide-helper", diff --git a/src/Appwrite/Event/Messaging.php b/src/Appwrite/Event/Messaging.php index 9201799355..f8ea9e472b 100644 --- a/src/Appwrite/Event/Messaging.php +++ b/src/Appwrite/Event/Messaging.php @@ -8,13 +8,13 @@ use Utopia\Queue\Client; class Messaging extends Event { + protected string $type = ''; protected ?string $messageId = null; protected ?Document $message = null; protected ?array $recipients = null; protected ?string $scheduledAt = null; protected ?string $providerType = null; - public function __construct(protected Connection $connection) { parent::__construct($connection); @@ -24,6 +24,29 @@ class Messaging extends Event ->setClass(Event::MESSAGING_CLASS_NAME); } + /** + * Sets type for the build event. + * + * @param string $type Can be `MESSAGE_TYPE_INTERNAL` or `MESSAGE_TYPE_EXTERNAL`. + * @return self + */ + public function setType(string $type): self + { + $this->type = $type; + + return $this; + } + + /** + * Returns set type for the function event. + * + * @return string + */ + public function getType(): string + { + return $this->type; + } + /** * Sets recipient for the messaging event. * @@ -162,6 +185,7 @@ class Messaging extends Event $client = new Client($this->queue, $this->connection); return $client->enqueue([ + 'type' => $this->type, 'project' => $this->project, 'user' => $this->user, 'messageId' => $this->messageId, diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index eba88480c4..b07e31fc3f 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -92,8 +92,8 @@ class Exception extends \Exception public const USER_OAUTH2_BAD_REQUEST = 'user_oauth2_bad_request'; public const USER_OAUTH2_UNAUTHORIZED = 'user_oauth2_unauthorized'; public const USER_OAUTH2_PROVIDER_ERROR = 'user_oauth2_provider_error'; - public const USER_EMAIL_ALREADY_VERIFIED = 'user_email_alread_verified'; - public const USER_PHONE_ALREADY_VERIFIED = 'user_phone_already_verified'; + public const USER_EMAIL_ALREADY_VERIFIED = 'user_email_already_verified'; + public const USER_PHONE_ALREADY_VERIFIED = 'user_phone_already_verified'; public const USER_TARGET_NOT_FOUND = 'user_target_not_found'; public const USER_TARGET_ALREADY_EXISTS = 'user_target_already_exists'; @@ -130,18 +130,19 @@ class Exception extends \Exception public const STORAGE_INVALID_CONTENT_RANGE = 'storage_invalid_content_range'; public const STORAGE_INVALID_RANGE = 'storage_invalid_range'; public const STORAGE_INVALID_APPWRITE_ID = 'storage_invalid_appwrite_id'; + public const STORAGE_FILE_NOT_PUBLIC = 'storage_file_not_public'; /** VCS */ - public const INSTALLATION_NOT_FOUND = 'installation_not_found'; - public const PROVIDER_REPOSITORY_NOT_FOUND = 'provider_repository_not_found'; - public const REPOSITORY_NOT_FOUND = 'repository_not_found'; - public const PROVIDER_CONTRIBUTION_CONFLICT = 'provider_contribution_conflict'; - public const GENERAL_PROVIDER_FAILURE = 'general_provider_failure'; + public const INSTALLATION_NOT_FOUND = 'installation_not_found'; + public const PROVIDER_REPOSITORY_NOT_FOUND = 'provider_repository_not_found'; + public const REPOSITORY_NOT_FOUND = 'repository_not_found'; + public const PROVIDER_CONTRIBUTION_CONFLICT = 'provider_contribution_conflict'; + public const GENERAL_PROVIDER_FAILURE = 'general_provider_failure'; /** Functions */ public const FUNCTION_NOT_FOUND = 'function_not_found'; public const FUNCTION_RUNTIME_UNSUPPORTED = 'function_runtime_unsupported'; - public const FUNCTION_ENTRYPOINT_MISSING = 'function_entrypoint_missing'; + public const FUNCTION_ENTRYPOINT_MISSING = 'function_entrypoint_missing'; /** Deployments */ public const DEPLOYMENT_NOT_FOUND = 'deployment_not_found'; @@ -204,6 +205,8 @@ class Exception extends \Exception public const PROJECT_TEMPLATE_DEFAULT_DELETION = 'project_template_default_deletion'; + public const PROJECT_REGION_UNSUPPORTED = 'project_region_unsupported'; + /** Webhooks */ public const WEBHOOK_NOT_FOUND = 'webhook_not_found'; @@ -212,10 +215,10 @@ class Exception extends \Exception public const ROUTER_DOMAIN_NOT_CONFIGURED = 'router_domain_not_configured'; /** Proxy */ - public const RULE_RESOURCE_NOT_FOUND = 'rule_resource_not_found'; - public const RULE_NOT_FOUND = 'rule_not_found'; - public const RULE_ALREADY_EXISTS = 'rule_already_exists'; - public const RULE_VERIFICATION_FAILED = 'rule_verification_failed'; + public const RULE_RESOURCE_NOT_FOUND = 'rule_resource_not_found'; + public const RULE_NOT_FOUND = 'rule_not_found'; + public const RULE_ALREADY_EXISTS = 'rule_already_exists'; + public const RULE_VERIFICATION_FAILED = 'rule_verification_failed'; /** Keys */ public const KEY_NOT_FOUND = 'key_not_found'; @@ -232,53 +235,52 @@ class Exception extends \Exception public const GRAPHQL_TOO_MANY_QUERIES = 'graphql_too_many_queries'; /** Migrations */ - public const MIGRATION_NOT_FOUND = 'migration_not_found'; - public const MIGRATION_ALREADY_EXISTS = 'migration_already_exists'; - public const MIGRATION_IN_PROGRESS = 'migration_in_progress'; - public const MIGRATION_PROVIDER_ERROR = 'migration_provider_error'; + public const MIGRATION_NOT_FOUND = 'migration_not_found'; + public const MIGRATION_ALREADY_EXISTS = 'migration_already_exists'; + public const MIGRATION_IN_PROGRESS = 'migration_in_progress'; + public const MIGRATION_PROVIDER_ERROR = 'migration_provider_error'; /** Realtime */ - public const REALTIME_MESSAGE_FORMAT_INVALID = 'realtime_message_format_invalid'; - public const REALTIME_TOO_MANY_MESSAGES = 'realtime_too_many_messages'; - public const REALTIME_POLICY_VIOLATION = 'realtime_policy_violation'; + public const REALTIME_MESSAGE_FORMAT_INVALID = 'realtime_message_format_invalid'; + public const REALTIME_TOO_MANY_MESSAGES = 'realtime_too_many_messages'; + public const REALTIME_POLICY_VIOLATION = 'realtime_policy_violation'; /** Health */ - public const HEALTH_QUEUE_SIZE_EXCEEDED = 'health_queue_size_exceeded'; - public const HEALTH_CERTIFICATE_EXPIRED = 'health_certificate_expired'; - public const HEALTH_INVALID_HOST = 'health_invalid_host'; + public const HEALTH_QUEUE_SIZE_EXCEEDED = 'health_queue_size_exceeded'; + public const HEALTH_CERTIFICATE_EXPIRED = 'health_certificate_expired'; + public const HEALTH_INVALID_HOST = 'health_invalid_host'; /** Provider */ - public const PROVIDER_NOT_FOUND = 'provider_not_found'; - public const PROVIDER_ALREADY_EXISTS = 'provider_already_exists'; - public const PROVIDER_INCORRECT_TYPE = 'provider_incorrect_type'; - - public const PROVIDER_MISSING_CREDENTIALS = 'provider_missing_credentials'; + public const PROVIDER_NOT_FOUND = 'provider_not_found'; + public const PROVIDER_ALREADY_EXISTS = 'provider_already_exists'; + public const PROVIDER_INCORRECT_TYPE = 'provider_incorrect_type'; + public const PROVIDER_MISSING_CREDENTIALS = 'provider_missing_credentials'; /** Topic */ - public const TOPIC_NOT_FOUND = 'topic_not_found'; - public const TOPIC_ALREADY_EXISTS = 'topic_already_exists'; + public const TOPIC_NOT_FOUND = 'topic_not_found'; + public const TOPIC_ALREADY_EXISTS = 'topic_already_exists'; /** Subscriber */ - public const SUBSCRIBER_NOT_FOUND = 'subscriber_not_found'; - public const SUBSCRIBER_ALREADY_EXISTS = 'subscriber_already_exists'; + public const SUBSCRIBER_NOT_FOUND = 'subscriber_not_found'; + public const SUBSCRIBER_ALREADY_EXISTS = 'subscriber_already_exists'; /** Message */ - public const MESSAGE_NOT_FOUND = 'message_not_found'; - public const MESSAGE_MISSING_TARGET = 'message_missing_target'; - public const MESSAGE_ALREADY_SENT = 'message_already_sent'; - public const MESSAGE_ALREADY_PROCESSING = 'message_already_processing'; - public const MESSAGE_ALREADY_FAILED = 'message_already_failed'; - public const MESSAGE_ALREADY_SCHEDULED = 'message_already_scheduled'; - public const MESSAGE_TARGET_NOT_EMAIL = 'message_target_not_email'; - public const MESSAGE_TARGET_NOT_SMS = 'message_target_not_sms'; - public const MESSAGE_TARGET_NOT_PUSH = 'message_target_not_push'; - public const MESSAGE_MISSING_SCHEDULE = 'message_missing_schedule'; + public const MESSAGE_NOT_FOUND = 'message_not_found'; + public const MESSAGE_MISSING_TARGET = 'message_missing_target'; + public const MESSAGE_ALREADY_SENT = 'message_already_sent'; + public const MESSAGE_ALREADY_PROCESSING = 'message_already_processing'; + public const MESSAGE_ALREADY_FAILED = 'message_already_failed'; + public const MESSAGE_ALREADY_SCHEDULED = 'message_already_scheduled'; + public const MESSAGE_TARGET_NOT_EMAIL = 'message_target_not_email'; + public const MESSAGE_TARGET_NOT_SMS = 'message_target_not_sms'; + public const MESSAGE_TARGET_NOT_PUSH = 'message_target_not_push'; + public const MESSAGE_MISSING_SCHEDULE = 'message_missing_schedule'; /** Targets */ public const TARGET_PROVIDER_INVALID_TYPE = 'target_provider_invalid_type'; /** Schedules */ - public const SCHEDULE_NOT_FOUND = 'schedule_not_found'; + public const SCHEDULE_NOT_FOUND = 'schedule_not_found'; protected string $type = ''; diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index f0aa559a7f..fbb2320a34 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -64,8 +64,13 @@ abstract class ScheduleBase extends Action $getSchedule = function (Document $schedule) use ($dbForConsole, $getProjectDB): array { $project = $dbForConsole->getDocument('projects', $schedule->getAttribute('projectId')); + $collectionId = match ($schedule->getAttribute('resourceType')) { + 'function' => 'functions', + 'message' => 'messages' + }; + $resource = $getProjectDB($project)->getDocument( - $schedule->getAttribute('resourceCollection'), + $collectionId, $schedule->getAttribute('resourceId') ); @@ -108,7 +113,12 @@ abstract class ScheduleBase extends Action try { $this->schedules[$document['resourceId']] = $getSchedule($document); } catch (\Throwable $th) { - Console::error("Failed to load schedule for project {$document['projectId']} {$document['resourceCollection']} {$document['resourceId']}"); + $collectionId = match ($document->getAttribute('resourceType')) { + 'function' => 'functions', + 'message' => 'messages' + }; + + Console::error("Failed to load schedule for project {$document['projectId']} {$collectionId} {$document['resourceId']}"); Console::error($th->getMessage()); } } diff --git a/src/Appwrite/Platform/Tasks/ScheduleMessages.php b/src/Appwrite/Platform/Tasks/ScheduleMessages.php index cc641b434a..bc9b6d37d2 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleMessages.php +++ b/src/Appwrite/Platform/Tasks/ScheduleMessages.php @@ -50,6 +50,7 @@ class ScheduleMessages extends ScheduleBase $queueForMessaging = new Messaging($connection); $queueForMessaging + ->setType(MESSAGE_SEND_TYPE_EXTERNAL) ->setMessageId($schedule['resourceId']) ->setProject($schedule['project']) ->trigger(); diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index d23cfbff94..5b2e5be761 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -25,6 +25,7 @@ use Utopia\Database\Helpers\ID; use Utopia\Logger\Log; use Utopia\Platform\Action; use Utopia\Queue\Message; +use Utopia\Storage\Device; use Utopia\Storage\Device\Local; use Utopia\VCS\Adapter\Git\GitHub; @@ -49,9 +50,9 @@ class Builds extends Action ->inject('queueForUsage') ->inject('cache') ->inject('dbForProject') - ->inject('getFunctionsDevice') + ->inject('deviceForFunctions') ->inject('log') - ->callback(fn($message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Usage $usage, Cache $cache, Database $dbForProject, callable $getFunctionsDevice, Log $log) => $this->action($message, $dbForConsole, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $getFunctionsDevice, $log)); + ->callback(fn($message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Usage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log) => $this->action($message, $dbForConsole, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $log)); } /** @@ -62,12 +63,12 @@ class Builds extends Action * @param Usage $queueForUsage * @param Cache $cache * @param Database $dbForProject - * @param callable $getFunctionsDevice + * @param Device $deviceForFunctions * @param Log $log * @return void * @throws \Utopia\Database\Exception */ - public function action(Message $message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, callable $getFunctionsDevice, Log $log): void + public function action(Message $message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log): void { $payload = $message->getPayload() ?? []; @@ -89,7 +90,7 @@ class Builds extends Action case BUILD_TYPE_RETRY: Console::info('Creating build for deployment: ' . $deployment->getId()); $github = new GitHub($cache); - $this->buildDeployment($getFunctionsDevice, $queueForFunctions, $queueForEvents, $queueForUsage, $dbForConsole, $dbForProject, $github, $project, $resource, $deployment, $template, $log); + $this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForUsage, $dbForConsole, $dbForProject, $github, $project, $resource, $deployment, $template, $log); break; default: @@ -98,7 +99,7 @@ class Builds extends Action } /** - * @param callable $getFunctionsDevice + * @param Device $deviceForFunctions * @param Func $queueForFunctions * @param Event $queueForEvents * @param Usage $queueForUsage @@ -114,7 +115,7 @@ class Builds extends Action * @throws \Utopia\Database\Exception * @throws Exception */ - protected function buildDeployment(callable $getFunctionsDevice, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForConsole, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, Log $log): void + protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForConsole, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, Log $log): void { $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); @@ -156,7 +157,6 @@ class Builds extends Action $durationStart = \microtime(true); $buildId = $deployment->getAttribute('buildId', ''); $isNewBuild = empty($buildId); - $deviceFunctions = $getFunctionsDevice($project->getId()); if ($isNewBuild) { $buildId = ID::unique(); @@ -170,7 +170,7 @@ class Builds extends Action 'path' => '', 'runtime' => $function->getAttribute('runtime'), 'source' => $deployment->getAttribute('path', ''), - 'sourceType' => strtolower($deviceFunctions->getType()), + 'sourceType' => strtolower($deviceForFunctions->getType()), 'logs' => '', 'endTime' => null, 'duration' => 0, @@ -188,7 +188,7 @@ class Builds extends Action $installationId = $deployment->getAttribute('installationId', ''); $providerRepositoryId = $deployment->getAttribute('providerRepositoryId', ''); $providerCommitHash = $deployment->getAttribute('providerCommitHash', ''); - $isVcsEnabled = $providerRepositoryId ? true : false; + $isVcsEnabled = !empty($providerRepositoryId); $owner = ''; $repositoryName = ''; @@ -311,10 +311,8 @@ class Builds extends Action Console::execute('tar --exclude code.tar.gz -czf ' . $tmpPathFile . ' -C /tmp/builds/' . \escapeshellcmd($buildId) . '/code' . (empty($rootDirectory) ? '' : '/' . $rootDirectory) . ' .', '', $stdout, $stderr); - $deviceFunctions = $getFunctionsDevice($project->getId()); - - $path = $deviceFunctions->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); - $result = $localDevice->transfer($tmpPathFile, $path, $deviceFunctions); + $path = $deviceForFunctions->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); + $result = $localDevice->transfer($tmpPathFile, $path, $deviceForFunctions); if (!$result) { throw new \Exception("Unable to move file"); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 542f7f41db..d5e5b40d18 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -44,22 +44,22 @@ class Deletes extends Action ->inject('message') ->inject('dbForConsole') ->inject('getProjectDB') - ->inject('getFilesDevice') - ->inject('getFunctionsDevice') - ->inject('getBuildsDevice') - ->inject('getCacheDevice') + ->inject('deviceForFiles') + ->inject('deviceForFunctions') + ->inject('deviceForBuilds') + ->inject('deviceForCache') ->inject('abuseRetention') ->inject('executionRetention') ->inject('auditRetention') ->inject('log') - ->callback(fn ($message, $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => $this->action($message, $dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $abuseRetention, $executionRetention, $auditRetention, $log)); + ->callback(fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => $this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $abuseRetention, $executionRetention, $auditRetention, $log)); } /** * @throws Exception * @throws Throwable */ - public function action(Message $message, Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -87,13 +87,13 @@ class Deletes extends Action $this->deleteCollection($getProjectDB, $document, $project); break; case DELETE_TYPE_PROJECTS: - $this->deleteProject($dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $document); + $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $document); break; case DELETE_TYPE_FUNCTIONS: - $this->deleteFunction($dbForConsole, $getProjectDB, $getFunctionsDevice, $getBuildsDevice, $document, $project); + $this->deleteFunction($dbForConsole, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project); break; case DELETE_TYPE_DEPLOYMENTS: - $this->deleteDeployment($getProjectDB, $getFunctionsDevice, $getBuildsDevice, $document, $project); + $this->deleteDeployment($getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project); break; case DELETE_TYPE_USERS: $this->deleteUser($getProjectDB, $document, $project); @@ -101,11 +101,11 @@ class Deletes extends Action case DELETE_TYPE_TEAMS: $this->deleteMemberships($getProjectDB, $document, $project); if ($project->getId() === 'console') { - $this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $document); + $this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $document); } break; case DELETE_TYPE_BUCKETS: - $this->deleteBucket($getProjectDB, $getFilesDevice, $document, $project); + $this->deleteBucket($getProjectDB, $deviceForFiles, $document, $project); break; case DELETE_TYPE_INSTALLATIONS: $this->deleteInstallation($dbForConsole, $getProjectDB, $document, $project); @@ -203,8 +203,13 @@ class Deletes extends Action return; } + $collectionId = match ($document->getAttribute('resourceType')) { + 'function' => 'functions', + 'message' => 'messages' + }; + $resource = $getProjectDB($project)->getDocument( - $document->getAttribute('resourceCollection'), + $collectionId, $document->getAttribute('resourceId') ); @@ -511,14 +516,14 @@ class Deletes extends Action * @throws Restricted * @throws Structure */ - private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, Document $document): void + private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, Document $document): void { $projects = $dbForConsole->find('projects', [ Query::equal('teamInternalId', [$document->getInternalId()]) ]); foreach ($projects as $project) { - $this->deleteProject($dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $project); + $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $project); $dbForConsole->deleteDocument('projects', $project->getId()); } } @@ -526,17 +531,17 @@ class Deletes extends Action /** * @param Database $dbForConsole * @param callable $getProjectDB - * @param callable $getFilesDevice - * @param callable $getFunctionsDevice - * @param callable $getBuildsDevice - * @param callable $getCacheDevice + * @param Device $deviceForFiles + * @param Device $deviceForFunctions + * @param Device $deviceForBuilds + * @param Device $deviceForCache * @param Document $document * @return void * @throws Exception * @throws Authorization * @throws \Utopia\Database\Exception */ - private function deleteProject(Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, Document $document): void + private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, Document $document): void { $projectId = $document->getId(); $projectInternalId = $document->getInternalId(); @@ -602,15 +607,10 @@ class Deletes extends Action } // Delete all storage directories - $uploads = $getFilesDevice($projectId); - $functions = $getFunctionsDevice($projectId); - $builds = $getBuildsDevice($projectId); - $cache = $getCacheDevice($projectId); - - $uploads->delete($uploads->getRoot(), true); - $functions->delete($functions->getRoot(), true); - $builds->delete($builds->getRoot(), true); - $cache->delete($cache->getRoot(), true); + $deviceForFiles->delete($deviceForFiles->getRoot(), true); + $deviceForFunctions->delete($deviceForFunctions->getRoot(), true); + $deviceForBuilds->delete($deviceForBuilds->getRoot(), true); + $deviceForCache->delete($deviceForCache->getRoot(), true); } /** @@ -772,14 +772,14 @@ class Deletes extends Action /** * @param callable $getProjectDB - * @param callable $getFunctionsDevice - * @param callable $getBuildsDevice + * @param Device $deviceForFunctions + * @param Device $deviceForBuilds * @param Document $document function document * @param Document $project * @return void * @throws Exception */ - private function deleteFunction(Database $dbForConsole, callable $getProjectDB, callable $getFunctionsDevice, callable $getBuildsDevice, Document $document, Document $project): void + private function deleteFunction(Database $dbForConsole, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, Document $document, Document $project): void { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); @@ -811,25 +811,25 @@ class Deletes extends Action * Delete Deployments */ Console::info("Deleting deployments for function " . $functionId); - $functionsStorage = $getFunctionsDevice($projectId); + $deploymentInternalIds = []; $this->deleteByGroup('deployments', [ Query::equal('resourceInternalId', [$functionInternalId]) - ], $dbForProject, function (Document $document) use ($functionsStorage, &$deploymentInternalIds) { + ], $dbForProject, function (Document $document) use ($deviceForFunctions, &$deploymentInternalIds) { $deploymentInternalIds[] = $document->getInternalId(); - $this->deleteDeploymentFiles($functionsStorage, $document); + $this->deleteDeploymentFiles($deviceForFunctions, $document); }); /** * Delete builds */ Console::info("Deleting builds for function " . $functionId); - $buildsStorage = $getBuildsDevice($projectId); + foreach ($deploymentInternalIds as $deploymentInternalId) { $this->deleteByGroup('builds', [ Query::equal('deploymentInternalId', [$deploymentInternalId]) - ], $dbForProject, function (Document $document) use ($buildsStorage) { - $this->deleteBuildFiles($buildsStorage, $document); + ], $dbForProject, function (Document $document) use ($deviceForBuilds) { + $this->deleteBuildFiles($deviceForBuilds, $document); }); } @@ -929,14 +929,14 @@ class Deletes extends Action /** * @param callable $getProjectDB - * @param callable $getFunctionsDevice - * @param callable $getBuildsDevice + * @param Device $deviceForFunctions + * @param Device $deviceForBuilds * @param Document $document * @param Document $project * @return void * @throws Exception */ - private function deleteDeployment(callable $getProjectDB, callable $getFunctionsDevice, callable $getBuildsDevice, Document $document, Document $project): void + private function deleteDeployment(callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, Document $document, Document $project): void { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); @@ -946,18 +946,17 @@ class Deletes extends Action /** * Delete deployment files */ - $functionsStorage = $getFunctionsDevice($projectId); - $this->deleteDeploymentFiles($functionsStorage, $document); + $this->deleteDeploymentFiles($deviceForFunctions, $document); /** * Delete builds */ Console::info("Deleting builds for deployment " . $deploymentId); - $buildsStorage = $getBuildsDevice($projectId); + $this->deleteByGroup('builds', [ Query::equal('deploymentInternalId', [$deploymentInternalId]) - ], $dbForProject, function (Document $document) use ($buildsStorage) { - $this->deleteBuildFiles($buildsStorage, $document); + ], $dbForProject, function (Document $document) use ($deviceForBuilds) { + $this->deleteBuildFiles($deviceForBuilds, $document); }); /** @@ -1101,21 +1100,18 @@ class Deletes extends Action /** * @param callable $getProjectDB - * @param callable $getFilesDevice + * @param Device $deviceForFiles * @param Document $document * @param Document $project * @return void */ - private function deleteBucket(callable $getProjectDB, callable $getFilesDevice, Document $document, Document $project): void + private function deleteBucket(callable $getProjectDB, Device $deviceForFiles, Document $document, Document $project): void { - $projectId = $project->getId(); $dbForProject = $getProjectDB($project); $dbForProject->deleteCollection('bucket_' . $document->getInternalId()); - $device = $getFilesDevice($projectId); - - $device->deletePath($document->getId()); + $deviceForFiles->deletePath($document->getId()); } /** diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 083eae4e0a..77b46f8c12 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -2,11 +2,14 @@ namespace Appwrite\Platform\Workers; +use Appwrite\Auth\Auth; use Appwrite\Event\Usage; use Appwrite\Extend\Exception; use Appwrite\Messaging\Status as MessageStatus; use Utopia\App; use Utopia\CLI\Console; +use Utopia\Config\Config; +use Utopia\Database\Validator\Authorization; use Utopia\DSN\DSN; use Utopia\Database\Database; use Utopia\Database\DateTime; @@ -29,10 +32,13 @@ use Utopia\Messaging\Adapter\SMS\Textmagic; use Utopia\Messaging\Adapter\SMS\Twilio; use Utopia\Messaging\Adapter\SMS\Vonage; use Utopia\Messaging\Messages\Email; +use Utopia\Messaging\Messages\Email\Attachment; use Utopia\Messaging\Messages\Push; use Utopia\Messaging\Messages\SMS; use Utopia\Platform\Action; use Utopia\Queue\Message; +use Utopia\Storage\Device; +use Utopia\Storage\Storage; use function Swoole\Coroutine\batch; @@ -44,7 +50,7 @@ class Messaging extends Action } /** - * @throws Exception + * @throws \Exception */ public function __construct() { @@ -53,49 +59,61 @@ class Messaging extends Action ->inject('message') ->inject('log') ->inject('dbForProject') + ->inject('deviceForFiles') + ->inject('deviceForLocalFiles') ->inject('queueForUsage') - ->callback(fn(Message $message, Log $log, Database $dbForProject, Usage $queueForUsage) => $this->action($message, $log, $dbForProject, $queueForUsage)); + ->callback(fn(Message $message, Log $log, Database $dbForProject, Device $deviceForFiles, Device $deviceForLocalFiles, Usage $queueForUsage) => $this->action($message, $log, $dbForProject, $deviceForFiles, $deviceForLocalFiles, $queueForUsage)); } /** * @param Message $message * @param Log $log * @param Database $dbForProject + * @param callable $getLocalCache * @param Usage $queueForUsage * @return void - * @throws Exception + * @throws \Exception */ - public function action(Message $message, Log $log, Database $dbForProject, Usage $queueForUsage): void - { + public function action( + Message $message, + Log $log, + Database $dbForProject, + Device $deviceForFiles, + Device $deviceForLocalFiles, + Usage $queueForUsage + ): void { $payload = $message->getPayload() ?? []; if (empty($payload)) { throw new Exception('Missing payload'); } + $type = $payload['type'] ?? ''; + $project = new Document($payload['project'] ?? []); - if ( - !\is_null($payload['message']) - && !\is_null($payload['recipients']) - && $payload['providerType'] === MESSAGE_TYPE_SMS - ) { - // Message was triggered internally - $this->processInternalSMSMessage( - new Document($payload['message']), - new Document($payload['project'] ?? []), - $payload['recipients'], - $queueForUsage, - $log, - ); - } else { - $message = $dbForProject->getDocument('messages', $payload['messageId']); + switch ($type) { + case MESSAGE_SEND_TYPE_INTERNAL: + $message = new Document($payload['message'] ?? []); + $recipients = $payload['recipients'] ?? []; - $this->processMessage($dbForProject, $message); + $this->sendInternalSMSMessage($message, $project, $recipients, $queueForUsage, $log); + break; + case MESSAGE_SEND_TYPE_EXTERNAL: + $message = $dbForProject->getDocument('messages', $payload['messageId']); + + $this->sendExternalMessage($dbForProject, $message, $deviceForFiles, $deviceForLocalFiles,); + break; + default: + throw new Exception('Unknown message type: ' . $type); } } - private function processMessage(Database $dbForProject, Document $message): void - { + private function sendExternalMessage( + Database $dbForProject, + Document $message, + Device $deviceForFiles, + Device $deviceForLocalFiles, + ): void { $topicIds = $message->getAttribute('topics', []); $targetIds = $message->getAttribute('targets', []); $userIds = $message->getAttribute('users', []); @@ -199,8 +217,8 @@ class Messaging extends Action /** * @var array $results */ - $results = batch(\array_map(function ($providerId) use ($identifiers, $providers, $fallback, $message, $dbForProject) { - return function () use ($providerId, $identifiers, $providers, $fallback, $message, $dbForProject) { + $results = batch(\array_map(function ($providerId) use ($identifiers, $providers, $fallback, $message, $dbForProject, $deviceForFiles, $deviceForLocalFiles) { + return function () use ($providerId, $identifiers, $providers, $fallback, $message, $dbForProject, $deviceForFiles, $deviceForLocalFiles) { if (\array_key_exists($providerId, $providers)) { $provider = $providers[$providerId]; } else { @@ -216,9 +234,9 @@ class Messaging extends Action $identifiers = $identifiers[$providerId]; $adapter = match ($provider->getAttribute('type')) { - MESSAGE_TYPE_SMS => $this->sms($provider), - MESSAGE_TYPE_PUSH => $this->push($provider), - MESSAGE_TYPE_EMAIL => $this->email($provider), + MESSAGE_TYPE_SMS => $this->getSmsAdapter($provider), + MESSAGE_TYPE_PUSH => $this->getPushAdapter($provider), + MESSAGE_TYPE_EMAIL => $this->getEmailAdapter($provider), default => throw new Exception(Exception::PROVIDER_INCORRECT_TYPE) }; @@ -226,17 +244,17 @@ class Messaging extends Action $batches = \array_chunk($identifiers, $maxBatchSize); $batchIndex = 0; - return batch(\array_map(function ($batch) use ($message, $provider, $adapter, &$batchIndex, $dbForProject) { - return function () use ($batch, $message, $provider, $adapter, &$batchIndex, $dbForProject) { + return batch(\array_map(function ($batch) use ($message, $provider, $adapter, &$batchIndex, $dbForProject, $deviceForFiles, $deviceForLocalFiles) { + return function () use ($batch, $message, $provider, $adapter, &$batchIndex, $dbForProject, $deviceForFiles, $deviceForLocalFiles) { $deliveredTotal = 0; $deliveryErrors = []; $messageData = clone $message; $messageData->setAttribute('to', $batch); $data = match ($provider->getAttribute('type')) { - MESSAGE_TYPE_SMS => $this->buildSMSMessage($messageData, $provider), + MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider), MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData), - MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider), + MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $deviceForLocalFiles), default => throw new Exception(Exception::PROVIDER_INCORRECT_TYPE) }; @@ -310,9 +328,40 @@ class Messaging extends Action $message->setAttribute('deliveredAt', DateTime::now()); $dbForProject->updateDocument('messages', $message->getId(), $message); + + // Delete any attachments that were downloaded to the local cache + if ($provider->getAttribute('type') === MESSAGE_TYPE_EMAIL) { + if ($deviceForFiles->getType() === Storage::DEVICE_LOCAL) { + return; + } + + $data = $message->getAttribute('data'); + $attachments = $data['attachments'] ?? []; + + foreach ($attachments as $attachment) { + $bucketId = $attachment['bucketId']; + $fileId = $attachment['fileId']; + + $bucket = $dbForProject->getDocument('buckets', $bucketId); + if ($bucket->isEmpty()) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); + if ($file->isEmpty()) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); + } + + $path = $file->getAttribute('path', ''); + + if ($deviceForLocalFiles->exists($path)) { + $deviceForLocalFiles->delete($path); + } + } + } } - private function processInternalSMSMessage(Document $message, Document $project, array $recipients, Usage $queueForUsage, Log $log): void + private function sendInternalSMSMessage(Document $message, Document $project, array $recipients, Usage $queueForUsage, Log $log): void { if (empty(App::getEnv('_APP_SMS_PROVIDER')) || empty(App::getEnv('_APP_SMS_FROM'))) { throw new \Exception('Skipped SMS processing. Missing "_APP_SMS_PROVIDER" or "_APP_SMS_FROM" environment variables.'); @@ -375,7 +424,7 @@ class Messaging extends Action ] ]); - $adapter = $this->sms($provider); + $adapter = $this->getSmsAdapter($provider); $maxBatchSize = $adapter->getMaxMessagesPerRequest(); $batches = \array_chunk($recipients, $maxBatchSize); @@ -385,7 +434,7 @@ class Messaging extends Action return function () use ($batch, $message, $provider, $adapter, $batchIndex, $project, $queueForUsage) { $message->setAttribute('to', $batch); - $data = $this->buildSMSMessage($message, $provider); + $data = $this->buildSmsMessage($message, $provider); try { $adapter->send($data); @@ -401,11 +450,7 @@ class Messaging extends Action }, $batches)); } - public function shutdown(): void - { - } - - private function sms(Document $provider): ?SMSAdapter + private function getSmsAdapter(Document $provider): ?SMSAdapter { $credentials = $provider->getAttribute('credentials'); @@ -420,7 +465,7 @@ class Messaging extends Action }; } - private function push(Document $provider): ?PushAdapter + private function getPushAdapter(Document $provider): ?PushAdapter { $credentials = $provider->getAttribute('credentials'); @@ -437,7 +482,7 @@ class Messaging extends Action }; } - private function email(Document $provider): ?EmailAdapter + private function getEmailAdapter(Document $provider): ?EmailAdapter { $credentials = $provider->getAttribute('credentials', []); $options = $provider->getAttribute('options', []); @@ -463,8 +508,13 @@ class Messaging extends Action }; } - private function buildEmailMessage(Database $dbForProject, Document $message, Document $provider): Email - { + private function buildEmailMessage( + Database $dbForProject, + Document $message, + Document $provider, + Device $deviceForFiles, + Device $deviceForLocalFiles, + ): Email { $fromName = $provider['options']['fromName'] ?? null; $fromEmail = $provider['options']['fromEmail'] ?? null; $replyToEmail = $provider['options']['replyToEmail'] ?? null; @@ -474,8 +524,9 @@ class Messaging extends Action $bccTargets = $data['bcc'] ?? []; $cc = []; $bcc = []; + $attachments = $data['attachments'] ?? []; - if (\count($ccTargets) > 0) { + if (!empty($ccTargets)) { $ccTargets = $dbForProject->find('targets', [ Query::equal('$id', $ccTargets), Query::limit(\count($ccTargets)), @@ -485,7 +536,7 @@ class Messaging extends Action } } - if (\count($bccTargets) > 0) { + if (!empty($bccTargets)) { $bccTargets = $dbForProject->find('targets', [ Query::equal('$id', $bccTargets), Query::limit(\count($bccTargets)), @@ -495,21 +546,77 @@ class Messaging extends Action } } + if (!empty($attachments)) { + foreach ($attachments as &$attachment) { + $bucketId = $attachment['bucketId']; + $fileId = $attachment['fileId']; + + $bucket = $dbForProject->getDocument('buckets', $bucketId); + if ($bucket->isEmpty()) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); + if ($file->isEmpty()) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); + } + + $mimes = Config::getParam('storage-mimes'); + $path = $file->getAttribute('path', ''); + + if (!$deviceForFiles->exists($path)) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); + } + + $contentType = 'text/plain'; + + if (\in_array($file->getAttribute('mimeType'), $mimes)) { + $contentType = $file->getAttribute('mimeType'); + } + + if ($deviceForFiles->getType() !== Storage::DEVICE_LOCAL) { + $deviceForFiles->transfer($path, $path, $deviceForLocalFiles); + } + + $attachment = new Attachment( + $file->getAttribute('name'), + $path, + $contentType + ); + } + } + $to = $message['to']; $subject = $data['subject']; $content = $data['content']; $html = $data['html'] ?? false; - return new Email($to, $subject, $content, $fromName, $fromEmail, $replyToName, $replyToEmail, $cc, $bcc, null, $html); + return new Email( + $to, + $subject, + $content, + $fromName, + $fromEmail, + $replyToName, + $replyToEmail, + $cc, + $bcc, + $attachments, + $html + ); } - private function buildSMSMessage(Document $message, Document $provider): SMS + private function buildSmsMessage(Document $message, Document $provider): SMS { $to = $message['to']; $content = $message['data']['content']; $from = $provider['options']['from']; - return new SMS($to, $content, $from); + return new SMS( + $to, + $content, + $from + ); } private function buildPushMessage(Document $message): Push @@ -519,12 +626,25 @@ class Messaging extends Action $body = $message['data']['body']; $data = $message['data']['data'] ?? null; $action = $message['data']['action'] ?? null; + $image = $message['data']['image'] ?? null; $sound = $message['data']['sound'] ?? null; $icon = $message['data']['icon'] ?? null; $color = $message['data']['color'] ?? null; $tag = $message['data']['tag'] ?? null; $badge = $message['data']['badge'] ?? null; - return new Push($to, $title, $body, $data, $action, $sound, $icon, $color, $tag, $badge); + return new Push( + $to, + $title, + $body, + $data, + $action, + $sound, + $image, + $icon, + $color, + $tag, + $badge + ); } } diff --git a/src/Appwrite/Specification/Format/OpenAPI3.php b/src/Appwrite/Specification/Format/OpenAPI3.php index 6672631ff3..b405ec6d17 100644 --- a/src/Appwrite/Specification/Format/OpenAPI3.php +++ b/src/Appwrite/Specification/Format/OpenAPI3.php @@ -451,6 +451,10 @@ class OpenAPI3 extends Format $node['format'] = 'int32'; } break; + case 'Appwrite\Utopia\Database\Validator\CompoundUID': + $node['schema']['type'] = $validator->getType(); + $node['schema']['x-example'] = '[ID1:ID2]'; + break; default: $node['schema']['type'] = 'string'; break; diff --git a/src/Appwrite/Specification/Format/Swagger2.php b/src/Appwrite/Specification/Format/Swagger2.php index 20aeb96222..b2ccef6c07 100644 --- a/src/Appwrite/Specification/Format/Swagger2.php +++ b/src/Appwrite/Specification/Format/Swagger2.php @@ -446,6 +446,10 @@ class Swagger2 extends Format $node['format'] = 'int32'; } break; + case 'Appwrite\Utopia\Database\Validator\CompoundUID': + $node['type'] = $validator->getType(); + $node['x-example'] = '[ID1:ID2]'; + break; default: $node['type'] = 'string'; break; diff --git a/src/Appwrite/Utopia/Database/Validator/CompoundUID.php b/src/Appwrite/Utopia/Database/Validator/CompoundUID.php new file mode 100644 index 0000000000..3f23500952 --- /dev/null +++ b/src/Appwrite/Utopia/Database/Validator/CompoundUID.php @@ -0,0 +1,58 @@ +isValid($id)) { + return false; + } + } + + return true; + } + + public function getType(): string + { + return self::TYPE_STRING; + } + + public static function parse(string $key): array + { + $parts = \explode(':', $key); + $result = []; + + foreach ($parts as $part) { + $result[] = $part; + } + + return $result; + } +} diff --git a/tests/e2e/Services/Account/AccountCustomClientTest.php b/tests/e2e/Services/Account/AccountCustomClientTest.php index 1169686ada..ba525f556e 100644 --- a/tests/e2e/Services/Account/AccountCustomClientTest.php +++ b/tests/e2e/Services/Account/AccountCustomClientTest.php @@ -1918,7 +1918,7 @@ class AccountCustomClientTest extends Scope $this->assertEquals($response['body']['users'][0]['email'], $email); } - + #[Retry(count: 2)] public function testCreatePhone(): array { $number = '+123456789'; @@ -1941,22 +1941,8 @@ class AccountCustomClientTest extends Scope $this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['expire'])); $userId = $response['body']['userId']; - $messageId = $response['body']['$id']; - /** - * Test for FAILURE - */ - $response = $this->client->call(Client::METHOD_POST, '/account/tokens/phone', array_merge([ - 'origin' => 'http://localhost', - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ]), [ - 'userId' => ID::unique() - ]); - - $this->assertEquals(400, $response['headers']['status-code']); - - \sleep(5); + \sleep(7); $smsRequest = $this->getLastRequest(); @@ -1972,6 +1958,19 @@ class AccountCustomClientTest extends Scope $data['id'] = $userId; $data['number'] = $number; + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_POST, '/account/tokens/phone', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ]), [ + 'userId' => ID::unique() + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + return $data; } diff --git a/tests/e2e/Services/Messaging/MessagingBase.php b/tests/e2e/Services/Messaging/MessagingBase.php index 956ac5f68a..25bf985692 100644 --- a/tests/e2e/Services/Messaging/MessagingBase.php +++ b/tests/e2e/Services/Messaging/MessagingBase.php @@ -444,6 +444,17 @@ trait MessagingBase $this->assertEquals($target['body']['userId'], $response['body']['target']['userId']); $this->assertEquals($target['body']['providerType'], $response['body']['target']['providerType']); + // Test duplicate subscribers not allowed + $failure = $this->client->call(Client::METHOD_POST, '/messaging/topics/' . $topics['public']['$id'] . '/subscribers', \array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'subscriberId' => ID::unique(), + 'targetId' => $target['body']['$id'], + ]); + + $this->assertEquals(409, $failure['headers']['status-code']); + $topic = $this->client->call(Client::METHOD_GET, '/messaging/topics/' . $topics['public']['$id'], [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], diff --git a/tests/unit/Utopia/Database/Validator/CompoundUIDTest.php b/tests/unit/Utopia/Database/Validator/CompoundUIDTest.php new file mode 100644 index 0000000000..b443cf590b --- /dev/null +++ b/tests/unit/Utopia/Database/Validator/CompoundUIDTest.php @@ -0,0 +1,37 @@ +object = new CompoundUID(); + } + + public function tearDown(): void + { + } + + public function testValues(): void + { + $this->assertEquals($this->object->isValid('123:456'), true); + $this->assertEquals($this->object->isValid('123'), false); + $this->assertEquals($this->object->isValid('123:_456'), false); + $this->assertEquals($this->object->isValid('dasda asdasd'), false); + $this->assertEquals($this->object->isValid('dasda:asdasd'), true); + $this->assertEquals($this->object->isValid('_asdas:dasdas'), false); + $this->assertEquals($this->object->isValid('as$$5da:sdasdas'), false); + $this->assertEquals($this->object->isValid(false), false); + $this->assertEquals($this->object->isValid(null), false); + $this->assertEquals($this->object->isValid('socialAccountForYoutubeAndRestSubscribers:12345'), false); + $this->assertEquals($this->object->isValid('socialAccountForYoutubeAndRSubscriber:12345'), false); + $this->assertEquals($this->object->isValid('socialAccount:ForYoutubeSubscribe'), true); + $this->assertEquals($this->object->isValid('socialAccountForYoutubeSubscribe:socialAccountForYoutubeSubscribe'), true); + } +}