diff --git a/app/cli.php b/app/cli.php index 504e4fb5e6..2ad37c0ce2 100644 --- a/app/cli.php +++ b/app/cli.php @@ -194,6 +194,9 @@ CLI::setResource('publisher', function (Group $pools) { CLI::setResource('publisherDatabases', function (BrokerPool $publisher) { return $publisher; }, ['publisher']); +CLI::setResource('publisherFunctions', function (BrokerPool $publisher) { + return $publisher; +}, ['publisher']); CLI::setResource('publisherMigrations', function (BrokerPool $publisher) { return $publisher; }, ['publisher']); diff --git a/app/config/templates/site.php b/app/config/templates/site.php index 4ae6f61607..95a2161902 100644 --- a/app/config/templates/site.php +++ b/app/config/templates/site.php @@ -473,13 +473,13 @@ return [ 'frameworks' => [ getFramework('FLUTTER', [ 'providerRootDirectory' => './', - 'buildCommand' => 'bash build.sh', + 'buildCommand' => 'bash prepare-env.sh && flutter build web', ]), ], 'vcsProvider' => 'github', 'providerRepositoryId' => 'starter-for-flutter', 'providerOwner' => 'appwrite', - 'providerVersion' => '0.1.*', + 'providerVersion' => '0.2.*', 'variables' => [ [ 'name' => 'APPWRITE_PUBLIC_ENDPOINT', diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 39ebae9590..32b77433f0 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -686,12 +686,12 @@ App::get('/v1/health/queue/functions') contentType: ContentType::JSON )) ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) - ->inject('publisher') + ->inject('publisherFunctions') ->inject('response') - ->action(function (int|string $threshold, Publisher $publisher, Response $response) { + ->action(function (int|string $threshold, Publisher $publisherFunctions, Response $response) { $threshold = \intval($threshold); - $size = $publisher->getQueueSize(new Queue(Event::FUNCTIONS_QUEUE_NAME)); + $size = $publisherFunctions->getQueueSize(new Queue(Event::FUNCTIONS_QUEUE_NAME)); if ($size >= $threshold) { throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 3376f4857c..993f8a03c9 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -643,15 +643,20 @@ App::get('/v1/users') $cursor->setValue($cursorDocument); } - $filterQueries = Query::groupByType($queries)['filters']; - try { - $users = $dbForProject->find('users', $queries); - $total = $dbForProject->count('users', $filterQueries, APP_LIMIT_COUNT); - } catch (OrderException $e) { - throw new Exception(Exception::DATABASE_QUERY_ORDER_NULL, "The order attribute '{$e->getAttribute()}' had a null value. Cursor pagination requires all documents order attribute values are non-null."); - } catch (QueryException $e) { - throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); - } + $users = []; + $total = 0; + + $dbForProject->skipFilters(function () use ($dbForProject, $queries, &$users, &$total) { + try { + $users = $dbForProject->find('users', $queries); + $total = $dbForProject->count('users', $queries, APP_LIMIT_COUNT); + } catch (OrderException $e) { + throw new Exception(Exception::DATABASE_QUERY_ORDER_NULL, "The order attribute '{$e->getAttribute()}' had a null value. Cursor pagination requires all documents order attribute values are non-null."); + } catch (QueryException $e) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); + } + }, ['subQueryAuthenticators', 'subQuerySessions', 'subQueryTokens', 'subQueryChallenges', 'subQueryMemberships']); + $response->dynamic(new Document([ 'users' => $users, 'total' => $total, diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index c2f5c415a0..1fed4362b5 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -29,6 +29,7 @@ use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Helpers\Role; use Utopia\Database\Validator\Authorization; +use Utopia\Queue\Broker\Pool as BrokerPool; use Utopia\Queue\Publisher; use Utopia\System\System; use Utopia\Telemetry\Adapter as Telemetry; @@ -408,6 +409,7 @@ App::init() ->inject('project') ->inject('user') ->inject('publisher') + ->inject('publisherFunctions') ->inject('queueForEvents') ->inject('queueForMessaging') ->inject('queueForAudits') @@ -423,7 +425,7 @@ App::init() ->inject('plan') ->inject('devKey') ->inject('telemetry') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Publisher $publisher, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, StatsUsage $queueForStatsUsage, Database $dbForProject, callable $timelimit, Document $resourceToken, string $mode, ?Key $apiKey, array $plan, Document $devKey, Telemetry $telemetry) use ($usageDatabaseListener, $eventDatabaseListener) { + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Publisher $publisher, BrokerPool $publisherFunctions, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, StatsUsage $queueForStatsUsage, Database $dbForProject, callable $timelimit, Document $resourceToken, string $mode, ?Key $apiKey, array $plan, Document $devKey, Telemetry $telemetry) use ($usageDatabaseListener, $eventDatabaseListener) { $route = $utopia->getRoute(); @@ -535,7 +537,7 @@ App::init() // Clone the queues, to prevent events triggered by the database listener // from overwriting the events that are supposed to be triggered in the shutdown hook. $queueForEventsClone = new Event($publisher); - $queueForFunctions = new Func($publisher); + $queueForFunctions = new Func($publisherFunctions); $queueForWebhooks = new Webhook($publisher); $queueForRealtime = new Realtime(); diff --git a/app/init/resources.php b/app/init/resources.php index 4d1e0444c5..8dcfc8ceb2 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -83,6 +83,9 @@ App::setResource('publisher', function (Group $pools) { App::setResource('publisherDatabases', function (BrokerPool $publisher) { return $publisher; }, ['publisher']); +App::setResource('publisherFunctions', function (BrokerPool $publisher) { + return $publisher; +}, ['publisher']); App::setResource('publisherMigrations', function (BrokerPool $publisher) { return $publisher; }, ['publisher']); @@ -95,6 +98,9 @@ App::setResource('consumer', function (Group $pools) { App::setResource('consumerDatabases', function (BrokerPool $consumer) { return $consumer; }, ['consumer']); +App::setResource('consumerFunctions', function (BrokerPool $consumer) { + return $consumer; +}, ['consumer']); App::setResource('consumerMigrations', function (BrokerPool $consumer) { return $consumer; }, ['consumer']); diff --git a/app/worker.php b/app/worker.php index 90f3368fe7..cc8aca2d8e 100644 --- a/app/worker.php +++ b/app/worker.php @@ -251,6 +251,10 @@ Server::setResource('publisherDatabases', function (BrokerPool $publisher) { return $publisher; }, ['publisher']); +Server::setResource('publisherFunctions', function (BrokerPool $publisher) { + return $publisher; +}, ['publisher']); + Server::setResource('publisherMigrations', function (BrokerPool $publisher) { return $publisher; }, ['publisher']); @@ -267,6 +271,10 @@ Server::setResource('consumerDatabases', function (BrokerPool $consumer) { return $consumer; }, ['consumer']); +Server::setResource('consumerFunctions', function (BrokerPool $consumer) { + return $consumer; +}, ['consumer']); + Server::setResource('consumerMigrations', function (BrokerPool $consumer) { return $consumer; }, ['consumer']); diff --git a/composer.lock b/composer.lock index 154d3df907..88b3d0f739 100644 --- a/composer.lock +++ b/composer.lock @@ -4118,7 +4118,7 @@ "ext-curl": "*", "ext-openssl": "*", "php": ">=8.1", - "utopia-php/database": "0.*.*", + "utopia-php/database": "0.71.*", "utopia-php/dsn": "0.2.*", "utopia-php/framework": "0.33.*", "utopia-php/storage": "0.18.*" diff --git a/docker-compose.yml b/docker-compose.yml index 331daea522..cd693c18ed 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -216,7 +216,7 @@ services: appwrite-console: <<: *x-logging container_name: appwrite-console - image: appwrite/console:6.2.0 + image: appwrite/console:7.0.0-qa.8 restart: unless-stopped networks: - appwrite diff --git a/src/Appwrite/Platform/Modules/Databases/Workers/Databases.php b/src/Appwrite/Platform/Modules/Databases/Workers/Databases.php index 7e5e9eb19f..22f1e6a2f2 100644 --- a/src/Appwrite/Platform/Modules/Databases/Workers/Databases.php +++ b/src/Appwrite/Platform/Modules/Databases/Workers/Databases.php @@ -225,9 +225,11 @@ class Databases extends Action if (! $relatedCollection->isEmpty()) { $dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $relatedCollection->getId()); + $dbForProject->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $relatedCollection->getSequence()); } $dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $collectionId); + $dbForProject->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $collection->getSequence()); } } @@ -383,9 +385,11 @@ class Databases extends Action } } finally { $dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $collectionId); + $dbForProject->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $collection->getSequence()); if (! $relatedCollection->isEmpty()) { $dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $relatedCollection->getId()); + $dbForProject->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $relatedCollection->getSequence()); } } } @@ -444,6 +448,7 @@ class Databases extends Action } finally { $this->trigger($database, $collection, $project, $event, $queueForRealtime, null, $index); $dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $collectionId); + $dbForProject->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $collection->getSequence()); } } @@ -500,6 +505,7 @@ class Databases extends Action } finally { $this->trigger($database, $collection, $project, $event, $queueForRealtime, null, $index); $dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $collection->getId()); + $dbForProject->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $collection->getSequence()); } } diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index 222051a67f..1710f0163d 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -26,6 +26,7 @@ abstract class ScheduleBase extends Action protected BrokerPool $publisher; protected BrokerPool $publisherMigrations; + protected BrokerPool $publisherFunctions; private ?Histogram $collectSchedulesTelemetryDuration = null; private ?Gauge $collectSchedulesTelemetryCount = null; @@ -45,6 +46,7 @@ abstract class ScheduleBase extends Action ->desc("Execute {$type}s scheduled in Appwrite") ->inject('publisher') ->inject('publisherMigrations') + ->inject('publisherFunctions') ->inject('dbForPlatform') ->inject('getProjectDB') ->inject('telemetry') @@ -67,13 +69,14 @@ abstract class ScheduleBase extends Action * 2. Create timer that sync all changes from 'schedules' collection to local copy. Only reading changes thanks to 'resourceUpdatedAt' attribute * 3. Create timer that prepares coroutines for soon-to-execute schedules. When it's ready, coroutine sleeps until exact time before sending request to worker. */ - public function action(BrokerPool $publisher, BrokerPool $publisherMigrations, Database $dbForPlatform, callable $getProjectDB, Telemetry $telemetry): void + public function action(BrokerPool $publisher, BrokerPool $publisherMigrations, BrokerPool $publisherFunctions, Database $dbForPlatform, callable $getProjectDB, Telemetry $telemetry): void { Console::title(\ucfirst(static::getSupportedResource()) . ' scheduler V1'); Console::success(APP_NAME . ' ' . \ucfirst(static::getSupportedResource()) . ' scheduler v1 has started'); $this->publisher = $publisher; $this->publisherMigrations = $publisherMigrations; + $this->publisherFunctions = $publisherFunctions; $this->scheduleTelemetryCount = $telemetry->createGauge('task.schedule.count'); $this->collectSchedulesTelemetryDuration = $telemetry->createHistogram('task.schedule.collect_schedules.duration', 's'); diff --git a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php index 96a5a05f0e..14a4259e17 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php @@ -30,7 +30,7 @@ class ScheduleExecutions extends ScheduleBase { $intervalEnd = (new \DateTime())->modify('+' . self::ENQUEUE_TIMER . ' seconds'); - $queueForFunctions = new Func($this->publisher); + $queueForFunctions = new Func($this->publisherFunctions); foreach ($this->schedules as $schedule) { if (!$schedule['active']) { diff --git a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php index 43f1025c08..6f072425e4 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php @@ -90,7 +90,7 @@ class ScheduleFunctions extends ScheduleBase $this->updateProjectAccess($schedule['project'], $dbForPlatform); - $queueForFunctions = new Func($this->publisher); + $queueForFunctions = new Func($this->publisherFunctions); $queueForFunctions ->setType('schedule') diff --git a/src/Appwrite/Platform/Workers/Mails.php b/src/Appwrite/Platform/Workers/Mails.php index 117b689863..f07cfcf699 100644 --- a/src/Appwrite/Platform/Workers/Mails.php +++ b/src/Appwrite/Platform/Workers/Mails.php @@ -6,6 +6,7 @@ use Appwrite\Template\Template; use Exception; use PHPMailer\PHPMailer\PHPMailer; use Swoole\Runtime; +use Utopia\CLI\Console; use Utopia\Logger\Log; use Utopia\Platform\Action; use Utopia\Queue\Message; @@ -148,6 +149,17 @@ class Mails extends Action $mail->AltBody = \strip_tags($mail->AltBody); $mail->AltBody = \trim($mail->AltBody); + if (\str_contains($mail->Body, 'buttonText') || \str_contains($mail->AltBody, 'buttonText')) { + Console::warning('Email might contain placeholder. Logs relevant to verify and isolate the issue:'); + var_dump($mail->Body); + var_dump($mail->AltBody); + \var_dump($message->getPayload()); + \var_dump($message->getPid()); + \var_dump($message->getQueue()); + \var_dump($message->getTimestamp()); + Console::warning('End of placeholder detection report.'); + } + $replyTo = System::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM); $replyToName = \urldecode(System::getEnv('_APP_SYSTEM_EMAIL_NAME', APP_NAME . ' Server')); diff --git a/tests/e2e/Services/Messaging/MessagingBase.php b/tests/e2e/Services/Messaging/MessagingBase.php index dc5ddb9d70..728d3f0082 100644 --- a/tests/e2e/Services/Messaging/MessagingBase.php +++ b/tests/e2e/Services/Messaging/MessagingBase.php @@ -3,6 +3,7 @@ namespace Tests\E2E\Services\Messaging; use Appwrite\Messaging\Status as MessageStatus; +use Appwrite\Tests\Retry; use CURLFile; use Tests\E2E\Client; use Utopia\Database\DateTime; @@ -1190,6 +1191,7 @@ trait MessagingBase $this->assertEquals(MessageStatus::FAILED, $message['body']['status']); } + #[Retry(count: 3)] public function testUpdateScheduledAt(): void { // Create user diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index 1fa17ffa95..3166688db1 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -1049,7 +1049,6 @@ trait MigrationsBase $this->assertEquals('Appwrite', $migration['body']['destination']); $this->assertContains(Resource::TYPE_ROW, $migration['body']['resources']); $this->assertEmpty($migration['body']['statusCounters']); - $errorJson = $migration['body']['errors'][0]; $errorData = json_decode($errorJson, true); @@ -1082,7 +1081,6 @@ trait MigrationsBase $this->assertEquals('Appwrite', $migration['body']['destination']); $this->assertContains(Resource::TYPE_ROW, $migration['body']['resources']); $this->assertEmpty($migration['body']['statusCounters']); - $errorJson = $migration['body']['errors'][0]; $errorData = json_decode($errorJson, true);