diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index e0e26781cf..90e080d5fa 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -14,6 +14,7 @@ use Utopia\Registry\Registry; use Utopia\Storage\Device; use Utopia\Storage\Device\Local; use Utopia\Storage\Storage; +use Utopia\Validator\Integer; use Utopia\Validator\Text; App::get('/v1/health') @@ -344,11 +345,20 @@ App::get('/v1/health/queue/webhooks') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (Connection $queue, Response $response) { + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); + $client = new Client(Event::WEBHOOK_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/logs') @@ -362,11 +372,20 @@ App::get('/v1/health/queue/logs') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (Connection $queue, Response $response) { + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); + $client = new Client(Event::AUDITS_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/certificates') @@ -380,11 +399,20 @@ App::get('/v1/health/queue/certificates') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (Connection $queue, Response $response) { + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); + $client = new Client(Event::CERTIFICATES_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/builds') @@ -398,11 +426,20 @@ App::get('/v1/health/queue/builds') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (Connection $queue, Response $response) { + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); + $client = new Client(Event::BUILDS_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/databases') @@ -417,11 +454,20 @@ App::get('/v1/health/queue/databases') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) ->param('name', 'database_db_main', new Text(256), 'Queue name for which to check the queue size', true) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (string $name, Connection $queue, Response $response) { + ->action(function (string $name, int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); + $client = new Client($name, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/deletes') @@ -435,11 +481,20 @@ App::get('/v1/health/queue/deletes') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (Connection $queue, Response $response) { + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); + $client = new Client(Event::DELETE_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/mails') @@ -453,11 +508,20 @@ App::get('/v1/health/queue/mails') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (Connection $queue, Response $response) { + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); + $client = new Client(Event::MAILS_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/messaging') @@ -471,11 +535,20 @@ App::get('/v1/health/queue/messaging') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (Connection $queue, Response $response) { + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); + $client = new Client(Event::MESSAGING_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/migrations') @@ -489,11 +562,20 @@ App::get('/v1/health/queue/migrations') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (Connection $queue, Response $response) { + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); + $client = new Client(Event::MIGRATIONS_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/functions') @@ -507,11 +589,20 @@ App::get('/v1/health/queue/functions') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (Connection $queue, Response $response) { + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); + $client = new Client(Event::FUNCTIONS_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/storage/local') diff --git a/tests/e2e/Services/Health/HealthCustomServerTest.php b/tests/e2e/Services/Health/HealthCustomServerTest.php index eafc961e8b..8fa9faadd2 100644 --- a/tests/e2e/Services/Health/HealthCustomServerTest.php +++ b/tests/e2e/Services/Health/HealthCustomServerTest.php @@ -138,6 +138,15 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/webhooks?threshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } @@ -155,6 +164,15 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/logs?threshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } @@ -172,6 +190,15 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/certificates?threshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } @@ -189,6 +216,15 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/functions?threshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } @@ -206,6 +242,15 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/builds?threshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } @@ -225,6 +270,18 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/databases', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'database_db_main', + 'threshold' => '0' + ]); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } @@ -242,6 +299,15 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/deletes?threshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } @@ -259,6 +325,15 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/mails?threshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } @@ -276,6 +351,15 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/messaging?threshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } @@ -293,6 +377,15 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/migrations?threshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(500, $response['headers']['status-code']); + return []; }