Merge pull request #7123 from appwrite/feat-health-tresholds

Implement health tresholds
2026-05-23 00:49:02 +00:00 · 2023-11-13 08:57:51 -05:00 · 2023-11-13 08:57:51 -05:00 · be6b31b43f
commit be6b31b43f
parent 573a035f92 967854d34a
2 changed files with 204 additions and 20 deletions
--- a/app/controllers/api/health.php
+++ b/app/controllers/api/health.php
@ -14,6 +14,7 @@ use Utopia\Registry\Registry;
 use Utopia\Storage\Device;
 use Utopia\Storage\Device\Local;
 use Utopia\Storage\Storage;
+use Utopia\Validator\Integer;
 use Utopia\Validator\Text;

 App::get('/v1/health')
@ -344,11 +345,20 @@ App::get('/v1/health/queue/webhooks')
    ->label('sdk.response.code', Response::STATUS_CODE_OK)
    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
    ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
    ->inject('queue')
    ->inject('response')
-    ->action(function (Connection $queue, Response $response) {
+    ->action(function (int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
        $client = new Client(Event::WEBHOOK_QUEUE_NAME, $queue);
-        $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
    }, ['response']);

 App::get('/v1/health/queue/logs')
@ -362,11 +372,20 @@ App::get('/v1/health/queue/logs')
    ->label('sdk.response.code', Response::STATUS_CODE_OK)
    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
    ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
    ->inject('queue')
    ->inject('response')
-    ->action(function (Connection $queue, Response $response) {
+    ->action(function (int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
        $client = new Client(Event::AUDITS_QUEUE_NAME, $queue);
-        $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
    }, ['response']);

 App::get('/v1/health/queue/certificates')
@ -380,11 +399,20 @@ App::get('/v1/health/queue/certificates')
    ->label('sdk.response.code', Response::STATUS_CODE_OK)
    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
    ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
    ->inject('queue')
    ->inject('response')
-    ->action(function (Connection $queue, Response $response) {
+    ->action(function (int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
        $client = new Client(Event::CERTIFICATES_QUEUE_NAME, $queue);
-        $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
    }, ['response']);

 App::get('/v1/health/queue/builds')
@ -398,11 +426,20 @@ App::get('/v1/health/queue/builds')
    ->label('sdk.response.code', Response::STATUS_CODE_OK)
    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
    ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
    ->inject('queue')
    ->inject('response')
-    ->action(function (Connection $queue, Response $response) {
+    ->action(function (int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
        $client = new Client(Event::BUILDS_QUEUE_NAME, $queue);
-        $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
    }, ['response']);

 App::get('/v1/health/queue/databases')
@ -417,11 +454,20 @@ App::get('/v1/health/queue/databases')
    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
    ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
    ->param('name', 'database_db_main', new Text(256), 'Queue name for which to check the queue size', true)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
    ->inject('queue')
    ->inject('response')
-    ->action(function (string $name, Connection $queue, Response $response) {
+    ->action(function (string $name, int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
        $client = new Client($name, $queue);
-        $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
    }, ['response']);

 App::get('/v1/health/queue/deletes')
@ -435,11 +481,20 @@ App::get('/v1/health/queue/deletes')
    ->label('sdk.response.code', Response::STATUS_CODE_OK)
    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
    ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
    ->inject('queue')
    ->inject('response')
-    ->action(function (Connection $queue, Response $response) {
+    ->action(function (int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
        $client = new Client(Event::DELETE_QUEUE_NAME, $queue);
-        $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
    }, ['response']);

 App::get('/v1/health/queue/mails')
@ -453,11 +508,20 @@ App::get('/v1/health/queue/mails')
    ->label('sdk.response.code', Response::STATUS_CODE_OK)
    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
    ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
    ->inject('queue')
    ->inject('response')
-    ->action(function (Connection $queue, Response $response) {
+    ->action(function (int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
        $client = new Client(Event::MAILS_QUEUE_NAME, $queue);
-        $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
    }, ['response']);

 App::get('/v1/health/queue/messaging')
@ -471,11 +535,20 @@ App::get('/v1/health/queue/messaging')
    ->label('sdk.response.code', Response::STATUS_CODE_OK)
    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
    ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
    ->inject('queue')
    ->inject('response')
-    ->action(function (Connection $queue, Response $response) {
+    ->action(function (int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
        $client = new Client(Event::MESSAGING_QUEUE_NAME, $queue);
-        $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
    }, ['response']);

 App::get('/v1/health/queue/migrations')
@ -489,11 +562,20 @@ App::get('/v1/health/queue/migrations')
    ->label('sdk.response.code', Response::STATUS_CODE_OK)
    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
    ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
    ->inject('queue')
    ->inject('response')
-    ->action(function (Connection $queue, Response $response) {
+    ->action(function (int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
        $client = new Client(Event::MIGRATIONS_QUEUE_NAME, $queue);
-        $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
    }, ['response']);

 App::get('/v1/health/queue/functions')
@ -507,11 +589,20 @@ App::get('/v1/health/queue/functions')
    ->label('sdk.response.code', Response::STATUS_CODE_OK)
    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
    ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
    ->inject('queue')
    ->inject('response')
-    ->action(function (Connection $queue, Response $response) {
+    ->action(function (int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
        $client = new Client(Event::FUNCTIONS_QUEUE_NAME, $queue);
-        $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
    }, ['response']);

 App::get('/v1/health/storage/local')
--- a/tests/e2e/Services/Health/HealthCustomServerTest.php
+++ b/tests/e2e/Services/Health/HealthCustomServerTest.php
@ -138,6 +138,15 @@ class HealthCustomServerTest extends Scope
        $this->assertIsInt($response['body']['size']);
        $this->assertLessThan(100, $response['body']['size']);

+        /**
+         * Test for FAILURE
+         */
+        $response = $this->client->call(Client::METHOD_GET, '/health/queue/webhooks?threshold=0', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), []);
+        $this->assertEquals(500, $response['headers']['status-code']);
+
        return [];
    }

@ -155,6 +164,15 @@ class HealthCustomServerTest extends Scope
        $this->assertIsInt($response['body']['size']);
        $this->assertLessThan(100, $response['body']['size']);

+        /**
+         * Test for FAILURE
+         */
+        $response = $this->client->call(Client::METHOD_GET, '/health/queue/logs?threshold=0', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), []);
+        $this->assertEquals(500, $response['headers']['status-code']);
+
        return [];
    }

@ -172,6 +190,15 @@ class HealthCustomServerTest extends Scope
        $this->assertIsInt($response['body']['size']);
        $this->assertLessThan(100, $response['body']['size']);

+        /**
+         * Test for FAILURE
+         */
+        $response = $this->client->call(Client::METHOD_GET, '/health/queue/certificates?threshold=0', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), []);
+        $this->assertEquals(500, $response['headers']['status-code']);
+
        return [];
    }

@ -189,6 +216,15 @@ class HealthCustomServerTest extends Scope
        $this->assertIsInt($response['body']['size']);
        $this->assertLessThan(100, $response['body']['size']);

+        /**
+         * Test for FAILURE
+         */
+        $response = $this->client->call(Client::METHOD_GET, '/health/queue/functions?threshold=0', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), []);
+        $this->assertEquals(500, $response['headers']['status-code']);
+
        return [];
    }

@ -206,6 +242,15 @@ class HealthCustomServerTest extends Scope
        $this->assertIsInt($response['body']['size']);
        $this->assertLessThan(100, $response['body']['size']);

+        /**
+         * Test for FAILURE
+         */
+        $response = $this->client->call(Client::METHOD_GET, '/health/queue/builds?threshold=0', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), []);
+        $this->assertEquals(500, $response['headers']['status-code']);
+
        return [];
    }

@ -225,6 +270,18 @@ class HealthCustomServerTest extends Scope
        $this->assertIsInt($response['body']['size']);
        $this->assertLessThan(100, $response['body']['size']);

+        /**
+         * Test for FAILURE
+         */
+        $response = $this->client->call(Client::METHOD_GET, '/health/queue/databases', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'name' => 'database_db_main',
+            'threshold' => '0'
+        ]);
+        $this->assertEquals(500, $response['headers']['status-code']);
+
        return [];
    }

@ -242,6 +299,15 @@ class HealthCustomServerTest extends Scope
        $this->assertIsInt($response['body']['size']);
        $this->assertLessThan(100, $response['body']['size']);

+        /**
+         * Test for FAILURE
+         */
+        $response = $this->client->call(Client::METHOD_GET, '/health/queue/deletes?threshold=0', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), []);
+        $this->assertEquals(500, $response['headers']['status-code']);
+
        return [];
    }

@ -259,6 +325,15 @@ class HealthCustomServerTest extends Scope
        $this->assertIsInt($response['body']['size']);
        $this->assertLessThan(100, $response['body']['size']);

+        /**
+         * Test for FAILURE
+         */
+        $response = $this->client->call(Client::METHOD_GET, '/health/queue/mails?threshold=0', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), []);
+        $this->assertEquals(500, $response['headers']['status-code']);
+
        return [];
    }

@ -276,6 +351,15 @@ class HealthCustomServerTest extends Scope
        $this->assertIsInt($response['body']['size']);
        $this->assertLessThan(100, $response['body']['size']);

+        /**
+         * Test for FAILURE
+         */
+        $response = $this->client->call(Client::METHOD_GET, '/health/queue/messaging?threshold=0', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), []);
+        $this->assertEquals(500, $response['headers']['status-code']);
+
        return [];
    }

@ -293,6 +377,15 @@ class HealthCustomServerTest extends Scope
        $this->assertIsInt($response['body']['size']);
        $this->assertLessThan(100, $response['body']['size']);

+        /**
+         * Test for FAILURE
+         */
+        $response = $this->client->call(Client::METHOD_GET, '/health/queue/migrations?threshold=0', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), []);
+        $this->assertEquals(500, $response['headers']['status-code']);
+
        return [];
    }