feat: Add additional alert threshold types (#2122)

## Summary

This PR adds new types of alert thresholds: >, <=, =, and !=.

### Screenshots or video

https://github.com/user-attachments/assets/159bffc4-87e5-41af-b59b-51d4bc88d6ed

### How to test locally or on Vercel

This must be tested locally, since alerts are not supported in the preview environment.

To see the notification content, run an echo server locally and create a webhook that targets it (http://localhost:3000):

```bash
npx http-echo-server
```

### References



- Linear Issue: Closes HDX-3988
- Related PRs:
This commit is contained in:
Drew Davis 2026-04-16 11:47:04 -04:00 committed by GitHub
parent 71e1441257
commit d3a61f9bb9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 1972 additions and 91 deletions

View file

@ -0,0 +1,8 @@
---
"@hyperdx/common-utils": patch
"@hyperdx/api": patch
"@hyperdx/app": patch
"@hyperdx/cli": patch
---
feat: Add additional alert threshold types

View file

@ -70,7 +70,11 @@
"type": "string",
"enum": [
"above",
"below"
"below",
"above_exclusive",
"below_or_equal",
"equal",
"not_equal"
],
"description": "Threshold comparison direction."
},

View file

@ -3,6 +3,7 @@ import {
validateRawSqlForAlert,
} from '@hyperdx/common-utils/dist/core/utils';
import { isRawSqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards';
import { AlertThresholdType } from '@hyperdx/common-utils/dist/types';
import { sign, verify } from 'jsonwebtoken';
import { groupBy } from 'lodash';
import ms from 'ms';
@ -13,7 +14,6 @@ import Alert, {
AlertChannel,
AlertInterval,
AlertSource,
AlertThresholdType,
IAlert,
} from '@/models/alert';
import Dashboard, { IDashboard } from '@/models/dashboard';

View file

@ -1,5 +1,6 @@
import { createNativeClient } from '@hyperdx/common-utils/dist/clickhouse/node';
import {
AlertThresholdType,
BuilderSavedChartConfig,
DisplayType,
RawSqlSavedChartConfig,
@ -15,7 +16,7 @@ import { AlertInput } from '@/controllers/alerts';
import { getTeam } from '@/controllers/team';
import { findUserByEmail } from '@/controllers/user';
import { mongooseConnection } from '@/models';
import { AlertInterval, AlertSource, AlertThresholdType } from '@/models/alert';
import { AlertInterval, AlertSource } from '@/models/alert';
import Server from '@/server';
import logger from '@/utils/logger';
import { MetricModel } from '@/utils/logParser';

View file

@ -1,14 +1,13 @@
import { ALERT_INTERVAL_TO_MINUTES } from '@hyperdx/common-utils/dist/types';
import {
ALERT_INTERVAL_TO_MINUTES,
AlertThresholdType,
} from '@hyperdx/common-utils/dist/types';
export { AlertThresholdType } from '@hyperdx/common-utils/dist/types';
import mongoose, { Schema } from 'mongoose';
import type { ObjectId } from '.';
import Team from './team';
export enum AlertThresholdType {
ABOVE = 'above',
BELOW = 'below',
}
export enum AlertState {
ALERT = 'ALERT',
DISABLED = 'DISABLED',

View file

@ -1,4 +1,7 @@
import { DisplayType } from '@hyperdx/common-utils/dist/types';
import {
AlertThresholdType,
DisplayType,
} from '@hyperdx/common-utils/dist/types';
import {
getLoggedInAgent,
@ -11,11 +14,7 @@ import {
randomMongoId,
RAW_SQL_ALERT_TEMPLATE,
} from '@/fixtures';
import Alert, {
AlertSource,
AlertState,
AlertThresholdType,
} from '@/models/alert';
import Alert, { AlertSource, AlertState } from '@/models/alert';
import AlertHistory from '@/models/alertHistory';
import Webhook, { WebhookDocument, WebhookService } from '@/models/webhook';

View file

@ -34,7 +34,7 @@ import { alertSchema, objectIdSchema } from '@/utils/zod';
* description: Evaluation interval.
* AlertThresholdType:
* type: string
* enum: [above, below]
* enum: [above, below, above_exclusive, below_or_equal, equal, not_equal]
* description: Threshold comparison direction.
* AlertSource:
* type: string

View file

@ -0,0 +1,284 @@
// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing
exports[`buildAlertMessageTemplateTitle saved search alerts ALERT state above threshold=5 alertValue=10 1`] = `"🚨 Alert for "My Search" - 10 lines found"`;
exports[`buildAlertMessageTemplateTitle saved search alerts ALERT state above_exclusive threshold=5 alertValue=10 1`] = `"🚨 Alert for "My Search" - 10 lines found"`;
exports[`buildAlertMessageTemplateTitle saved search alerts ALERT state below threshold=5 alertValue=2 1`] = `"🚨 Alert for "My Search" - 2 lines found"`;
exports[`buildAlertMessageTemplateTitle saved search alerts ALERT state below_or_equal threshold=5 alertValue=3 1`] = `"🚨 Alert for "My Search" - 3 lines found"`;
exports[`buildAlertMessageTemplateTitle saved search alerts ALERT state equal threshold=5 alertValue=5 1`] = `"🚨 Alert for "My Search" - 5 lines found"`;
exports[`buildAlertMessageTemplateTitle saved search alerts ALERT state not_equal threshold=5 alertValue=10 1`] = `"🚨 Alert for "My Search" - 10 lines found"`;
exports[`buildAlertMessageTemplateTitle saved search alerts OK state (resolved) above threshold=5 okValue=3 1`] = `"✅ Alert for "My Search" - 3 lines found"`;
exports[`buildAlertMessageTemplateTitle saved search alerts OK state (resolved) above_exclusive threshold=5 okValue=3 1`] = `"✅ Alert for "My Search" - 3 lines found"`;
exports[`buildAlertMessageTemplateTitle saved search alerts OK state (resolved) below threshold=5 okValue=10 1`] = `"✅ Alert for "My Search" - 10 lines found"`;
exports[`buildAlertMessageTemplateTitle saved search alerts OK state (resolved) below_or_equal threshold=5 okValue=10 1`] = `"✅ Alert for "My Search" - 10 lines found"`;
exports[`buildAlertMessageTemplateTitle saved search alerts OK state (resolved) equal threshold=5 okValue=10 1`] = `"✅ Alert for "My Search" - 10 lines found"`;
exports[`buildAlertMessageTemplateTitle saved search alerts OK state (resolved) not_equal threshold=5 okValue=5 1`] = `"✅ Alert for "My Search" - 5 lines found"`;
exports[`buildAlertMessageTemplateTitle tile alerts ALERT state above threshold=5 alertValue=10 1`] = `"🚨 Alert for "Test Chart" in "My Dashboard" - 10 meets or exceeds 5"`;
exports[`buildAlertMessageTemplateTitle tile alerts ALERT state above_exclusive threshold=5 alertValue=10 1`] = `"🚨 Alert for "Test Chart" in "My Dashboard" - 10 exceeds 5"`;
exports[`buildAlertMessageTemplateTitle tile alerts ALERT state below threshold=5 alertValue=2 1`] = `"🚨 Alert for "Test Chart" in "My Dashboard" - 2 falls below 5"`;
exports[`buildAlertMessageTemplateTitle tile alerts ALERT state below_or_equal threshold=5 alertValue=3 1`] = `"🚨 Alert for "Test Chart" in "My Dashboard" - 3 falls to or below 5"`;
exports[`buildAlertMessageTemplateTitle tile alerts ALERT state decimal threshold 1`] = `"🚨 Alert for "Test Chart" in "My Dashboard" - 10.1 meets or exceeds 1.5"`;
exports[`buildAlertMessageTemplateTitle tile alerts ALERT state equal threshold=5 alertValue=5 1`] = `"🚨 Alert for "Test Chart" in "My Dashboard" - 5 equals 5"`;
exports[`buildAlertMessageTemplateTitle tile alerts ALERT state integer threshold rounds value 1`] = `"🚨 Alert for "Test Chart" in "My Dashboard" - 11 meets or exceeds 5"`;
exports[`buildAlertMessageTemplateTitle tile alerts ALERT state not_equal threshold=5 alertValue=10 1`] = `"🚨 Alert for "Test Chart" in "My Dashboard" - 10 does not equal 5"`;
exports[`buildAlertMessageTemplateTitle tile alerts OK state (resolved) above threshold=5 okValue=3 1`] = `"✅ Alert for "Test Chart" in "My Dashboard" - 3 falls below 5"`;
exports[`buildAlertMessageTemplateTitle tile alerts OK state (resolved) above_exclusive threshold=5 okValue=3 1`] = `"✅ Alert for "Test Chart" in "My Dashboard" - 3 falls to or below 5"`;
exports[`buildAlertMessageTemplateTitle tile alerts OK state (resolved) below threshold=5 okValue=10 1`] = `"✅ Alert for "Test Chart" in "My Dashboard" - 10 meets or exceeds 5"`;
exports[`buildAlertMessageTemplateTitle tile alerts OK state (resolved) below_or_equal threshold=5 okValue=10 1`] = `"✅ Alert for "Test Chart" in "My Dashboard" - 10 exceeds 5"`;
exports[`buildAlertMessageTemplateTitle tile alerts OK state (resolved) equal threshold=5 okValue=10 1`] = `"✅ Alert for "Test Chart" in "My Dashboard" - 10 does not equal 5"`;
exports[`buildAlertMessageTemplateTitle tile alerts OK state (resolved) not_equal threshold=5 okValue=5 1`] = `"✅ Alert for "Test Chart" in "My Dashboard" - 5 equals 5"`;
exports[`renderAlertTemplate saved search alerts ALERT state above threshold=5 alertValue=10 1`] = `
"
10 lines found, which meets or exceeds the threshold of 5 lines
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
\`\`\`
"2023-03-17 22:14:01","error","Failed to connect to database"
"2023-03-17 22:13:45","error","Connection timeout after 30s"
"2023-03-17 22:12:30","error","Retry limit exceeded"
\`\`\`"
`;
exports[`renderAlertTemplate saved search alerts ALERT state above_exclusive threshold=5 alertValue=10 1`] = `
"
10 lines found, which exceeds the threshold of 5 lines
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
\`\`\`
"2023-03-17 22:14:01","error","Failed to connect to database"
"2023-03-17 22:13:45","error","Connection timeout after 30s"
"2023-03-17 22:12:30","error","Retry limit exceeded"
\`\`\`"
`;
exports[`renderAlertTemplate saved search alerts ALERT state below threshold=5 alertValue=2 1`] = `
"
2 lines found, which falls below the threshold of 5 lines
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
\`\`\`
"2023-03-17 22:14:01","error","Failed to connect to database"
"2023-03-17 22:13:45","error","Connection timeout after 30s"
"2023-03-17 22:12:30","error","Retry limit exceeded"
\`\`\`"
`;
exports[`renderAlertTemplate saved search alerts ALERT state below_or_equal threshold=5 alertValue=3 1`] = `
"
3 lines found, which falls to or below the threshold of 5 lines
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
\`\`\`
"2023-03-17 22:14:01","error","Failed to connect to database"
"2023-03-17 22:13:45","error","Connection timeout after 30s"
"2023-03-17 22:12:30","error","Retry limit exceeded"
\`\`\`"
`;
exports[`renderAlertTemplate saved search alerts ALERT state equal threshold=5 alertValue=5 1`] = `
"
5 lines found, which equals the threshold of 5 lines
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
\`\`\`
"2023-03-17 22:14:01","error","Failed to connect to database"
"2023-03-17 22:13:45","error","Connection timeout after 30s"
"2023-03-17 22:12:30","error","Retry limit exceeded"
\`\`\`"
`;
exports[`renderAlertTemplate saved search alerts ALERT state not_equal threshold=5 alertValue=10 1`] = `
"
10 lines found, which does not equal the threshold of 5 lines
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
\`\`\`
"2023-03-17 22:14:01","error","Failed to connect to database"
"2023-03-17 22:13:45","error","Connection timeout after 30s"
"2023-03-17 22:12:30","error","Retry limit exceeded"
\`\`\`"
`;
exports[`renderAlertTemplate saved search alerts ALERT state with group 1`] = `
"Group: "http"
10 lines found, which meets or exceeds the threshold of 5 lines
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
\`\`\`
"2023-03-17 22:14:01","error","Failed to connect to database"
"2023-03-17 22:13:45","error","Connection timeout after 30s"
"2023-03-17 22:12:30","error","Retry limit exceeded"
\`\`\`"
`;
exports[`renderAlertTemplate saved search alerts OK state (resolved) above threshold=5 okValue=3 1`] = `
"The alert has been resolved.
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
"
`;
exports[`renderAlertTemplate saved search alerts OK state (resolved) above_exclusive threshold=5 okValue=3 1`] = `
"The alert has been resolved.
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
"
`;
exports[`renderAlertTemplate saved search alerts OK state (resolved) below threshold=5 okValue=10 1`] = `
"The alert has been resolved.
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
"
`;
exports[`renderAlertTemplate saved search alerts OK state (resolved) below_or_equal threshold=5 okValue=10 1`] = `
"The alert has been resolved.
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
"
`;
exports[`renderAlertTemplate saved search alerts OK state (resolved) equal threshold=5 okValue=10 1`] = `
"The alert has been resolved.
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
"
`;
exports[`renderAlertTemplate saved search alerts OK state (resolved) not_equal threshold=5 okValue=5 1`] = `
"The alert has been resolved.
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
"
`;
exports[`renderAlertTemplate saved search alerts OK state (resolved) with group 1`] = `
"Group: "http" - The alert has been resolved.
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
"
`;
exports[`renderAlertTemplate tile alerts ALERT state above threshold=5 alertValue=10 1`] = `
"
10 meets or exceeds 5
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
"
`;
exports[`renderAlertTemplate tile alerts ALERT state above_exclusive threshold=5 alertValue=10 1`] = `
"
10 exceeds 5
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
"
`;
exports[`renderAlertTemplate tile alerts ALERT state below threshold=5 alertValue=2 1`] = `
"
2 falls below 5
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
"
`;
exports[`renderAlertTemplate tile alerts ALERT state below_or_equal threshold=5 alertValue=3 1`] = `
"
3 falls to or below 5
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
"
`;
exports[`renderAlertTemplate tile alerts ALERT state decimal threshold 1`] = `
"
10.1 meets or exceeds 1.5
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
"
`;
exports[`renderAlertTemplate tile alerts ALERT state equal threshold=5 alertValue=5 1`] = `
"
5 equals 5
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
"
`;
exports[`renderAlertTemplate tile alerts ALERT state integer threshold rounds value 1`] = `
"
11 meets or exceeds 5
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
"
`;
exports[`renderAlertTemplate tile alerts ALERT state not_equal threshold=5 alertValue=10 1`] = `
"
10 does not equal 5
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
"
`;
exports[`renderAlertTemplate tile alerts ALERT state with group 1`] = `
"Group: "us-east-1"
10 meets or exceeds 5
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
"
`;
exports[`renderAlertTemplate tile alerts OK state (resolved) above threshold=5 okValue=3 1`] = `
"The alert has been resolved.
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
"
`;
exports[`renderAlertTemplate tile alerts OK state (resolved) above_exclusive threshold=5 okValue=3 1`] = `
"The alert has been resolved.
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
"
`;
exports[`renderAlertTemplate tile alerts OK state (resolved) below threshold=5 okValue=10 1`] = `
"The alert has been resolved.
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
"
`;
exports[`renderAlertTemplate tile alerts OK state (resolved) below_or_equal threshold=5 okValue=10 1`] = `
"The alert has been resolved.
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
"
`;
exports[`renderAlertTemplate tile alerts OK state (resolved) equal threshold=5 okValue=10 1`] = `
"The alert has been resolved.
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
"
`;
exports[`renderAlertTemplate tile alerts OK state (resolved) not_equal threshold=5 okValue=5 1`] = `
"The alert has been resolved.
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
"
`;
exports[`renderAlertTemplate tile alerts OK state (resolved) with group 1`] = `
"Group: "us-east-1" - The alert has been resolved.
Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM)
"
`;

View file

@ -0,0 +1,402 @@
import {
AlertState,
AlertThresholdType,
SourceKind,
} from '@hyperdx/common-utils/dist/types';
import mongoose from 'mongoose';
import { makeTile } from '@/fixtures';
import { AlertSource } from '@/models/alert';
import { loadProvider } from '@/tasks/checkAlerts/providers';
import {
AlertMessageTemplateDefaultView,
buildAlertMessageTemplateTitle,
renderAlertTemplate,
} from '@/tasks/checkAlerts/template';
let alertProvider: any;
beforeAll(async () => {
alertProvider = await loadProvider();
});
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
const mockMetadata = {
getColumn: jest.fn().mockImplementation(({ column }) => {
const columnMap = {
Timestamp: { name: 'Timestamp', type: 'DateTime' },
Body: { name: 'Body', type: 'String' },
SeverityText: { name: 'SeverityText', type: 'String' },
ServiceName: { name: 'ServiceName', type: 'String' },
};
return Promise.resolve(columnMap[column]);
}),
getColumns: jest.fn().mockResolvedValue([]),
getMapKeys: jest.fn().mockResolvedValue([]),
getMapValues: jest.fn().mockResolvedValue([]),
getAllFields: jest.fn().mockResolvedValue([]),
getTableMetadata: jest.fn().mockResolvedValue({}),
getClickHouseSettings: jest.fn().mockReturnValue({}),
setClickHouseSettings: jest.fn(),
getSkipIndices: jest.fn().mockResolvedValue([]),
getSetting: jest.fn().mockResolvedValue(undefined),
} as any;
const sampleLogsCsv = [
'"2023-03-17 22:14:01","error","Failed to connect to database"',
'"2023-03-17 22:13:45","error","Connection timeout after 30s"',
'"2023-03-17 22:12:30","error","Retry limit exceeded"',
].join('\n');
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
const mockClickhouseClient = {
query: jest.fn().mockResolvedValue({
json: jest.fn().mockResolvedValue({ data: [] }),
text: jest.fn().mockResolvedValue(sampleLogsCsv),
}),
} as any;
const startTime = new Date('2023-03-17T22:10:00.000Z');
const endTime = new Date('2023-03-17T22:15:00.000Z');
const makeSearchView = (
overrides: Partial<AlertMessageTemplateDefaultView> & {
thresholdType?: AlertThresholdType;
threshold?: number;
value?: number;
group?: string;
} = {},
): AlertMessageTemplateDefaultView => ({
alert: {
thresholdType: overrides.thresholdType ?? AlertThresholdType.ABOVE,
threshold: overrides.threshold ?? 5,
source: AlertSource.SAVED_SEARCH,
channel: { type: null },
interval: '1m',
},
source: {
id: 'fake-source-id',
kind: SourceKind.Log,
team: 'team-123',
from: { databaseName: 'default', tableName: 'otel_logs' },
timestampValueExpression: 'Timestamp',
connection: 'connection-123',
name: 'Logs',
defaultTableSelectExpression: 'Timestamp, Body',
},
savedSearch: {
_id: 'fake-saved-search-id' as any,
team: 'team-123' as any,
id: 'fake-saved-search-id',
name: 'My Search',
select: 'Body',
where: 'Body: "error"',
whereLanguage: 'lucene',
orderBy: 'timestamp',
source: 'fake-source-id' as any,
tags: ['test'],
createdAt: new Date(),
updatedAt: new Date(),
},
attributes: {},
granularity: '1m',
group: overrides.group,
isGroupedAlert: false,
startTime,
endTime,
value: overrides.value ?? 10,
});
const testTile = makeTile({ id: 'test-tile-id' });
const makeTileView = (
overrides: Partial<AlertMessageTemplateDefaultView> & {
thresholdType?: AlertThresholdType;
threshold?: number;
value?: number;
group?: string;
} = {},
): AlertMessageTemplateDefaultView => ({
alert: {
thresholdType: overrides.thresholdType ?? AlertThresholdType.ABOVE,
threshold: overrides.threshold ?? 5,
source: AlertSource.TILE,
channel: { type: null },
interval: '1m',
tileId: 'test-tile-id',
},
dashboard: {
_id: new mongoose.Types.ObjectId(),
id: 'id-123',
name: 'My Dashboard',
tiles: [testTile],
team: 'team-123' as any,
tags: ['test'],
createdAt: new Date(),
updatedAt: new Date(),
},
attributes: {},
granularity: '5 minute',
group: overrides.group,
isGroupedAlert: false,
startTime,
endTime,
value: overrides.value ?? 10,
});
const render = (view: AlertMessageTemplateDefaultView, state: AlertState) =>
renderAlertTemplate({
alertProvider,
clickhouseClient: mockClickhouseClient,
metadata: mockMetadata,
state,
template: null,
title: 'Test Alert Title',
view,
teamWebhooksById: new Map(),
});
interface AlertCase {
thresholdType: AlertThresholdType;
threshold: number;
alertValue: number; // value that would trigger the alert
okValue: number; // value that would resolve the alert
}
const alertCases: AlertCase[] = [
{
thresholdType: AlertThresholdType.ABOVE,
threshold: 5,
alertValue: 10,
okValue: 3,
},
{
thresholdType: AlertThresholdType.ABOVE_EXCLUSIVE,
threshold: 5,
alertValue: 10,
okValue: 3,
},
{
thresholdType: AlertThresholdType.BELOW,
threshold: 5,
alertValue: 2,
okValue: 10,
},
{
thresholdType: AlertThresholdType.BELOW_OR_EQUAL,
threshold: 5,
alertValue: 3,
okValue: 10,
},
{
thresholdType: AlertThresholdType.EQUAL,
threshold: 5,
alertValue: 5,
okValue: 10,
},
{
thresholdType: AlertThresholdType.NOT_EQUAL,
threshold: 5,
alertValue: 10,
okValue: 5,
},
];
describe('renderAlertTemplate', () => {
describe('saved search alerts', () => {
describe('ALERT state', () => {
it.each(alertCases)(
'$thresholdType threshold=$threshold alertValue=$alertValue',
async ({ thresholdType, threshold, alertValue }) => {
const result = await render(
makeSearchView({ thresholdType, threshold, value: alertValue }),
AlertState.ALERT,
);
expect(result).toMatchSnapshot();
},
);
it('with group', async () => {
const result = await render(
makeSearchView({ group: 'http' }),
AlertState.ALERT,
);
expect(result).toMatchSnapshot();
});
});
describe('OK state (resolved)', () => {
it.each(alertCases)(
'$thresholdType threshold=$threshold okValue=$okValue',
async ({ thresholdType, threshold, okValue }) => {
const result = await render(
makeSearchView({ thresholdType, threshold, value: okValue }),
AlertState.OK,
);
expect(result).toMatchSnapshot();
},
);
it('with group', async () => {
const result = await render(
makeSearchView({ group: 'http' }),
AlertState.OK,
);
expect(result).toMatchSnapshot();
});
});
});
describe('tile alerts', () => {
describe('ALERT state', () => {
it.each(alertCases)(
'$thresholdType threshold=$threshold alertValue=$alertValue',
async ({ thresholdType, threshold, alertValue }) => {
const result = await render(
makeTileView({ thresholdType, threshold, value: alertValue }),
AlertState.ALERT,
);
expect(result).toMatchSnapshot();
},
);
it('with group', async () => {
const result = await render(
makeTileView({ group: 'us-east-1' }),
AlertState.ALERT,
);
expect(result).toMatchSnapshot();
});
it('decimal threshold', async () => {
const result = await render(
makeTileView({
thresholdType: AlertThresholdType.ABOVE,
threshold: 1.5,
value: 10.123,
}),
AlertState.ALERT,
);
expect(result).toMatchSnapshot();
});
it('integer threshold rounds value', async () => {
const result = await render(
makeTileView({
thresholdType: AlertThresholdType.ABOVE,
threshold: 5,
value: 10.789,
}),
AlertState.ALERT,
);
expect(result).toMatchSnapshot();
});
});
describe('OK state (resolved)', () => {
it.each(alertCases)(
'$thresholdType threshold=$threshold okValue=$okValue',
async ({ thresholdType, threshold, okValue }) => {
const result = await render(
makeTileView({ thresholdType, threshold, value: okValue }),
AlertState.OK,
);
expect(result).toMatchSnapshot();
},
);
it('with group', async () => {
const result = await render(
makeTileView({ group: 'us-east-1' }),
AlertState.OK,
);
expect(result).toMatchSnapshot();
});
});
});
});
describe('buildAlertMessageTemplateTitle', () => {
describe('saved search alerts', () => {
describe('ALERT state', () => {
it.each(alertCases)(
'$thresholdType threshold=$threshold alertValue=$alertValue',
({ thresholdType, threshold, alertValue }) => {
const result = buildAlertMessageTemplateTitle({
view: makeSearchView({
thresholdType,
threshold,
value: alertValue,
}),
state: AlertState.ALERT,
});
expect(result).toMatchSnapshot();
},
);
});
describe('OK state (resolved)', () => {
it.each(alertCases)(
'$thresholdType threshold=$threshold okValue=$okValue',
({ thresholdType, threshold, okValue }) => {
const result = buildAlertMessageTemplateTitle({
view: makeSearchView({ thresholdType, threshold, value: okValue }),
state: AlertState.OK,
});
expect(result).toMatchSnapshot();
},
);
});
});
describe('tile alerts', () => {
describe('ALERT state', () => {
it.each(alertCases)(
'$thresholdType threshold=$threshold alertValue=$alertValue',
({ thresholdType, threshold, alertValue }) => {
const result = buildAlertMessageTemplateTitle({
view: makeTileView({ thresholdType, threshold, value: alertValue }),
state: AlertState.ALERT,
});
expect(result).toMatchSnapshot();
},
);
it('decimal threshold', () => {
const result = buildAlertMessageTemplateTitle({
view: makeTileView({
thresholdType: AlertThresholdType.ABOVE,
threshold: 1.5,
value: 10.123,
}),
state: AlertState.ALERT,
});
expect(result).toMatchSnapshot();
});
it('integer threshold rounds value', () => {
const result = buildAlertMessageTemplateTitle({
view: makeTileView({
thresholdType: AlertThresholdType.ABOVE,
threshold: 5,
value: 10.789,
}),
state: AlertState.ALERT,
});
expect(result).toMatchSnapshot();
});
});
describe('OK state (resolved)', () => {
it.each(alertCases)(
'$thresholdType threshold=$threshold okValue=$okValue',
({ thresholdType, threshold, okValue }) => {
const result = buildAlertMessageTemplateTitle({
view: makeTileView({ thresholdType, threshold, value: okValue }),
state: AlertState.OK,
});
expect(result).toMatchSnapshot();
},
);
});
});
});

View file

@ -248,7 +248,7 @@ describe('Single Invocation Alert Test', () => {
// Verify the message body contains the search link
const messageBody = webhookPayload.blocks[0].text.text;
expect(messageBody).toContain('lines found');
expect(messageBody).toContain('expected less than 1 lines');
expect(messageBody).toContain('meets or exceeds the threshold of 1 lines');
expect(messageBody).toContain('http://app:8080/search/');
expect(messageBody).toContain('from=');
expect(messageBody).toContain('to=');

View file

@ -27,6 +27,7 @@ import {
isRawSqlSavedChartConfig,
} from '@hyperdx/common-utils/dist/guards';
import {
AlertThresholdType,
BuilderChartConfigWithOptDateRange,
ChartConfigWithOptDateRange,
DisplayType,
@ -42,7 +43,7 @@ import ms from 'ms';
import { serializeError } from 'serialize-error';
import { ALERT_HISTORY_QUERY_CONCURRENCY } from '@/controllers/alertHistory';
import { AlertState, AlertThresholdType, IAlert } from '@/models/alert';
import { AlertState, IAlert } from '@/models/alert';
import AlertHistory, { IAlertHistory } from '@/models/alertHistory';
import { IDashboard } from '@/models/dashboard';
import { ISavedSearch } from '@/models/savedSearch';
@ -141,13 +142,20 @@ export const doesExceedThreshold = (
threshold: number,
value: number,
) => {
const isThresholdTypeAbove = thresholdType === AlertThresholdType.ABOVE;
if (isThresholdTypeAbove && value >= threshold) {
return true;
} else if (!isThresholdTypeAbove && value < threshold) {
return true;
switch (thresholdType) {
case AlertThresholdType.ABOVE:
return value >= threshold;
case AlertThresholdType.BELOW:
return value < threshold;
case AlertThresholdType.ABOVE_EXCLUSIVE:
return value > threshold;
case AlertThresholdType.BELOW_OR_EQUAL:
return value <= threshold;
case AlertThresholdType.EQUAL:
return value === threshold;
case AlertThresholdType.NOT_EQUAL:
return value !== threshold;
}
return false;
};
const normalizeScheduleOffsetMinutes = ({
@ -643,6 +651,9 @@ const parseAlertData = (
for (const [k, v] of Object.entries(data)) {
if (meta.valueColumnNames.has(k)) {
// Due to output_format_json_quote_64bit_integers=1, 64-bit integers will be returned as strings.
// Parse them as integers to ensure correct threshold comparison.
// Floats are not returned as strings (unless output_format_json_quote_64bit_floats=1, which is not the default).
value = isString(v) ? parseInt(v) : v;
} else if (meta.type !== 'time_series' || k !== meta.timestampColumnName) {
extraFields.push(`${k}:${v}`);

View file

@ -8,6 +8,7 @@ import {
} from '@hyperdx/common-utils/dist/core/utils';
import {
AlertChannelType,
AlertThresholdType,
ChartConfigWithOptDateRange,
DisplayType,
pickSampleWeightExpressionProps,
@ -24,7 +25,7 @@ import { z } from 'zod';
import * as config from '@/config';
import { AlertInput } from '@/controllers/alerts';
import { AlertSource, AlertState, AlertThresholdType } from '@/models/alert';
import { AlertSource, AlertState } from '@/models/alert';
import { IDashboard } from '@/models/dashboard';
import { ISavedSearch } from '@/models/savedSearch';
import { ISource } from '@/models/source';
@ -42,6 +43,44 @@ import { truncateString } from '@/utils/common';
import logger from '@/utils/logger';
import * as slack from '@/utils/slack';
const describeThresholdViolation = (
thresholdType: AlertThresholdType,
): string => {
switch (thresholdType) {
case AlertThresholdType.ABOVE:
return 'meets or exceeds';
case AlertThresholdType.ABOVE_EXCLUSIVE:
return 'exceeds';
case AlertThresholdType.BELOW:
return 'falls below';
case AlertThresholdType.BELOW_OR_EQUAL:
return 'falls to or below';
case AlertThresholdType.EQUAL:
return 'equals';
case AlertThresholdType.NOT_EQUAL:
return 'does not equal';
}
};
const describeThresholdResolution = (
thresholdType: AlertThresholdType,
): string => {
switch (thresholdType) {
case AlertThresholdType.ABOVE:
return 'falls below';
case AlertThresholdType.ABOVE_EXCLUSIVE:
return 'falls to or below';
case AlertThresholdType.BELOW:
return 'meets or exceeds';
case AlertThresholdType.BELOW_OR_EQUAL:
return 'exceeds';
case AlertThresholdType.EQUAL:
return 'does not equal';
case AlertThresholdType.NOT_EQUAL:
return 'equals';
}
};
const MAX_MESSAGE_LENGTH = 500;
const NOTIFY_FN_NAME = '__hdx_notify_channel__';
const IS_MATCH_FN_NAME = 'is_match';
@ -377,12 +416,8 @@ export const buildAlertMessageTemplateTitle = ({
? handlebars.compile(template)(view)
: `Alert for "${tile.config.name}" in "${dashboard.name}" - ${formattedValue} ${
doesExceedThreshold(alert.thresholdType, alert.threshold, value)
? alert.thresholdType === AlertThresholdType.ABOVE
? 'exceeds'
: 'falls below'
: alert.thresholdType === AlertThresholdType.ABOVE
? 'falls below'
: 'exceeds'
? describeThresholdViolation(alert.thresholdType)
: describeThresholdResolution(alert.thresholdType)
} ${alert.threshold}`;
return `${emoji}${baseTitle}`;
}
@ -649,11 +684,7 @@ ${targetTemplate}`;
}
rawTemplateBody = `${group ? `Group: "${group}"` : ''}
${value} lines found, expected ${
alert.thresholdType === AlertThresholdType.ABOVE
? 'less than'
: 'greater than'
} ${alert.threshold} lines\n${timeRangeMessage}
${value} lines found, which ${describeThresholdViolation(alert.thresholdType)} the threshold of ${alert.threshold} lines\n${timeRangeMessage}
${targetTemplate}
\`\`\`
${truncatedResults}
@ -666,12 +697,8 @@ ${truncatedResults}
rawTemplateBody = `${group ? `Group: "${group}"` : ''}
${formattedValue} ${
doesExceedThreshold(alert.thresholdType, alert.threshold, value)
? alert.thresholdType === AlertThresholdType.ABOVE
? 'exceeds'
: 'falls below'
: alert.thresholdType === AlertThresholdType.ABOVE
? 'falls below'
: 'exceeds'
? describeThresholdViolation(alert.thresholdType)
: describeThresholdResolution(alert.thresholdType)
} ${alert.threshold}\n${timeRangeMessage}
${targetTemplate}`;
}

View file

@ -1,4 +1,5 @@
import {
AlertThresholdType,
BuilderSavedChartConfig,
DashboardFilter,
DisplayType,
@ -12,7 +13,6 @@ import {
AlertDocument,
AlertInterval,
AlertState,
AlertThresholdType,
} from '@/models/alert';
import type { DashboardDocument } from '@/models/dashboard';
import { SeriesTile } from '@/routers/external-api/v2/utils/dashboards';

View file

@ -1,5 +1,6 @@
import {
AggregateFunctionSchema,
AlertThresholdType,
DashboardFilterSchema,
MetricsDataType,
NumberFormatSchema,
@ -11,7 +12,7 @@ import {
import { Types } from 'mongoose';
import { z } from 'zod';
import { AlertSource, AlertThresholdType } from '@/models/alert';
import { AlertSource } from '@/models/alert';
export const objectIdSchema = z.string().refine(val => {
return Types.ObjectId.isValid(val);

View file

@ -244,16 +244,33 @@ const AlertForm = ({
Send to
</Text>
<AlertChannelForm control={control} type={channelType} />
{groupBy && thresholdType === AlertThresholdType.BELOW && (
{groupBy &&
(thresholdType === AlertThresholdType.BELOW ||
thresholdType === AlertThresholdType.BELOW_OR_EQUAL ||
thresholdType === AlertThresholdType.EQUAL ||
thresholdType === AlertThresholdType.NOT_EQUAL) && (
<MantineAlert
icon={<IconInfoCircleFilled size={16} />}
color="gray"
py="xs"
>
<Text size="sm" opacity={0.7}>
Warning: Alerts with this threshold type and a &quot;grouped
by&quot; value will not alert for periods with no data for a
group.
</Text>
</MantineAlert>
)}
{(thresholdType === AlertThresholdType.EQUAL ||
thresholdType === AlertThresholdType.NOT_EQUAL) && (
<MantineAlert
icon={<IconInfoCircleFilled size={16} />}
bg="dark"
color="gray"
py="xs"
>
<Text size="sm" opacity={0.7}>
Warning: Alerts with a &quot;Below (&lt;)&quot; threshold and a
&quot;grouped by&quot; value will not alert for periods with no
data for a group.
Note: Floating-point query results are not rounded during
equality comparison.
</Text>
</MantineAlert>
)}

View file

@ -2,6 +2,7 @@ import React from 'react';
import { aliasMapToWithClauses } from '@hyperdx/common-utils/dist/core/utils';
import {
AlertInterval,
AlertThresholdType,
Filter,
getSampleWeightExpression,
isLogSource,
@ -25,7 +26,7 @@ type AlertPreviewChartProps = {
filters?: Filter[] | null;
interval: AlertInterval;
groupBy?: string;
thresholdType: 'above' | 'below';
thresholdType: AlertThresholdType;
threshold: number;
select?: string | null;
};

View file

@ -9,6 +9,7 @@ import {
import { Label, ReferenceArea, ReferenceLine } from 'recharts';
import {
type AlertChannelType,
AlertThresholdType,
WebhookService,
} from '@hyperdx/common-utils/dist/types';
import { Button, ComboboxData, Group, Modal, Select } from '@mantine/core';
@ -145,10 +146,16 @@ export const getAlertReferenceLines = ({
threshold,
// TODO: zScore
}: {
thresholdType: 'above' | 'below';
thresholdType: AlertThresholdType;
threshold: number;
}) => {
if (threshold != null && thresholdType === 'below') {
if (threshold == null) {
return null;
}
if (
thresholdType === AlertThresholdType.BELOW ||
thresholdType === AlertThresholdType.BELOW_OR_EQUAL
) {
return (
<ReferenceArea
y1={0}
@ -160,7 +167,10 @@ export const getAlertReferenceLines = ({
/>
);
}
if (threshold != null && thresholdType === 'above') {
if (
thresholdType === AlertThresholdType.ABOVE ||
thresholdType === AlertThresholdType.ABOVE_EXCLUSIVE
) {
return (
<ReferenceArea
y1={threshold}
@ -171,22 +181,20 @@ export const getAlertReferenceLines = ({
/>
);
}
if (threshold != null) {
return (
<ReferenceLine
y={threshold}
label={
<Label
value="Alert Threshold"
fill={'white'}
fontSize={11}
opacity={0.7}
/>
}
stroke="red"
strokeDasharray="3 3"
/>
);
}
return null;
// For 'equal' and 'not_equal', show a reference line at the threshold
return (
<ReferenceLine
y={threshold}
label={
<Label
value="Alert Threshold"
fill={'white'}
fontSize={11}
opacity={0.7}
/>
}
stroke="red"
strokeDasharray="3 3"
/>
);
};

View file

@ -4,8 +4,10 @@ import {
UseFormSetValue,
useWatch,
} from 'react-hook-form';
import { AlertThresholdType } from '@hyperdx/common-utils/dist/types';
import {
ActionIcon,
Alert,
Badge,
Box,
Collapse,
@ -21,6 +23,7 @@ import { useDisclosure } from '@mantine/hooks';
import {
IconChevronDown,
IconHelpCircle,
IconInfoCircleFilled,
IconTrash,
} from '@tabler/icons-react';
@ -58,6 +61,7 @@ export function TileAlertEditor({
const [opened, { toggle }] = useDisclosure(true);
const alertChannelType = useWatch({ control, name: 'alert.channel.type' });
const alertThresholdType = useWatch({ control, name: 'alert.thresholdType' });
const alertScheduleOffsetMinutes = useWatch({
control,
name: 'alert.scheduleOffsetMinutes',
@ -210,6 +214,18 @@ export function TileAlertEditor({
type={alertChannelType}
namePrefix="alert."
/>
{(alertThresholdType === AlertThresholdType.EQUAL ||
alertThresholdType === AlertThresholdType.NOT_EQUAL) && (
<Alert
icon={<IconInfoCircleFilled size={16} />}
color="gray"
py="xs"
mt="md"
>
Note: Floating-point query results are not rounded during equality
comparison.
</Alert>
)}
</Box>
</Collapse>
</Paper>

View file

@ -83,11 +83,19 @@ export function extendDateRangeToInterval(
export const ALERT_THRESHOLD_TYPE_OPTIONS: Record<string, string> = {
above: 'At least (≥)',
below: 'Below (<)',
above_exclusive: 'Above (>)',
below_or_equal: 'At most (≤)',
equal: 'Equal to (=)',
not_equal: 'Not equal to (≠)',
};
export const TILE_ALERT_THRESHOLD_TYPE_OPTIONS: Record<string, string> = {
above: 'is at least (≥)',
below: 'falls below (<)',
above_exclusive: 'is above (>)',
below_or_equal: 'is at most (≤)',
equal: 'equals (=)',
not_equal: 'does not equal (≠)',
};
export const ALERT_INTERVAL_OPTIONS: Record<AlertInterval, string> = {

View file

@ -25,6 +25,7 @@ import {
} from '@hyperdx/common-utils/dist/core/metadata';
import { loadSession, saveSession, clearSession } from '@/utils/config';
import { AlertThresholdType } from '@hyperdx/common-utils/dist/types';
// ------------------------------------------------------------------
// API Client (session management + REST calls)
@ -418,7 +419,7 @@ export interface AlertItem {
scheduleOffsetMinutes?: number;
scheduleStartAt?: string | null;
threshold: number;
thresholdType: 'above' | 'below';
thresholdType: AlertThresholdType;
channel: { type?: string | null };
state?: 'ALERT' | 'OK' | 'INSUFFICIENT_DATA' | 'DISABLED';
source?: 'saved_search' | 'tile';

View file

@ -278,6 +278,10 @@ export type WebhookApiData = Omit<IWebhook, 'team'>;
export enum AlertThresholdType {
ABOVE = 'above',
BELOW = 'below',
ABOVE_EXCLUSIVE = 'above_exclusive',
BELOW_OR_EQUAL = 'below_or_equal',
EQUAL = 'equal',
NOT_EQUAL = 'not_equal',
}
export enum AlertState {