feat: Add silence alerts feature (#1464)

Fixes: HDX-2734
This commit is contained in:
Tom Alexander 2025-12-12 16:56:09 -05:00 committed by GitHub
parent ae4c8765e7
commit 96f0539e7a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 632 additions and 19 deletions

View file

@ -0,0 +1,6 @@
---
"@hyperdx/api": patch
"@hyperdx/app": patch
---
feat: Add silence alerts feature

View file

@ -174,4 +174,124 @@ describe('alerts router', () => {
expect(alert.dashboard).toBeDefined();
}
});
it('can silence an alert', async () => {
const { agent, user } = await getLoggedInAgent(server);
const dashboard = await agent
.post('/dashboards')
.send(MOCK_DASHBOARD)
.expect(200);
const alert = await agent
.post('/alerts')
.send(
makeAlertInput({
dashboardId: dashboard.body.id,
tileId: dashboard.body.tiles[0].id,
}),
)
.expect(200);
const mutedUntil = new Date(Date.now() + 3600000).toISOString(); // 1 hour from now
await agent
.post(`/alerts/${alert.body.data._id}/silenced`)
.send({ mutedUntil })
.expect(200);
// Verify the alert was silenced
const alertFromDb = await Alert.findById(alert.body.data._id);
expect(alertFromDb).toBeDefined();
expect(alertFromDb!.silenced).toBeDefined();
expect(alertFromDb!.silenced!.by).toEqual(user._id);
expect(alertFromDb!.silenced!.at).toBeDefined();
expect(new Date(alertFromDb!.silenced!.until).toISOString()).toBe(
mutedUntil,
);
});
it('can unsilence an alert', async () => {
const { agent, user } = await getLoggedInAgent(server);
const dashboard = await agent
.post('/dashboards')
.send(MOCK_DASHBOARD)
.expect(200);
const alert = await agent
.post('/alerts')
.send(
makeAlertInput({
dashboardId: dashboard.body.id,
tileId: dashboard.body.tiles[0].id,
}),
)
.expect(200);
// First silence the alert
const mutedUntil = new Date(Date.now() + 3600000).toISOString();
await agent
.post(`/alerts/${alert.body.data._id}/silenced`)
.send({ mutedUntil })
.expect(200);
// Verify it was silenced
let alertFromDb = await Alert.findById(alert.body.data._id);
expect(alertFromDb!.silenced).toBeDefined();
// Now unsilence it
await agent.delete(`/alerts/${alert.body.data._id}/silenced`).expect(200);
// Verify it was unsilenced
alertFromDb = await Alert.findById(alert.body.data._id);
expect(alertFromDb).toBeDefined();
expect(alertFromDb!.silenced).toBeUndefined();
});
it('returns silenced info in GET /alerts', async () => {
const { agent } = await getLoggedInAgent(server);
const dashboard = await agent
.post('/dashboards')
.send(MOCK_DASHBOARD)
.expect(200);
const alert = await agent
.post('/alerts')
.send(
makeAlertInput({
dashboardId: dashboard.body.id,
tileId: dashboard.body.tiles[0].id,
}),
)
.expect(200);
// Silence the alert
const mutedUntil = new Date(Date.now() + 3600000).toISOString();
await agent
.post(`/alerts/${alert.body.data._id}/silenced`)
.send({ mutedUntil })
.expect(200);
// Get alerts and verify silenced info is returned
const alerts = await agent.get('/alerts').expect(200);
expect(alerts.body.data.length).toBe(1);
const silencedAlert = alerts.body.data[0];
expect(silencedAlert.silenced).toBeDefined();
expect(silencedAlert.silenced.by).toBeDefined(); // Should contain email
expect(silencedAlert.silenced.at).toBeDefined();
expect(silencedAlert.silenced.until).toBeDefined();
});
it('prevents silencing an alert that does not exist', async () => {
const { agent } = await getLoggedInAgent(server);
const fakeId = randomMongoId();
const mutedUntil = new Date(Date.now() + 3600000).toISOString();
await agent
.post(`/alerts/${fakeId}/silenced`)
.send({ mutedUntil })
.expect(404); // Should fail because alert doesn't exist
});
it('prevents unsilencing an alert that does not exist', async () => {
const { agent } = await getLoggedInAgent(server);
const fakeId = randomMongoId();
await agent.delete(`/alerts/${fakeId}/silenced`).expect(404); // Should fail
});
});

View file

@ -135,7 +135,12 @@ router.post(
'/:id/silenced',
validateRequest({
body: z.object({
mutedUntil: z.string().datetime(),
mutedUntil: z
.string()
.datetime()
.refine(val => new Date(val) > new Date(), {
message: 'mutedUntil must be in the future',
}),
}),
params: z.object({
id: objectIdSchema,
@ -150,7 +155,7 @@ router.post(
const alert = await getAlertById(req.params.id, teamId);
if (!alert) {
throw new Error('Alert not found');
return res.status(404).json({ error: 'Alert not found' });
}
alert.silenced = {
by: req.user._id,
@ -182,7 +187,7 @@ router.delete(
const alert = await getAlertById(req.params.id, teamId);
if (!alert) {
throw new Error('Alert not found');
return res.status(404).json({ error: 'Alert not found' });
}
alert.silenced = undefined;
await alert.save();

View file

@ -3604,6 +3604,249 @@ describe('checkAlerts', () => {
// Verify that webhook was called for the alert
expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(1);
});
it('should not fire notifications when alert is silenced', async () => {
const {
team,
webhook,
connection,
source,
savedSearch,
teamWebhooksById,
clickhouseClient,
} = await setupSavedSearchAlertTest();
const now = new Date('2023-11-16T22:12:00.000Z');
const eventMs = new Date('2023-11-16T22:05:00.000Z');
await bulkInsertLogs([
{
ServiceName: 'api',
Timestamp: eventMs,
SeverityText: 'error',
Body: 'Oh no! Something went wrong!',
},
{
ServiceName: 'api',
Timestamp: eventMs,
SeverityText: 'error',
Body: 'Oh no! Something went wrong!',
},
{
ServiceName: 'api',
Timestamp: eventMs,
SeverityText: 'error',
Body: 'Oh no! Something went wrong!',
},
]);
const details = await createAlertDetails(
team,
source,
{
source: AlertSource.SAVED_SEARCH,
channel: {
type: 'webhook',
webhookId: webhook._id.toString(),
},
interval: '5m',
thresholdType: AlertThresholdType.ABOVE,
threshold: 1,
savedSearchId: savedSearch.id,
},
{
taskType: AlertTaskType.SAVED_SEARCH,
savedSearch,
},
);
// Silence the alert until 1 hour from now
const alertDoc = await Alert.findById(details.alert.id);
alertDoc!.silenced = {
at: new Date(),
until: new Date(Date.now() + 3600000), // 1 hour from now
};
await alertDoc!.save();
// Update the details.alert object to reflect the silenced state
// (simulates what would happen if the alert was silenced before task queuing)
details.alert.silenced = alertDoc!.silenced;
// Process the alert - should skip firing because it's silenced
await processAlertAtTime(
now,
details,
clickhouseClient,
connection.id,
alertProvider,
teamWebhooksById,
);
// Verify webhook was NOT called
expect(slack.postMessageToWebhook).not.toHaveBeenCalled();
// Verify alert state was still updated
expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT');
});
it('should fire notifications when silenced period has expired', async () => {
const {
team,
webhook,
connection,
source,
savedSearch,
teamWebhooksById,
clickhouseClient,
} = await setupSavedSearchAlertTest();
const now = new Date('2023-11-16T22:12:00.000Z');
const eventMs = new Date('2023-11-16T22:05:00.000Z');
await bulkInsertLogs([
{
ServiceName: 'api',
Timestamp: eventMs,
SeverityText: 'error',
Body: 'Oh no! Something went wrong!',
},
{
ServiceName: 'api',
Timestamp: eventMs,
SeverityText: 'error',
Body: 'Oh no! Something went wrong!',
},
{
ServiceName: 'api',
Timestamp: eventMs,
SeverityText: 'error',
Body: 'Oh no! Something went wrong!',
},
]);
const details = await createAlertDetails(
team,
source,
{
source: AlertSource.SAVED_SEARCH,
channel: {
type: 'webhook',
webhookId: webhook._id.toString(),
},
interval: '5m',
thresholdType: AlertThresholdType.ABOVE,
threshold: 1,
savedSearchId: savedSearch.id,
},
{
taskType: AlertTaskType.SAVED_SEARCH,
savedSearch,
},
);
// Silence the alert but set expiry to the past
const alertDoc = await Alert.findById(details.alert.id);
alertDoc!.silenced = {
at: new Date(Date.now() - 7200000), // 2 hours ago
until: new Date(Date.now() - 3600000), // 1 hour ago (expired)
};
await alertDoc!.save();
// Update the details.alert object to reflect the expired silenced state
details.alert.silenced = alertDoc!.silenced;
// Process the alert - should fire because silence has expired
await processAlertAtTime(
now,
details,
clickhouseClient,
connection.id,
alertProvider,
teamWebhooksById,
);
// Verify webhook WAS called
expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(1);
expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT');
});
it('should fire notifications when alert is unsilenced', async () => {
const {
team,
webhook,
connection,
source,
savedSearch,
teamWebhooksById,
clickhouseClient,
} = await setupSavedSearchAlertTest();
const now = new Date('2023-11-16T22:12:00.000Z');
const eventMs = new Date('2023-11-16T22:05:00.000Z');
await bulkInsertLogs([
{
ServiceName: 'api',
Timestamp: eventMs,
SeverityText: 'error',
Body: 'Oh no! Something went wrong!',
},
{
ServiceName: 'api',
Timestamp: eventMs,
SeverityText: 'error',
Body: 'Oh no! Something went wrong!',
},
{
ServiceName: 'api',
Timestamp: eventMs,
SeverityText: 'error',
Body: 'Oh no! Something went wrong!',
},
]);
const details = await createAlertDetails(
team,
source,
{
source: AlertSource.SAVED_SEARCH,
channel: {
type: 'webhook',
webhookId: webhook._id.toString(),
},
interval: '5m',
thresholdType: AlertThresholdType.ABOVE,
threshold: 1,
savedSearchId: savedSearch.id,
},
{
taskType: AlertTaskType.SAVED_SEARCH,
savedSearch,
},
);
// Alert is unsilenced (no silenced field)
const alertDoc = await Alert.findById(details.alert.id);
alertDoc!.silenced = undefined;
await alertDoc!.save();
// Update the details.alert object to reflect the unsilenced state
details.alert.silenced = undefined;
// Process the alert - should fire normally
await processAlertAtTime(
now,
details,
clickhouseClient,
connection.id,
alertProvider,
teamWebhooksById,
);
// Verify webhook WAS called
expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(1);
expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT');
});
});
describe('getPreviousAlertHistories', () => {

View file

@ -101,6 +101,11 @@ const fireChannelEvent = async ({
throw new Error('Team not found');
}
// KNOWN LIMITATION: Alert data (including silenced state) is fetched when the
// task is queued via AlertProvider, not when it processes. If a user silences
// an alert after it's queued but before it processes, this execution may still
// send a notification. Subsequent alert checks will respect the silenced state.
// This trade-off maintains architectural separation from direct database access.
if ((alert.silenced?.until?.getTime() ?? 0) > Date.now()) {
logger.info(
{

View file

@ -2,19 +2,35 @@ import * as React from 'react';
import Head from 'next/head';
import Link from 'next/link';
import cx from 'classnames';
import { formatRelative } from 'date-fns';
import type { Duration } from 'date-fns';
import { add, formatRelative } from 'date-fns';
import {
AlertHistory,
AlertSource,
AlertState,
} from '@hyperdx/common-utils/dist/types';
import { Alert, Badge, Container, Group, Stack, Tooltip } from '@mantine/core';
import {
Alert,
Badge,
Button,
Container,
Group,
Menu,
Stack,
Tooltip,
} from '@mantine/core';
import { notifications } from '@mantine/notifications';
import { IconBell } from '@tabler/icons-react';
import { useQueryClient } from '@tanstack/react-query';
import { ErrorBoundary } from '@/components/ErrorBoundary';
import { PageHeader } from '@/components/PageHeader';
import { isAlertSilenceExpired } from './utils/alerts';
import api from './api';
import { withAppNav } from './layout';
import type { AlertsPageItem } from './types';
import { FormatTime } from './useFormatTime';
import styles from '../styles/AlertsPage.module.scss';
@ -73,6 +89,179 @@ function AlertHistoryCard({
const HISTORY_ITEMS = 18;
function AckAlert({ alert }: { alert: AlertsPageItem }) {
const queryClient = useQueryClient();
const silenceAlert = api.useSilenceAlert();
const unsilenceAlert = api.useUnsilenceAlert();
const mutateOptions = React.useMemo(
() => ({
onSuccess: () => {
queryClient.invalidateQueries({ queryKey: ['alerts'] });
},
onError: (error: any) => {
const status = error?.response?.status;
let message = 'Failed to silence alert, please try again later.';
if (status === 404) {
message = 'Alert not found.';
} else if (status === 400) {
message =
'Invalid request. Please ensure the silence duration is valid.';
}
notifications.show({
color: 'red',
message,
});
},
}),
[queryClient],
);
const handleUnsilenceAlert = React.useCallback(() => {
unsilenceAlert.mutate(alert._id || '', mutateOptions);
}, [alert._id, mutateOptions, unsilenceAlert]);
const isNoLongerMuted = React.useMemo(() => {
return isAlertSilenceExpired(alert.silenced);
}, [alert.silenced]);
const handleSilenceAlert = React.useCallback(
(duration: Duration) => {
const mutedUntil = add(new Date(), duration);
silenceAlert.mutate(
{
alertId: alert._id || '',
mutedUntil: mutedUntil.toISOString(),
},
mutateOptions,
);
},
[alert._id, mutateOptions, silenceAlert],
);
if (alert.silenced?.at) {
return (
<ErrorBoundary message="Failed to load alert acknowledgment menu">
<Menu>
<Menu.Target>
<Button
size="compact-sm"
variant="light"
color={
isNoLongerMuted
? 'var(--color-bg-warning)'
: 'var(--color-bg-success)'
}
leftSection={<IconBell size={16} />}
>
Ack&apos;d
</Button>
</Menu.Target>
<Menu.Dropdown>
<Menu.Label py={6}>
Acknowledged{' '}
{alert.silenced?.by ? (
<>
by <strong>{alert.silenced?.by}</strong>
</>
) : null}{' '}
on <br />
<FormatTime value={alert.silenced?.at} />
.<br />
</Menu.Label>
<Menu.Label py={6}>
{isNoLongerMuted ? (
'Alert resumed.'
) : (
<>
Resumes <FormatTime value={alert.silenced.until} />.
</>
)}
</Menu.Label>
<Menu.Item
lh="1"
py={8}
color="orange"
onClick={handleUnsilenceAlert}
disabled={unsilenceAlert.isPending}
>
{isNoLongerMuted ? 'Unacknowledge' : 'Resume alert'}
</Menu.Item>
</Menu.Dropdown>
</Menu>
</ErrorBoundary>
);
}
if (alert.state === 'ALERT') {
return (
<ErrorBoundary message="Failed to load alert acknowledgment menu">
<Menu disabled={silenceAlert.isPending}>
<Menu.Target>
<Button size="compact-sm" variant="default">
Ack
</Button>
</Menu.Target>
<Menu.Dropdown>
<Menu.Label lh="1" py={6}>
Acknowledge and silence for
</Menu.Label>
<Menu.Item
lh="1"
py={8}
onClick={() =>
handleSilenceAlert({
minutes: 30,
})
}
>
30 minutes
</Menu.Item>
<Menu.Item
lh="1"
py={8}
onClick={() =>
handleSilenceAlert({
hours: 1,
})
}
>
1 hour
</Menu.Item>
<Menu.Item
lh="1"
py={8}
onClick={() =>
handleSilenceAlert({
hours: 6,
})
}
>
6 hours
</Menu.Item>
<Menu.Item
lh="1"
py={8}
onClick={() =>
handleSilenceAlert({
hours: 24,
})
}
>
24 hours
</Menu.Item>
</Menu.Dropdown>
</Menu>
</ErrorBoundary>
);
}
return null;
}
function AlertHistoryCardList({
history,
alertUrl,
@ -189,7 +378,7 @@ function AlertDetails({ alert }: { alert: AlertsPageItem }) {
}, [alert]);
return (
<div data-testid={`alert-card-${alert.id}`} className={styles.alertRow}>
<div data-testid={`alert-card-${alert._id}`} className={styles.alertRow}>
<Group>
{alert.state === AlertState.ALERT && (
<Badge variant="light" color="red">
@ -206,7 +395,7 @@ function AlertDetails({ alert }: { alert: AlertsPageItem }) {
<Stack gap={2}>
<div>
<Link
data-testid={`alert-link-${alert.id}`}
data-testid={`alert-link-${alert._id}`}
href={alertUrl}
className={styles.alertLink}
title={linkTitle}
@ -232,6 +421,7 @@ function AlertDetails({ alert }: { alert: AlertsPageItem }) {
<Group>
<AlertHistoryCardList history={alert.history} alertUrl={alertUrl} />
<AckAlert alert={alert} />
</Group>
</div>
);

View file

@ -9,6 +9,7 @@ import {
import dynamic from 'next/dynamic';
import Head from 'next/head';
import { useRouter } from 'next/router';
import { formatRelative } from 'date-fns';
import produce from 'immer';
import { parseAsString, useQueryState } from 'nuqs';
import { ErrorBoundary } from 'react-error-boundary';
@ -45,7 +46,7 @@ import {
} from '@mantine/core';
import { useHover } from '@mantine/hooks';
import { notifications } from '@mantine/notifications';
import { IconFilterEdit, IconPlayerPlay } from '@tabler/icons-react';
import { IconBell, IconFilterEdit, IconPlayerPlay } from '@tabler/icons-react';
import { ContactSupportText } from '@/components/ContactSupportText';
import EditTimeChartForm from '@/components/DBEditTimeChartForm';
@ -207,6 +208,18 @@ const Tile = forwardRef(
return 'red';
}, [alert]);
const alertTooltip = useMemo(() => {
if (!alert) {
return 'Add alert';
}
let tooltip = `Has alert and is in ${alert.state} state`;
if (alert.silenced?.at) {
const silencedAt = new Date(alert.silenced.at);
tooltip += `. Ack'd ${formatRelative(silencedAt, new Date())}`;
}
return tooltip;
}, [alert]);
const { data: me } = api.useMe();
return (
@ -235,22 +248,24 @@ const Tile = forwardRef(
<Flex gap="0px">
{chart.config.displayType === DisplayType.Line && (
<Indicator
size={5}
size={alert?.state === AlertState.OK ? 6 : 8}
zIndex={1}
color={alertIndicatorColor}
processing={alert?.state === AlertState.ALERT}
label={!alert && <span className="fs-8">+</span>}
mr={4}
>
<Button
data-testid={`tile-alerts-button-${chart.id}`}
variant="subtle"
color="gray"
size="xxs"
onClick={onEditClick}
title="Alerts"
>
<i className="bi bi-bell fs-7"></i>
</Button>
<Tooltip label={alertTooltip} withArrow>
<Button
data-testid={`tile-alerts-button-${chart.id}`}
variant="subtle"
color="gray"
size="xxs"
onClick={onEditClick}
>
<IconBell size={16} />
</Button>
</Tooltip>
</Indicator>
)}

View file

@ -92,6 +92,23 @@ const api = {
}),
});
},
useSilenceAlert() {
return useMutation<any, Error, { alertId: string; mutedUntil: string }>({
mutationFn: async ({ alertId, mutedUntil }) =>
server(`alerts/${alertId}/silenced`, {
method: 'POST',
json: { mutedUntil },
}),
});
},
useUnsilenceAlert() {
return useMutation<any, Error, string>({
mutationFn: async (alertId: string) =>
server(`alerts/${alertId}/silenced`, {
method: 'DELETE',
}),
});
},
useDashboards(options?: UseQueryOptions<any, Error>) {
return useQuery({
queryKey: [`dashboards`],

View file

@ -51,6 +51,7 @@ export type LogStreamModel = KeyValuePairs & {
};
export type AlertsPageItem = Alert & {
_id: string;
history: AlertHistory[];
dashboard?: ServerDashboard;
savedSearch?: SavedSearch;

View file

@ -121,3 +121,14 @@ export const DEFAULT_TILE_ALERT: z.infer<typeof ChartAlertBaseSchema> = {
webhookId: '',
},
};
/**
* Checks if an alert's silence period has expired.
* @param silenced - The alert's silenced state containing the until timestamp
* @returns true if the silence period has expired, false otherwise
*/
export function isAlertSilenceExpired(silenced?: {
until: string | Date;
}): boolean {
return silenced ? new Date() > new Date(silenced.until) : false;
}