diff --git a/.changeset/healthy-spoons-shop.md b/.changeset/healthy-spoons-shop.md
new file mode 100644
index 00000000..54113e9d
--- /dev/null
+++ b/.changeset/healthy-spoons-shop.md
@@ -0,0 +1,6 @@
+---
+"@hyperdx/api": patch
+"@hyperdx/app": patch
+---
+
+feat: Add silence alerts feature
diff --git a/packages/api/src/routers/api/__tests__/alerts.test.ts b/packages/api/src/routers/api/__tests__/alerts.test.ts
index 5d37a3d6..3accf375 100644
--- a/packages/api/src/routers/api/__tests__/alerts.test.ts
+++ b/packages/api/src/routers/api/__tests__/alerts.test.ts
@@ -174,4 +174,124 @@ describe('alerts router', () => {
expect(alert.dashboard).toBeDefined();
}
});
+
+ it('can silence an alert', async () => {
+ const { agent, user } = await getLoggedInAgent(server);
+ const dashboard = await agent
+ .post('/dashboards')
+ .send(MOCK_DASHBOARD)
+ .expect(200);
+ const alert = await agent
+ .post('/alerts')
+ .send(
+ makeAlertInput({
+ dashboardId: dashboard.body.id,
+ tileId: dashboard.body.tiles[0].id,
+ }),
+ )
+ .expect(200);
+
+ const mutedUntil = new Date(Date.now() + 3600000).toISOString(); // 1 hour from now
+ await agent
+ .post(`/alerts/${alert.body.data._id}/silenced`)
+ .send({ mutedUntil })
+ .expect(200);
+
+ // Verify the alert was silenced
+ const alertFromDb = await Alert.findById(alert.body.data._id);
+ expect(alertFromDb).toBeDefined();
+ expect(alertFromDb!.silenced).toBeDefined();
+ expect(alertFromDb!.silenced!.by).toEqual(user._id);
+ expect(alertFromDb!.silenced!.at).toBeDefined();
+ expect(new Date(alertFromDb!.silenced!.until).toISOString()).toBe(
+ mutedUntil,
+ );
+ });
+
+ it('can unsilence an alert', async () => {
+ const { agent, user } = await getLoggedInAgent(server);
+ const dashboard = await agent
+ .post('/dashboards')
+ .send(MOCK_DASHBOARD)
+ .expect(200);
+ const alert = await agent
+ .post('/alerts')
+ .send(
+ makeAlertInput({
+ dashboardId: dashboard.body.id,
+ tileId: dashboard.body.tiles[0].id,
+ }),
+ )
+ .expect(200);
+
+ // First silence the alert
+ const mutedUntil = new Date(Date.now() + 3600000).toISOString();
+ await agent
+ .post(`/alerts/${alert.body.data._id}/silenced`)
+ .send({ mutedUntil })
+ .expect(200);
+
+ // Verify it was silenced
+ let alertFromDb = await Alert.findById(alert.body.data._id);
+ expect(alertFromDb!.silenced).toBeDefined();
+
+ // Now unsilence it
+ await agent.delete(`/alerts/${alert.body.data._id}/silenced`).expect(200);
+
+ // Verify it was unsilenced
+ alertFromDb = await Alert.findById(alert.body.data._id);
+ expect(alertFromDb).toBeDefined();
+ expect(alertFromDb!.silenced).toBeUndefined();
+ });
+
+ it('returns silenced info in GET /alerts', async () => {
+ const { agent } = await getLoggedInAgent(server);
+ const dashboard = await agent
+ .post('/dashboards')
+ .send(MOCK_DASHBOARD)
+ .expect(200);
+ const alert = await agent
+ .post('/alerts')
+ .send(
+ makeAlertInput({
+ dashboardId: dashboard.body.id,
+ tileId: dashboard.body.tiles[0].id,
+ }),
+ )
+ .expect(200);
+
+ // Silence the alert
+ const mutedUntil = new Date(Date.now() + 3600000).toISOString();
+ await agent
+ .post(`/alerts/${alert.body.data._id}/silenced`)
+ .send({ mutedUntil })
+ .expect(200);
+
+ // Get alerts and verify silenced info is returned
+ const alerts = await agent.get('/alerts').expect(200);
+ expect(alerts.body.data.length).toBe(1);
+ const silencedAlert = alerts.body.data[0];
+ expect(silencedAlert.silenced).toBeDefined();
+ expect(silencedAlert.silenced.by).toBeDefined(); // Should contain email
+ expect(silencedAlert.silenced.at).toBeDefined();
+ expect(silencedAlert.silenced.until).toBeDefined();
+ });
+
+ it('prevents silencing an alert that does not exist', async () => {
+ const { agent } = await getLoggedInAgent(server);
+ const fakeId = randomMongoId();
+ const mutedUntil = new Date(Date.now() + 3600000).toISOString();
+
+ await agent
+ .post(`/alerts/${fakeId}/silenced`)
+ .send({ mutedUntil })
+ .expect(404); // Should fail because alert doesn't exist
+ });
+
+ it('prevents unsilencing an alert that does not exist', async () => {
+ const { agent } = await getLoggedInAgent(server);
+ const fakeId = randomMongoId();
+
+ await agent.delete(`/alerts/${fakeId}/silenced`).expect(404); // Should fail
+ });
});
diff --git a/packages/api/src/routers/api/alerts.ts b/packages/api/src/routers/api/alerts.ts
index 9c14fbbf..62eeb2a5 100644
--- a/packages/api/src/routers/api/alerts.ts
+++ b/packages/api/src/routers/api/alerts.ts
@@ -135,7 +135,12 @@ router.post(
'/:id/silenced',
validateRequest({
body: z.object({
- mutedUntil: z.string().datetime(),
+ mutedUntil: z
+ .string()
+ .datetime()
+ .refine(val => new Date(val) > new Date(), {
+ message: 'mutedUntil must be in the future',
+ }),
}),
params: z.object({
id: objectIdSchema,
@@ -150,7 +155,7 @@ router.post(
const alert = await getAlertById(req.params.id, teamId);
if (!alert) {
- throw new Error('Alert not found');
+ return res.status(404).json({ error: 'Alert not found' });
}
alert.silenced = {
by: req.user._id,
@@ -182,7 +187,7 @@ router.delete(
const alert = await getAlertById(req.params.id, teamId);
if (!alert) {
- throw new Error('Alert not found');
+ return res.status(404).json({ error: 'Alert not found' });
}
alert.silenced = undefined;
await alert.save();
diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.test.ts
index 9e688eb7..dd484360 100644
--- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.test.ts
+++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.test.ts
@@ -3604,6 +3604,249 @@ describe('checkAlerts', () => {
// Verify that webhook was called for the alert
expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(1);
});
+
+ it('should not fire notifications when alert is silenced', async () => {
+ const {
+ team,
+ webhook,
+ connection,
+ source,
+ savedSearch,
+ teamWebhooksById,
+ clickhouseClient,
+ } = await setupSavedSearchAlertTest();
+
+ const now = new Date('2023-11-16T22:12:00.000Z');
+ const eventMs = new Date('2023-11-16T22:05:00.000Z');
+
+ await bulkInsertLogs([
+ {
+ ServiceName: 'api',
+ Timestamp: eventMs,
+ SeverityText: 'error',
+ Body: 'Oh no! Something went wrong!',
+ },
+ {
+ ServiceName: 'api',
+ Timestamp: eventMs,
+ SeverityText: 'error',
+ Body: 'Oh no! Something went wrong!',
+ },
+ {
+ ServiceName: 'api',
+ Timestamp: eventMs,
+ SeverityText: 'error',
+ Body: 'Oh no! Something went wrong!',
+ },
+ ]);
+
+ const details = await createAlertDetails(
+ team,
+ source,
+ {
+ source: AlertSource.SAVED_SEARCH,
+ channel: {
+ type: 'webhook',
+ webhookId: webhook._id.toString(),
+ },
+ interval: '5m',
+ thresholdType: AlertThresholdType.ABOVE,
+ threshold: 1,
+ savedSearchId: savedSearch.id,
+ },
+ {
+ taskType: AlertTaskType.SAVED_SEARCH,
+ savedSearch,
+ },
+ );
+
+ // Silence the alert until 1 hour from now
+ const alertDoc = await Alert.findById(details.alert.id);
+ alertDoc!.silenced = {
+ at: new Date(),
+ until: new Date(Date.now() + 3600000), // 1 hour from now
+ };
+ await alertDoc!.save();
+
+ // Update the details.alert object to reflect the silenced state
+ // (simulates what would happen if the alert was silenced before task queuing)
+ details.alert.silenced = alertDoc!.silenced;
+
+ // Process the alert - should skip firing because it's silenced
+ await processAlertAtTime(
+ now,
+ details,
+ clickhouseClient,
+ connection.id,
+ alertProvider,
+ teamWebhooksById,
+ );
+
+ // Verify webhook was NOT called
+ expect(slack.postMessageToWebhook).not.toHaveBeenCalled();
+
+ // Verify alert state was still updated
+ expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT');
+ });
+
+ it('should fire notifications when silenced period has expired', async () => {
+ const {
+ team,
+ webhook,
+ connection,
+ source,
+ savedSearch,
+ teamWebhooksById,
+ clickhouseClient,
+ } = await setupSavedSearchAlertTest();
+
+ const now = new Date('2023-11-16T22:12:00.000Z');
+ const eventMs = new Date('2023-11-16T22:05:00.000Z');
+
+ await bulkInsertLogs([
+ {
+ ServiceName: 'api',
+ Timestamp: eventMs,
+ SeverityText: 'error',
+ Body: 'Oh no! Something went wrong!',
+ },
+ {
+ ServiceName: 'api',
+ Timestamp: eventMs,
+ SeverityText: 'error',
+ Body: 'Oh no! Something went wrong!',
+ },
+ {
+ ServiceName: 'api',
+ Timestamp: eventMs,
+ SeverityText: 'error',
+ Body: 'Oh no! Something went wrong!',
+ },
+ ]);
+
+ const details = await createAlertDetails(
+ team,
+ source,
+ {
+ source: AlertSource.SAVED_SEARCH,
+ channel: {
+ type: 'webhook',
+ webhookId: webhook._id.toString(),
+ },
+ interval: '5m',
+ thresholdType: AlertThresholdType.ABOVE,
+ threshold: 1,
+ savedSearchId: savedSearch.id,
+ },
+ {
+ taskType: AlertTaskType.SAVED_SEARCH,
+ savedSearch,
+ },
+ );
+
+ // Silence the alert but set expiry to the past
+ const alertDoc = await Alert.findById(details.alert.id);
+ alertDoc!.silenced = {
+ at: new Date(Date.now() - 7200000), // 2 hours ago
+ until: new Date(Date.now() - 3600000), // 1 hour ago (expired)
+ };
+ await alertDoc!.save();
+
+ // Update the details.alert object to reflect the expired silenced state
+ details.alert.silenced = alertDoc!.silenced;
+
+ // Process the alert - should fire because silence has expired
+ await processAlertAtTime(
+ now,
+ details,
+ clickhouseClient,
+ connection.id,
+ alertProvider,
+ teamWebhooksById,
+ );
+
+ // Verify webhook WAS called
+ expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(1);
+ expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT');
+ });
+
+ it('should fire notifications when alert is unsilenced', async () => {
+ const {
+ team,
+ webhook,
+ connection,
+ source,
+ savedSearch,
+ teamWebhooksById,
+ clickhouseClient,
+ } = await setupSavedSearchAlertTest();
+
+ const now = new Date('2023-11-16T22:12:00.000Z');
+ const eventMs = new Date('2023-11-16T22:05:00.000Z');
+
+ await bulkInsertLogs([
+ {
+ ServiceName: 'api',
+ Timestamp: eventMs,
+ SeverityText: 'error',
+ Body: 'Oh no! Something went wrong!',
+ },
+ {
+ ServiceName: 'api',
+ Timestamp: eventMs,
+ SeverityText: 'error',
+ Body: 'Oh no! Something went wrong!',
+ },
+ {
+ ServiceName: 'api',
+ Timestamp: eventMs,
+ SeverityText: 'error',
+ Body: 'Oh no! Something went wrong!',
+ },
+ ]);
+
+ const details = await createAlertDetails(
+ team,
+ source,
+ {
+ source: AlertSource.SAVED_SEARCH,
+ channel: {
+ type: 'webhook',
+ webhookId: webhook._id.toString(),
+ },
+ interval: '5m',
+ thresholdType: AlertThresholdType.ABOVE,
+ threshold: 1,
+ savedSearchId: savedSearch.id,
+ },
+ {
+ taskType: AlertTaskType.SAVED_SEARCH,
+ savedSearch,
+ },
+ );
+
+ // Alert is unsilenced (no silenced field)
+ const alertDoc = await Alert.findById(details.alert.id);
+ alertDoc!.silenced = undefined;
+ await alertDoc!.save();
+
+ // Update the details.alert object to reflect the unsilenced state
+ details.alert.silenced = undefined;
+
+ // Process the alert - should fire normally
+ await processAlertAtTime(
+ now,
+ details,
+ clickhouseClient,
+ connection.id,
+ alertProvider,
+ teamWebhooksById,
+ );
+
+ // Verify webhook WAS called
+ expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(1);
+ expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT');
+ });
});
describe('getPreviousAlertHistories', () => {
diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts
index 0347c245..bdb8292c 100644
--- a/packages/api/src/tasks/checkAlerts/index.ts
+++ b/packages/api/src/tasks/checkAlerts/index.ts
@@ -101,6 +101,11 @@ const fireChannelEvent = async ({
throw new Error('Team not found');
}
+ // KNOWN LIMITATION: Alert data (including silenced state) is fetched when the
+ // task is queued via AlertProvider, not when it processes. If a user silences
+ // an alert after it's queued but before it processes, this execution may still
+ // send a notification. Subsequent alert checks will respect the silenced state.
+ // This trade-off maintains architectural separation from direct database access.
if ((alert.silenced?.until?.getTime() ?? 0) > Date.now()) {
logger.info(
{
diff --git a/packages/app/src/AlertsPage.tsx b/packages/app/src/AlertsPage.tsx
index 6f101d78..e7ab030f 100644
--- a/packages/app/src/AlertsPage.tsx
+++ b/packages/app/src/AlertsPage.tsx
@@ -2,19 +2,35 @@ import * as React from 'react';
import Head from 'next/head';
import Link from 'next/link';
import cx from 'classnames';
-import { formatRelative } from 'date-fns';
+import type { Duration } from 'date-fns';
+import { add, formatRelative } from 'date-fns';
import {
AlertHistory,
AlertSource,
AlertState,
} from '@hyperdx/common-utils/dist/types';
-import { Alert, Badge, Container, Group, Stack, Tooltip } from '@mantine/core';
+import {
+ Alert,
+ Badge,
+ Button,
+ Container,
+ Group,
+ Menu,
+ Stack,
+ Tooltip,
+} from '@mantine/core';
+import { notifications } from '@mantine/notifications';
+import { IconBell } from '@tabler/icons-react';
+import { useQueryClient } from '@tanstack/react-query';
+import { ErrorBoundary } from '@/components/ErrorBoundary';
import { PageHeader } from '@/components/PageHeader';
+import { isAlertSilenceExpired } from './utils/alerts';
import api from './api';
import { withAppNav } from './layout';
import type { AlertsPageItem } from './types';
+import { FormatTime } from './useFormatTime';
import styles from '../styles/AlertsPage.module.scss';
@@ -73,6 +89,179 @@ function AlertHistoryCard({
const HISTORY_ITEMS = 18;
+function AckAlert({ alert }: { alert: AlertsPageItem }) {
+ const queryClient = useQueryClient();
+ const silenceAlert = api.useSilenceAlert();
+ const unsilenceAlert = api.useUnsilenceAlert();
+
+ const mutateOptions = React.useMemo(
+ () => ({
+ onSuccess: () => {
+ queryClient.invalidateQueries({ queryKey: ['alerts'] });
+ },
+ onError: (error: any) => {
+ const status = error?.response?.status;
+ let message = 'Failed to silence alert, please try again later.';
+
+ if (status === 404) {
+ message = 'Alert not found.';
+ } else if (status === 400) {
+ message =
+ 'Invalid request. Please ensure the silence duration is valid.';
+ }
+
+ notifications.show({
+ color: 'red',
+ message,
+ });
+ },
+ }),
+ [queryClient],
+ );
+
+ const handleUnsilenceAlert = React.useCallback(() => {
+ unsilenceAlert.mutate(alert._id || '', mutateOptions);
+ }, [alert._id, mutateOptions, unsilenceAlert]);
+
+ const isNoLongerMuted = React.useMemo(() => {
+ return isAlertSilenceExpired(alert.silenced);
+ }, [alert.silenced]);
+
+ const handleSilenceAlert = React.useCallback(
+ (duration: Duration) => {
+ const mutedUntil = add(new Date(), duration);
+ silenceAlert.mutate(
+ {
+ alertId: alert._id || '',
+ mutedUntil: mutedUntil.toISOString(),
+ },
+ mutateOptions,
+ );
+ },
+ [alert._id, mutateOptions, silenceAlert],
+ );
+
+ if (alert.silenced?.at) {
+ return (
+
+
+
+ );
+ }
+
+ if (alert.state === 'ALERT') {
+ return (
+
+
+
+ );
+ }
+
+ return null;
+}
+
function AlertHistoryCardList({
history,
alertUrl,
@@ -189,7 +378,7 @@ function AlertDetails({ alert }: { alert: AlertsPageItem }) {
}, [alert]);
return (
-
+
{alert.state === AlertState.ALERT && (
@@ -206,7 +395,7 @@ function AlertDetails({ alert }: { alert: AlertsPageItem }) {
);
diff --git a/packages/app/src/DBDashboardPage.tsx b/packages/app/src/DBDashboardPage.tsx
index 63e0183a..5acff88a 100644
--- a/packages/app/src/DBDashboardPage.tsx
+++ b/packages/app/src/DBDashboardPage.tsx
@@ -9,6 +9,7 @@ import {
import dynamic from 'next/dynamic';
import Head from 'next/head';
import { useRouter } from 'next/router';
+import { formatRelative } from 'date-fns';
import produce from 'immer';
import { parseAsString, useQueryState } from 'nuqs';
import { ErrorBoundary } from 'react-error-boundary';
@@ -45,7 +46,7 @@ import {
} from '@mantine/core';
import { useHover } from '@mantine/hooks';
import { notifications } from '@mantine/notifications';
-import { IconFilterEdit, IconPlayerPlay } from '@tabler/icons-react';
+import { IconBell, IconFilterEdit, IconPlayerPlay } from '@tabler/icons-react';
import { ContactSupportText } from '@/components/ContactSupportText';
import EditTimeChartForm from '@/components/DBEditTimeChartForm';
@@ -207,6 +208,18 @@ const Tile = forwardRef(
return 'red';
}, [alert]);
+ const alertTooltip = useMemo(() => {
+ if (!alert) {
+ return 'Add alert';
+ }
+ let tooltip = `Has alert and is in ${alert.state} state`;
+ if (alert.silenced?.at) {
+ const silencedAt = new Date(alert.silenced.at);
+ tooltip += `. Ack'd ${formatRelative(silencedAt, new Date())}`;
+ }
+ return tooltip;
+ }, [alert]);
+
const { data: me } = api.useMe();
return (
@@ -235,22 +248,24 @@ const Tile = forwardRef(
{chart.config.displayType === DisplayType.Line && (
+}
mr={4}
>
-
+
+
+
)}
diff --git a/packages/app/src/api.ts b/packages/app/src/api.ts
index 35e20e02..edf9a996 100644
--- a/packages/app/src/api.ts
+++ b/packages/app/src/api.ts
@@ -92,6 +92,23 @@ const api = {
}),
});
},
+ useSilenceAlert() {
+ return useMutation({
+ mutationFn: async ({ alertId, mutedUntil }) =>
+ server(`alerts/${alertId}/silenced`, {
+ method: 'POST',
+ json: { mutedUntil },
+ }),
+ });
+ },
+ useUnsilenceAlert() {
+ return useMutation({
+ mutationFn: async (alertId: string) =>
+ server(`alerts/${alertId}/silenced`, {
+ method: 'DELETE',
+ }),
+ });
+ },
useDashboards(options?: UseQueryOptions) {
return useQuery({
queryKey: [`dashboards`],
diff --git a/packages/app/src/types.ts b/packages/app/src/types.ts
index 85222d14..8365a6a2 100644
--- a/packages/app/src/types.ts
+++ b/packages/app/src/types.ts
@@ -51,6 +51,7 @@ export type LogStreamModel = KeyValuePairs & {
};
export type AlertsPageItem = Alert & {
+ _id: string;
history: AlertHistory[];
dashboard?: ServerDashboard;
savedSearch?: SavedSearch;
diff --git a/packages/app/src/utils/alerts.ts b/packages/app/src/utils/alerts.ts
index 7c87c82f..3e492d26 100644
--- a/packages/app/src/utils/alerts.ts
+++ b/packages/app/src/utils/alerts.ts
@@ -121,3 +121,14 @@ export const DEFAULT_TILE_ALERT: z.infer = {
webhookId: '',
},
};
+
+/**
+ * Checks if an alert's silence period has expired.
+ * @param silenced - The alert's silenced state containing the until timestamp
+ * @returns true if the silence period has expired, false otherwise
+ */
+export function isAlertSilenceExpired(silenced?: {
+ until: string | Date;
+}): boolean {
+ return silenced ? new Date() > new Date(silenced.until) : false;
+}