mirror of
https://github.com/hyperdxio/hyperdx
synced 2026-04-21 13:37:15 +00:00
feat: adjust alert template title and body to reflect alert state (#1339)
Currently, the resolved alert will have the same title and body message as the alerting one, which is misleading Ref: HDX-2786 ## Slack ### ALERT <img width="723" height="358" alt="Screenshot 2025-11-09 at 10 02 52 PM" src="https://github.com/user-attachments/assets/b1c6f563-f095-457e-9a70-01c8149796c4" /> ### RESOLVED <img width="650" height="117" alt="Screenshot 2025-11-09 at 10 26 01 PM" src="https://github.com/user-attachments/assets/07ef1e7d-8ee5-4604-92cf-4811a0a5c811" /> ## incident.io ### ALERT <img width="1432" height="398" alt="Screenshot 2025-11-09 at 11 07 30 PM" src="https://github.com/user-attachments/assets/30e25eb3-32b2-4f51-934d-b28e75dd5cf7" /> ### RESOLVED <img width="1427" height="305" alt="Screenshot 2025-11-09 at 11 08 56 PM" src="https://github.com/user-attachments/assets/913a5b99-bb07-47ae-bec9-6b0814e4b400" />
This commit is contained in:
Warren
GitHub
parent
b33db7660b
commit
840d73076c
4 changed files with 159 additions and 24 deletions
5
.changeset/spotty-yaks-sniff.md
Normal file
5
.changeset/spotty-yaks-sniff.md
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
"@hyperdx/api": minor
|
||||
---
|
||||
|
||||
feat: adjust alert template title and body to reflect alert state
|
||||
|
|
@ -43,6 +43,7 @@ import {
|
|||
buildAlertMessageTemplateHdxLink,
|
||||
buildAlertMessageTemplateTitle,
|
||||
getDefaultExternalAction,
|
||||
isAlertResolved,
|
||||
renderAlertTemplate,
|
||||
translateExternalActionsToInternal,
|
||||
} from '@/tasks/checkAlerts/template';
|
||||
|
|
@ -229,16 +230,70 @@ describe('checkAlerts', () => {
|
|||
buildAlertMessageTemplateTitle({
|
||||
view: defaultSearchView,
|
||||
}),
|
||||
).toMatchInlineSnapshot(`"Alert for \\"My Search\\" - 10 lines found"`);
|
||||
).toMatchInlineSnapshot(
|
||||
`"🚨 Alert for \\"My Search\\" - 10 lines found"`,
|
||||
);
|
||||
expect(
|
||||
buildAlertMessageTemplateTitle({
|
||||
view: defaultChartView,
|
||||
}),
|
||||
).toMatchInlineSnapshot(
|
||||
`"Alert for \\"Test Chart\\" in \\"My Dashboard\\" - 5 exceeds 1"`,
|
||||
`"🚨 Alert for \\"Test Chart\\" in \\"My Dashboard\\" - 5 exceeds 1"`,
|
||||
);
|
||||
});
|
||||
|
||||
it('buildAlertMessageTemplateTitle with state parameter', () => {
|
||||
// Test ALERT state (should have 🚨 emoji)
|
||||
expect(
|
||||
buildAlertMessageTemplateTitle({
|
||||
view: defaultSearchView,
|
||||
state: AlertState.ALERT,
|
||||
}),
|
||||
).toMatchInlineSnapshot(
|
||||
`"🚨 Alert for \\"My Search\\" - 10 lines found"`,
|
||||
);
|
||||
expect(
|
||||
buildAlertMessageTemplateTitle({
|
||||
view: defaultChartView,
|
||||
state: AlertState.ALERT,
|
||||
}),
|
||||
).toMatchInlineSnapshot(
|
||||
`"🚨 Alert for \\"Test Chart\\" in \\"My Dashboard\\" - 5 exceeds 1"`,
|
||||
);
|
||||
|
||||
// Test OK state (should have ✅ emoji)
|
||||
expect(
|
||||
buildAlertMessageTemplateTitle({
|
||||
view: defaultSearchView,
|
||||
state: AlertState.OK,
|
||||
}),
|
||||
).toMatchInlineSnapshot(
|
||||
`"✅ Alert for \\"My Search\\" - 10 lines found"`,
|
||||
);
|
||||
expect(
|
||||
buildAlertMessageTemplateTitle({
|
||||
view: defaultChartView,
|
||||
state: AlertState.OK,
|
||||
}),
|
||||
).toMatchInlineSnapshot(
|
||||
`"✅ Alert for \\"Test Chart\\" in \\"My Dashboard\\" - 5 exceeds 1"`,
|
||||
);
|
||||
});
|
||||
|
||||
it('isAlertResolved', () => {
|
||||
// Test OK state returns true
|
||||
expect(isAlertResolved(AlertState.OK)).toBe(true);
|
||||
|
||||
// Test ALERT state returns false
|
||||
expect(isAlertResolved(AlertState.ALERT)).toBe(false);
|
||||
|
||||
// Test INSUFFICIENT_DATA state returns false
|
||||
expect(isAlertResolved(AlertState.INSUFFICIENT_DATA)).toBe(false);
|
||||
|
||||
// Test DISABLED state returns false
|
||||
expect(isAlertResolved(AlertState.DISABLED)).toBe(false);
|
||||
});
|
||||
|
||||
it('getDefaultExternalAction', () => {
|
||||
expect(
|
||||
getDefaultExternalAction({
|
||||
|
|
@ -372,7 +427,7 @@ describe('checkAlerts', () => {
|
|||
},
|
||||
},
|
||||
},
|
||||
title: 'Alert for "My Search" - 10 lines found',
|
||||
title: '🚨 Alert for "My Search" - 10 lines found',
|
||||
teamWebhooksById: new Map<string, typeof webhook>([
|
||||
[webhook._id.toString(), webhook],
|
||||
]),
|
||||
|
|
@ -382,12 +437,12 @@ describe('checkAlerts', () => {
|
|||
1,
|
||||
'https://hooks.slack.com/services/123',
|
||||
{
|
||||
text: 'Alert for "My Search" - 10 lines found',
|
||||
text: '🚨 Alert for "My Search" - 10 lines found',
|
||||
blocks: [
|
||||
{
|
||||
text: {
|
||||
text: [
|
||||
'*<http://app:8080/search/fake-saved-search-id?from=1679091183103&to=1679091239103&isLive=false | Alert for "My Search" - 10 lines found>*',
|
||||
'*<http://app:8080/search/fake-saved-search-id?from=1679091183103&to=1679091239103&isLive=false | 🚨 Alert for "My Search" - 10 lines found>*',
|
||||
'Group: "http"',
|
||||
'10 lines found, expected less than 1 lines',
|
||||
'Time Range (UTC): [Mar 17 10:13:03 PM - Mar 17 10:13:59 PM)',
|
||||
|
|
@ -432,7 +487,7 @@ describe('checkAlerts', () => {
|
|||
webhookName: 'My_Webhook',
|
||||
},
|
||||
},
|
||||
title: 'Alert for "My Search" - 10 lines found',
|
||||
title: '🚨 Alert for "My Search" - 10 lines found',
|
||||
teamWebhooksById: new Map<string, typeof webhook>([
|
||||
[webhook._id.toString(), webhook],
|
||||
]),
|
||||
|
|
@ -442,12 +497,12 @@ describe('checkAlerts', () => {
|
|||
1,
|
||||
'https://hooks.slack.com/services/123',
|
||||
{
|
||||
text: 'Alert for "My Search" - 10 lines found',
|
||||
text: '🚨 Alert for "My Search" - 10 lines found',
|
||||
blocks: [
|
||||
{
|
||||
text: {
|
||||
text: [
|
||||
'*<http://app:8080/search/fake-saved-search-id?from=1679091183103&to=1679091239103&isLive=false | Alert for "My Search" - 10 lines found>*',
|
||||
'*<http://app:8080/search/fake-saved-search-id?from=1679091183103&to=1679091239103&isLive=false | 🚨 Alert for "My Search" - 10 lines found>*',
|
||||
'Group: "http"',
|
||||
'10 lines found, expected less than 1 lines',
|
||||
'Time Range (UTC): [Mar 17 10:13:03 PM - Mar 17 10:13:59 PM)',
|
||||
|
|
@ -517,7 +572,7 @@ describe('checkAlerts', () => {
|
|||
},
|
||||
},
|
||||
},
|
||||
title: 'Alert for "My Search" - 10 lines found',
|
||||
title: '🚨 Alert for "My Search" - 10 lines found',
|
||||
teamWebhooksById,
|
||||
});
|
||||
|
||||
|
|
@ -541,7 +596,7 @@ describe('checkAlerts', () => {
|
|||
host: 'web2',
|
||||
},
|
||||
},
|
||||
title: 'Alert for "My Search" - 10 lines found',
|
||||
title: '🚨 Alert for "My Search" - 10 lines found',
|
||||
teamWebhooksById,
|
||||
});
|
||||
|
||||
|
|
@ -549,12 +604,12 @@ describe('checkAlerts', () => {
|
|||
expect(slack.postMessageToWebhook).toHaveBeenCalledWith(
|
||||
'https://hooks.slack.com/services/123',
|
||||
{
|
||||
text: 'Alert for "My Search" - 10 lines found',
|
||||
text: '🚨 Alert for "My Search" - 10 lines found',
|
||||
blocks: [
|
||||
{
|
||||
text: {
|
||||
text: [
|
||||
'*<http://app:8080/search/fake-saved-search-id?from=1679091183103&to=1679091239103&isLive=false | Alert for "My Search" - 10 lines found>*',
|
||||
'*<http://app:8080/search/fake-saved-search-id?from=1679091183103&to=1679091239103&isLive=false | 🚨 Alert for "My Search" - 10 lines found>*',
|
||||
'Group: "http"',
|
||||
'10 lines found, expected less than 1 lines',
|
||||
'Time Range (UTC): [Mar 17 10:13:03 PM - Mar 17 10:13:59 PM)',
|
||||
|
|
@ -578,12 +633,12 @@ describe('checkAlerts', () => {
|
|||
expect(slack.postMessageToWebhook).toHaveBeenCalledWith(
|
||||
'https://hooks.slack.com/services/456',
|
||||
{
|
||||
text: 'Alert for "My Search" - 10 lines found',
|
||||
text: '🚨 Alert for "My Search" - 10 lines found',
|
||||
blocks: [
|
||||
{
|
||||
text: {
|
||||
text: [
|
||||
'*<http://app:8080/search/fake-saved-search-id?from=1679091183103&to=1679091239103&isLive=false | Alert for "My Search" - 10 lines found>*',
|
||||
'*<http://app:8080/search/fake-saved-search-id?from=1679091183103&to=1679091239103&isLive=false | 🚨 Alert for "My Search" - 10 lines found>*',
|
||||
'Group: "http"',
|
||||
'10 lines found, expected less than 1 lines',
|
||||
'Time Range (UTC): [Mar 17 10:13:03 PM - Mar 17 10:13:59 PM)',
|
||||
|
|
@ -605,6 +660,63 @@ describe('checkAlerts', () => {
|
|||
},
|
||||
);
|
||||
});
|
||||
|
||||
it('renderAlertTemplate - resolved alert with simplified message', async () => {
|
||||
const team = await createTeam({ name: 'My Team' });
|
||||
const webhook = await new Webhook({
|
||||
team: team._id,
|
||||
service: 'slack',
|
||||
url: 'https://hooks.slack.com/services/123',
|
||||
name: 'My_Webhook',
|
||||
}).save();
|
||||
|
||||
await renderAlertTemplate({
|
||||
alertProvider,
|
||||
clickhouseClient: {} as any,
|
||||
metadata: {} as any,
|
||||
state: AlertState.OK, // Resolved state
|
||||
template: '@webhook-My_Webhook',
|
||||
view: {
|
||||
...defaultSearchView,
|
||||
alert: {
|
||||
...defaultSearchView.alert,
|
||||
channel: {
|
||||
type: null, // using template instead
|
||||
},
|
||||
},
|
||||
},
|
||||
title: '✅ Alert for "My Search" - 10 lines found',
|
||||
teamWebhooksById: new Map<string, typeof webhook>([
|
||||
[webhook._id.toString(), webhook],
|
||||
]),
|
||||
});
|
||||
|
||||
expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(1);
|
||||
expect(slack.postMessageToWebhook).toHaveBeenCalledWith(
|
||||
'https://hooks.slack.com/services/123',
|
||||
{
|
||||
text: '✅ Alert for "My Search" - 10 lines found',
|
||||
blocks: [
|
||||
{
|
||||
text: {
|
||||
text: expect.stringContaining('The alert has been resolved'),
|
||||
type: 'mrkdwn',
|
||||
},
|
||||
type: 'section',
|
||||
},
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
// Verify the message includes the time range but not detailed logs
|
||||
const callArgs = (slack.postMessageToWebhook as any).mock.calls[0][1];
|
||||
const messageText = callArgs.blocks[0].text.text;
|
||||
expect(messageText).toContain('The alert has been resolved');
|
||||
expect(messageText).toContain('Time Range (UTC):');
|
||||
expect(messageText).toContain('Group: "http"');
|
||||
// Should NOT contain detailed log data
|
||||
expect(messageText).not.toContain('lines found, expected');
|
||||
});
|
||||
});
|
||||
|
||||
describe('processAlert', () => {
|
||||
|
|
@ -915,7 +1027,7 @@ describe('checkAlerts', () => {
|
|||
1,
|
||||
'https://hooks.slack.com/services/123',
|
||||
{
|
||||
text: 'Alert for "My Search" - 3 lines found',
|
||||
text: '🚨 Alert for "My Search" - 3 lines found',
|
||||
blocks: [
|
||||
{
|
||||
text: expect.any(Object),
|
||||
|
|
@ -928,7 +1040,7 @@ describe('checkAlerts', () => {
|
|||
2,
|
||||
'https://hooks.slack.com/services/123',
|
||||
{
|
||||
text: 'Alert for "My Search" - 1 lines found',
|
||||
text: '🚨 Alert for "My Search" - 1 lines found',
|
||||
blocks: [
|
||||
{
|
||||
text: expect.any(Object),
|
||||
|
|
@ -1089,12 +1201,12 @@ describe('checkAlerts', () => {
|
|||
1,
|
||||
'https://hooks.slack.com/services/123',
|
||||
{
|
||||
text: 'Alert for "Logs Count" in "My Dashboard" - 3 exceeds 1',
|
||||
text: '🚨 Alert for "Logs Count" in "My Dashboard" - 3 exceeds 1',
|
||||
blocks: [
|
||||
{
|
||||
text: {
|
||||
text: [
|
||||
`*<http://app:8080/dashboards/${dashboard._id}?from=1700170200000&granularity=5+minute&to=1700174700000 | Alert for "Logs Count" in "My Dashboard" - 3 exceeds 1>*`,
|
||||
`*<http://app:8080/dashboards/${dashboard._id}?from=1700170200000&granularity=5+minute&to=1700174700000 | 🚨 Alert for "Logs Count" in "My Dashboard" - 3 exceeds 1>*`,
|
||||
'',
|
||||
'3 exceeds 1',
|
||||
'Time Range (UTC): [Nov 16 10:05:00 PM - Nov 16 10:10:00 PM)',
|
||||
|
|
@ -1281,7 +1393,7 @@ describe('checkAlerts', () => {
|
|||
expect(fetchMock).toHaveBeenCalledWith('https://webhook.site/123', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({
|
||||
text: `http://app:8080/dashboards/${dashboard.id}?from=1700170200000&granularity=5+minute&to=1700174700000 | Alert for "Logs Count" in "My Dashboard" - 3 exceeds 1`,
|
||||
text: `http://app:8080/dashboards/${dashboard.id}?from=1700170200000&granularity=5+minute&to=1700174700000 | 🚨 Alert for "Logs Count" in "My Dashboard" - 3 exceeds 1`,
|
||||
}),
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
|
|
@ -2087,12 +2199,12 @@ describe('checkAlerts', () => {
|
|||
1,
|
||||
'https://hooks.slack.com/services/123',
|
||||
{
|
||||
text: 'Alert for "CPU" in "My Dashboard" - 6.25 exceeds 1',
|
||||
text: '🚨 Alert for "CPU" in "My Dashboard" - 6.25 exceeds 1',
|
||||
blocks: [
|
||||
{
|
||||
text: {
|
||||
text: [
|
||||
`*<http://app:8080/dashboards/${dashboard._id}?from=1700170200000&granularity=5+minute&to=1700174700000 | Alert for "CPU" in "My Dashboard" - 6.25 exceeds 1>*`,
|
||||
`*<http://app:8080/dashboards/${dashboard._id}?from=1700170200000&granularity=5+minute&to=1700174700000 | 🚨 Alert for "CPU" in "My Dashboard" - 6.25 exceeds 1>*`,
|
||||
'',
|
||||
'6.25 exceeds 1',
|
||||
'Time Range (UTC): [Nov 16 10:05:00 PM - Nov 16 10:10:00 PM)',
|
||||
|
|
|
|||
|
|
@ -147,6 +147,7 @@ const fireChannelEvent = async ({
|
|||
title: buildAlertMessageTemplateTitle({
|
||||
template: alert.name,
|
||||
view: templateView,
|
||||
state,
|
||||
}),
|
||||
template: alert.message,
|
||||
view: templateView,
|
||||
|
|
|
|||
|
|
@ -83,6 +83,10 @@ interface Message {
|
|||
eventId: string;
|
||||
}
|
||||
|
||||
export const isAlertResolved = (state?: AlertState): boolean => {
|
||||
return state === AlertState.OK;
|
||||
};
|
||||
|
||||
export const notifyChannel = async ({
|
||||
channel,
|
||||
message,
|
||||
|
|
@ -304,20 +308,27 @@ export const buildAlertMessageTemplateHdxLink = (
|
|||
export const buildAlertMessageTemplateTitle = ({
|
||||
template,
|
||||
view,
|
||||
state,
|
||||
}: {
|
||||
template?: string | null;
|
||||
view: AlertMessageTemplateDefaultView;
|
||||
state?: AlertState;
|
||||
}) => {
|
||||
const { alert, dashboard, savedSearch, value } = view;
|
||||
const handlebars = createHandlebarsWithHelpers();
|
||||
|
||||
// Add emoji prefix based on alert state
|
||||
const emoji = isAlertResolved(state) ? '✅ ' : '🚨 ';
|
||||
|
||||
if (alert.source === AlertSource.SAVED_SEARCH) {
|
||||
if (savedSearch == null) {
|
||||
throw new Error(`Source is ${alert.source} but savedSearch is null`);
|
||||
}
|
||||
// TODO: using template engine to render the title
|
||||
return template
|
||||
const baseTitle = template
|
||||
? handlebars.compile(template)(view)
|
||||
: `Alert for "${savedSearch.name}" - ${value} lines found`;
|
||||
return `${emoji}${baseTitle}`;
|
||||
} else if (alert.source === AlertSource.TILE) {
|
||||
if (dashboard == null) {
|
||||
throw new Error(`Source is ${alert.source} but dashboard is null`);
|
||||
|
|
@ -328,7 +339,7 @@ export const buildAlertMessageTemplateTitle = ({
|
|||
`Tile with id ${alert.tileId} not found in dashboard ${dashboard.name}`,
|
||||
);
|
||||
}
|
||||
return template
|
||||
const baseTitle = template
|
||||
? handlebars.compile(template)(view)
|
||||
: `Alert for "${tile.config.name}" in "${dashboard.name}" - ${value} ${
|
||||
doesExceedThreshold(alert.thresholdType, alert.threshold, value)
|
||||
|
|
@ -339,6 +350,7 @@ export const buildAlertMessageTemplateTitle = ({
|
|||
? 'falls below'
|
||||
: 'exceeds'
|
||||
} ${alert.threshold}`;
|
||||
return `${emoji}${baseTitle}`;
|
||||
}
|
||||
|
||||
throw new Error(`Unsupported alert source: ${(alert as any).source}`);
|
||||
|
|
@ -527,9 +539,14 @@ export const renderAlertTemplate = async ({
|
|||
})})`;
|
||||
let rawTemplateBody;
|
||||
|
||||
// For resolved alerts, use a simple message instead of fetching data
|
||||
if (isAlertResolved(state)) {
|
||||
rawTemplateBody = `${group ? `Group: "${group}" - ` : ''}The alert has been resolved.\n${timeRangeMessage}
|
||||
${targetTemplate}`;
|
||||
}
|
||||
// TODO: support advanced routing with template engine
|
||||
// users should be able to use '@' syntax to trigger alerts
|
||||
if (alert.source === AlertSource.SAVED_SEARCH) {
|
||||
else if (alert.source === AlertSource.SAVED_SEARCH) {
|
||||
if (savedSearch == null) {
|
||||
throw new Error(`Source is ${alert.source} but savedSearch is null`);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue