feat: alert template message pt4 (#338)

Introduce conditional alert routing helper `#is_match`
2026-04-21 13:37:15 +00:00 · 2024-03-10 22:48:38 -07:00 · 2024-03-10 22:48:38 -07:00 · b454003335
commit b454003335
parent 0e365bfff2
5 changed files with 262 additions and 17 deletions
--- a/.changeset/green-bulldogs-behave.md
+++ b/.changeset/green-bulldogs-behave.md
@ -0,0 +1,6 @@
+---
+'@hyperdx/api': patch
+'@hyperdx/app': patch
+---
+
+feat: introduce conditional alert routing helper #is_match
--- a/packages/api/src/clickhouse/tests/clickhouse.test.ts
+++ b/packages/api/src/clickhouse/tests/clickhouse.test.ts
@ -1271,6 +1271,9 @@ Array [
    expect(data).toMatchInlineSnapshot(`
 Array [
  Object {
+    "attributes": Object {
+      "testGroup": "group2",
+    },
    "data": 777,
    "group": Array [
      "group2",
@ -1278,6 +1281,9 @@ Array [
    "ts_bucket": 1641340800,
  },
  Object {
+    "attributes": Object {
+      "testGroup": "group1",
+    },
    "data": 77,
    "group": Array [
      "group1",
@ -1285,6 +1291,9 @@ Array [
    "ts_bucket": 1641340800,
  },
  Object {
+    "attributes": Object {
+      "testGroup": "group1",
+    },
    "data": 7,
    "group": Array [
      "group1",
--- a/packages/api/src/clickhouse/index.ts
+++ b/packages/api/src/clickhouse/index.ts
@ -1818,21 +1818,39 @@ export const getMultiSeriesChartLegacyFormat = async ({

  const flatData = result.data.flatMap(row => {
    if (seriesReturnType === 'column') {
-      return series.map((_, i) => {
+      return series.map((s, i) => {
+        const groupBy =
+          s.type === 'number' ? [] : 'groupBy' in s ? s.groupBy : [];
+        const attributes = groupBy.reduce((acc, curVal, curIndex) => {
+          acc[curVal] = row.group[curIndex];
+          return acc;
+        }, {} as Record<string, string>);
        return {
-          ts_bucket: row.ts_bucket,
-          group: row.group,
+          attributes,
          data: row[`series_${i}.data`],
+          group: row.group,
+          ts_bucket: row.ts_bucket,
        };
      });
    }

    // Ratio only has 1 series
+    const groupBy =
+      series[0].type === 'number'
+        ? []
+        : 'groupBy' in series[0]
+        ? series[0].groupBy
+        : [];
+    const attributes = groupBy.reduce((acc, curVal, curIndex) => {
+      acc[curVal] = row.group[curIndex];
+      return acc;
+    }, {} as Record<string, string>);
    return [
      {
-        ts_bucket: row.ts_bucket,
-        group: row.group,
+        attributes,
        data: row['series_0.data'],
+        group: row.group,
+        ts_bucket: row.ts_bucket,
      },
    ];
  });
@ -2550,8 +2568,9 @@ export const checkAlert = async ({
    `
      SELECT 
        ?
-        count(*) as data,
-        toUnixTimestamp(toStartOfInterval(timestamp, INTERVAL ?)) as ts_bucket
+        count(*) AS data,
+        any(_string_attributes) AS attributes,
+        toUnixTimestamp(toStartOfInterval(timestamp, INTERVAL ?)) AS ts_bucket
      FROM ??
      WHERE ? AND (?)
      GROUP BY ?
@ -2596,7 +2615,12 @@ export const checkAlert = async ({
    },
  });
  const result = await rows.json<
-    ResponseJSON<{ data: string; group?: string; ts_bucket: number }>
+    ResponseJSON<{
+      data: string;
+      group?: string;
+      ts_bucket: number;
+      attributes: Record<string, string>;
+    }>
  >();
  logger.info({
    message: 'checkAlert',
--- a/packages/api/src/tasks/tests/checkAlerts.test.ts
+++ b/packages/api/src/tasks/tests/checkAlerts.test.ts
@ -283,9 +283,16 @@ describe('checkAlerts', () => {
      ).toMatchInlineSnapshot(
        `"{{__hdx_notify_channel__ channel=\\"action\\" id=\\"id-with-multiple-dashes\\"}}"`,
      );
+
+      // custom template id
+      expect(
+        translateExternalActionsToInternal('@action-{{action_id}}'),
+      ).toMatchInlineSnapshot(
+        `"{{__hdx_notify_channel__ channel=\\"action\\" id=\\"{{action_id}}\\"}}"`,
+      );
    });

-    it('renderAlertTemplate', async () => {
+    it('renderAlertTemplate - custom body with single action', async () => {
      jest
        .spyOn(slack, 'postMessageToWebhook')
        .mockResolvedValueOnce(null as any);
@ -356,6 +363,180 @@ describe('checkAlerts', () => {
        },
      );
    });
+
+    it('renderAlertTemplate - single action with custom action id', async () => {
+      jest
+        .spyOn(slack, 'postMessageToWebhook')
+        .mockResolvedValueOnce(null as any);
+      jest.spyOn(clickhouse, 'getLogBatch').mockResolvedValueOnce({
+        data: [
+          {
+            timestamp: '2023-11-16T22:10:00.000Z',
+            severity_text: 'error',
+            body: 'Oh no! Something went wrong!',
+          },
+          {
+            timestamp: '2023-11-16T22:15:00.000Z',
+            severity_text: 'info',
+            body: 'All good!',
+          },
+        ],
+      } as any);
+
+      const team = await createTeam({ name: 'My Team' });
+      await new Webhook({
+        team: team._id,
+        service: 'slack',
+        url: 'https://hooks.slack.com/services/123',
+        name: 'My_Webhook',
+      }).save();
+
+      await renderAlertTemplate({
+        template: 'Custom body @slack_webhook-{{attributes.webhookName}}', // partial name should work
+        view: {
+          ...defaultSearchView,
+          alert: {
+            ...defaultSearchView.alert,
+            channel: {
+              type: null, // using template instead
+            },
+          },
+          attributes: {
+            webhookName: 'My_Webhook',
+          },
+          team: {
+            id: team._id.toString(),
+            logStreamTableVersion: team.logStreamTableVersion,
+          },
+        },
+        title: 'Alert for "My Search" - 10 lines found',
+      });
+
+      expect(slack.postMessageToWebhook).toHaveBeenNthCalledWith(
+        1,
+        'https://hooks.slack.com/services/123',
+        {
+          text: 'Alert for "My Search" - 10 lines found',
+          blocks: [
+            {
+              text: {
+                text: [
+                  '*<http://localhost:9090/search/id-123?from=1679091183103&to=1679091239103&q=level%3Aerror+span_name%3A%22http%22 | Alert for "My Search" - 10 lines found>*',
+                  'Group: "http"',
+                  '10 lines found, expected less than 1 lines',
+                  'Custom body ',
+                  '```',
+                  'Nov 16 22:10:00Z [error] Oh no! Something went wrong!',
+                  'Nov 16 22:15:00Z [info] All good!',
+                  '```',
+                ].join('\n'),
+                type: 'mrkdwn',
+              },
+              type: 'section',
+            },
+          ],
+        },
+      );
+    });
+
+    it('renderAlertTemplate - #is_match with single action', async () => {
+      jest
+        .spyOn(slack, 'postMessageToWebhook')
+        .mockResolvedValueOnce(null as any);
+      jest.spyOn(clickhouse, 'getLogBatch').mockResolvedValueOnce({
+        data: [
+          {
+            timestamp: '2023-11-16T22:10:00.000Z',
+            severity_text: 'error',
+            body: 'Oh no! Something went wrong!',
+          },
+          {
+            timestamp: '2023-11-16T22:15:00.000Z',
+            severity_text: 'info',
+            body: 'All good!',
+          },
+        ],
+      } as any);
+
+      const team = await createTeam({ name: 'My Team' });
+      await new Webhook({
+        team: team._id,
+        service: 'slack',
+        url: 'https://hooks.slack.com/services/123',
+        name: 'My_Webhook',
+      }).save();
+
+      await renderAlertTemplate({
+        template:
+          '{{#is_match "host" "web"}} @slack_webhook-My_Web {{/is_match}}', // partial name should work
+        view: {
+          ...defaultSearchView,
+          alert: {
+            ...defaultSearchView.alert,
+            channel: {
+              type: null, // using template instead
+            },
+          },
+          team: {
+            id: team._id.toString(),
+            logStreamTableVersion: team.logStreamTableVersion,
+          },
+          attributes: {
+            host: 'web',
+          },
+        },
+        title: 'Alert for "My Search" - 10 lines found',
+      });
+
+      // @slack_webhook should not be called
+      await renderAlertTemplate({
+        template:
+          '{{#is_match "host" "web"}} @slack_webhook-My_Web {{/is_match}}', // partial name should work
+        view: {
+          ...defaultSearchView,
+          alert: {
+            ...defaultSearchView.alert,
+            channel: {
+              type: null, // using template instead
+            },
+          },
+          team: {
+            id: team._id.toString(),
+            logStreamTableVersion: team.logStreamTableVersion,
+          },
+          attributes: {
+            host: 'web2',
+          },
+        },
+        title: 'Alert for "My Search" - 10 lines found',
+      });
+
+      expect(slack.postMessageToWebhook).toHaveBeenNthCalledWith(
+        1,
+        'https://hooks.slack.com/services/123',
+        {
+          text: 'Alert for "My Search" - 10 lines found',
+          blocks: [
+            {
+              text: {
+                text: [
+                  '*<http://localhost:9090/search/id-123?from=1679091183103&to=1679091239103&q=level%3Aerror+span_name%3A%22http%22 | Alert for "My Search" - 10 lines found>*',
+                  'Group: "http"',
+                  '10 lines found, expected less than 1 lines',
+                  '',
+                  '```',
+                  'Nov 16 22:10:00Z [error] Oh no! Something went wrong!',
+                  'Nov 16 22:15:00Z [info] All good!',
+                  '```',
+                ].join('\n'),
+                type: 'mrkdwn',
+              },
+              type: 'section',
+            },
+          ],
+        },
+      );
+    });
  });

  describe('processAlert', () => {
--- a/packages/api/src/tasks/checkAlerts.ts
+++ b/packages/api/src/tasks/checkAlerts.ts
@ -3,7 +3,7 @@
 // --------------------------------------------------------
 import * as fns from 'date-fns';
 import * as fnsTz from 'date-fns-tz';
-import Handlebars from 'handlebars';
+import Handlebars, { HelperOptions } from 'handlebars';
 import { escapeRegExp, isString } from 'lodash';
 import mongoose from 'mongoose';
 import ms from 'ms';
@ -34,6 +34,7 @@ type EnhancedDashboard = Omit<IDashboard, 'team'> & { team: ITeam };

 const MAX_MESSAGE_LENGTH = 500;
 const NOTIFY_FN_NAME = '__hdx_notify_channel__';
+const IS_MATCH_FN_NAME = 'is_match';

 const getLogViewEnhanced = async (logViewId: ObjectId) => {
  const logView = await LogView.findById(logViewId).populate<{
@ -113,6 +114,7 @@ export const doesExceedThreshold = (
 type AlertMessageTemplateDefaultView = {
  // FIXME: do we want to include groupBy in the external alert schema?
  alert: z.infer<typeof externalAlertSchema> & { groupBy?: string };
+  attributes: Record<string, string>;
  dashboard: ReturnType<
    typeof translateDashboardDocumentToExternalDashboard
  > | null;
@ -386,7 +388,8 @@ export const getDefaultExternalAction = (

 export const translateExternalActionsToInternal = (template: string) => {
  // ex: @webhook-1234_5678 -> "{{NOTIFY_FN_NAME channel="webhook" id="1234_5678}}"
-  return template.replace(/(?: |^)@([a-zA-Z0-9.@_-]+)/g, (match, input) => {
+  // ex: @webhook-{{attributes.webhookId}} -> "{{NOTIFY_FN_NAME channel="webhook" id="{{attributes.webhookId}}"}}"
+  return template.replace(/(?: |^)@([a-zA-Z0-9.{}@_-]+)/g, (match, input) => {
    const prefix = match.startsWith(' ') ? ' ' : '';
    const [channel, ...ids] = input.split('-');
    const id = ids.join('-');
@ -407,6 +410,7 @@ export const renderAlertTemplate = async ({
 }) => {
  const {
    alert,
+    attributes,
    dashboard,
    endTime,
    group,
@ -427,26 +431,43 @@ export const renderAlertTemplate = async ({

  const _hb = Handlebars.create();
  _hb.registerHelper(NOTIFY_FN_NAME, () => null);
+  _hb.registerHelper(IS_MATCH_FN_NAME, () => null);
  const hb = PromisedHandlebars(Handlebars);
  const registerHelpers = (rawTemplateBody: string) => {
+    const attributesMap = new Map(Object.entries(attributes ?? {}));
+
+    hb.registerHelper(
+      IS_MATCH_FN_NAME,
+      function (
+        targetKey: string,
+        targetValue: string,
+        options: HelperOptions,
+      ) {
+        if (attributesMap.get(targetKey) === targetValue) {
+          options.fn(this);
+        }
+      },
+    );
+
    hb.registerHelper(
      NOTIFY_FN_NAME,
      async (options: { hash: Record<string, string> }) => {
        const { channel, id } = options.hash;
-        // const [channel, id] = input.split('-');
        if (channel !== 'slack_webhook') {
          throw new Error(`Unsupported channel type: ${channel}`);
        }
-        // rendered body template
-        const compiledTemplateBody = _hb.compile(rawTemplateBody);
+        // render id template
+        const renderedId = _hb.compile(id)(view);
+        // render body template
+        const renderedBody = _hb.compile(rawTemplateBody)(view);

        await notifyChannel({
          channel,
-          id,
+          id: renderedId,
          message: {
            hdxLink: buildAlertMessageTemplateHdxLink(view),
            title,
-            body: compiledTemplateBody(view),
+            body: renderedBody,
          },
          teamId: team.id,
        });
@ -533,6 +554,7 @@ ${targetTemplate}`;

 const fireChannelEvent = async ({
  alert,
+  attributes,
  dashboard,
  endTime,
  group,
@ -542,10 +564,11 @@ const fireChannelEvent = async ({
  windowSizeInMins,
 }: {
  alert: AlertDocument;
-  logView: Awaited<ReturnType<typeof getLogViewEnhanced>> | null;
+  attributes: Record<string, string>; // TODO: support other types than string
  dashboard: EnhancedDashboard | null;
  endTime: Date;
  group?: string;
+  logView: Awaited<ReturnType<typeof getLogViewEnhanced>> | null;
  startTime: Date;
  totalCount: number;
  windowSizeInMins: number;
@ -559,6 +582,7 @@ const fireChannelEvent = async ({
      ...translateAlertDocumentToExternalAlert(alert),
      groupBy: alert.groupBy,
    },
+    attributes,
    dashboard: dashboard
      ? translateDashboardDocumentToExternalDashboard({
          _id: dashboard._id,
@ -783,6 +807,7 @@ export const processAlert = async (now: Date, alert: AlertDocument) => {
          try {
            await fireChannelEvent({
              alert,
+              attributes: checkData.attributes,
              dashboard: targetDashboard,
              endTime: fns.addMinutes(bucketStart, windowSizeInMins),
              group: Array.isArray(checkData.group)