fix: correct handling of gauge metrics in renderChartConfig (#654)

2026-04-21 13:37:15 +00:00 · 2025-03-05 16:06:57 -08:00 · 2025-03-05 16:06:57 -08:00 · cd0e4fd71c
commit cd0e4fd71c
parent a8a1f81124
6 changed files with 290 additions and 28 deletions
--- a/.changeset/tasty-bats-refuse.md
+++ b/.changeset/tasty-bats-refuse.md
@ -0,0 +1,7 @@
+---
+"@hyperdx/common-utils": patch
+"@hyperdx/api": patch
+"@hyperdx/app": patch
+---
+
+fix: correct handling of gauge metrics in renderChartConfig
--- a/packages/api/src/clickhouse/tests/snapshots/renderChartConfig.test.ts.snap
+++ b/packages/api/src/clickhouse/tests/snapshots/renderChartConfig.test.ts.snap
@ -34,6 +34,83 @@ Array [
 ]
 `;

+exports[`renderChartConfig Query Metrics single avg gauge with group-by 1`] = `
+Array [
+  Object {
+    "__hdx_time_bucket": "2022-01-05T00:00:00Z",
+    "arrayElement(ResourceAttributes, 'host')": "host2",
+    "avg(toFloat64OrNull(toString(LastValue)))": 4,
+  },
+  Object {
+    "__hdx_time_bucket": "2022-01-05T00:00:00Z",
+    "arrayElement(ResourceAttributes, 'host')": "host1",
+    "avg(toFloat64OrNull(toString(LastValue)))": 6.25,
+  },
+  Object {
+    "__hdx_time_bucket": "2022-01-05T00:05:00Z",
+    "arrayElement(ResourceAttributes, 'host')": "host2",
+    "avg(toFloat64OrNull(toString(LastValue)))": 4,
+  },
+  Object {
+    "__hdx_time_bucket": "2022-01-05T00:05:00Z",
+    "arrayElement(ResourceAttributes, 'host')": "host1",
+    "avg(toFloat64OrNull(toString(LastValue)))": 80,
+  },
+]
+`;
+
+exports[`renderChartConfig Query Metrics single avg gauge with where 1`] = `
+Array [
+  Object {
+    "__hdx_time_bucket": "2022-01-05T00:00:00Z",
+    "avg(toFloat64OrNull(toString(LastValue)))": 6.25,
+  },
+  Object {
+    "__hdx_time_bucket": "2022-01-05T00:05:00Z",
+    "avg(toFloat64OrNull(toString(LastValue)))": 80,
+  },
+]
+`;
+
+exports[`renderChartConfig Query Metrics single max/avg/sum gauge 1`] = `
+Array [
+  Object {
+    "__hdx_time_bucket": "2022-01-05T00:00:00Z",
+    "avg(toFloat64OrNull(toString(LastValue)))": 5.125,
+  },
+  Object {
+    "__hdx_time_bucket": "2022-01-05T00:05:00Z",
+    "avg(toFloat64OrNull(toString(LastValue)))": 42,
+  },
+]
+`;
+
+exports[`renderChartConfig Query Metrics single max/avg/sum gauge 2`] = `
+Array [
+  Object {
+    "__hdx_time_bucket": "2022-01-05T00:00:00Z",
+    "max(toFloat64OrNull(toString(LastValue)))": 6.25,
+  },
+  Object {
+    "__hdx_time_bucket": "2022-01-05T00:05:00Z",
+    "max(toFloat64OrNull(toString(LastValue)))": 80,
+  },
+]
+`;
+
+exports[`renderChartConfig Query Metrics single max/avg/sum gauge 3`] = `
+Array [
+  Object {
+    "__hdx_time_bucket": "2022-01-05T00:00:00Z",
+    "sum(toFloat64OrNull(toString(LastValue)))": 10.25,
+  },
+  Object {
+    "__hdx_time_bucket": "2022-01-05T00:05:00Z",
+    "sum(toFloat64OrNull(toString(LastValue)))": 84,
+  },
+]
+`;
+
 exports[`renderChartConfig Query Metrics single sum rate 1`] = `
 Array [
  Object {
--- a/packages/api/src/clickhouse/tests/renderChartConfig.test.ts
+++ b/packages/api/src/clickhouse/tests/renderChartConfig.test.ts
@ -340,15 +340,8 @@ describe('renderChartConfig', () => {
      ]);
    });

-    it.skip('gauge (last value)', async () => {
-      // IMPLEMENT ME (last_value aggregation)
-    });
-
-    // FIXME: gauge avg doesn't work as expected (it should pull the average of the last value)
-    // in this case
-    // (6.25 + 4) / 2, (80 + 4) / 2
-    it('single avg gauge', async () => {
-      const query = await renderChartConfig(
+    it('single max/avg/sum gauge', async () => {
+      const avgQuery = await renderChartConfig(
        {
          select: [
            {
@ -372,6 +365,115 @@ describe('renderChartConfig', () => {
        },
        metadata,
      );
+      expect(await queryData(avgQuery)).toMatchSnapshot();
+      const maxQuery = await renderChartConfig(
+        {
+          select: [
+            {
+              aggFn: 'max',
+              metricName: 'test.cpu',
+              metricType: MetricsDataType.Gauge,
+              valueExpression: 'Value',
+            },
+          ],
+          from: metricSource.from,
+          where: '',
+          metricTables: {
+            sum: DEFAULT_METRICS_TABLE.SUM,
+            gauge: DEFAULT_METRICS_TABLE.GAUGE,
+            histogram: DEFAULT_METRICS_TABLE.HISTOGRAM,
+          },
+          dateRange: [new Date(now), new Date(now + ms('10m'))],
+          granularity: '5 minute',
+          timestampValueExpression: metricSource.timestampValueExpression,
+          connection: connection.id,
+        },
+        metadata,
+      );
+      expect(await queryData(maxQuery)).toMatchSnapshot();
+      const sumQuery = await renderChartConfig(
+        {
+          select: [
+            {
+              aggFn: 'sum',
+              metricName: 'test.cpu',
+              metricType: MetricsDataType.Gauge,
+              valueExpression: 'Value',
+            },
+          ],
+          from: metricSource.from,
+          where: '',
+          metricTables: {
+            sum: DEFAULT_METRICS_TABLE.SUM,
+            gauge: DEFAULT_METRICS_TABLE.GAUGE,
+            histogram: DEFAULT_METRICS_TABLE.HISTOGRAM,
+          },
+          dateRange: [new Date(now), new Date(now + ms('10m'))],
+          granularity: '5 minute',
+          timestampValueExpression: metricSource.timestampValueExpression,
+          connection: connection.id,
+        },
+        metadata,
+      );
+      expect(await queryData(sumQuery)).toMatchSnapshot();
+    });
+
+    it('single avg gauge with where', async () => {
+      const query = await renderChartConfig(
+        {
+          select: [
+            {
+              aggFn: 'avg',
+              metricName: 'test.cpu',
+              metricType: MetricsDataType.Gauge,
+              valueExpression: 'Value',
+            },
+          ],
+          from: metricSource.from,
+          where: `ResourceAttributes['host'] = 'host1'`,
+          whereLanguage: 'sql',
+          metricTables: {
+            sum: DEFAULT_METRICS_TABLE.SUM,
+            gauge: DEFAULT_METRICS_TABLE.GAUGE,
+            histogram: DEFAULT_METRICS_TABLE.HISTOGRAM,
+          },
+          dateRange: [new Date(now), new Date(now + ms('10m'))],
+          granularity: '5 minute',
+          timestampValueExpression: metricSource.timestampValueExpression,
+          connection: connection.id,
+        },
+        metadata,
+      );
+      expect(await queryData(query)).toMatchSnapshot();
+    });
+
+    it('single avg gauge with group-by', async () => {
+      const query = await renderChartConfig(
+        {
+          select: [
+            {
+              aggFn: 'avg',
+              metricName: 'test.cpu',
+              metricType: MetricsDataType.Gauge,
+              valueExpression: 'Value',
+            },
+          ],
+          from: metricSource.from,
+          where: '',
+          metricTables: {
+            sum: DEFAULT_METRICS_TABLE.SUM,
+            gauge: DEFAULT_METRICS_TABLE.GAUGE,
+            histogram: DEFAULT_METRICS_TABLE.HISTOGRAM,
+          },
+          dateRange: [new Date(now), new Date(now + ms('10m'))],
+          granularity: '5 minute',
+          groupBy: `ResourceAttributes['host']`,
+          timestampValueExpression: metricSource.timestampValueExpression,
+          connection: connection.id,
+        },
+        metadata,
+      );
+      expect(await queryData(query)).toMatchSnapshot();
    });

    it('single sum rate', async () => {
--- a/packages/common-utils/src/tests/snapshots/renderChartConfig.test.ts.snap
+++ b/packages/common-utils/src/tests/snapshots/renderChartConfig.test.ts.snap
@ -0,0 +1,29 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`renderChartConfig should generate sql for a single gauge metric 1`] = `
+"WITH Bucketed AS (
+            SELECT
+              toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS \`__hdx_time_bucket2\`,
+              ScopeAttributes,
+              ResourceAttributes,
+              Attributes,
+              cityHash64(mapConcat(ScopeAttributes, ResourceAttributes, Attributes)) AS AttributesHash,
+              last_value(Value) AS LastValue,
+              any(ResourceSchemaUrl) AS ResourceSchemaUrl,
+              any(ScopeName) AS ScopeName,
+              any(ScopeVersion) AS ScopeVersion,
+              any(ScopeDroppedAttrCount) AS ScopeDroppedAttrCount,
+              any(ScopeSchemaUrl) AS ScopeSchemaUrl,
+              any(ServiceName) AS ServiceName,
+              any(MetricDescription) AS MetricDescription,
+              any(MetricUnit) AS MetricUnit,
+              any(StartTimeUnix) AS StartTimeUnix,
+              any(Flags) AS Flags
+            FROM default.otel_metrics_gauge
+            WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1765670400000)) AND ((MetricName = 'nodejs.event_loop.utilization'))
+            GROUP BY ScopeAttributes, ResourceAttributes, Attributes, __hdx_time_bucket2
+            ORDER BY AttributesHash, __hdx_time_bucket2
+          ) SELECT quantile(0.95)(toFloat64OrNull(toString(LastValue))),toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (__hdx_time_bucket2 >= fromUnixTimestamp64Milli(1739318400000) AND __hdx_time_bucket2 <= fromUnixTimestamp64Milli(1765670400000)) GROUP BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` WITH FILL FROM toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(1739318400000), INTERVAL 1 minute))
+      TO toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(1765670400000), INTERVAL 1 minute))
+      STEP 60 LIMIT 10"
+`;
--- a/packages/common-utils/src/tests/renderChartConfig.test.ts
+++ b/packages/common-utils/src/tests/renderChartConfig.test.ts
@ -17,7 +17,7 @@ describe('renderChartConfig', () => {
        { name: 'timestamp', type: 'DateTime' },
        { name: 'value', type: 'Float64' },
      ]),
-      getMaterializedColumnsLookupTable: jest.fn().mockResolvedValue({}),
+      getMaterializedColumnsLookupTable: jest.fn().mockResolvedValue(null),
      getColumn: jest.fn().mockResolvedValue({ type: 'DateTime' }),
    } as unknown as Metadata;
  });
@ -57,16 +57,7 @@ describe('renderChartConfig', () => {

    const generatedSql = await renderChartConfig(config, mockMetadata);
    const actual = parameterizedQueryToSql(generatedSql);
-    expect(actual).toBe(
-      'SELECT quantile(0.95)(toFloat64OrNull(toString(Value))),toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS `__hdx_time_bucket`' +
-        ' FROM default.otel_metrics_gauge WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1765670400000)) AND' +
-        " (MetricName = 'nodejs.event_loop.utilization') GROUP BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS `__hdx_time_bucket`" +
-        ' ORDER BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS `__hdx_time_bucket`' +
-        ' WITH FILL FROM toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(1739318400000), INTERVAL 1 minute))\n' +
-        '      TO toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(1765670400000), INTERVAL 1 minute))\n' +
-        '      STEP 60' +
-        ' LIMIT 10',
-    );
+    expect(actual).toMatchSnapshot();
  });

  it('should generate sql for a single sum metric', async () => {
--- a/packages/common-utils/src/renderChartConfig.ts
+++ b/packages/common-utils/src/renderChartConfig.ts
@ -38,6 +38,7 @@ function determineTableName(select: SelectSQLStatement): string {
  return '';
 }

+const DEFAULT_METRIC_TABLE_TIME_COLUMN = 'TimeUnix';
 export const FIXED_TIME_BUCKET_EXPR_ALIAS = '__hdx_time_bucket';

 export function isUsingGroupBy(
@ -784,9 +785,10 @@ function renderFill(
  return undefined;
 }

-function translateMetricChartConfig(
+async function translateMetricChartConfig(
  chartConfig: ChartConfigWithOptDateRange,
-): ChartConfigWithOptDateRangeEx {
+  metadata: Metadata,
+): Promise<ChartConfigWithOptDateRangeEx> {
  const metricTables = chartConfig.metricTables;
  if (!metricTables) {
    return chartConfig;
@ -800,20 +802,74 @@ function translateMetricChartConfig(

  const { metricType, metricName, ..._select } = select[0]; // Initial impl only supports one metric select per chart config
  if (metricType === MetricsDataType.Gauge && metricName) {
+    const timeBucketCol = '__hdx_time_bucket2';
+    const timeExpr = timeBucketExpr({
+      interval: chartConfig.granularity || 'auto',
+      timestampValueExpression:
+        chartConfig.timestampValueExpression ||
+        DEFAULT_METRIC_TABLE_TIME_COLUMN,
+      dateRange: chartConfig.dateRange,
+      alias: timeBucketCol,
+    });
+
+    const where = await renderWhere(
+      {
+        ...chartConfig,
+        from: {
+          ...from,
+          tableName: metricTables[MetricsDataType.Gauge],
+        },
+        filters: [
+          {
+            type: 'sql',
+            condition: `MetricName = '${metricName}'`,
+          },
+        ],
+      },
+      metadata,
+    );
+
    return {
      ...restChartConfig,
+      with: [
+        {
+          name: 'Bucketed',
+          sql: chSql`
+            SELECT
+              ${timeExpr},
+              ScopeAttributes,
+              ResourceAttributes,
+              Attributes,
+              cityHash64(mapConcat(ScopeAttributes, ResourceAttributes, Attributes)) AS AttributesHash,
+              last_value(Value) AS LastValue,
+              any(ResourceSchemaUrl) AS ResourceSchemaUrl,
+              any(ScopeName) AS ScopeName,
+              any(ScopeVersion) AS ScopeVersion,
+              any(ScopeDroppedAttrCount) AS ScopeDroppedAttrCount,
+              any(ScopeSchemaUrl) AS ScopeSchemaUrl,
+              any(ServiceName) AS ServiceName,
+              any(MetricDescription) AS MetricDescription,
+              any(MetricUnit) AS MetricUnit,
+              any(StartTimeUnix) AS StartTimeUnix,
+              any(Flags) AS Flags
+            FROM ${renderFrom({ from: { ...from, tableName: metricTables[MetricsDataType.Gauge] } })}
+            WHERE ${where}
+            GROUP BY ScopeAttributes, ResourceAttributes, Attributes, ${timeBucketCol}
+            ORDER BY AttributesHash, ${timeBucketCol}
+          `,
+        },
+      ],
      select: [
        {
          ..._select,
-          valueExpression: 'Value',
+          valueExpression: 'LastValue',
        },
      ],
      from: {
-        ...from,
-        tableName: metricTables[MetricsDataType.Gauge],
+        databaseName: '',
+        tableName: 'Bucketed',
      },
-      where: `MetricName = '${metricName}'`,
-      whereLanguage: 'sql',
+      timestampValueExpression: timeBucketCol,
    };
  } else if (metricType === MetricsDataType.Sum && metricName) {
    return {
@ -921,7 +977,7 @@ export async function renderChartConfig(
  // but goes through the same generation process
  const chartConfig =
    rawChartConfig.metricTables != null
-      ? translateMetricChartConfig(rawChartConfig)
+      ? await translateMetricChartConfig(rawChartConfig, metadata)
      : rawChartConfig;

  const withClauses = renderWith(chartConfig, metadata);