Add rate aggFn support for sum metrics (#77)

2026-04-21 13:37:15 +00:00 · 2023-10-31 00:06:17 -07:00 · 2023-10-31 00:06:17 -07:00 · 8cb0eac332
commit 8cb0eac332
parent 4d24bfac0a
7 changed files with 376 additions and 98 deletions
--- a/.changeset/empty-ears-kneel.md
+++ b/.changeset/empty-ears-kneel.md
@ -0,0 +1,6 @@
+---
+'@hyperdx/api': patch
+'@hyperdx/app': patch
+---
+
+Add rate function for sum metrics
--- a/packages/api/src/clickhouse/index.ts
+++ b/packages/api/src/clickhouse/index.ts
@ -44,15 +44,23 @@ export type SortOrder = 'asc' | 'desc' | null;

 export enum AggFn {
  Avg = 'avg',
+  AvgRate = 'avg_rate',
  Count = 'count',
  CountDistinct = 'count_distinct',
  Max = 'max',
+  MaxRate = 'max_rate',
  Min = 'min',
+  MinRate = 'min_rate',
  P50 = 'p50',
+  P50Rate = 'p50_rate',
  P90 = 'p90',
+  P90Rate = 'p90_rate',
  P95 = 'p95',
+  P95Rate = 'p95_rate',
  P99 = 'p99',
+  P99Rate = 'p99_rate',
  Sum = 'sum',
+  SumRate = 'sum_rate',
 }

 export enum Granularity {
@ -621,6 +629,19 @@ export const getMetricsTags = async (teamId: string) => {
  return result;
 };

+const isRateAggFn = (aggFn: AggFn) => {
+  return (
+    aggFn === AggFn.SumRate ||
+    aggFn === AggFn.AvgRate ||
+    aggFn === AggFn.MaxRate ||
+    aggFn === AggFn.MinRate ||
+    aggFn === AggFn.P50Rate ||
+    aggFn === AggFn.P90Rate ||
+    aggFn === AggFn.P95Rate ||
+    aggFn === AggFn.P99Rate
+  );
+};
+
 export const getMetricsChart = async ({
  aggFn,
  dataType,
@ -663,78 +684,100 @@ export const getMetricsChart = async ({
      : 'name AS group',
  ];

-  switch (dataType) {
-    case 'Gauge':
-      selectClause.push(
-        aggFn === AggFn.Count
-          ? 'COUNT(value) as data'
-          : aggFn === AggFn.Sum
-          ? `SUM(value) as data`
-          : aggFn === AggFn.Avg
-          ? `AVG(value) as data`
-          : aggFn === AggFn.Max
-          ? `MAX(value) as data`
-          : aggFn === AggFn.Min
-          ? `MIN(value) as data`
-          : `quantile(${
-              aggFn === AggFn.P50
-                ? '0.5'
-                : aggFn === AggFn.P90
-                ? '0.90'
-                : aggFn === AggFn.P95
-                ? '0.95'
-                : '0.99'
-            })(value) as data`,
-      );
-      break;
-    case 'Sum':
-      selectClause.push(
-        aggFn === AggFn.Count
-          ? 'COUNT(delta) as data'
-          : aggFn === AggFn.Sum
-          ? `SUM(delta) as data`
-          : aggFn === AggFn.Avg
-          ? `AVG(delta) as data`
-          : aggFn === AggFn.Max
-          ? `MAX(delta) as data`
-          : aggFn === AggFn.Min
-          ? `MIN(delta) as data`
-          : `quantile(${
-              aggFn === AggFn.P50
-                ? '0.5'
-                : aggFn === AggFn.P90
-                ? '0.90'
-                : aggFn === AggFn.P95
-                ? '0.95'
-                : '0.99'
-            })(delta) as data`,
-      );
-      break;
-    default:
-      logger.error(`Unsupported data type: ${dataType}`);
-      break;
+  const isRate = isRateAggFn(aggFn);
+
+  if (dataType === 'Gauge' || dataType === 'Sum') {
+    selectClause.push(
+      aggFn === AggFn.Count
+        ? 'COUNT(value) as data'
+        : aggFn === AggFn.Sum
+        ? `SUM(value) as data`
+        : aggFn === AggFn.Avg
+        ? `AVG(value) as data`
+        : aggFn === AggFn.Max
+        ? `MAX(value) as data`
+        : aggFn === AggFn.Min
+        ? `MIN(value) as data`
+        : aggFn === AggFn.SumRate
+        ? `SUM(rate) as data`
+        : aggFn === AggFn.AvgRate
+        ? `AVG(rate) as data`
+        : aggFn === AggFn.MaxRate
+        ? `MAX(rate) as data`
+        : aggFn === AggFn.MinRate
+        ? `MIN(rate) as data`
+        : `quantile(${
+            aggFn === AggFn.P50 || aggFn === AggFn.P50Rate
+              ? '0.5'
+              : aggFn === AggFn.P90 || aggFn === AggFn.P90Rate
+              ? '0.90'
+              : aggFn === AggFn.P95 || aggFn === AggFn.P95Rate
+              ? '0.95'
+              : '0.99'
+          })(${isRate ? 'rate' : 'value'}) as data`,
+    );
+  } else {
+    logger.error(`Unsupported data type: ${dataType}`);
  }

+  const rateMetricSource = SqlString.format(
+    `
+SELECT
+  if(
+    runningDifference(value) < 0
+    OR neighbor(_string_attributes, -1, _string_attributes) != _string_attributes,
+    nan,
+    runningDifference(value)
+  ) AS rate,
+  ts_bucket as timestamp,
+  _string_attributes,
+  min_name as name
+FROM
+  (
+    SELECT
+      toStartOfInterval(timestamp, INTERVAL ?) as ts_bucket,
+      min(value) as value,
+      _string_attributes,
+      min(name) as min_name
+    FROM
+      ??
+    WHERE
+      name = ?
+      AND data_type = ?
+      AND (?)
+    GROUP BY
+      _string_attributes,
+      ts_bucket
+    ORDER BY
+      _string_attributes,
+      ts_bucket ASC
+  )
+`.trim(),
+    [granularity, tableName, name, dataType, SqlString.raw(whereClause)],
+  );
+
+  const gaugeMetricSource = SqlString.format(
+    `
+SELECT 
+  timestamp,
+  name,
+  value,
+  _string_attributes
+FROM ??
+WHERE name = ?
+AND data_type = ?
+AND (?)
+ORDER BY _timestamp_sort_key ASC
+`.trim(),
+    [tableName, name, dataType, SqlString.raw(whereClause)],
+  );
+
  // TODO: support other data types like Sum, Histogram, etc.
  const query = SqlString.format(
    `
-      WITH metrcis AS (
-        SELECT *, runningDifference(value) AS delta
-        FROM (
-          SELECT 
-            timestamp,
-            name,
-            value,
-            _string_attributes
-          FROM ??
-          WHERE name = ?
-          AND data_type = ?
-          AND (?)
-          ORDER BY _timestamp_sort_key ASC
-        )
-      )
+      WITH metrics AS (?)
      SELECT ?
-      FROM metrcis
+      FROM metrics
      GROUP BY group, ts_bucket
      ORDER BY ts_bucket ASC
      WITH FILL
@ -743,10 +786,7 @@ export const getMetricsChart = async ({
        STEP ?
    `,
    [
-      tableName,
-      name,
-      dataType,
-      SqlString.raw(whereClause),
+      SqlString.raw(isRate ? rateMetricSource : gaugeMetricSource),
      SqlString.raw(selectClause.join(',')),
      startTime / 1000,
      granularity,
@ -798,6 +838,10 @@ export const getLogsChart = async ({
  tableVersion: number | undefined;
  teamId: string;
 }) => {
+  if (isRateAggFn(aggFn)) {
+    throw new Error('Rate is not supported in logs chart');
+  }
+
  const tableName = getLogStreamTableName(tableVersion, teamId);
  const whereClause = await buildSearchQueryWhereCondition({
    endTime,
--- a/packages/app/src/ChartPage.tsx
+++ b/packages/app/src/ChartPage.tsx
@ -1,7 +1,7 @@
 import Head from 'next/head';
 import { Button } from 'react-bootstrap';
 import { useQueryParam, StringParam, withDefault } from 'use-query-params';
-import { useState } from 'react';
+import { useCallback, useEffect, useState } from 'react';
 import { encodeArray, decodeArray } from 'serialize-query-params';
 import produce from 'immer';

@ -86,6 +86,20 @@ export default function GraphPage() {
      }),
    );
  };
+  const setFieldAndAggFn = (
+    index: number,
+    field: string | undefined,
+    fn: AggFn,
+  ) => {
+    setChartSeries(
+      produce(chartSeries, series => {
+        if (series?.[index] != null) {
+          series[index].field = field;
+          series[index].aggFn = fn;
+        }
+      }),
+    );
+  };
  const setWhere = (index: number, where: string) => {
    setChartSeries(
      produce(chartSeries, series => {
@ -132,7 +146,7 @@ export default function GraphPage() {
      ],
    });

-  const onRunQuery = () => {
+  const onRunQuery = useCallback(() => {
    onSearch(displayedTimeInputValue);
    const dateRange = parseTimeRangeInput(displayedTimeInputValue);

@ -150,7 +164,7 @@ export default function GraphPage() {
    } else {
      toast.error('Invalid time range');
    }
-  };
+  }, [chartSeries, displayedTimeInputValue, granularity, onSearch]);

  return (
    <div className="LogViewerPage d-flex" style={{ height: '100vh' }}>
@ -173,6 +187,19 @@ export default function GraphPage() {
                where={series.where}
                groupBy={series.groupBy[0]}
                field={series.field}
+                setFieldAndAggFn={(field, aggFn) =>
+                  setFieldAndAggFn(index, field, aggFn)
+                }
+                setTableAndAggFn={(table, aggFn) => {
+                  setChartSeries(
+                    produce(chartSeries, series => {
+                      if (series?.[index] != null) {
+                        series[index].table = table;
+                        series[index].aggFn = aggFn;
+                      }
+                    }),
+                  );
+                }}
                setTable={table => setTable(index, table)}
                setWhere={where => setWhere(index, where)}
                setAggFn={fn => setAggFn(index, fn)}
--- a/packages/app/src/ChartUtils.tsx
+++ b/packages/app/src/ChartUtils.tsx
@ -1,4 +1,3 @@
-import { Form, InputGroup } from 'react-bootstrap';
 import { useMemo, useRef } from 'react';
 import Select from 'react-select';
 import AsyncSelect from 'react-select/async';
@ -7,6 +6,7 @@ import api from './api';
 import { add } from 'date-fns';
 import SearchInput from './SearchInput';
 import MetricTagFilterInput from './MetricTagFilterInput';
+import Checkbox from './Checkbox';

 export const SORT_ORDER = [
  { value: 'asc' as const, label: 'Ascending' },
@ -160,6 +160,102 @@ export function MetricTagSelect({
  );
 }

+export function MetricSelect({
+  aggFn,
+  isRate,
+  metricName,
+  setAggFn,
+  setFieldAndAggFn,
+  setMetricName,
+}: {
+  aggFn: AggFn;
+  isRate: boolean;
+  metricName: string | undefined | null;
+  setAggFn: (fn: AggFn) => void;
+  setFieldAndAggFn: (field: string | undefined, fn: AggFn) => void;
+  setMetricName: (value: string | undefined) => void;
+}) {
+  // TODO: Dedup with metric rate checkbox
+  const { data: metricTagsData } = api.useMetricsTags();
+
+  const aggFnWithMaybeRate = (aggFn: AggFn, isRate: boolean) => {
+    if (isRate) {
+      if (aggFn.includes('_rate')) {
+        return aggFn;
+      } else {
+        return `${aggFn}_rate` as AggFn;
+      }
+    } else {
+      if (aggFn.includes('_rate')) {
+        return aggFn.replace('_rate', '') as AggFn;
+      } else {
+        return aggFn;
+      }
+    }
+  };
+
+  return (
+    <>
+      <div className="ms-3 flex-grow-1">
+        <MetricNameSelect
+          value={metricName}
+          setValue={name => {
+            const metricType = metricTagsData?.data?.find(
+              metric => metric.name === name,
+            )?.data_type;
+
+            const newAggFn = aggFnWithMaybeRate(aggFn, metricType === 'Sum');
+
+            setFieldAndAggFn(name, newAggFn);
+          }}
+        />
+      </div>
+      <div className="ms-3 flex-shrink-1">
+        <MetricRateSelect
+          metricName={metricName}
+          isRate={isRate}
+          setIsRate={(isRate: boolean) => {
+            setAggFn(aggFnWithMaybeRate(aggFn, isRate));
+          }}
+        />
+      </div>
+    </>
+  );
+}
+
+export function MetricRateSelect({
+  metricName,
+  isRate,
+  setIsRate,
+}: {
+  isRate: boolean;
+  setIsRate: (isRate: boolean) => void;
+  metricName: string | undefined | null;
+}) {
+  const { data: metricTagsData } = api.useMetricsTags();
+
+  const metricType = useMemo(() => {
+    return metricTagsData?.data?.find(metric => metric.name === metricName)
+      ?.data_type;
+  }, [metricTagsData, metricName]);
+
+  return (
+    <>
+      {metricType === 'Sum' ? (
+        <Checkbox
+          title="Convert the sum metric into change over time (rate)"
+          id="metric-use-rate"
+          className="text-nowrap"
+          labelClassName="fs-7"
+          checked={isRate}
+          onChange={() => setIsRate(!isRate)}
+          label="Use Rate"
+        />
+      ) : null}
+    </>
+  );
+}
+
 export function MetricNameSelect({
  value,
  setValue,
@ -248,38 +344,79 @@ export function FieldSelect({
  );
 }

-export type AggFn = (typeof AGG_FNS)[number]['value'];
+export type AggFn =
+  | 'avg_rate'
+  | 'avg'
+  | 'count_distinct'
+  | 'count'
+  | 'max_rate'
+  | 'max'
+  | 'min_rate'
+  | 'min'
+  | 'p50_rate'
+  | 'p50'
+  | 'p90_rate'
+  | 'p90'
+  | 'p95_rate'
+  | 'p95'
+  | 'p99_rate'
+  | 'p99'
+  | 'sum_rate'
+  | 'sum';

 export function ChartSeriesForm({
-  sortOrder,
-  table,
  aggFn,
-  where,
  field,
  groupBy,
+  setAggFn,
+  setField,
+  setFieldAndAggFn,
+  setTableAndAggFn,
+  setGroupBy,
  setSortOrder,
  setTable,
-  setAggFn,
  setWhere,
-  setField,
-  setGroupBy,
+  sortOrder,
+  table,
+  where,
 }: {
-  sortOrder?: string;
-  table: string;
  aggFn: AggFn;
-  where: string;
  field: string | undefined;
  groupBy: string | undefined;
-  setTable: (table: string) => void;
  setAggFn: (fn: AggFn) => void;
-  setWhere: (where: string) => void;
  setField: (field: string | undefined) => void;
+  setFieldAndAggFn: (field: string | undefined, fn: AggFn) => void;
+  setTableAndAggFn: (table: string, fn: AggFn) => void;
  setGroupBy: (groupBy: string | undefined) => void;
  setSortOrder?: (sortOrder: SortOrder) => void;
+  setTable: (table: string) => void;
+  setWhere: (where: string) => void;
+  sortOrder?: string;
+  table: string;
+  where: string;
 }) {
  const labelWidth = 350;
  const searchInputRef = useRef<HTMLInputElement>(null);

+  const isRate = useMemo(() => {
+    return aggFn.includes('_rate');
+  }, [aggFn]);
+  const _setAggFn = (fn: AggFn, _isRate?: boolean) => {
+    if (_isRate ?? isRate) {
+      if (fn.includes('_rate')) {
+        setAggFn(fn);
+      } else {
+        setAggFn(`${fn}_rate` as AggFn);
+      }
+    } else {
+      if (fn.includes('_rate')) {
+        setAggFn(fn.replace('_rate', '') as AggFn);
+      } else {
+        setAggFn(fn);
+      }
+    }
+  };
+
  return (
    <div>
      <div className="d-flex align-items-center">
@ -288,7 +425,16 @@ export function ChartSeriesForm({
            options={TABLES}
            className="ds-select"
            value={TABLES.find(v => v.value === table)}
-            onChange={opt => setTable(opt?.value ?? 'logs')}
+            onChange={opt => {
+              const val = opt?.value ?? 'logs';
+              if (val === 'logs') {
+                setTableAndAggFn('logs', 'count');
+              } else if (val === 'metrics') {
+                // TODO: This should set rate if metric field is a sum
+                // or we should just reset the field if changing tables
+                setTableAndAggFn('metrics', 'max');
+              }
+            }}
            classNamePrefix="ds-react-select"
          />
        </div>
@ -298,15 +444,17 @@ export function ChartSeriesForm({
              options={AGG_FNS}
              className="ds-select"
              value={AGG_FNS.find(v => v.value === aggFn)}
-              onChange={opt => setAggFn(opt?.value ?? 'count')}
+              onChange={opt => _setAggFn(opt?.value ?? 'count')}
              classNamePrefix="ds-react-select"
            />
          ) : (
            <Select
              options={METRIC_AGG_FNS}
              className="ds-select"
-              value={METRIC_AGG_FNS.find(v => v.value === aggFn)}
-              onChange={opt => setAggFn(opt?.value ?? 'sum')}
+              value={METRIC_AGG_FNS.find(
+                v => v.value === aggFn.replace('_rate', ''),
+              )}
+              onChange={opt => _setAggFn(opt?.value ?? 'sum')}
              classNamePrefix="ds-react-select"
            />
          )}
@ -326,8 +474,15 @@ export function ChartSeriesForm({
          </div>
        ) : null}
        {table === 'metrics' ? (
-          <div className="ms-3 flex-grow-1">
-            <MetricNameSelect value={field} setValue={setField} />
+          <div className="d-flex align-items-center align-middle flex-grow-1">
+            <MetricSelect
+              metricName={field}
+              setMetricName={setField}
+              isRate={isRate}
+              setAggFn={setAggFn}
+              setFieldAndAggFn={setFieldAndAggFn}
+              aggFn={aggFn}
+            />
          </div>
        ) : null}
      </div>
--- a/packages/app/src/Checkbox.tsx
+++ b/packages/app/src/Checkbox.tsx
@ -6,6 +6,7 @@ export default function Checkbox({
  onChange,
  label,
  disabled,
+  title,
 }: {
  id: string;
  className?: string;
@ -14,6 +15,7 @@ export default function Checkbox({
  onChange: () => void;
  label: React.ReactNode;
  disabled?: boolean;
+  title?: string;
 }) {
  return (
    <span className={`d-flex align-items-center ${className ?? ''}`}>
@ -26,6 +28,7 @@ export default function Checkbox({
        disabled={disabled}
      />
      <label
+        title={title}
        htmlFor={id}
        className={`fs-7 cursor-pointer ${labelClassName ?? ''}`}
      >
--- a/packages/app/src/EditChartForm.tsx
+++ b/packages/app/src/EditChartForm.tsx
@ -548,6 +548,16 @@ export const EditTableChartForm = ({
            }),
          )
        }
+        setTableAndAggFn={(table, aggFn) => {
+          setEditedChart(
+            produce(editedChart, draft => {
+              if (draft.series[0].type === CHART_TYPE) {
+                draft.series[0].table = table;
+                draft.series[0].aggFn = aggFn;
+              }
+            }),
+          );
+        }}
        setWhere={where =>
          setEditedChart(
            produce(editedChart, draft => {
@ -579,6 +589,16 @@ export const EditTableChartForm = ({
            }),
          )
        }
+        setFieldAndAggFn={(field, aggFn) => {
+          setEditedChart(
+            produce(editedChart, draft => {
+              if (draft.series[0].type === CHART_TYPE) {
+                draft.series[0].field = field;
+                draft.series[0].aggFn = aggFn;
+              }
+            }),
+          );
+        }}
        setSortOrder={sortOrder =>
          setEditedChart(
            produce(editedChart, draft => {
@ -795,11 +815,13 @@ export const EditLineChartForm = ({
  onSave: (chart: Chart) => void;
  onClose: () => void;
 }) => {
+  const CHART_TYPE = 'time';
+
  const [editedChart, setEditedChart] = useState<Chart | undefined>(chart);

  const chartConfig = useMemo(
    () =>
-      editedChart != null && editedChart.series?.[0]?.type === 'time'
+      editedChart != null && editedChart.series?.[0]?.type === CHART_TYPE
        ? {
            table: editedChart.series[0].table ?? 'logs',
            aggFn: editedChart.series[0].aggFn,
@ -855,7 +877,7 @@ export const EditLineChartForm = ({
        setTable={table =>
          setEditedChart(
            produce(editedChart, draft => {
-              if (draft.series[0].type === 'time') {
+              if (draft.series[0].type === CHART_TYPE) {
                draft.series[0].table = table;
              }
            }),
@ -864,7 +886,7 @@ export const EditLineChartForm = ({
        setAggFn={aggFn =>
          setEditedChart(
            produce(editedChart, draft => {
-              if (draft.series[0].type === 'time') {
+              if (draft.series[0].type === CHART_TYPE) {
                draft.series[0].aggFn = aggFn;
              }
            }),
@ -873,7 +895,7 @@ export const EditLineChartForm = ({
        setWhere={where =>
          setEditedChart(
            produce(editedChart, draft => {
-              if (draft.series[0].type === 'time') {
+              if (draft.series[0].type === CHART_TYPE) {
                draft.series[0].where = where;
              }
            }),
@ -882,7 +904,7 @@ export const EditLineChartForm = ({
        setGroupBy={groupBy =>
          setEditedChart(
            produce(editedChart, draft => {
-              if (draft.series[0].type === 'time') {
+              if (draft.series[0].type === CHART_TYPE) {
                if (groupBy != undefined) {
                  draft.series[0].groupBy[0] = groupBy;
                } else {
@ -895,12 +917,32 @@ export const EditLineChartForm = ({
        setField={field =>
          setEditedChart(
            produce(editedChart, draft => {
-              if (draft.series[0].type === 'time') {
+              if (draft.series[0].type === CHART_TYPE) {
                draft.series[0].field = field;
              }
            }),
          )
        }
+        setTableAndAggFn={(table, aggFn) => {
+          setEditedChart(
+            produce(editedChart, draft => {
+              if (draft.series[0].type === CHART_TYPE) {
+                draft.series[0].table = table;
+                draft.series[0].aggFn = aggFn;
+              }
+            }),
+          );
+        }}
+        setFieldAndAggFn={(field, aggFn) => {
+          setEditedChart(
+            produce(editedChart, draft => {
+              if (draft.series[0].type === CHART_TYPE) {
+                draft.series[0].field = field;
+                draft.series[0].aggFn = aggFn;
+              }
+            }),
+          );
+        }}
      />
      <div className="d-flex justify-content-between my-3 ps-2">
        <Button
--- a/packages/app/src/api.ts
+++ b/packages/app/src/api.ts
@ -63,6 +63,7 @@ const api = {
      {
        data: {
          name: string;
+          data_type: string;
          tags: Record<string, string>[];
        }[];
      },