feat: add Summary and Exponential Histogram metrics to source form (#832)

.changeset/heavy-chicken-share.md

@@ -0,0 +1,7 @@
+---
+"@hyperdx/common-utils": patch
+"@hyperdx/api": patch
+"@hyperdx/app": patch
+---
+
+feat: add summary and exponential histogram metrics to the source form and database storage

packages/api/src/clickhouse/__tests__/renderChartConfig.test.ts

@@ -32,6 +32,8 @@ const TEST_METRIC_TABLES = {
   sum: DEFAULT_METRICS_TABLE.SUM,
   gauge: DEFAULT_METRICS_TABLE.GAUGE,
   histogram: DEFAULT_METRICS_TABLE.HISTOGRAM,
+  summary: DEFAULT_METRICS_TABLE.SUMMARY,
+  'exponential histogram': DEFAULT_METRICS_TABLE.EXPONENTIAL_HISTOGRAM,
 };
 
 describe('renderChartConfig', () => {

packages/api/src/fixtures.ts

@@ -28,6 +28,8 @@ export const DEFAULT_METRICS_TABLE = {
   GAUGE: 'otel_metrics_gauge',
   SUM: 'otel_metrics_sum',
   HISTOGRAM: 'otel_metrics_histogram',
+  SUMMARY: 'otel_metrics_summary',
+  EXPONENTIAL_HISTOGRAM: 'otel_metrics_exponential_histogram',
 };
 
 const connectClickhouse = async () => {

packages/api/src/models/source.ts

@@ -70,6 +70,8 @@ export const Source = mongoose.model<ISource>(
           [MetricsDataType.Gauge]: String,
           [MetricsDataType.Histogram]: String,
           [MetricsDataType.Sum]: String,
+          [MetricsDataType.Summary]: String,
+          [MetricsDataType.ExponentialHistogram]: String,
         },
         default: undefined,
       },

packages/app/src/components/OnboardingModal.tsx

@@ -1,5 +1,5 @@
 import { useEffect, useState } from 'react';
-import { SourceKind } from '@hyperdx/common-utils/dist/types';
+import { MetricsDataType, SourceKind } from '@hyperdx/common-utils/dist/types';
 import { Button, Divider, Modal, Text } from '@mantine/core';
 import { notifications } from '@mantine/notifications';
 
@@ -111,9 +111,12 @@ export default function OnboardingModal({
                     timestampValueExpression: 'TimeUnix',
                     serviceNameExpression: 'ServiceName',
                     metricTables: {
-                      gauge: 'otel_metrics_gauge',
-                      histogram: 'otel_metrics_histogram',
-                      sum: 'otel_metrics_sum',
+                      [MetricsDataType.Gauge]: 'otel_metrics_gauge',
+                      [MetricsDataType.Histogram]: 'otel_metrics_histogram',
+                      [MetricsDataType.Sum]: 'otel_metrics_sum',
+                      [MetricsDataType.Summary]: 'otel_metrics_summary',
+                      [MetricsDataType.ExponentialHistogram]:
+                        'otel_metrics_exponential_histogram',
                     },
                     resourceAttributesExpression: 'ResourceAttributes',
                   },

packages/app/src/components/SourceForm.tsx

@@ -79,7 +79,7 @@ function FormRow({
         }}
       >
         {typeof label === 'string' ? (
-          <Text c="gray.6" size="sm">
+          <Text tt="capitalize" c="gray.6" size="sm">
             {label}
           </Text>
         ) : (
@@ -821,11 +821,16 @@ export function MetricTableModelForm({
             name={`from.databaseName`}
           />
         </FormRow>
-        {Object.keys(MetricsDataType).map(metricType => (
+        {Object.values(MetricsDataType).map(metricType => (
           <FormRow
             key={metricType.toLowerCase()}
             label={`${metricType} Table`}
-            helpText={`Table containing ${metricType.toLowerCase()} metrics data`}
+            helpText={
+              metricType === MetricsDataType.ExponentialHistogram ||
+              metricType === MetricsDataType.Summary
+                ? `Table containing ${metricType.toLowerCase()} metrics data. Note: not yet fully supported by HyperDX`
+                : `Table containing ${metricType.toLowerCase()} metrics data`
+            }
           >
             <DBTableSelectControlled
               connectionId={connectionId}

packages/app/src/source.ts

@@ -316,6 +316,8 @@ export function getDurationSecondsExpression(source: TSource) {
   return `(${source.durationExpression})/1e${source.durationPrecision ?? 9}`;
 }
 
+// defined in https://github.com/open-telemetry/opentelemetry-proto/blob/cfbf9357c03bf4ac150a3ab3bcbe4cc4ed087362/opentelemetry/proto/metrics/v1/metrics.proto
+// NOTE: We don't follow the standard perfectly, we enforce the required fields + a few more (ServiceName, MetricName, and ResourceAttributes primarily)
 const ReqMetricTableColumns = {
   [MetricsDataType.Gauge]: [
     'TimeUnix',
@@ -344,6 +346,34 @@ const ReqMetricTableColumns = {
     'Attributes',
     'ResourceAttributes',
   ],
+  [MetricsDataType.Summary]: [
+    'Attributes',
+    'TimeUnix',
+    'Count',
+    'Sum',
+    'ValueAtQuantiles.Quantile',
+    'ValueAtQuantiles.Value',
+    'Flags',
+    'ServiceName',
+    'MetricName',
+    'ResourceAttributes',
+  ],
+  [MetricsDataType.ExponentialHistogram]: [
+    'Attributes',
+    'TimeUnix',
+    'Count',
+    'Sum',
+    'Scale',
+    'ZeroCount',
+    'PositiveOffset',
+    'PositiveBucketCounts',
+    'NegativeOffset',
+    'NegativeBucketCounts',
+    'Flags',
+    'ServiceName',
+    'MetricName',
+    'ResourceAttributes',
+  ],
 };
 
 export async function isValidMetricTable({

packages/common-utils/src/__tests__/renderChartConfig.test.ts

@@ -31,6 +31,8 @@ describe('renderChartConfig', () => {
         gauge: 'otel_metrics_gauge',
         histogram: 'otel_metrics_histogram',
         sum: 'otel_metrics_sum',
+        summary: 'otel_metrics_summary',
+        'exponential histogram': 'otel_metrics_exponential_histogram',
       },
       from: {
         databaseName: 'default',
@@ -69,6 +71,8 @@ describe('renderChartConfig', () => {
         gauge: 'otel_metrics_gauge',
         histogram: 'otel_metrics_histogram',
         sum: 'otel_metrics_sum',
+        summary: 'otel_metrics_summary',
+        'exponential histogram': 'otel_metrics_exponential_histogram',
       },
       from: {
         databaseName: 'default',
@@ -105,6 +109,8 @@ describe('renderChartConfig', () => {
         gauge: 'otel_metrics_gauge',
         histogram: 'otel_metrics_histogram',
         sum: 'otel_metrics_sum',
+        summary: 'otel_metrics_summary',
+        'exponential histogram': 'otel_metrics_exponential_histogram',
       },
       from: {
         databaseName: 'default',
@@ -133,6 +139,8 @@ describe('renderChartConfig', () => {
           gauge: 'otel_metrics_gauge',
           histogram: 'otel_metrics_histogram',
           sum: 'otel_metrics_sum',
+          summary: 'otel_metrics_summary',
+          'exponential histogram': 'otel_metrics_exponential_histogram',
         },
         from: {
           databaseName: 'default',
@@ -167,6 +175,8 @@ describe('renderChartConfig', () => {
           gauge: 'otel_metrics_gauge',
           histogram: 'otel_metrics_histogram',
           sum: 'otel_metrics_sum',
+          summary: 'otel_metrics_summary',
+          'exponential histogram': 'otel_metrics_exponential_histogram',
         },
         from: {
           databaseName: 'default',
@@ -202,6 +212,8 @@ describe('renderChartConfig', () => {
           gauge: 'otel_metrics_gauge',
           histogram: 'otel_metrics_histogram',
           sum: 'otel_metrics_sum',
+          summary: 'otel_metrics_summary',
+          'exponential histogram': 'otel_metrics_exponential_histogram',
         },
         from: {
           databaseName: 'default',

packages/common-utils/src/types.ts

@@ -5,6 +5,8 @@ export enum MetricsDataType {
   Gauge = 'gauge',
   Histogram = 'histogram',
   Sum = 'sum',
+  Summary = 'summary',
+  ExponentialHistogram = 'exponential histogram',
 }
 
 // --------------------------
@@ -20,11 +22,15 @@ export enum DisplayType {
   Markdown = 'markdown',
 }
 
-export const MetricTableSchema = z.object({
-  [MetricsDataType.Gauge]: z.string(),
-  [MetricsDataType.Histogram]: z.string(),
-  [MetricsDataType.Sum]: z.string(),
-});
+export const MetricTableSchema = z.object(
+  Object.values(MetricsDataType).reduce(
+    (acc, key) => ({
+      ...acc,
+      [key]: z.string(),
+    }),
+    {} as Record<MetricsDataType, z.ZodString>,
+  ),
+);
 
 export type MetricTable = z.infer<typeof MetricTableSchema>;