Support custom resource attributes for Hyperdx frontend app's internal telemetry (#2067)

.changeset/tidy-pets-shout.md

@@ -0,0 +1,6 @@
+---
+"@hyperdx/api": patch
+"@hyperdx/app": patch
+---
+
+fix: filter logs with ID field

docker/otel-collector/schema/seed/00002_otel_logs.sql

@@ -25,6 +25,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs
   `__hdx_materialized_k8s.pod.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.name'] CODEC(ZSTD(1)),
   `__hdx_materialized_k8s.pod.uid` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.uid'] CODEC(ZSTD(1)),
   `__hdx_materialized_deployment.environment.name` LowCardinality(String) MATERIALIZED ResourceAttributes['deployment.environment.name'] CODEC(ZSTD(1)),
+  `__hdx_id` UInt16 MATERIALIZED toUInt16(rand()),
   INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
   INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
   INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,

packages/api/migrations/ch/000003_add_hdx_id_to_otel_logs.down.sql

@@ -0,0 +1 @@
+ALTER TABLE ${DATABASE}.otel_logs DROP COLUMN IF EXISTS `__hdx_id`;

packages/api/migrations/ch/000003_add_hdx_id_to_otel_logs.up.sql

@@ -0,0 +1,3 @@
+ALTER TABLE ${DATABASE}.otel_logs
+    ADD COLUMN IF NOT EXISTS `__hdx_id` UInt16
+    MATERIALIZED toUInt16(rand());

packages/app/pages/_app.tsx

@@ -3,6 +3,7 @@ import type { NextPage } from 'next';
 import type { AppProps } from 'next/app';
 import Head from 'next/head';
 import { NextAdapter } from 'next-query-params';
+import { env } from 'next-runtime-env';
 import randomUUID from 'crypto-randomuuid';
 import { enableMapSet } from 'immer';
 import { QueryParamProvider } from 'use-query-params';
@@ -16,7 +17,7 @@ import {
 import { ReactQueryDevtools } from '@tanstack/react-query-devtools';
 
 import { DynamicFavicon } from '@/components/DynamicFavicon';
-import { IS_LOCAL_MODE } from '@/config';
+import { IS_LOCAL_MODE, parseResourceAttributes } from '@/config';
 import {
   DEFAULT_FONT_VAR,
   FONT_VAR_MAP,
@@ -136,12 +137,19 @@ export default function MyApp({ Component, pageProps }: AppPropsWithLayout) {
       .then(res => res.json())
       .then((_jsonData?: NextApiConfigResponseData) => {
         if (_jsonData?.apiKey) {
+          const frontendAttrs = parseResourceAttributes(
+            env('NEXT_PUBLIC_OTEL_RESOURCE_ATTRIBUTES') ?? '',
+          );
           HyperDX.init({
             apiKey: _jsonData.apiKey,
             consoleCapture: true,
             maskAllInputs: true,
             maskAllText: true,
+            // service.version is applied last so it always reflects the
+            // NEXT_PUBLIC_APP_VERSION and cannot be overridden by
+            // NEXT_PUBLIC_OTEL_RESOURCE_ATTRIBUTES.
             otelResourceAttributes: {
+              ...frontendAttrs,
               'service.version': process.env.NEXT_PUBLIC_APP_VERSION,
             },
             service: _jsonData.serviceName,

packages/app/src/__tests__/config.test.ts

@@ -0,0 +1,83 @@
+import { parseResourceAttributes } from '@/config';
+
+describe('parseResourceAttributes', () => {
+  it('parses a standard comma-separated string', () => {
+    const raw =
+      'service.namespace=observability,deployment.environment=prod,k8s.cluster.name=us-west-2';
+    expect(parseResourceAttributes(raw)).toEqual({
+      'service.namespace': 'observability',
+      'deployment.environment': 'prod',
+      'k8s.cluster.name': 'us-west-2',
+    });
+  });
+
+  it('returns an empty object for an empty string', () => {
+    expect(parseResourceAttributes('')).toEqual({});
+  });
+
+  it('handles a single key=value pair', () => {
+    expect(parseResourceAttributes('foo=bar')).toEqual({ foo: 'bar' });
+  });
+
+  it('handles values containing equals signs', () => {
+    expect(parseResourceAttributes('url=https://example.com?a=1')).toEqual({
+      url: 'https://example.com?a=1',
+    });
+  });
+
+  it('skips malformed entries without an equals sign', () => {
+    expect(parseResourceAttributes('good=value,badentry,ok=yes')).toEqual({
+      good: 'value',
+      ok: 'yes',
+    });
+  });
+
+  it('skips entries where key is empty (leading equals)', () => {
+    expect(parseResourceAttributes('=nokey,valid=value')).toEqual({
+      valid: 'value',
+    });
+  });
+
+  it('handles trailing commas gracefully', () => {
+    expect(parseResourceAttributes('a=1,b=2,')).toEqual({ a: '1', b: '2' });
+  });
+
+  it('handles leading commas gracefully', () => {
+    expect(parseResourceAttributes(',a=1,b=2')).toEqual({ a: '1', b: '2' });
+  });
+
+  it('allows empty values', () => {
+    expect(parseResourceAttributes('key=')).toEqual({ key: '' });
+  });
+
+  it('last value wins for duplicate keys', () => {
+    expect(parseResourceAttributes('k=first,k=second')).toEqual({
+      k: 'second',
+    });
+  });
+
+  it('decodes percent-encoded commas in values', () => {
+    expect(parseResourceAttributes('tags=a%2Cb%2Cc')).toEqual({
+      tags: 'a,b,c',
+    });
+  });
+
+  it('decodes percent-encoded equals in values', () => {
+    expect(parseResourceAttributes('expr=x%3D1')).toEqual({
+      expr: 'x=1',
+    });
+  });
+
+  it('decodes percent-encoded keys', () => {
+    expect(parseResourceAttributes('my%2Ekey=value')).toEqual({
+      'my.key': 'value',
+    });
+  });
+
+  it('round-trips values with both encoded commas and equals', () => {
+    expect(parseResourceAttributes('q=a%3D1%2Cb%3D2,other=plain')).toEqual({
+      q: 'a=1,b=2',
+      other: 'plain',
+    });
+  });
+});

packages/app/src/__tests__/source.test.ts

@@ -1,3 +1,5 @@
+import { ColumnMeta } from '@hyperdx/common-utils/dist/clickhouse';
+import { Metadata } from '@hyperdx/common-utils/dist/core/metadata';
 import {
   SourceKind,
   TLogSource,
@@ -9,6 +11,7 @@ import {
   getEventBody,
   getSourceValidationNotificationId,
   getTraceDurationNumberFormat,
+  inferTableSourceConfig,
   useSources,
 } from '../source';
 
@@ -258,3 +261,86 @@ describe('useSources validation notifications', () => {
     );
   });
 });
+
+describe('inferTableSourceConfig', () => {
+  const col = (name: string, type = 'String'): ColumnMeta => ({
+    name,
+    type,
+    codec_expression: '',
+    comment: '',
+    default_expression: '',
+    default_type: '',
+    ttl_expression: '',
+  });
+
+  const OTEL_LOG_COLUMNS = [
+    col('Timestamp', "DateTime64(9, 'UTC')"),
+    col('TimestampTime', 'DateTime'),
+    col('Body'),
+    col('SeverityText'),
+    col('TraceId'),
+    col('SpanId'),
+    col('ServiceName'),
+    col('LogAttributes', 'Map(String, String)'),
+    col('ResourceAttributes', 'Map(String, String)'),
+  ];
+
+  const baseArgs = {
+    databaseName: 'default',
+    tableName: 'otel_logs',
+    connectionId: 'test-conn',
+  };
+
+  function mockMetadata(columns: ColumnMeta[]): Metadata {
+    return {
+      getColumns: jest.fn().mockResolvedValue(columns),
+      getTableMetadata: jest.fn().mockResolvedValue({
+        primary_key: 'ServiceName, TimestampTime',
+      }),
+    } as unknown as Metadata;
+  }
+
+  it('should set uniqueRowIdExpression when __hdx_id column exists on otel log table', async () => {
+    const columns = [...OTEL_LOG_COLUMNS, col('__hdx_id', 'UInt16')];
+    const result = await inferTableSourceConfig({
+      ...baseArgs,
+      kind: SourceKind.Log,
+      metadata: mockMetadata(columns),
+    });
+    expect(result).toHaveProperty('uniqueRowIdExpression', '__hdx_id');
+  });
+
+  it('should not set uniqueRowIdExpression when __hdx_id column is missing', async () => {
+    const result = await inferTableSourceConfig({
+      ...baseArgs,
+      kind: SourceKind.Log,
+      metadata: mockMetadata(OTEL_LOG_COLUMNS),
+    });
+    expect(result).not.toHaveProperty('uniqueRowIdExpression');
+  });
+
+  it('should not set uniqueRowIdExpression for trace sources even if __hdx_id exists', async () => {
+    const OTEL_TRACE_COLUMNS = [
+      col('Timestamp', "DateTime64(9, 'UTC')"),
+      col('SpanName'),
+      col('Duration', 'UInt64'),
+      col('SpanKind'),
+      col('TraceId'),
+      col('SpanId'),
+      col('ParentSpanId'),
+      col('ServiceName'),
+      col('SpanAttributes', 'Map(String, String)'),
+      col('ResourceAttributes', 'Map(String, String)'),
+      col('StatusCode'),
+      col('StatusMessage'),
+      col('__hdx_id', 'UInt16'),
+    ];
+    const result = await inferTableSourceConfig({
+      ...baseArgs,
+      tableName: 'otel_traces',
+      kind: SourceKind.Trace,
+      metadata: mockMetadata(OTEL_TRACE_COLUMNS),
+    });
+    expect(result).not.toHaveProperty('uniqueRowIdExpression');
+  });
+});

packages/app/src/components/DBRowTable.tsx

@@ -1374,14 +1374,15 @@ export const RawLogTable = memo(
   },
 );
 
-export function appendSelectWithPrimaryAndPartitionKey(
+export function appendSelectWithAdditionalKeys(
   select: SelectList,
   primaryKeys: string,
   partitionKey: string,
+  extraKeys: string[] = [],
 ): { select: SelectList; additionalKeysLength: number } {
   const partitionKeyArr = extractColumnReferencesFromKey(partitionKey);
   const primaryKeyArr = extractColumnReferencesFromKey(primaryKeys);
-  const allKeys = new Set([...partitionKeyArr, ...primaryKeyArr]);
+  const allKeys = new Set([...partitionKeyArr, ...primaryKeyArr, ...extraKeys]);
   if (typeof select === 'string') {
     const selectSplit = splitAndTrimWithBracket(select);
     const selectColumns = new Set(selectSplit);
@@ -1407,8 +1408,9 @@ function getSelectLength(select: SelectList): number {
   }
 }
 
-export function useConfigWithPrimaryAndPartitionKey(
+export function useConfigWithAdditionalSelect(
   config: BuilderChartConfigWithDateRange,
+  sourceId?: string,
 ) {
   const { data: tableMetadata } = useTableMetadata({
     databaseName: config.from.databaseName,
@@ -1416,6 +1418,9 @@ export function useConfigWithPrimaryAndPartitionKey(
     connectionId: config.connection,
   });
 
+  // We're only interested in `uniqueRowIdExpression` for logs.
+  const { data: source } = useSource({ id: sourceId, kinds: [SourceKind.Log] });
+
   const primaryKey = tableMetadata?.primary_key;
   const partitionKey = tableMetadata?.partition_key;
 
@@ -1424,14 +1429,14 @@ export function useConfigWithPrimaryAndPartitionKey(
       return undefined;
     }
 
-    const { select, additionalKeysLength } =
-      appendSelectWithPrimaryAndPartitionKey(
-        config.select,
-        primaryKey,
-        partitionKey,
-      );
+    const { select, additionalKeysLength } = appendSelectWithAdditionalKeys(
+      config.select,
+      primaryKey,
+      partitionKey,
+      source?.uniqueRowIdExpression ? [source.uniqueRowIdExpression] : [],
+    );
     return { ...config, select, additionalKeysLength };
-  }, [primaryKey, partitionKey, config]);
+  }, [primaryKey, partitionKey, config, source]);
 
   return mergedConfig;
 }
@@ -1564,7 +1569,7 @@ function DBSqlRowTableComponent({
     return base;
   }, [me, config, orderByArray]);
 
-  const mergedConfig = useConfigWithPrimaryAndPartitionKey(mergedConfigObj);
+  const mergedConfig = useConfigWithAdditionalSelect(mergedConfigObj, sourceId);
 
   const { data, fetchNextPage, hasNextPage, isFetching, isError, error } =
     useOffsetPaginatedQuery(mergedConfig ?? config, {

packages/app/src/components/Sources/SourceForm.tsx

@@ -1216,6 +1216,20 @@ function LogTableModelForm(props: TableModelProps) {
             disableKeywordAutocomplete
           />
         </FormRow>
+        <FormRow
+          label={'Unique Row Identifier Expression'}
+          helpText="Expression used to disambiguate rows with identical visible column values."
+        >
+          <SQLInlineEditorControlled
+            tableConnection={{
+              databaseName,
+              tableName,
+              connectionId,
+            }}
+            control={control}
+            name="uniqueRowIdExpression"
+          />
+        </FormRow>
         <Divider />
         <FormRow
           label={'Correlated Metric Source'}
@@ -1256,21 +1270,6 @@ function LogTableModelForm(props: TableModelProps) {
         </FormRow>
 
         <Divider />
-        {/* <FormRow
-          label={'Unique Row ID Expression'}
-          helpText="Unique identifier for a given row, will be primary key if not specified. Used for showing full row details in search results."
-        >
-          <SQLInlineEditorControlled
-            tableConnection={{
-              databaseName,
-              tableName,
-              connectionId,
-            }}
-            control={control}
-            name="uniqueRowIdExpression"
-            placeholder="Timestamp, ServiceName, Body"
-          />
-        </FormRow> */}
         {/* <FormRow label={'Table Filter Expression'}>
           <SQLInlineEditorControlled
             tableConnection={{

packages/app/src/components/__tests__/DBRowTable.test.tsx

@@ -2,7 +2,7 @@ import { screen } from '@testing-library/react';
 import userEvent from '@testing-library/user-event';
 
 import {
-  appendSelectWithPrimaryAndPartitionKey,
+  appendSelectWithAdditionalKeys,
   RawLogTable,
 } from '@/components/DBRowTable';
 import { RowWhereResult } from '@/hooks/useRowWhere';
@@ -145,9 +145,9 @@ describe('RawLogTable', () => {
   });
 });
 
-describe('appendSelectWithPrimaryAndPartitionKey', () => {
+describe('appendSelectWithAdditionalKeys', () => {
   it('should extract columns from partition key with nested function call', () => {
-    const result = appendSelectWithPrimaryAndPartitionKey(
+    const result = appendSelectWithAdditionalKeys(
       'col1, col2',
       'id, created_at',
       ' toStartOfInterval(timestamp, toIntervalDay(3))',
@@ -159,7 +159,7 @@ describe('appendSelectWithPrimaryAndPartitionKey', () => {
   });
 
   it('should extract no columns from empty primary key and partition key', () => {
-    const result = appendSelectWithPrimaryAndPartitionKey('col1, col2', '', '');
+    const result = appendSelectWithAdditionalKeys('col1, col2', '', '', []);
     expect(result).toEqual({
       additionalKeysLength: 0,
       select: 'col1,col2',
@@ -167,7 +167,7 @@ describe('appendSelectWithPrimaryAndPartitionKey', () => {
   });
 
   it('should extract columns from complex primary key', () => {
-    const result = appendSelectWithPrimaryAndPartitionKey(
+    const result = appendSelectWithAdditionalKeys(
       'col1, col2',
       'id, timestamp, toStartOfInterval(timestamp2, toIntervalDay(3))',
       "toStartOfInterval(timestamp, toIntervalDay(3)), date_diff('DAY', col3, col4), now(), toDate(col5 + INTERVAL 1 DAY)",
@@ -179,7 +179,7 @@ describe('appendSelectWithPrimaryAndPartitionKey', () => {
   });
 
   it('should extract map columns', () => {
-    const result = appendSelectWithPrimaryAndPartitionKey(
+    const result = appendSelectWithAdditionalKeys(
       'col1, col2',
       `map['key']`,
       `map2['key'], map1['key3 ']`,
@@ -191,7 +191,7 @@ describe('appendSelectWithPrimaryAndPartitionKey', () => {
   });
 
   it('should extract map columns', () => {
-    const result = appendSelectWithPrimaryAndPartitionKey(
+    const result = appendSelectWithAdditionalKeys(
       'col1, col2',
       ``,
       `map2['key.2']`,
@@ -203,7 +203,7 @@ describe('appendSelectWithPrimaryAndPartitionKey', () => {
   });
 
   it('should extract array columns', () => {
-    const result = appendSelectWithPrimaryAndPartitionKey(
+    const result = appendSelectWithAdditionalKeys(
       'col1, col2',
       `array[1]`,
       `array[2], array[3]`,
@@ -215,7 +215,7 @@ describe('appendSelectWithPrimaryAndPartitionKey', () => {
   });
 
   it('should extract json columns', () => {
-    const result = appendSelectWithPrimaryAndPartitionKey(
+    const result = appendSelectWithAdditionalKeys(
       'col1, col2',
       `json.b`,
       `json.a, json.b.c, toStartOfDay(timestamp, json_2.d)`,
@@ -227,7 +227,7 @@ describe('appendSelectWithPrimaryAndPartitionKey', () => {
   });
 
   it('should extract json columns with type specifiers', () => {
-    const result = appendSelectWithPrimaryAndPartitionKey(
+    const result = appendSelectWithAdditionalKeys(
       'col1, col2',
       `json.b.:Int64`,
       `toStartOfDay(json.a.b.:DateTime)`,
@@ -239,7 +239,7 @@ describe('appendSelectWithPrimaryAndPartitionKey', () => {
   });
 
   it('should skip json columns with hard-to-parse type specifiers', () => {
-    const result = appendSelectWithPrimaryAndPartitionKey(
+    const result = appendSelectWithAdditionalKeys(
       'col1, col2',
       `json.b.:Array(String), col3`,
       ``,
@@ -251,7 +251,7 @@ describe('appendSelectWithPrimaryAndPartitionKey', () => {
   });
 
   it('should skip nested map references', () => {
-    const result = appendSelectWithPrimaryAndPartitionKey(
+    const result = appendSelectWithAdditionalKeys(
       'col1, col2',
       `map['key']['key2'], col3`,
       ``,
@@ -261,4 +261,53 @@ describe('appendSelectWithPrimaryAndPartitionKey', () => {
       select: `col1,col2,col3`,
     });
   });
+
+  it('should append extraKeys to string select', () => {
+    const result = appendSelectWithAdditionalKeys('col1, col2', 'id', '', [
+      '__hdx_id',
+    ]);
+    expect(result).toEqual({
+      additionalKeysLength: 2,
+      select: 'col1,col2,id,__hdx_id',
+    });
+  });
+
+  it('should not duplicate extraKeys already in select', () => {
+    const result = appendSelectWithAdditionalKeys('col1, __hdx_id', 'id', '', [
+      '__hdx_id',
+    ]);
+    expect(result).toEqual({
+      additionalKeysLength: 1,
+      select: 'col1,__hdx_id,id',
+    });
+  });
+
+  it('should deduplicate extraKeys that overlap with primary/partition keys', () => {
+    const result = appendSelectWithAdditionalKeys('col1, col2', 'id', '', [
+      'id',
+      '__hdx_id',
+    ]);
+    expect(result).toEqual({
+      additionalKeysLength: 2,
+      select: 'col1,col2,id,__hdx_id',
+    });
+  });
+
+  it('should append extraKeys to array-style select', () => {
+    const result = appendSelectWithAdditionalKeys(
+      [{ valueExpression: 'col1' }, { valueExpression: 'col2' }],
+      'id',
+      '',
+      ['__hdx_id'],
+    );
+    expect(result).toEqual({
+      additionalKeysLength: 2,
+      select: [
+        { valueExpression: 'col1' },
+        { valueExpression: 'col2' },
+        { valueExpression: 'id' },
+        { valueExpression: '__hdx_id' },
+      ],
+    });
+  });
 });

packages/app/src/config.ts

@@ -14,6 +14,25 @@ export const HDX_SERVICE_NAME =
   process.env.NEXT_PUBLIC_OTEL_SERVICE_NAME ?? 'hdx-oss-dev-app';
 export const HDX_EXPORTER_ENABLED =
   (process.env.HDX_EXPORTER_ENABLED ?? 'true') === 'true';
+
+export function parseResourceAttributes(raw: string): Record<string, string> {
+  return raw
+    .split(',')
+    .filter(Boolean)
+    .reduce(
+      (acc, pair) => {
+        const idx = pair.indexOf('=');
+        if (idx > 0) {
+          acc[decodeURIComponent(pair.slice(0, idx))] = decodeURIComponent(
+            pair.slice(idx + 1),
+          );
+        }
+        return acc;
+      },
+      {} as Record<string, string>,
+    );
+}
+
 export const HDX_COLLECTOR_URL =
   process.env.NEXT_PUBLIC_OTEL_EXPORTER_OTLP_ENDPOINT ??
   process.env.OTEL_EXPORTER_OTLP_ENDPOINT ??

packages/app/src/hooks/usePatterns.tsx

@@ -5,7 +5,7 @@ import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/type
 import { useQuery } from '@tanstack/react-query';
 
 import { timeBucketByGranularity, toStartOfInterval } from '@/ChartUtils';
-import { useConfigWithPrimaryAndPartitionKey } from '@/components/DBRowTable';
+import { useConfigWithAdditionalSelect } from '@/components/DBRowTable';
 import { useQueriedChartConfig } from '@/hooks/useChartConfig';
 import { getFirstTimestampValueExpression } from '@/source';
 
@@ -134,7 +134,7 @@ function usePatterns({
   statusCodeExpression?: string;
   enabled?: boolean;
 }) {
-  const configWithPrimaryAndPartitionKey = useConfigWithPrimaryAndPartitionKey({
+  const configWithPrimaryAndPartitionKey = useConfigWithAdditionalSelect({
     ...config,
     // TODO: User-configurable pattern columns and non-pattern/group by columns
     select: [

packages/app/src/source.ts

@@ -348,6 +348,12 @@ export async function inferTableSourceConfig({
   // Check if SpanEvents column is available
   const hasSpanEvents = columns.some(col => col.name === 'Events.Timestamp');
 
+  // Column name used to disambiguate log rows with identical visible column values.
+  // This is a MATERIALIZED column (random UInt16) added to otel_logs to ensure
+  // the WHERE clause generated for row detail queries uniquely identifies a row.
+  const HDX_ID_COLUMN = '__hdx_id';
+  const hasHdxIdColumn = columns.some(c => c.name === HDX_ID_COLUMN);
+
   return {
     ...baseConfig,
     ...(isOtelLogSchema
@@ -365,6 +371,7 @@ export async function inferTableSourceConfig({
           traceIdExpression: 'TraceId',
 
           severityTextExpression: 'SeverityText',
+          ...(hasHdxIdColumn ? { uniqueRowIdExpression: HDX_ID_COLUMN } : {}),
         }
       : {}),
     ...(isOtelSpanSchema