diff --git a/.changeset/funny-zebras-collect.md b/.changeset/funny-zebras-collect.md
new file mode 100644
index 00000000..90efed4c
--- /dev/null
+++ b/.changeset/funny-zebras-collect.md
@@ -0,0 +1,6 @@
+---
+"@hyperdx/common-utils": minor
+"@hyperdx/app": minor
+---
+
+feat: Add auto-detecting and creating OTel sources during onboarding
diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx
index 2964b32f..3fcf81d6 100644
--- a/packages/app/src/DBSearchPage.tsx
+++ b/packages/app/src/DBSearchPage.tsx
@@ -1789,8 +1789,8 @@ function DBSearchPage() {
- Please start by selecting a database, table, and timestamp
- column above to view data.
+ Please start by selecting a source and then click the play
+ button to query data.
diff --git a/packages/app/src/components/OnboardingModal.tsx b/packages/app/src/components/OnboardingModal.tsx
index 87db00cc..eebf744a 100644
--- a/packages/app/src/components/OnboardingModal.tsx
+++ b/packages/app/src/components/OnboardingModal.tsx
@@ -1,17 +1,20 @@
import { memo, useCallback, useEffect, useState } from 'react';
import {
MetricsDataType,
+ MetricTable,
SourceKind,
TSource,
} from '@hyperdx/common-utils/dist/types';
-import { Button, Divider, Modal, Text } from '@mantine/core';
+import { Button, Divider, Flex, Loader, Modal, Text } from '@mantine/core';
import { notifications } from '@mantine/notifications';
import { IconArrowLeft } from '@tabler/icons-react';
import { ConnectionForm } from '@/components/ConnectionForm';
import { IS_LOCAL_MODE } from '@/config';
import { useConnections, useCreateConnection } from '@/connection';
+import { useMetadataWithSettings } from '@/hooks/useMetadata';
import {
+ inferTableSourceConfig,
useCreateSource,
useDeleteSource,
useSources,
@@ -19,6 +22,7 @@ import {
} from '@/source';
import { TableSourceForm } from './Sources/SourceForm';
+import { SourcesList } from './Sources/SourcesList';
async function addOtelDemoSources({
connectionId,
@@ -213,25 +217,304 @@ function OnboardingModalComponent({
connections?.length === 0
? 'connection'
: sources?.length === 0 && requireSource
- ? 'source'
+ ? 'auto-detect'
: undefined;
- const [_step, setStep] = useState<'connection' | 'source' | undefined>(
- undefined,
- );
+ const [_step, setStep] = useState<
+ 'connection' | 'auto-detect' | 'source' | 'closed' | undefined
+ >(undefined);
- const step = _step ?? startStep;
+ const step = _step;
useEffect(() => {
- if (step === 'source' && sources != null && sources.length > 0) {
- setStep(undefined);
+ if (startStep != null && step == null) {
+ setStep(startStep);
}
- }, [step, sources]);
+ }, [startStep, step]);
const createSourceMutation = useCreateSource();
const createConnectionMutation = useCreateConnection();
const updateSourceMutation = useUpdateSource();
const deleteSourceMutation = useDeleteSource();
+ const metadata = useMetadataWithSettings();
+
+ const [isAutoDetecting, setIsAutoDetecting] = useState(false);
+ // We should only try to auto-detect once
+ const [hasAutodetected, setHasAutodetected] = useState(false);
+ const [autoDetectedSources, setAutoDetectedSources] = useState([]);
+
+ const handleAutoDetectSources = useCallback(
+ async (connectionId: string) => {
+ try {
+ setIsAutoDetecting(true);
+ setHasAutodetected(true);
+
+ // Try to detect OTEL tables
+ const otelTables = await metadata.getOtelTables({ connectionId });
+
+ if (!otelTables) {
+ // No tables detected, go to manual source setup
+ setStep('source');
+ return;
+ }
+
+ const createdSources: TSource[] = [];
+
+ // Create Log Source if available
+ if (otelTables.tables.logs) {
+ const inferredConfig = await inferTableSourceConfig({
+ databaseName: otelTables.database,
+ tableName: otelTables.tables.logs,
+ connectionId,
+ metadata,
+ });
+
+ if (inferredConfig.timestampValueExpression != null) {
+ const logSource = await createSourceMutation.mutateAsync({
+ source: {
+ kind: SourceKind.Log,
+ name: 'Logs',
+ connection: connectionId,
+ from: {
+ databaseName: otelTables.database,
+ tableName: otelTables.tables.logs,
+ },
+ ...inferredConfig,
+ timestampValueExpression:
+ inferredConfig.timestampValueExpression,
+ },
+ });
+ createdSources.push(logSource);
+ } else {
+ console.error(
+ 'Log source was found but missing required fields',
+ inferredConfig,
+ );
+ }
+ }
+
+ // Create Trace Source if available
+ if (otelTables.tables.traces) {
+ const inferredConfig = await inferTableSourceConfig({
+ databaseName: otelTables.database,
+ tableName: otelTables.tables.traces,
+ connectionId,
+ metadata,
+ });
+
+ if (inferredConfig.timestampValueExpression != null) {
+ const traceSource = await createSourceMutation.mutateAsync({
+ source: {
+ kind: SourceKind.Trace,
+ name: 'Traces',
+ connection: connectionId,
+ from: {
+ databaseName: otelTables.database,
+ tableName: otelTables.tables.traces,
+ },
+ ...inferredConfig,
+ // Help typescript understand it's not null
+ timestampValueExpression:
+ inferredConfig.timestampValueExpression,
+ },
+ });
+ createdSources.push(traceSource);
+ } else {
+ console.error(
+ 'Trace source was found but missing required fields',
+ inferredConfig,
+ );
+ }
+ }
+
+ // Create Metrics Source if any metrics tables are available
+ const hasMetrics = Object.values(otelTables.tables.metrics).some(
+ t => t != null,
+ );
+ if (hasMetrics) {
+ const metricTables: MetricTable = {
+ [MetricsDataType.Gauge]: '',
+ [MetricsDataType.Histogram]: '',
+ [MetricsDataType.Sum]: '',
+ [MetricsDataType.Summary]: '',
+ [MetricsDataType.ExponentialHistogram]: '',
+ };
+ if (otelTables.tables.metrics.gauge) {
+ metricTables[MetricsDataType.Gauge] =
+ otelTables.tables.metrics.gauge;
+ }
+ if (otelTables.tables.metrics.histogram) {
+ metricTables[MetricsDataType.Histogram] =
+ otelTables.tables.metrics.histogram;
+ }
+ if (otelTables.tables.metrics.sum) {
+ metricTables[MetricsDataType.Sum] = otelTables.tables.metrics.sum;
+ }
+ if (otelTables.tables.metrics.summary) {
+ metricTables[MetricsDataType.Summary] =
+ otelTables.tables.metrics.summary;
+ }
+ if (otelTables.tables.metrics.expHistogram) {
+ metricTables[MetricsDataType.ExponentialHistogram] =
+ otelTables.tables.metrics.expHistogram;
+ }
+
+ const metricsSource = await createSourceMutation.mutateAsync({
+ source: {
+ kind: SourceKind.Metric,
+ name: 'Metrics',
+ connection: connectionId,
+ from: {
+ databaseName: otelTables.database,
+ tableName: '',
+ },
+ timestampValueExpression: 'TimeUnix',
+ serviceNameExpression: 'ServiceName',
+ metricTables,
+ resourceAttributesExpression: 'ResourceAttributes',
+ },
+ });
+ createdSources.push(metricsSource);
+ }
+
+ // Create Session Source if available
+ if (otelTables.tables.sessions) {
+ const inferredConfig = await inferTableSourceConfig({
+ databaseName: otelTables.database,
+ tableName: otelTables.tables.sessions,
+ connectionId,
+ metadata,
+ });
+ const traceSource = createdSources.find(
+ s => s.kind === SourceKind.Trace,
+ );
+
+ if (
+ inferredConfig.timestampValueExpression != null &&
+ traceSource != null
+ ) {
+ const sessionSource = await createSourceMutation.mutateAsync({
+ source: {
+ kind: SourceKind.Session,
+ name: 'Sessions',
+ connection: connectionId,
+ from: {
+ databaseName: otelTables.database,
+ tableName: otelTables.tables.sessions,
+ },
+ ...inferredConfig,
+ timestampValueExpression:
+ inferredConfig.timestampValueExpression,
+ traceSourceId: traceSource.id, // this is required for session source creation
+ },
+ });
+ createdSources.push(sessionSource);
+ } else {
+ console.error(
+ 'Session source was found but missing required fields',
+ inferredConfig,
+ );
+ }
+ }
+
+ if (createdSources.length === 0) {
+ console.error('No sources created due to missing required fields');
+ // No sources created, go to manual source setup
+ setStep('source');
+ return;
+ }
+
+ // Update sources to link them together
+ const logSource = createdSources.find(s => s.kind === SourceKind.Log);
+ const traceSource = createdSources.find(
+ s => s.kind === SourceKind.Trace,
+ );
+ const metricsSource = createdSources.find(
+ s => s.kind === SourceKind.Metric,
+ );
+ const sessionSource = createdSources.find(
+ s => s.kind === SourceKind.Session,
+ );
+
+ const updatePromises = [];
+
+ if (logSource) {
+ updatePromises.push(
+ updateSourceMutation.mutateAsync({
+ source: {
+ ...logSource,
+ ...(traceSource ? { traceSourceId: traceSource.id } : {}),
+ ...(metricsSource ? { metricSourceId: metricsSource.id } : {}),
+ ...(sessionSource ? { sessionSourceId: sessionSource.id } : {}),
+ },
+ }),
+ );
+ }
+
+ if (traceSource) {
+ updatePromises.push(
+ updateSourceMutation.mutateAsync({
+ source: {
+ ...traceSource,
+ ...(logSource ? { logSourceId: logSource.id } : {}),
+ ...(metricsSource ? { metricSourceId: metricsSource.id } : {}),
+ ...(sessionSource ? { sessionSourceId: sessionSource.id } : {}),
+ },
+ }),
+ );
+ }
+
+ await Promise.all(updatePromises);
+
+ setAutoDetectedSources(createdSources);
+ notifications.show({
+ title: 'Success',
+ message: `Automatically detected and created ${createdSources.length} source${createdSources.length > 1 ? 's' : ''}.`,
+ });
+ setStep('closed');
+ } catch (err) {
+ console.error('Error auto-detecting sources:', err);
+ notifications.show({
+ color: 'red',
+ title: 'Error',
+ message:
+ 'Failed to auto-detect telemetry sources. Please set up manually.',
+ });
+ // Fall back to manual source setup
+ setStep('source');
+ } finally {
+ setIsAutoDetecting(false);
+ }
+ },
+ [
+ metadata,
+ createSourceMutation,
+ updateSourceMutation,
+ setStep,
+ setAutoDetectedSources,
+ ],
+ );
+
+ // Trigger auto-detection when entering the auto-detect step
+ useEffect(() => {
+ if (
+ step === 'auto-detect' && // we should be trying to auto detect
+ sources?.length === 0 && // no sources yet
+ connections && // we need connections
+ connections.length > 0 &&
+ isAutoDetecting === false && // make sure we aren't currently auto detecting
+ hasAutodetected === false // only call it once
+ ) {
+ handleAutoDetectSources(connections[0].id);
+ }
+ }, [
+ step,
+ connections,
+ handleAutoDetectSources,
+ isAutoDetecting,
+ sources,
+ hasAutodetected,
+ ]);
const handleDemoServerClick = useCallback(async () => {
try {
@@ -370,7 +653,7 @@ function OnboardingModalComponent({
title: 'Success',
message: 'Connected to HyperDX demo server.',
});
- setStep(undefined);
+ setStep('closed');
} catch (err) {
console.error(err);
notifications.show({
@@ -391,7 +674,7 @@ function OnboardingModalComponent({
return (
{}}
title="Welcome to HyperDX"
size="xl"
@@ -412,7 +695,11 @@ function OnboardingModalComponent({
password: '',
}}
onSave={() => {
- setStep('source');
+ if (hasAutodetected) {
+ setStep('source');
+ } else {
+ setStep('auto-detect');
+ }
}}
isNew={true}
/>
@@ -421,7 +708,12 @@ function OnboardingModalComponent({
connection={connections[0]}
isNew={false}
onSave={() => {
- setStep('source');
+ // If we've already auto-detected, just go to manual source setup
+ if (hasAutodetected) {
+ setStep('source');
+ } else {
+ setStep('auto-detect');
+ }
}}
showCancelButton={false}
showDeleteButton={false}
@@ -443,6 +735,86 @@ function OnboardingModalComponent({
)}
+ {step === 'auto-detect' && (
+ <>
+ {isAutoDetecting ? (
+ <>
+
+
+ Detecting available tables...
+
+ {
+ setIsAutoDetecting(false);
+ setStep('source');
+ }}
+ >
+ Skip and setup manually
+
+
+
+ ) : autoDetectedSources.length > 0 ? (
+ <>
+ setStep('connection')}
+ p="xs"
+ mb="md"
+ >
+
+
+ We automatically detected and created{' '}
+ {autoDetectedSources.length} source
+ {autoDetectedSources.length > 1 ? 's' : ''} from your
+ connection. You can review, edit, or continue.
+
+
+ {
+ setStep('source');
+ }}
+ >
+ Add more sources
+
+ {
+ setStep('closed');
+ }}
+ >
+ Continue
+
+
+
+ ) : (
+
+ {/* We don't expect users to hit this - but this allows them to get unstuck if they do */}
+
+ No OTel tables detected automatically, please setup sources
+ manually.
+
+ {
+ setStep('source');
+ }}
+ >
+ Continue
+
+
+ )}
+
+ )}
{step === 'source' && (
<>
{
- setStep(undefined);
+ setStep('closed');
}}
/>
diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts
index 2c31d929..a5f2f9fd 100644
--- a/packages/common-utils/src/__tests__/metadata.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.test.ts
@@ -557,4 +557,161 @@ describe('Metadata', () => {
},
);
});
+
+ describe('getOtelTables', () => {
+ beforeEach(() => {
+ mockCache.getOrFetch.mockImplementation((key, queryFn) => queryFn());
+ });
+
+ it('should return null when no OTEL tables are found', async () => {
+ (mockClickhouseClient.query as jest.Mock).mockResolvedValue({
+ json: jest.fn().mockResolvedValue({
+ data: [],
+ }),
+ });
+
+ const result = await metadata.getOtelTables({
+ connectionId: 'test_connection',
+ });
+
+ expect(result).toBeNull();
+ });
+
+ it('should return a coherent set of tables from a single database', async () => {
+ (mockClickhouseClient.query as jest.Mock).mockResolvedValue({
+ json: jest.fn().mockResolvedValue({
+ data: [
+ { database: 'default', name: 'otel_logs' },
+ { database: 'default', name: 'otel_traces' },
+ { database: 'default', name: 'hyperdx_sessions' },
+ { database: 'default', name: 'otel_metrics_gauge' },
+ { database: 'default', name: 'otel_metrics_sum' },
+ { database: 'default', name: 'otel_metrics_histogram' },
+ ],
+ }),
+ });
+
+ const result = await metadata.getOtelTables({
+ connectionId: 'test_connection',
+ });
+
+ expect(result).toEqual({
+ database: 'default',
+ tables: {
+ logs: 'otel_logs',
+ traces: 'otel_traces',
+ sessions: 'hyperdx_sessions',
+ metrics: {
+ gauge: 'otel_metrics_gauge',
+ sum: 'otel_metrics_sum',
+ histogram: 'otel_metrics_histogram',
+ summary: undefined,
+ expHistogram: undefined,
+ },
+ },
+ });
+ });
+
+ it('should select the database with the most complete set when multiple databases exist', async () => {
+ (mockClickhouseClient.query as jest.Mock).mockResolvedValue({
+ json: jest.fn().mockResolvedValue({
+ data: [
+ { database: 'default', name: 'hyperdx_sessions' },
+ { database: 'default', name: 'otel_logs' },
+ { database: 'default', name: 'otel_metrics_gauge' },
+ { database: 'default', name: 'otel_metrics_histogram' },
+ { database: 'default', name: 'otel_metrics_sum' },
+ { database: 'default', name: 'otel_metrics_summary' },
+ { database: 'default', name: 'otel_traces' },
+ { database: 'otel_json', name: 'hyperdx_sessions' },
+ { database: 'otel_json', name: 'otel_logs' },
+ { database: 'otel_json', name: 'otel_metrics_gauge' },
+ { database: 'otel_json', name: 'otel_metrics_histogram' },
+ { database: 'otel_json', name: 'otel_metrics_sum' },
+ { database: 'otel_json', name: 'otel_metrics_summary' },
+ { database: 'otel_json', name: 'otel_traces' },
+ ],
+ }),
+ });
+
+ const result = await metadata.getOtelTables({
+ connectionId: 'test_connection',
+ });
+
+ expect(result).toBeDefined();
+ expect(result?.database).toBe('default'); // Both have same score, first one wins
+ expect(result?.tables.logs).toBe('otel_logs');
+ expect(result?.tables.traces).toBe('otel_traces');
+ expect(result?.tables.sessions).toBe('hyperdx_sessions');
+ });
+
+ it('should prioritize database with logs and traces over one with only metrics', async () => {
+ (mockClickhouseClient.query as jest.Mock).mockResolvedValue({
+ json: jest.fn().mockResolvedValue({
+ data: [
+ { database: 'metrics_db', name: 'otel_metrics_gauge' },
+ { database: 'metrics_db', name: 'otel_metrics_sum' },
+ { database: 'full_db', name: 'otel_logs' },
+ { database: 'full_db', name: 'otel_traces' },
+ ],
+ }),
+ });
+
+ const result = await metadata.getOtelTables({
+ connectionId: 'test_connection',
+ });
+
+ expect(result?.database).toBe('full_db');
+ });
+
+ it('should use cache when retrieving OTEL tables', async () => {
+ mockCache.getOrFetch.mockReset();
+
+ const mockResult = {
+ database: 'default',
+ tables: {
+ logs: 'otel_logs',
+ traces: 'otel_traces',
+ sessions: 'hyperdx_sessions',
+ metrics: {
+ gauge: 'otel_metrics_gauge',
+ sum: undefined,
+ histogram: undefined,
+ summary: undefined,
+ expHistogram: undefined,
+ },
+ },
+ };
+
+ mockCache.getOrFetch.mockImplementation((key, queryFn) => {
+ if (key === 'test_connection.otelTables') {
+ return Promise.resolve(mockResult);
+ }
+ return queryFn();
+ });
+
+ const result = await metadata.getOtelTables({
+ connectionId: 'test_connection',
+ });
+
+ expect(mockCache.getOrFetch).toHaveBeenCalledWith(
+ 'test_connection.otelTables',
+ expect.any(Function),
+ );
+ expect(mockClickhouseClient.query).not.toHaveBeenCalled();
+ expect(result).toEqual(mockResult);
+ });
+
+ it('should return null when permissions error occurs', async () => {
+ (mockClickhouseClient.query as jest.Mock).mockRejectedValue(
+ new Error('Not enough privileges'),
+ );
+
+ const result = await metadata.getOtelTables({
+ connectionId: 'test_connection',
+ });
+
+ expect(result).toBeNull();
+ });
+ });
});
diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index 56c4109f..fd610805 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -6,6 +6,7 @@ import {
ChSql,
chSql,
ColumnMeta,
+ concatChSql,
convertCHDataTypeToJSType,
filterColumnMetaByType,
JSDataType,
@@ -717,6 +718,145 @@ export class Metadata {
);
}
+ /**
+ * Inspects the ClickHouse connection for OpenTelemetry telemetry tables.
+ * Returns one coherent set of tables from the same database.
+ *
+ * When multiple databases contain the same table schema, this function prioritizes
+ * returning a complete set from a single database rather than mixing tables from different databases.
+ */
+ async getOtelTables({ connectionId }: { connectionId: string }): Promise<{
+ database: string;
+ tables: {
+ logs?: string;
+ traces?: string;
+ sessions?: string;
+ metrics: {
+ gauge?: string;
+ sum?: string;
+ summary?: string;
+ histogram?: string;
+ expHistogram?: string;
+ };
+ };
+ } | null> {
+ return this.cache.getOrFetch(`${connectionId}.otelTables`, async () => {
+ const OTEL_TABLE_NAMES = [
+ 'otel_logs',
+ 'otel_traces',
+ 'hyperdx_sessions',
+ 'otel_metrics_gauge',
+ 'otel_metrics_sum',
+ 'otel_metrics_summary',
+ 'otel_metrics_histogram',
+ 'otel_metrics_exp_histogram',
+ ];
+
+ const tableNameParams = OTEL_TABLE_NAMES.map(
+ t => chSql`${{ String: t }}`,
+ );
+
+ const sql = chSql`
+ SELECT
+ database,
+ name
+ FROM system.tables
+ WHERE (database != 'system')
+ AND (name IN (${concatChSql(',', tableNameParams)}))
+ ORDER BY database, name
+ `;
+
+ try {
+ const json = await this.clickhouseClient
+ .query<'JSON'>({
+ connectionId,
+ query: sql.sql,
+ query_params: sql.params,
+ clickhouse_settings: this.getClickHouseSettings(),
+ })
+ .then(res => res.json<{ database: string; name: string }>());
+
+ if (json.data.length === 0) {
+ return null;
+ }
+
+ // Group tables by database
+ const tablesByDatabase = new Map>();
+ for (const row of json.data) {
+ if (!tablesByDatabase.has(row.database)) {
+ tablesByDatabase.set(row.database, new Set());
+ }
+ tablesByDatabase.get(row.database)!.add(row.name);
+ }
+
+ // Find the database with the most complete set of tables
+ let bestDatabase = '';
+ let bestScore = 0;
+
+ for (const [database, tables] of tablesByDatabase.entries()) {
+ // Score based on number of essential tables present
+ let score = 0;
+ if (tables.has('otel_logs')) score += 10;
+ if (tables.has('otel_traces')) score += 10;
+ if (tables.has('hyperdx_sessions')) score += 5;
+ if (tables.has('otel_metrics_gauge')) score += 2;
+ if (tables.has('otel_metrics_sum')) score += 2;
+ if (tables.has('otel_metrics_histogram')) score += 2;
+ if (tables.has('otel_metrics_summary')) score += 1;
+ if (tables.has('otel_metrics_exp_histogram')) score += 1;
+
+ if (score > bestScore) {
+ bestScore = score;
+ bestDatabase = database;
+ }
+ }
+
+ if (!bestDatabase) {
+ return null;
+ }
+
+ const selectedTables = tablesByDatabase.get(bestDatabase)!;
+
+ return {
+ database: bestDatabase,
+ tables: {
+ logs: selectedTables.has('otel_logs') ? 'otel_logs' : undefined,
+ traces: selectedTables.has('otel_traces')
+ ? 'otel_traces'
+ : undefined,
+ sessions: selectedTables.has('hyperdx_sessions')
+ ? 'hyperdx_sessions'
+ : undefined,
+ metrics: {
+ gauge: selectedTables.has('otel_metrics_gauge')
+ ? 'otel_metrics_gauge'
+ : undefined,
+ sum: selectedTables.has('otel_metrics_sum')
+ ? 'otel_metrics_sum'
+ : undefined,
+ summary: selectedTables.has('otel_metrics_summary')
+ ? 'otel_metrics_summary'
+ : undefined,
+ histogram: selectedTables.has('otel_metrics_histogram')
+ ? 'otel_metrics_histogram'
+ : undefined,
+ expHistogram: selectedTables.has('otel_metrics_exp_histogram')
+ ? 'otel_metrics_exp_histogram'
+ : undefined,
+ },
+ },
+ };
+ } catch (e) {
+ if (e instanceof Error && e.message.includes('Not enough privileges')) {
+ console.warn('Not enough privileges to fetch tables:', e);
+ return null;
+ }
+
+ throw e;
+ }
+ });
+ }
+
/**
* Parses a ClickHouse index expression to check if it uses the tokens() function.
* Returns the inner expression if tokens() is found.