Elgato_dark/hyperdx
1
0
Fork 0
mirror of https://github.com/hyperdxio/hyperdx synced 2026-04-21 13:37:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

Auto Source Creation (#1692)

Browse source
This commit is contained in:
Mike Shi 2026-02-03 15:03:22 -08:00 committed by GitHub
parent 288213013c
commit f44923ba58
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 691 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.changeset/funny-zebras-collect.md Normal file
View file

@ -0,0 +1,6 @@
---
"@hyperdx/common-utils": minor
"@hyperdx/app": minor
---
feat: Add auto-detecting and creating OTel sources during onboarding

4
packages/app/src/DBSearchPage.tsx
View file

@ -1789,8 +1789,8 @@ function DBSearchPage() {
<Paper shadow="xs" p="xl" h="100%">
<Center mih={100} h="100%">
<Text size="sm">
Please start by selecting a database, table, and timestamp
column above to view data.
Please start by selecting a source and then click the play
button to query data.
</Text>
</Center>
</Paper>

400
packages/app/src/components/OnboardingModal.tsx
View file

@ -1,17 +1,20 @@
import { memo, useCallback, useEffect, useState } from 'react';
import {
MetricsDataType,
MetricTable,
SourceKind,
TSource,
} from '@hyperdx/common-utils/dist/types';
import { Button, Divider, Modal, Text } from '@mantine/core';
import { Button, Divider, Flex, Loader, Modal, Text } from '@mantine/core';
import { notifications } from '@mantine/notifications';
import { IconArrowLeft } from '@tabler/icons-react';
import { ConnectionForm } from '@/components/ConnectionForm';
import { IS_LOCAL_MODE } from '@/config';
import { useConnections, useCreateConnection } from '@/connection';
import { useMetadataWithSettings } from '@/hooks/useMetadata';
import {
inferTableSourceConfig,
useCreateSource,
useDeleteSource,
useSources,
@ -19,6 +22,7 @@ import {
} from '@/source';
import { TableSourceForm } from './Sources/SourceForm';
import { SourcesList } from './Sources/SourcesList';
async function addOtelDemoSources({
connectionId,
@ -213,25 +217,304 @@ function OnboardingModalComponent({
connections?.length === 0
? 'connection'
: sources?.length === 0 && requireSource
? 'source'
? 'auto-detect'
: undefined;
const [_step, setStep] = useState<'connection' | 'source' | undefined>(
undefined,
);
const [_step, setStep] = useState<
'connection' | 'auto-detect' | 'source' | 'closed' | undefined
>(undefined);
const step = _step ?? startStep;
const step = _step;
useEffect(() => {
if (step === 'source' && sources != null && sources.length > 0) {
setStep(undefined);
if (startStep != null && step == null) {
setStep(startStep);
}
}, [step, sources]);
}, [startStep, step]);
const createSourceMutation = useCreateSource();
const createConnectionMutation = useCreateConnection();
const updateSourceMutation = useUpdateSource();
const deleteSourceMutation = useDeleteSource();
const metadata = useMetadataWithSettings();
const [isAutoDetecting, setIsAutoDetecting] = useState(false);
// We should only try to auto-detect once
const [hasAutodetected, setHasAutodetected] = useState(false);
const [autoDetectedSources, setAutoDetectedSources] = useState<TSource[]>([]);
const handleAutoDetectSources = useCallback(
async (connectionId: string) => {
try {
setIsAutoDetecting(true);
setHasAutodetected(true);
// Try to detect OTEL tables
const otelTables = await metadata.getOtelTables({ connectionId });
if (!otelTables) {
// No tables detected, go to manual source setup
setStep('source');
return;
}
const createdSources: TSource[] = [];
// Create Log Source if available
if (otelTables.tables.logs) {
const inferredConfig = await inferTableSourceConfig({
databaseName: otelTables.database,
tableName: otelTables.tables.logs,
connectionId,
metadata,
});
if (inferredConfig.timestampValueExpression != null) {
const logSource = await createSourceMutation.mutateAsync({
source: {
kind: SourceKind.Log,
name: 'Logs',
connection: connectionId,
from: {
databaseName: otelTables.database,
tableName: otelTables.tables.logs,
},
...inferredConfig,
timestampValueExpression:
inferredConfig.timestampValueExpression,
},
});
createdSources.push(logSource);
} else {
console.error(
'Log source was found but missing required fields',
inferredConfig,
);
}
}
// Create Trace Source if available
if (otelTables.tables.traces) {
const inferredConfig = await inferTableSourceConfig({
databaseName: otelTables.database,
tableName: otelTables.tables.traces,
connectionId,
metadata,
});
if (inferredConfig.timestampValueExpression != null) {
const traceSource = await createSourceMutation.mutateAsync({
source: {
kind: SourceKind.Trace,
name: 'Traces',
connection: connectionId,
from: {
databaseName: otelTables.database,
tableName: otelTables.tables.traces,
},
...inferredConfig,
// Help typescript understand it's not null
timestampValueExpression:
inferredConfig.timestampValueExpression,
},
});
createdSources.push(traceSource);
} else {
console.error(
'Trace source was found but missing required fields',
inferredConfig,
);
}
}
// Create Metrics Source if any metrics tables are available
const hasMetrics = Object.values(otelTables.tables.metrics).some(
t => t != null,
);
if (hasMetrics) {
const metricTables: MetricTable = {
[MetricsDataType.Gauge]: '',
[MetricsDataType.Histogram]: '',
[MetricsDataType.Sum]: '',
[MetricsDataType.Summary]: '',
[MetricsDataType.ExponentialHistogram]: '',
};
if (otelTables.tables.metrics.gauge) {
metricTables[MetricsDataType.Gauge] =
otelTables.tables.metrics.gauge;
}
if (otelTables.tables.metrics.histogram) {
metricTables[MetricsDataType.Histogram] =
otelTables.tables.metrics.histogram;
}
if (otelTables.tables.metrics.sum) {
metricTables[MetricsDataType.Sum] = otelTables.tables.metrics.sum;
}
if (otelTables.tables.metrics.summary) {
metricTables[MetricsDataType.Summary] =
otelTables.tables.metrics.summary;
}
if (otelTables.tables.metrics.expHistogram) {
metricTables[MetricsDataType.ExponentialHistogram] =
otelTables.tables.metrics.expHistogram;
}
const metricsSource = await createSourceMutation.mutateAsync({
source: {
kind: SourceKind.Metric,
name: 'Metrics',
connection: connectionId,
from: {
databaseName: otelTables.database,
tableName: '',
},
timestampValueExpression: 'TimeUnix',
serviceNameExpression: 'ServiceName',
metricTables,
resourceAttributesExpression: 'ResourceAttributes',
},
});
createdSources.push(metricsSource);
}
// Create Session Source if available
if (otelTables.tables.sessions) {
const inferredConfig = await inferTableSourceConfig({
databaseName: otelTables.database,
tableName: otelTables.tables.sessions,
connectionId,
metadata,
});
const traceSource = createdSources.find(
s => s.kind === SourceKind.Trace,
);
if (
inferredConfig.timestampValueExpression != null &&
traceSource != null
) {
const sessionSource = await createSourceMutation.mutateAsync({
source: {
kind: SourceKind.Session,
name: 'Sessions',
connection: connectionId,
from: {
databaseName: otelTables.database,
tableName: otelTables.tables.sessions,
},
...inferredConfig,
timestampValueExpression:
inferredConfig.timestampValueExpression,
traceSourceId: traceSource.id, // this is required for session source creation
},
});
createdSources.push(sessionSource);
} else {
console.error(
'Session source was found but missing required fields',
inferredConfig,
);
}
}
if (createdSources.length === 0) {
console.error('No sources created due to missing required fields');
// No sources created, go to manual source setup
setStep('source');
return;
}
// Update sources to link them together
const logSource = createdSources.find(s => s.kind === SourceKind.Log);
const traceSource = createdSources.find(
s => s.kind === SourceKind.Trace,
);
const metricsSource = createdSources.find(
s => s.kind === SourceKind.Metric,
);
const sessionSource = createdSources.find(
s => s.kind === SourceKind.Session,
);
const updatePromises = [];
if (logSource) {
updatePromises.push(
updateSourceMutation.mutateAsync({
source: {
...logSource,
...(traceSource ? { traceSourceId: traceSource.id } : {}),
...(metricsSource ? { metricSourceId: metricsSource.id } : {}),
...(sessionSource ? { sessionSourceId: sessionSource.id } : {}),
},
}),
);
}
if (traceSource) {
updatePromises.push(
updateSourceMutation.mutateAsync({
source: {
...traceSource,
...(logSource ? { logSourceId: logSource.id } : {}),
...(metricsSource ? { metricSourceId: metricsSource.id } : {}),
...(sessionSource ? { sessionSourceId: sessionSource.id } : {}),
},
}),
);
}
await Promise.all(updatePromises);
setAutoDetectedSources(createdSources);
notifications.show({
title: 'Success',
message: `Automatically detected and created ${createdSources.length} source${createdSources.length > 1 ? 's' : ''}.`,
});
setStep('closed');
} catch (err) {
console.error('Error auto-detecting sources:', err);
notifications.show({
color: 'red',
title: 'Error',
message:
'Failed to auto-detect telemetry sources. Please set up manually.',
});
// Fall back to manual source setup
setStep('source');
} finally {
setIsAutoDetecting(false);
}
},
[
metadata,
createSourceMutation,
updateSourceMutation,
setStep,
setAutoDetectedSources,
],
);
// Trigger auto-detection when entering the auto-detect step
useEffect(() => {
if (
step === 'auto-detect' && // we should be trying to auto detect
sources?.length === 0 && // no sources yet
connections && // we need connections
connections.length > 0 &&
isAutoDetecting === false && // make sure we aren't currently auto detecting
hasAutodetected === false // only call it once
) {
handleAutoDetectSources(connections[0].id);
}
}, [
step,
connections,
handleAutoDetectSources,
isAutoDetecting,
sources,
hasAutodetected,
]);
const handleDemoServerClick = useCallback(async () => {
try {
@ -370,7 +653,7 @@ function OnboardingModalComponent({
title: 'Success',
message: 'Connected to HyperDX demo server.',
});
setStep(undefined);
setStep('closed');
} catch (err) {
console.error(err);
notifications.show({
@ -391,7 +674,7 @@ function OnboardingModalComponent({
return (
<Modal
data-testid="onboarding-modal"
opened={step != null}
opened={step != null && step !== 'closed'}
onClose={() => {}}
title="Welcome to HyperDX"
size="xl"
@ -412,7 +695,11 @@ function OnboardingModalComponent({
password: '',
}}
onSave={() => {
setStep('source');
if (hasAutodetected) {
setStep('source');
} else {
setStep('auto-detect');
}
}}
isNew={true}
/>
@ -421,7 +708,12 @@ function OnboardingModalComponent({
connection={connections[0]}
isNew={false}
onSave={() => {
setStep('source');
// If we've already auto-detected, just go to manual source setup
if (hasAutodetected) {
setStep('source');
} else {
setStep('auto-detect');
}
}}
showCancelButton={false}
showDeleteButton={false}
@ -443,6 +735,86 @@ function OnboardingModalComponent({
</Button>
</>
)}
{step === 'auto-detect' && (
<>
{isAutoDetecting ? (
<>
<Flex justify="center" align="center" direction="column" py="xl">
<Loader size="md" mb="md" />
<Text size="sm" c="dimmed" mb="md">
Detecting available tables...
</Text>
<Button
variant="subtle"
size="xs"
onClick={() => {
setIsAutoDetecting(false);
setStep('source');
}}
>
Skip and setup manually
</Button>
</Flex>
</>
) : autoDetectedSources.length > 0 ? (
<>
<Button
variant="subtle"
onClick={() => setStep('connection')}
p="xs"
mb="md"
>
<IconArrowLeft size={14} className="me-2" /> Back
</Button>
<Text size="sm" mb="md">
We automatically detected and created{' '}
{autoDetectedSources.length} source
{autoDetectedSources.length > 1 ? 's' : ''} from your
connection. You can review, edit, or continue.
</Text>
<SourcesList
withCard={false}
variant="default"
showEmptyState={false}
/>
<Flex justify="space-between" mt="md">
<Button
variant="subtle"
onClick={() => {
setStep('source');
}}
>
Add more sources
</Button>
<Button
variant="primary"
onClick={() => {
setStep('closed');
}}
>
Continue
</Button>
</Flex>
</>
) : (
<Flex justify="center" align="center" direction="column" py="xl">
{/* We don't expect users to hit this - but this allows them to get unstuck if they do */}
<Text size="sm" c="dimmed" mb="md">
No OTel tables detected automatically, please setup sources
manually.
</Text>
<Button
variant="primary"
onClick={() => {
setStep('source');
}}
>
Continue
</Button>
</Flex>
)}
</>
)}
{step === 'source' && (
<>
<Button
@ -460,7 +832,7 @@ function OnboardingModalComponent({
isNew
defaultName="Logs"
onCreate={() => {
setStep(undefined);
setStep('closed');
}}
/>
<Text size="xs" mt="lg">

157
packages/common-utils/src/__tests__/metadata.test.ts
View file

@ -557,4 +557,161 @@ describe('Metadata', () => {
},
);
});
describe('getOtelTables', () => {
beforeEach(() => {
mockCache.getOrFetch.mockImplementation((key, queryFn) => queryFn());
});
it('should return null when no OTEL tables are found', async () => {
(mockClickhouseClient.query as jest.Mock).mockResolvedValue({
json: jest.fn().mockResolvedValue({
data: [],
}),
});
const result = await metadata.getOtelTables({
connectionId: 'test_connection',
});
expect(result).toBeNull();
});
it('should return a coherent set of tables from a single database', async () => {
(mockClickhouseClient.query as jest.Mock).mockResolvedValue({
json: jest.fn().mockResolvedValue({
data: [
{ database: 'default', name: 'otel_logs' },
{ database: 'default', name: 'otel_traces' },
{ database: 'default', name: 'hyperdx_sessions' },
{ database: 'default', name: 'otel_metrics_gauge' },
{ database: 'default', name: 'otel_metrics_sum' },
{ database: 'default', name: 'otel_metrics_histogram' },
],
}),
});
const result = await metadata.getOtelTables({
connectionId: 'test_connection',
});
expect(result).toEqual({
database: 'default',
tables: {
logs: 'otel_logs',
traces: 'otel_traces',
sessions: 'hyperdx_sessions',
metrics: {
gauge: 'otel_metrics_gauge',
sum: 'otel_metrics_sum',
histogram: 'otel_metrics_histogram',
summary: undefined,
expHistogram: undefined,
},
},
});
});
it('should select the database with the most complete set when multiple databases exist', async () => {
(mockClickhouseClient.query as jest.Mock).mockResolvedValue({
json: jest.fn().mockResolvedValue({
data: [
{ database: 'default', name: 'hyperdx_sessions' },
{ database: 'default', name: 'otel_logs' },
{ database: 'default', name: 'otel_metrics_gauge' },
{ database: 'default', name: 'otel_metrics_histogram' },
{ database: 'default', name: 'otel_metrics_sum' },
{ database: 'default', name: 'otel_metrics_summary' },
{ database: 'default', name: 'otel_traces' },
{ database: 'otel_json', name: 'hyperdx_sessions' },
{ database: 'otel_json', name: 'otel_logs' },
{ database: 'otel_json', name: 'otel_metrics_gauge' },
{ database: 'otel_json', name: 'otel_metrics_histogram' },
{ database: 'otel_json', name: 'otel_metrics_sum' },
{ database: 'otel_json', name: 'otel_metrics_summary' },
{ database: 'otel_json', name: 'otel_traces' },
],
}),
});
const result = await metadata.getOtelTables({
connectionId: 'test_connection',
});
expect(result).toBeDefined();
expect(result?.database).toBe('default'); // Both have same score, first one wins
expect(result?.tables.logs).toBe('otel_logs');
expect(result?.tables.traces).toBe('otel_traces');
expect(result?.tables.sessions).toBe('hyperdx_sessions');
});
it('should prioritize database with logs and traces over one with only metrics', async () => {
(mockClickhouseClient.query as jest.Mock).mockResolvedValue({
json: jest.fn().mockResolvedValue({
data: [
{ database: 'metrics_db', name: 'otel_metrics_gauge' },
{ database: 'metrics_db', name: 'otel_metrics_sum' },
{ database: 'full_db', name: 'otel_logs' },
{ database: 'full_db', name: 'otel_traces' },
],
}),
});
const result = await metadata.getOtelTables({
connectionId: 'test_connection',
});
expect(result?.database).toBe('full_db');
});
it('should use cache when retrieving OTEL tables', async () => {
mockCache.getOrFetch.mockReset();
const mockResult = {
database: 'default',
tables: {
logs: 'otel_logs',
traces: 'otel_traces',
sessions: 'hyperdx_sessions',
metrics: {
gauge: 'otel_metrics_gauge',
sum: undefined,
histogram: undefined,
summary: undefined,
expHistogram: undefined,
},
},
};
mockCache.getOrFetch.mockImplementation((key, queryFn) => {
if (key === 'test_connection.otelTables') {
return Promise.resolve(mockResult);
}
return queryFn();
});
const result = await metadata.getOtelTables({
connectionId: 'test_connection',
});
expect(mockCache.getOrFetch).toHaveBeenCalledWith(
'test_connection.otelTables',
expect.any(Function),
);
expect(mockClickhouseClient.query).not.toHaveBeenCalled();
expect(result).toEqual(mockResult);
});
it('should return null when permissions error occurs', async () => {
(mockClickhouseClient.query as jest.Mock).mockRejectedValue(
new Error('Not enough privileges'),
);
const result = await metadata.getOtelTables({
connectionId: 'test_connection',
});
expect(result).toBeNull();
});
});
});

140
packages/common-utils/src/core/metadata.ts
View file

@ -6,6 +6,7 @@ import {
ChSql,
chSql,
ColumnMeta,
concatChSql,
convertCHDataTypeToJSType,
filterColumnMetaByType,
JSDataType,
@ -717,6 +718,145 @@ export class Metadata {
);
}
/**
* Inspects the ClickHouse connection for OpenTelemetry telemetry tables.
* Returns one coherent set of tables from the same database.
*
* When multiple databases contain the same table schema, this function prioritizes
* returning a complete set from a single database rather than mixing tables from different databases.
*/
async getOtelTables({ connectionId }: { connectionId: string }): Promise<{
database: string;
tables: {
logs?: string;
traces?: string;
sessions?: string;
metrics: {
gauge?: string;
sum?: string;
summary?: string;
histogram?: string;
expHistogram?: string;
};
};
} | null> {
return this.cache.getOrFetch(`${connectionId}.otelTables`, async () => {
const OTEL_TABLE_NAMES = [
'otel_logs',
'otel_traces',
'hyperdx_sessions',
'otel_metrics_gauge',
'otel_metrics_sum',
'otel_metrics_summary',
'otel_metrics_histogram',
'otel_metrics_exp_histogram',
];
const tableNameParams = OTEL_TABLE_NAMES.map(
t => chSql`${{ String: t }}`,
);
const sql = chSql`
SELECT
database,
name
FROM system.tables
WHERE (database != 'system')
AND (name IN (${concatChSql(',', tableNameParams)}))
ORDER BY database, name
`;
try {
const json = await this.clickhouseClient
.query<'JSON'>({
connectionId,
query: sql.sql,
query_params: sql.params,
clickhouse_settings: this.getClickHouseSettings(),
})
.then(res => res.json<{ database: string; name: string }>());
if (json.data.length === 0) {
return null;
}
// Group tables by database
const tablesByDatabase = new Map<string, Set<string>>();
for (const row of json.data) {
if (!tablesByDatabase.has(row.database)) {
tablesByDatabase.set(row.database, new Set());
}
tablesByDatabase.get(row.database)!.add(row.name);
}
// Find the database with the most complete set of tables
let bestDatabase = '';
let bestScore = 0;
for (const [database, tables] of tablesByDatabase.entries()) {
// Score based on number of essential tables present
let score = 0;
if (tables.has('otel_logs')) score += 10;
if (tables.has('otel_traces')) score += 10;
if (tables.has('hyperdx_sessions')) score += 5;
if (tables.has('otel_metrics_gauge')) score += 2;
if (tables.has('otel_metrics_sum')) score += 2;
if (tables.has('otel_metrics_histogram')) score += 2;
if (tables.has('otel_metrics_summary')) score += 1;
if (tables.has('otel_metrics_exp_histogram')) score += 1;
if (score > bestScore) {
bestScore = score;
bestDatabase = database;
}
}
if (!bestDatabase) {
return null;
}
const selectedTables = tablesByDatabase.get(bestDatabase)!;
return {
database: bestDatabase,
tables: {
logs: selectedTables.has('otel_logs') ? 'otel_logs' : undefined,
traces: selectedTables.has('otel_traces')
? 'otel_traces'
: undefined,
sessions: selectedTables.has('hyperdx_sessions')
? 'hyperdx_sessions'
: undefined,
metrics: {
gauge: selectedTables.has('otel_metrics_gauge')
? 'otel_metrics_gauge'
: undefined,
sum: selectedTables.has('otel_metrics_sum')
? 'otel_metrics_sum'
: undefined,
summary: selectedTables.has('otel_metrics_summary')
? 'otel_metrics_summary'
: undefined,
histogram: selectedTables.has('otel_metrics_histogram')
? 'otel_metrics_histogram'
: undefined,
expHistogram: selectedTables.has('otel_metrics_exp_histogram')
? 'otel_metrics_exp_histogram'
: undefined,
},
},
};
} catch (e) {
if (e instanceof Error && e.message.includes('Not enough privileges')) {
console.warn('Not enough privileges to fetch tables:', e);
return null;
}
throw e;
}
});
}
/**
* Parses a ClickHouse index expression to check if it uses the tokens() function.
* Returns the inner expression if tokens() is found.