Elgato_dark/hyperdx
1
0
Fork 0
mirror of https://github.com/hyperdxio/hyperdx synced 2026-04-21 13:37:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

fix: Refresh metadata after creating new connection in local mode (#1582)

Browse source
This commit is contained in:
Drew Davis 2026-01-13 07:55:12 -05:00 committed by GitHub
parent ddc7dd04ed
commit f39fcdac6a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 105 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.changeset/eleven-pears-pay.md Normal file
View file

@ -0,0 +1,5 @@
---
"@hyperdx/app": patch
---
fix: Refresh metadata after creating new connection in local mode

9
packages/app/src/components/KubeComponents.tsx
View file

@ -10,12 +10,12 @@ import {
SearchConditionLanguage,
TSource,
} from '@hyperdx/common-utils/dist/types';
import { Anchor, Badge, Group, Text, Timeline } from '@mantine/core';
import { Badge, Group, Text, Timeline } from '@mantine/core';
import { useQuery, UseQueryOptions } from '@tanstack/react-query';
import { useClickhouseClient } from '@/clickhouse';
import { getMetadata } from '@/metadata';
import { getDisplayedTimestampValueExpression, getEventBody } from '@/source';
import { useMetadataWithSettings } from '@/hooks/useMetadata';
import { getDisplayedTimestampValueExpression } from '@/source';
import { KubePhase } from '../types';
import { FormatTime } from '../useFormatTime';
@ -57,6 +57,7 @@ export const useV2LogBatch = <T = any,>(
options?: Omit<UseQueryOptions<any>, 'queryKey' | 'queryFn'>,
) => {
const clickhouseClient = useClickhouseClient();
const metadata = useMetadataWithSettings();
return useQuery<ResponseJSON<T>, Error>({
queryKey: [
'v2LogBatch',
@ -97,7 +98,7 @@ export const useV2LogBatch = <T = any,>(
},
orderBy: `${logSource.timestampValueExpression} ${order}`,
},
getMetadata(),
metadata,
);
const json = await clickhouseClient

18
packages/app/src/components/Sources/SourceForm.tsx
View file

@ -45,6 +45,7 @@ import {
import { SourceSelectControlled } from '@/components/SourceSelect';
import { IS_METRICS_ENABLED, IS_SESSIONS_ENABLED } from '@/config';
import { useConnections } from '@/connection';
import { useMetadataWithSettings } from '@/hooks/useMetadata';
import {
inferTableSourceConfig,
isValidMetricTable,
@ -544,6 +545,8 @@ function AggregatedColumnsFormSection({
const fromTableName = useWatch({ control, name: 'from.tableName' });
const prevMvTableNameRef = useRef(mvTableName);
const metadata = useMetadataWithSettings();
useEffect(() => {
(async () => {
try {
@ -569,6 +572,7 @@ function AggregatedColumnsFormSection({
tableName: fromTableName,
connectionId: connection,
},
metadata,
);
if (config) {
@ -603,6 +607,7 @@ function AggregatedColumnsFormSection({
mvIndex,
replaceAggregates,
setValue,
metadata,
]);
return (
@ -1278,6 +1283,7 @@ export function SessionTableModelForm({ control }: TableModelProps) {
const connectionId = useWatch({ control, name: 'connection' });
const tableName = useWatch({ control, name: 'from.tableName' });
const prevTableNameRef = useRef(tableName);
const metadata = useMetadataWithSettings();
useEffect(() => {
(async () => {
@ -1288,6 +1294,7 @@ export function SessionTableModelForm({ control }: TableModelProps) {
databaseName,
tableName,
connectionId,
metadata,
});
if (!isValid) {
@ -1305,7 +1312,7 @@ export function SessionTableModelForm({ control }: TableModelProps) {
});
}
})();
}, [tableName, databaseName, connectionId]);
}, [tableName, databaseName, connectionId, metadata]);
return (
<>
@ -1336,6 +1343,8 @@ export function MetricTableModelForm({ control, setValue }: TableModelProps) {
const metricTables = useWatch({ control, name: 'metricTables' });
const prevMetricTablesRef = useRef(metricTables);
const metadata = useMetadataWithSettings();
useEffect(() => {
for (const [_key, _value] of Object.entries(OTEL_CLICKHOUSE_EXPRESSIONS)) {
setValue(_key as any, _value);
@ -1361,6 +1370,7 @@ export function MetricTableModelForm({ control, setValue }: TableModelProps) {
tableName: newValue as string,
connectionId,
metricType: metricType as MetricsDataType,
metadata,
});
if (!isValid) {
notifications.show({
@ -1380,7 +1390,7 @@ export function MetricTableModelForm({ control, setValue }: TableModelProps) {
});
}
})();
}, [metricTables, databaseName, connectionId]);
}, [metricTables, databaseName, connectionId, metadata]);
return (
<>
@ -1495,6 +1505,8 @@ export function TableSourceForm({
});
const prevTableNameRef = useRef(watchedTableName);
const metadata = useMetadataWithSettings();
useEffect(() => {
(async () => {
try {
@ -1511,6 +1523,7 @@ export function TableSourceForm({
tableName:
watchedKind !== SourceKind.Metric ? watchedTableName : '',
connectionId: watchedConnection,
metadata,
});
if (Object.keys(config).length > 0) {
notifications.show({
@ -1537,6 +1550,7 @@ export function TableSourceForm({
watchedDatabaseName,
watchedKind,
resetField,
metadata,
]);
// Sets the default connection field to the first connection after the

8
packages/app/src/hdxMTViews.ts
View file

@ -4,6 +4,7 @@ import {
chSql,
parameterizedQueryToSql,
} from '@hyperdx/common-utils/dist/clickhouse';
import { Metadata } from '@hyperdx/common-utils/dist/core/metadata';
import {
FIXED_TIME_BUCKET_EXPR_ALIAS,
isNonEmptyWhereExpr,
@ -17,8 +18,6 @@ import {
SQLInterval,
} from '@hyperdx/common-utils/dist/types';
import { getMetadata } from '@/metadata';
const HDX_DATABASE = 'hyperdx'; // all materialized views should sit in this database
// a hashed select field used as a field name in the materialized view
@ -97,6 +96,7 @@ const buildMTViewDDL = (name: string, table: string, query: ChSql) => {
export const buildMTViewSelectQuery = async (
chartConfig: ChartConfigWithOptDateRange,
metadata: Metadata,
customGranularity?: SQLInterval,
) => {
const _config = {
@ -116,7 +116,7 @@ export const buildMTViewSelectQuery = async (
orderBy: undefined,
limit: undefined,
};
const mtViewSQL = await renderChartConfig(_config, getMetadata());
const mtViewSQL = await renderChartConfig(_config, metadata);
const mtViewSQLHash = objectHash.sha1(mtViewSQL);
const mtViewName = `${chartConfig.from.tableName}_mv_${mtViewSQLHash}`;
const renderMTViewConfig = {
@ -148,7 +148,7 @@ export const buildMTViewSelectQuery = async (
),
renderMTViewConfig: async () => {
try {
return await renderChartConfig(renderMTViewConfig, getMetadata());
return await renderChartConfig(renderMTViewConfig, metadata);
} catch (e) {
console.error('Failed to render MTView config', e);
return null;

11
packages/app/src/hooks/useChartConfig.tsx
View file

@ -29,7 +29,6 @@ import { useClickhouseClient } from '@/clickhouse';
import { IS_MTVIEWS_ENABLED } from '@/config';
import { buildMTViewSelectQuery } from '@/hdxMTViews';
import { useMetadataWithSettings } from '@/hooks/useMetadata';
import { getMetadata } from '@/metadata';
import { useSource } from '@/source';
import { generateTimeWindowsDescending } from '@/utils/searchWindows';
@ -144,7 +143,7 @@ async function* fetchDataInChunks({
if (IS_MTVIEWS_ENABLED) {
const { dataTableDDL, mtViewDDL, renderMTViewConfig } =
await buildMTViewSelectQuery(config);
await buildMTViewSelectQuery(config, metadata);
// TODO: show the DDLs in the UI so users can run commands manually
// eslint-disable-next-line no-console
console.log('dataTableDDL:', dataTableDDL);
@ -341,6 +340,8 @@ export function useRenderedSqlChartConfig(
) {
const { enabled = true } = options ?? {};
const metadata = useMetadataWithSettings();
const { data: mvOptimizationData, isLoading: isLoadingMVOptimization } =
useMVOptimizationExplanation(config, {
enabled: !!enabled,
@ -351,7 +352,7 @@ export function useRenderedSqlChartConfig(
queryKey: ['renderedSql', config],
queryFn: async () => {
const optimizedConfig = mvOptimizationData?.optimizedConfig ?? config;
const query = await renderChartConfig(optimizedConfig, getMetadata());
const query = await renderChartConfig(optimizedConfig, metadata);
return format(parameterizedQueryToSql(query));
},
...options,
@ -376,6 +377,8 @@ export function useAliasMapFromChartConfig(
? config.dateRange[1].getTime() - config.dateRange[0].getTime()
: undefined;
const metadata = useMetadataWithSettings();
return useQuery<Record<string, string>>({
// Only include config properties that affect SELECT structure and aliases.
// When adding new ChartConfig fields, check renderChartConfig.ts to see if they
@ -397,7 +400,7 @@ export function useAliasMapFromChartConfig(
return {};
}
const query = await renderChartConfig(config, getMetadata());
const query = await renderChartConfig(config, metadata);
const aliasMap = chSqlToAliasMap(query);

7
packages/app/src/hooks/useExplainQuery.tsx
View file

@ -3,7 +3,8 @@ import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
import { useQuery, UseQueryOptions } from '@tanstack/react-query';
import { useClickhouseClient } from '@/clickhouse';
import { getMetadata } from '@/metadata';
import { useMetadataWithSettings } from './useMetadata';
export function useExplainQuery(
_config: ChartConfigWithDateRange,
@ -14,10 +15,12 @@ export function useExplainQuery(
with: undefined,
};
const clickhouseClient = useClickhouseClient();
const metadata = useMetadataWithSettings();
const { data, isLoading, error } = useQuery({
queryKey: ['explain', config],
queryFn: async ({ signal }) => {
const query = await renderChartConfig(config, getMetadata());
const query = await renderChartConfig(config, metadata);
const response = await clickhouseClient.query<'JSONEachRow'>({
query: `EXPLAIN ESTIMATE ${query.sql}`,
query_params: query.params,

33
packages/app/src/hooks/useMetadata.tsx
View file

@ -1,4 +1,4 @@
import { useEffect, useRef } from 'react';
import { useEffect, useRef, useState } from 'react';
import objectHash from 'object-hash';
import {
ColumnMeta,
@ -14,19 +14,46 @@ import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
import {
keepPreviousData,
useQuery,
useQueryClient,
UseQueryOptions,
} from '@tanstack/react-query';
import api from '@/api';
import { IS_LOCAL_MODE } from '@/config';
import { LOCAL_STORE_CONNECTIONS_KEY } from '@/connection';
import { getMetadata } from '@/metadata';
import { toArray } from '@/utils';
// Hook to get metadata with proper settings applied
// TODO: replace all getMetadata calls with useMetadataWithSettings
export function useMetadataWithSettings() {
const metadata = getMetadata();
const [metadata, setMetadata] = useState(getMetadata());
const { data: me } = api.useMe();
const settingsApplied = useRef(false);
const queryClient = useQueryClient();
// Create a listener that triggers when connections are updated in local mode
useEffect(() => {
const isBrowser =
typeof window !== 'undefined' && typeof window.document !== 'undefined';
if (!isBrowser || !IS_LOCAL_MODE) return;
const createNewMetadata = (event: StorageEvent) => {
if (event.key === LOCAL_STORE_CONNECTIONS_KEY && event.newValue) {
// Create a new metadata instance with a new ClickHouse client,
// since the existing one will not have connection / auth info.
setMetadata(getMetadata());
settingsApplied.current = false;
// Clear react-query cache so that metadata is refetched with
// the new connection info, and error states are cleared.
queryClient.resetQueries();
}
};
window.addEventListener('storage', createNewMetadata);
return () => {
window.removeEventListener('storage', createNewMetadata);
};
}, [queryClient]);
useEffect(() => {
if (me?.team?.metadataMaxRowsToRead && !settingsApplied.current) {

14
packages/app/src/sessions.ts
View file

@ -11,12 +11,10 @@ import {
} from '@hyperdx/common-utils/dist/types';
import { useQuery, UseQueryOptions } from '@tanstack/react-query';
import { getMetadata } from '@/metadata';
import { usePrevious } from '@/utils';
import { useMetadataWithSettings } from './hooks/useMetadata';
import { getClickhouseClient, useClickhouseClient } from './clickhouse';
import { IS_LOCAL_MODE } from './config';
import { getLocalConnections } from './connection';
import { SESSION_TABLE_EXPRESSIONS, useSource } from './source';
export type Session = {
@ -54,6 +52,7 @@ export function useSessions(
const FIXED_SDK_ATTRIBUTES = ['teamId', 'teamName', 'userEmail', 'userName'];
const SESSIONS_CTE_NAME = 'sessions';
const clickhouseClient = useClickhouseClient();
const metadata = useMetadataWithSettings();
return useQuery<ResponseJSON<Session>, Error>({
queryKey: [
'sessions',
@ -141,7 +140,7 @@ export function useSessions(
connection: traceSource.connection,
groupBy: 'serviceName, sessionId',
},
getMetadata(),
metadata,
),
renderChartConfig(
{
@ -161,7 +160,7 @@ export function useSessions(
SESSION_TABLE_EXPRESSIONS.implicitColumnExpression,
connection: sessionSource.connection,
},
getMetadata(),
metadata,
),
renderChartConfig(
{
@ -179,7 +178,7 @@ export function useSessions(
implicitColumnExpression: traceSource.implicitColumnExpression,
connection: traceSource?.connection,
},
getMetadata(),
metadata,
),
]);
@ -291,6 +290,7 @@ export function useRRWebEventStream(
// @ts-ignore
const keepPreviousData = options?.keepPreviousData ?? false;
const shouldAbortPendingRequest = options?.shouldAbortPendingRequest ?? true;
const metadata = useMetadataWithSettings();
const [results, setResults] = useState<{ key: string; data: any[] }>({
key: '',
@ -331,7 +331,6 @@ export function useRRWebEventStream(
const MAX_LIMIT = 1e6;
const metadata = getMetadata();
const query = await renderChartConfig(
{
// FIXME: add mappings to session source
@ -465,6 +464,7 @@ export function useRRWebEventStream(
onEvent,
onEnd,
resultsKey,
metadata,
],
);

10
packages/app/src/source.ts
View file

@ -9,6 +9,7 @@ import {
filterColumnMetaByType,
JSDataType,
} from '@hyperdx/common-utils/dist/clickhouse';
import { Metadata } from '@hyperdx/common-utils/dist/core/metadata';
import {
hashCode,
splitAndTrimWithBracket,
@ -231,14 +232,15 @@ export async function inferTableSourceConfig({
databaseName,
tableName,
connectionId,
metadata,
}: {
databaseName: string;
tableName: string;
connectionId: string;
metadata: Metadata;
}): Promise<
Partial<Omit<TSource, 'id' | 'name' | 'from' | 'connection' | 'kind'>>
> {
const metadata = getMetadata();
const columns = await metadata.getColumns({
databaseName,
tableName,
@ -412,17 +414,18 @@ export async function isValidMetricTable({
tableName,
connectionId,
metricType,
metadata,
}: {
databaseName: string;
tableName?: string;
connectionId: string;
metricType: MetricsDataType;
metadata: Metadata;
}) {
if (!tableName) {
return false;
}
const metadata = getMetadata();
const columns = await metadata.getColumns({
databaseName,
tableName,
@ -438,16 +441,17 @@ export async function isValidSessionsTable({
databaseName,
tableName,
connectionId,
metadata,
}: {
databaseName: string;
tableName?: string;
connectionId: string;
metadata: Metadata;
}) {
if (!tableName) {
return false;
}
const metadata = getMetadata();
const columns = await metadata.getColumns({
databaseName,
tableName,

15
packages/app/src/utils/__tests__/materializedViews.test.ts
View file

@ -5,8 +5,6 @@ import {
TableMetadata,
} from '@hyperdx/common-utils/dist/core/metadata';
import { getMetadata } from '@/metadata';
import {
getSourceTableColumn,
inferMaterializedViewConfig,
@ -14,12 +12,6 @@ import {
parseSummedColumns,
} from '../materializedViews';
jest.mock('@/metadata', () => {
return {
getMetadata: jest.fn(),
};
});
function createMockColumnMeta({
name,
type,
@ -32,7 +24,6 @@ function createMockColumnMeta({
}
describe('inferMaterializedViewConfig', () => {
const mockGetMetadata = jest.mocked(getMetadata);
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
const mockMetadata: Metadata = {
getColumns: jest.fn(),
@ -163,7 +154,6 @@ describe('inferMaterializedViewConfig', () => {
};
beforeEach(() => {
mockGetMetadata.mockReturnValue(mockMetadata);
mockMetadata.getTableMetadata = jest
.fn()
.mockImplementation(({ tableName }) => {
@ -226,6 +216,7 @@ describe('inferMaterializedViewConfig', () => {
const actualConfig = await inferMaterializedViewConfig(
mvTableConnection,
sourceTableConnection,
mockMetadata,
);
expect(actualConfig).toEqual({
@ -275,6 +266,7 @@ describe('inferMaterializedViewConfig', () => {
const actualConfig = await inferMaterializedViewConfig(
mvTableConnection,
sourceTableConnection,
mockMetadata,
);
expect(actualConfig).toEqual({
@ -324,6 +316,7 @@ describe('inferMaterializedViewConfig', () => {
const actualConfig = await inferMaterializedViewConfig(
mvTableConnection,
sourceTableConnection,
mockMetadata,
);
expect(actualConfig).toEqual({
@ -373,6 +366,7 @@ describe('inferMaterializedViewConfig', () => {
const actualConfig = await inferMaterializedViewConfig(
mvTableConnection,
sourceTableConnection,
mockMetadata,
);
expect(actualConfig).toEqual({
@ -422,6 +416,7 @@ describe('inferMaterializedViewConfig', () => {
const actualConfig = await inferMaterializedViewConfig(
mvTableConnection,
sourceTableConnection,
mockMetadata,
);
expect(actualConfig).toBeUndefined();

26
packages/app/src/utils/materializedViews.ts
View file

@ -5,6 +5,7 @@ import {
JSDataType,
} from '@hyperdx/common-utils/dist/clickhouse';
import {
Metadata,
TableConnection,
TableMetadata,
} from '@hyperdx/common-utils/dist/core/metadata';
@ -96,13 +97,11 @@ function isSummingMergeTree(meta: TableMetadata) {
* Returns undefined if there are multiple materialized views targeting the given table,
* or if the target table is not an AggregatingMergeTree.
*/
async function getMetadataForMaterializedViewAndTable({
databaseName,
tableName,
connectionId,
}: TableConnection) {
async function getMetadataForMaterializedViewAndTable(
{ databaseName, tableName, connectionId }: TableConnection,
metadata: Metadata,
) {
try {
const metadata = getMetadata();
const givenMetadata = await metadata.getTableMetadata({
databaseName,
tableName,
@ -321,6 +320,7 @@ export function getSourceTableColumn(
export async function inferMaterializedViewConfig(
mvTableOrView: TableConnection,
sourceTable: TableConnection,
metadata: Metadata,
): Promise<MaterializedViewConfiguration | undefined> {
const { databaseName, tableName, connectionId } = mvTableOrView;
const { databaseName: sourceDatabaseName, tableName: sourceTableName } =
@ -330,18 +330,20 @@ export async function inferMaterializedViewConfig(
return undefined;
}
const meta = await getMetadataForMaterializedViewAndTable({
databaseName,
tableName,
connectionId,
});
const meta = await getMetadataForMaterializedViewAndTable(
{
databaseName,
tableName,
connectionId,
},
metadata,
);
if (!meta) {
return undefined;
}
const { mvMetadata, mvTableMetadata } = meta;
const metadata = getMetadata();
const [mvTableColumns, sourceTableColumns] = await Promise.all([
metadata.getColumns({