diff --git a/.changeset/eleven-pears-pay.md b/.changeset/eleven-pears-pay.md new file mode 100644 index 00000000..e7d6485a --- /dev/null +++ b/.changeset/eleven-pears-pay.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/app": patch +--- + +fix: Refresh metadata after creating new connection in local mode diff --git a/packages/app/src/components/KubeComponents.tsx b/packages/app/src/components/KubeComponents.tsx index 3afbda78..39e53be1 100644 --- a/packages/app/src/components/KubeComponents.tsx +++ b/packages/app/src/components/KubeComponents.tsx @@ -10,12 +10,12 @@ import { SearchConditionLanguage, TSource, } from '@hyperdx/common-utils/dist/types'; -import { Anchor, Badge, Group, Text, Timeline } from '@mantine/core'; +import { Badge, Group, Text, Timeline } from '@mantine/core'; import { useQuery, UseQueryOptions } from '@tanstack/react-query'; import { useClickhouseClient } from '@/clickhouse'; -import { getMetadata } from '@/metadata'; -import { getDisplayedTimestampValueExpression, getEventBody } from '@/source'; +import { useMetadataWithSettings } from '@/hooks/useMetadata'; +import { getDisplayedTimestampValueExpression } from '@/source'; import { KubePhase } from '../types'; import { FormatTime } from '../useFormatTime'; @@ -57,6 +57,7 @@ export const useV2LogBatch = ( options?: Omit, 'queryKey' | 'queryFn'>, ) => { const clickhouseClient = useClickhouseClient(); + const metadata = useMetadataWithSettings(); return useQuery, Error>({ queryKey: [ 'v2LogBatch', @@ -97,7 +98,7 @@ export const useV2LogBatch = ( }, orderBy: `${logSource.timestampValueExpression} ${order}`, }, - getMetadata(), + metadata, ); const json = await clickhouseClient diff --git a/packages/app/src/components/Sources/SourceForm.tsx b/packages/app/src/components/Sources/SourceForm.tsx index 78fc712c..c909db3a 100644 --- a/packages/app/src/components/Sources/SourceForm.tsx +++ b/packages/app/src/components/Sources/SourceForm.tsx @@ -45,6 +45,7 @@ import { import { SourceSelectControlled } from '@/components/SourceSelect'; import { IS_METRICS_ENABLED, IS_SESSIONS_ENABLED } from '@/config'; import { useConnections } from '@/connection'; +import { useMetadataWithSettings } from '@/hooks/useMetadata'; import { inferTableSourceConfig, isValidMetricTable, @@ -544,6 +545,8 @@ function AggregatedColumnsFormSection({ const fromTableName = useWatch({ control, name: 'from.tableName' }); const prevMvTableNameRef = useRef(mvTableName); + const metadata = useMetadataWithSettings(); + useEffect(() => { (async () => { try { @@ -569,6 +572,7 @@ function AggregatedColumnsFormSection({ tableName: fromTableName, connectionId: connection, }, + metadata, ); if (config) { @@ -603,6 +607,7 @@ function AggregatedColumnsFormSection({ mvIndex, replaceAggregates, setValue, + metadata, ]); return ( @@ -1278,6 +1283,7 @@ export function SessionTableModelForm({ control }: TableModelProps) { const connectionId = useWatch({ control, name: 'connection' }); const tableName = useWatch({ control, name: 'from.tableName' }); const prevTableNameRef = useRef(tableName); + const metadata = useMetadataWithSettings(); useEffect(() => { (async () => { @@ -1288,6 +1294,7 @@ export function SessionTableModelForm({ control }: TableModelProps) { databaseName, tableName, connectionId, + metadata, }); if (!isValid) { @@ -1305,7 +1312,7 @@ export function SessionTableModelForm({ control }: TableModelProps) { }); } })(); - }, [tableName, databaseName, connectionId]); + }, [tableName, databaseName, connectionId, metadata]); return ( <> @@ -1336,6 +1343,8 @@ export function MetricTableModelForm({ control, setValue }: TableModelProps) { const metricTables = useWatch({ control, name: 'metricTables' }); const prevMetricTablesRef = useRef(metricTables); + const metadata = useMetadataWithSettings(); + useEffect(() => { for (const [_key, _value] of Object.entries(OTEL_CLICKHOUSE_EXPRESSIONS)) { setValue(_key as any, _value); @@ -1361,6 +1370,7 @@ export function MetricTableModelForm({ control, setValue }: TableModelProps) { tableName: newValue as string, connectionId, metricType: metricType as MetricsDataType, + metadata, }); if (!isValid) { notifications.show({ @@ -1380,7 +1390,7 @@ export function MetricTableModelForm({ control, setValue }: TableModelProps) { }); } })(); - }, [metricTables, databaseName, connectionId]); + }, [metricTables, databaseName, connectionId, metadata]); return ( <> @@ -1495,6 +1505,8 @@ export function TableSourceForm({ }); const prevTableNameRef = useRef(watchedTableName); + const metadata = useMetadataWithSettings(); + useEffect(() => { (async () => { try { @@ -1511,6 +1523,7 @@ export function TableSourceForm({ tableName: watchedKind !== SourceKind.Metric ? watchedTableName : '', connectionId: watchedConnection, + metadata, }); if (Object.keys(config).length > 0) { notifications.show({ @@ -1537,6 +1550,7 @@ export function TableSourceForm({ watchedDatabaseName, watchedKind, resetField, + metadata, ]); // Sets the default connection field to the first connection after the diff --git a/packages/app/src/hdxMTViews.ts b/packages/app/src/hdxMTViews.ts index 9bb23495..71d1d09e 100644 --- a/packages/app/src/hdxMTViews.ts +++ b/packages/app/src/hdxMTViews.ts @@ -4,6 +4,7 @@ import { chSql, parameterizedQueryToSql, } from '@hyperdx/common-utils/dist/clickhouse'; +import { Metadata } from '@hyperdx/common-utils/dist/core/metadata'; import { FIXED_TIME_BUCKET_EXPR_ALIAS, isNonEmptyWhereExpr, @@ -17,8 +18,6 @@ import { SQLInterval, } from '@hyperdx/common-utils/dist/types'; -import { getMetadata } from '@/metadata'; - const HDX_DATABASE = 'hyperdx'; // all materialized views should sit in this database // a hashed select field used as a field name in the materialized view @@ -97,6 +96,7 @@ const buildMTViewDDL = (name: string, table: string, query: ChSql) => { export const buildMTViewSelectQuery = async ( chartConfig: ChartConfigWithOptDateRange, + metadata: Metadata, customGranularity?: SQLInterval, ) => { const _config = { @@ -116,7 +116,7 @@ export const buildMTViewSelectQuery = async ( orderBy: undefined, limit: undefined, }; - const mtViewSQL = await renderChartConfig(_config, getMetadata()); + const mtViewSQL = await renderChartConfig(_config, metadata); const mtViewSQLHash = objectHash.sha1(mtViewSQL); const mtViewName = `${chartConfig.from.tableName}_mv_${mtViewSQLHash}`; const renderMTViewConfig = { @@ -148,7 +148,7 @@ export const buildMTViewSelectQuery = async ( ), renderMTViewConfig: async () => { try { - return await renderChartConfig(renderMTViewConfig, getMetadata()); + return await renderChartConfig(renderMTViewConfig, metadata); } catch (e) { console.error('Failed to render MTView config', e); return null; diff --git a/packages/app/src/hooks/useChartConfig.tsx b/packages/app/src/hooks/useChartConfig.tsx index 98953228..b9a7f898 100644 --- a/packages/app/src/hooks/useChartConfig.tsx +++ b/packages/app/src/hooks/useChartConfig.tsx @@ -29,7 +29,6 @@ import { useClickhouseClient } from '@/clickhouse'; import { IS_MTVIEWS_ENABLED } from '@/config'; import { buildMTViewSelectQuery } from '@/hdxMTViews'; import { useMetadataWithSettings } from '@/hooks/useMetadata'; -import { getMetadata } from '@/metadata'; import { useSource } from '@/source'; import { generateTimeWindowsDescending } from '@/utils/searchWindows'; @@ -144,7 +143,7 @@ async function* fetchDataInChunks({ if (IS_MTVIEWS_ENABLED) { const { dataTableDDL, mtViewDDL, renderMTViewConfig } = - await buildMTViewSelectQuery(config); + await buildMTViewSelectQuery(config, metadata); // TODO: show the DDLs in the UI so users can run commands manually // eslint-disable-next-line no-console console.log('dataTableDDL:', dataTableDDL); @@ -341,6 +340,8 @@ export function useRenderedSqlChartConfig( ) { const { enabled = true } = options ?? {}; + const metadata = useMetadataWithSettings(); + const { data: mvOptimizationData, isLoading: isLoadingMVOptimization } = useMVOptimizationExplanation(config, { enabled: !!enabled, @@ -351,7 +352,7 @@ export function useRenderedSqlChartConfig( queryKey: ['renderedSql', config], queryFn: async () => { const optimizedConfig = mvOptimizationData?.optimizedConfig ?? config; - const query = await renderChartConfig(optimizedConfig, getMetadata()); + const query = await renderChartConfig(optimizedConfig, metadata); return format(parameterizedQueryToSql(query)); }, ...options, @@ -376,6 +377,8 @@ export function useAliasMapFromChartConfig( ? config.dateRange[1].getTime() - config.dateRange[0].getTime() : undefined; + const metadata = useMetadataWithSettings(); + return useQuery>({ // Only include config properties that affect SELECT structure and aliases. // When adding new ChartConfig fields, check renderChartConfig.ts to see if they @@ -397,7 +400,7 @@ export function useAliasMapFromChartConfig( return {}; } - const query = await renderChartConfig(config, getMetadata()); + const query = await renderChartConfig(config, metadata); const aliasMap = chSqlToAliasMap(query); diff --git a/packages/app/src/hooks/useExplainQuery.tsx b/packages/app/src/hooks/useExplainQuery.tsx index f390cc99..09229bac 100644 --- a/packages/app/src/hooks/useExplainQuery.tsx +++ b/packages/app/src/hooks/useExplainQuery.tsx @@ -3,7 +3,8 @@ import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import { useQuery, UseQueryOptions } from '@tanstack/react-query'; import { useClickhouseClient } from '@/clickhouse'; -import { getMetadata } from '@/metadata'; + +import { useMetadataWithSettings } from './useMetadata'; export function useExplainQuery( _config: ChartConfigWithDateRange, @@ -14,10 +15,12 @@ export function useExplainQuery( with: undefined, }; const clickhouseClient = useClickhouseClient(); + const metadata = useMetadataWithSettings(); + const { data, isLoading, error } = useQuery({ queryKey: ['explain', config], queryFn: async ({ signal }) => { - const query = await renderChartConfig(config, getMetadata()); + const query = await renderChartConfig(config, metadata); const response = await clickhouseClient.query<'JSONEachRow'>({ query: `EXPLAIN ESTIMATE ${query.sql}`, query_params: query.params, diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx index 9fe56dd5..c42ceac2 100644 --- a/packages/app/src/hooks/useMetadata.tsx +++ b/packages/app/src/hooks/useMetadata.tsx @@ -1,4 +1,4 @@ -import { useEffect, useRef } from 'react'; +import { useEffect, useRef, useState } from 'react'; import objectHash from 'object-hash'; import { ColumnMeta, @@ -14,19 +14,46 @@ import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import { keepPreviousData, useQuery, + useQueryClient, UseQueryOptions, } from '@tanstack/react-query'; import api from '@/api'; +import { IS_LOCAL_MODE } from '@/config'; +import { LOCAL_STORE_CONNECTIONS_KEY } from '@/connection'; import { getMetadata } from '@/metadata'; import { toArray } from '@/utils'; // Hook to get metadata with proper settings applied -// TODO: replace all getMetadata calls with useMetadataWithSettings export function useMetadataWithSettings() { - const metadata = getMetadata(); + const [metadata, setMetadata] = useState(getMetadata()); const { data: me } = api.useMe(); const settingsApplied = useRef(false); + const queryClient = useQueryClient(); + + // Create a listener that triggers when connections are updated in local mode + useEffect(() => { + const isBrowser = + typeof window !== 'undefined' && typeof window.document !== 'undefined'; + if (!isBrowser || !IS_LOCAL_MODE) return; + + const createNewMetadata = (event: StorageEvent) => { + if (event.key === LOCAL_STORE_CONNECTIONS_KEY && event.newValue) { + // Create a new metadata instance with a new ClickHouse client, + // since the existing one will not have connection / auth info. + setMetadata(getMetadata()); + settingsApplied.current = false; + // Clear react-query cache so that metadata is refetched with + // the new connection info, and error states are cleared. + queryClient.resetQueries(); + } + }; + + window.addEventListener('storage', createNewMetadata); + return () => { + window.removeEventListener('storage', createNewMetadata); + }; + }, [queryClient]); useEffect(() => { if (me?.team?.metadataMaxRowsToRead && !settingsApplied.current) { diff --git a/packages/app/src/sessions.ts b/packages/app/src/sessions.ts index c13c9918..e162a947 100644 --- a/packages/app/src/sessions.ts +++ b/packages/app/src/sessions.ts @@ -11,12 +11,10 @@ import { } from '@hyperdx/common-utils/dist/types'; import { useQuery, UseQueryOptions } from '@tanstack/react-query'; -import { getMetadata } from '@/metadata'; import { usePrevious } from '@/utils'; +import { useMetadataWithSettings } from './hooks/useMetadata'; import { getClickhouseClient, useClickhouseClient } from './clickhouse'; -import { IS_LOCAL_MODE } from './config'; -import { getLocalConnections } from './connection'; import { SESSION_TABLE_EXPRESSIONS, useSource } from './source'; export type Session = { @@ -54,6 +52,7 @@ export function useSessions( const FIXED_SDK_ATTRIBUTES = ['teamId', 'teamName', 'userEmail', 'userName']; const SESSIONS_CTE_NAME = 'sessions'; const clickhouseClient = useClickhouseClient(); + const metadata = useMetadataWithSettings(); return useQuery, Error>({ queryKey: [ 'sessions', @@ -141,7 +140,7 @@ export function useSessions( connection: traceSource.connection, groupBy: 'serviceName, sessionId', }, - getMetadata(), + metadata, ), renderChartConfig( { @@ -161,7 +160,7 @@ export function useSessions( SESSION_TABLE_EXPRESSIONS.implicitColumnExpression, connection: sessionSource.connection, }, - getMetadata(), + metadata, ), renderChartConfig( { @@ -179,7 +178,7 @@ export function useSessions( implicitColumnExpression: traceSource.implicitColumnExpression, connection: traceSource?.connection, }, - getMetadata(), + metadata, ), ]); @@ -291,6 +290,7 @@ export function useRRWebEventStream( // @ts-ignore const keepPreviousData = options?.keepPreviousData ?? false; const shouldAbortPendingRequest = options?.shouldAbortPendingRequest ?? true; + const metadata = useMetadataWithSettings(); const [results, setResults] = useState<{ key: string; data: any[] }>({ key: '', @@ -331,7 +331,6 @@ export function useRRWebEventStream( const MAX_LIMIT = 1e6; - const metadata = getMetadata(); const query = await renderChartConfig( { // FIXME: add mappings to session source @@ -465,6 +464,7 @@ export function useRRWebEventStream( onEvent, onEnd, resultsKey, + metadata, ], ); diff --git a/packages/app/src/source.ts b/packages/app/src/source.ts index 158775bb..e87ef2e2 100644 --- a/packages/app/src/source.ts +++ b/packages/app/src/source.ts @@ -9,6 +9,7 @@ import { filterColumnMetaByType, JSDataType, } from '@hyperdx/common-utils/dist/clickhouse'; +import { Metadata } from '@hyperdx/common-utils/dist/core/metadata'; import { hashCode, splitAndTrimWithBracket, @@ -231,14 +232,15 @@ export async function inferTableSourceConfig({ databaseName, tableName, connectionId, + metadata, }: { databaseName: string; tableName: string; connectionId: string; + metadata: Metadata; }): Promise< Partial> > { - const metadata = getMetadata(); const columns = await metadata.getColumns({ databaseName, tableName, @@ -412,17 +414,18 @@ export async function isValidMetricTable({ tableName, connectionId, metricType, + metadata, }: { databaseName: string; tableName?: string; connectionId: string; metricType: MetricsDataType; + metadata: Metadata; }) { if (!tableName) { return false; } - const metadata = getMetadata(); const columns = await metadata.getColumns({ databaseName, tableName, @@ -438,16 +441,17 @@ export async function isValidSessionsTable({ databaseName, tableName, connectionId, + metadata, }: { databaseName: string; tableName?: string; connectionId: string; + metadata: Metadata; }) { if (!tableName) { return false; } - const metadata = getMetadata(); const columns = await metadata.getColumns({ databaseName, tableName, diff --git a/packages/app/src/utils/__tests__/materializedViews.test.ts b/packages/app/src/utils/__tests__/materializedViews.test.ts index 0a978ecd..851901e7 100644 --- a/packages/app/src/utils/__tests__/materializedViews.test.ts +++ b/packages/app/src/utils/__tests__/materializedViews.test.ts @@ -5,8 +5,6 @@ import { TableMetadata, } from '@hyperdx/common-utils/dist/core/metadata'; -import { getMetadata } from '@/metadata'; - import { getSourceTableColumn, inferMaterializedViewConfig, @@ -14,12 +12,6 @@ import { parseSummedColumns, } from '../materializedViews'; -jest.mock('@/metadata', () => { - return { - getMetadata: jest.fn(), - }; -}); - function createMockColumnMeta({ name, type, @@ -32,7 +24,6 @@ function createMockColumnMeta({ } describe('inferMaterializedViewConfig', () => { - const mockGetMetadata = jest.mocked(getMetadata); // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion const mockMetadata: Metadata = { getColumns: jest.fn(), @@ -163,7 +154,6 @@ describe('inferMaterializedViewConfig', () => { }; beforeEach(() => { - mockGetMetadata.mockReturnValue(mockMetadata); mockMetadata.getTableMetadata = jest .fn() .mockImplementation(({ tableName }) => { @@ -226,6 +216,7 @@ describe('inferMaterializedViewConfig', () => { const actualConfig = await inferMaterializedViewConfig( mvTableConnection, sourceTableConnection, + mockMetadata, ); expect(actualConfig).toEqual({ @@ -275,6 +266,7 @@ describe('inferMaterializedViewConfig', () => { const actualConfig = await inferMaterializedViewConfig( mvTableConnection, sourceTableConnection, + mockMetadata, ); expect(actualConfig).toEqual({ @@ -324,6 +316,7 @@ describe('inferMaterializedViewConfig', () => { const actualConfig = await inferMaterializedViewConfig( mvTableConnection, sourceTableConnection, + mockMetadata, ); expect(actualConfig).toEqual({ @@ -373,6 +366,7 @@ describe('inferMaterializedViewConfig', () => { const actualConfig = await inferMaterializedViewConfig( mvTableConnection, sourceTableConnection, + mockMetadata, ); expect(actualConfig).toEqual({ @@ -422,6 +416,7 @@ describe('inferMaterializedViewConfig', () => { const actualConfig = await inferMaterializedViewConfig( mvTableConnection, sourceTableConnection, + mockMetadata, ); expect(actualConfig).toBeUndefined(); diff --git a/packages/app/src/utils/materializedViews.ts b/packages/app/src/utils/materializedViews.ts index 98811b90..11d2f7e1 100644 --- a/packages/app/src/utils/materializedViews.ts +++ b/packages/app/src/utils/materializedViews.ts @@ -5,6 +5,7 @@ import { JSDataType, } from '@hyperdx/common-utils/dist/clickhouse'; import { + Metadata, TableConnection, TableMetadata, } from '@hyperdx/common-utils/dist/core/metadata'; @@ -96,13 +97,11 @@ function isSummingMergeTree(meta: TableMetadata) { * Returns undefined if there are multiple materialized views targeting the given table, * or if the target table is not an AggregatingMergeTree. */ -async function getMetadataForMaterializedViewAndTable({ - databaseName, - tableName, - connectionId, -}: TableConnection) { +async function getMetadataForMaterializedViewAndTable( + { databaseName, tableName, connectionId }: TableConnection, + metadata: Metadata, +) { try { - const metadata = getMetadata(); const givenMetadata = await metadata.getTableMetadata({ databaseName, tableName, @@ -321,6 +320,7 @@ export function getSourceTableColumn( export async function inferMaterializedViewConfig( mvTableOrView: TableConnection, sourceTable: TableConnection, + metadata: Metadata, ): Promise { const { databaseName, tableName, connectionId } = mvTableOrView; const { databaseName: sourceDatabaseName, tableName: sourceTableName } = @@ -330,18 +330,20 @@ export async function inferMaterializedViewConfig( return undefined; } - const meta = await getMetadataForMaterializedViewAndTable({ - databaseName, - tableName, - connectionId, - }); + const meta = await getMetadataForMaterializedViewAndTable( + { + databaseName, + tableName, + connectionId, + }, + metadata, + ); if (!meta) { return undefined; } const { mvMetadata, mvTableMetadata } = meta; - const metadata = getMetadata(); const [mvTableColumns, sourceTableColumns] = await Promise.all([ metadata.getColumns({