From bc8c4eec9acdb49d34e00a339eb77fc12f24730c Mon Sep 17 00:00:00 2001 From: Karl Power <85935352+karl-power@users.noreply.github.com> Date: Wed, 21 Jan 2026 17:07:30 +0100 Subject: [PATCH] feat: allow applying session settings to queries (#1609) Closes HDX-3154 This PR adds a feature that allows the user to add settings to a source. These settings are then added to the end of every query that is rendered through the `renderChartConfig` function, along with any other chart specific settings. See: https://clickhouse.com/docs/sql-reference/statements/select#settings-in-select-query Most of the work was to pass the `source` or `source.querySettings` value through the code to the `renderChartConfig` calls and to update the related tests. There are also some UI changes in the `SourceForm` components. `SQLParser.Parser` from the `node-sql-parser` throws an error when it encounters a SETTINGS clause in a sql string, so a function was added to remove that clause from any sql that is passed to the parser. It assumes that the SETTINGS clause will always be at the end of the sql string, it removes any part of the string including and after the SETTINGS clause. https://github.com/user-attachments/assets/7ac3b852-2c86-4431-88bc-106f982343bb --- .changeset/hip-icons-build.md | 7 + .../renderChartConfig.test.ts.snap | 11 + .../__tests__/renderChartConfig.test.ts | 86 +++++- packages/api/src/models/source.ts | 11 + packages/api/src/routers/api/ai.ts | 1 + .../api/src/routers/external-api/v2/charts.ts | 1 + packages/api/src/tasks/checkAlerts/index.ts | 1 + .../api/src/tasks/checkAlerts/template.ts | 6 +- packages/api/src/tasks/pingPongTask.ts | 1 - packages/app/src/TeamPage.tsx | 4 +- packages/app/src/__tests__/utils.test.ts | 23 ++ packages/app/src/api.ts | 2 +- .../app/src/components/KubeComponents.tsx | 1 + .../app/src/components/Sources/SourceForm.tsx | 75 +++++- packages/app/src/hdxMTViews.ts | 10 +- .../useDashboardFilterValues.test.tsx | 10 + packages/app/src/hooks/useChartConfig.tsx | 38 ++- .../src/hooks/useDashboardFilterValues.tsx | 11 +- packages/app/src/hooks/useExplainQuery.tsx | 13 +- packages/app/src/hooks/useMetadata.tsx | 14 +- .../app/src/hooks/useOffsetPaginatedQuery.tsx | 23 +- packages/app/src/hooks/useServiceMap.tsx | 2 + packages/app/src/sessions.ts | 4 + packages/app/src/source.ts | 7 +- packages/app/src/utils.ts | 10 + .../renderChartConfig.test.ts.snap | 56 ++-- .../src/__tests__/metadata.int.test.ts | 19 +- .../src/__tests__/metadata.test.ts | 18 +- .../src/__tests__/renderChartConfig.test.ts | 248 +++++++++++++++--- .../common-utils/src/__tests__/utils.test.ts | 134 ++++++++++ .../__tests__/materializedViews.test.ts | 8 +- packages/common-utils/src/clickhouse/index.ts | 23 +- .../src/core/materializedViews.ts | 18 +- packages/common-utils/src/core/metadata.ts | 28 +- .../src/core/renderChartConfig.ts | 43 ++- packages/common-utils/src/core/utils.ts | 54 ++++ packages/common-utils/src/types.ts | 7 + 37 files changed, 902 insertions(+), 126 deletions(-) create mode 100644 .changeset/hip-icons-build.md diff --git a/.changeset/hip-icons-build.md b/.changeset/hip-icons-build.md new file mode 100644 index 00000000..8719773d --- /dev/null +++ b/.changeset/hip-icons-build.md @@ -0,0 +1,7 @@ +--- +"@hyperdx/common-utils": minor +"@hyperdx/api": minor +"@hyperdx/app": minor +--- + +feat: allow applying session settings to queries diff --git a/packages/api/src/clickhouse/__tests__/__snapshots__/renderChartConfig.test.ts.snap b/packages/api/src/clickhouse/__tests__/__snapshots__/renderChartConfig.test.ts.snap index 947c5950..d613b118 100644 --- a/packages/api/src/clickhouse/__tests__/__snapshots__/renderChartConfig.test.ts.snap +++ b/packages/api/src/clickhouse/__tests__/__snapshots__/renderChartConfig.test.ts.snap @@ -473,6 +473,17 @@ Array [ ] `; +exports[`renderChartConfig Query settings handles the the query settings 1`] = ` +Array [ + Object { + "Body": "Oh no! Something went wrong!", + }, + Object { + "Body": "This is a test message.", + }, +] +`; + exports[`renderChartConfig aggFn numeric agg functions should handle numeric values as strings 1`] = ` Array [ Object { diff --git a/packages/api/src/clickhouse/__tests__/renderChartConfig.test.ts b/packages/api/src/clickhouse/__tests__/renderChartConfig.test.ts index ac80c063..3b1bbcad 100644 --- a/packages/api/src/clickhouse/__tests__/renderChartConfig.test.ts +++ b/packages/api/src/clickhouse/__tests__/renderChartConfig.test.ts @@ -1,13 +1,12 @@ // TODO: we might want to move this test file to common-utils package -import { ChSql } from '@hyperdx/common-utils/dist/clickhouse'; +import { ChSql, chSql } from '@hyperdx/common-utils/dist/clickhouse'; import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig'; import { - AggregateFunctionSchema, - DerivedColumn, MetricsDataType, + QuerySettings, } from '@hyperdx/common-utils/dist/types'; import _ from 'lodash'; import ms from 'ms'; @@ -36,6 +35,13 @@ const TEST_METRIC_TABLES = { 'exponential histogram': DEFAULT_METRICS_TABLE.EXPONENTIAL_HISTOGRAM, }; +const querySettings: QuerySettings = [ + { setting: 'optimize_read_in_order', value: '0' }, + { setting: 'cast_keep_nullable', value: '1' }, + { setting: 'count_distinct_implementation', value: 'uniqCombined64' }, + { setting: 'async_insert_busy_timeout_min_ms', value: '20000' }, +]; + describe('renderChartConfig', () => { const server = getServer(); @@ -152,6 +158,7 @@ describe('renderChartConfig', () => { timestampValueExpression: 'ts', }, metadata, + querySettings, ); const res = await queryData(query); expect(res).toMatchSnapshot(); @@ -190,6 +197,7 @@ describe('renderChartConfig', () => { timestampValueExpression: 'ts', }, metadata, + querySettings, ); const res = await queryData(query); expect(res).toMatchSnapshot(); @@ -226,16 +234,10 @@ describe('renderChartConfig', () => { connection: connection.id, }, metadata, + querySettings, ); - const resp = await clickhouseClient - .query<'JSON'>({ - query: query.sql, - query_params: query.params, - format: 'JSON', - }) - .then(res => res.json() as any); - expect(resp.data).toMatchSnapshot(); + expect(await queryData(query)).toMatchSnapshot(); }); it('simple select + group by query logs', async () => { @@ -272,6 +274,7 @@ describe('renderChartConfig', () => { groupBy: 'ServiceName', }, metadata, + querySettings, ); expect(await queryData(query)).toMatchSnapshot(); }); @@ -342,6 +345,7 @@ describe('renderChartConfig', () => { connection: connection.id, }, metadata, + querySettings, ); expect(await queryData(avgQuery)).toMatchSnapshot(); const maxQuery = await renderChartConfig( @@ -363,6 +367,7 @@ describe('renderChartConfig', () => { connection: connection.id, }, metadata, + querySettings, ); expect(await queryData(maxQuery)).toMatchSnapshot(); const sumQuery = await renderChartConfig( @@ -384,6 +389,7 @@ describe('renderChartConfig', () => { connection: connection.id, }, metadata, + querySettings, ); expect(await queryData(sumQuery)).toMatchSnapshot(); }); @@ -409,6 +415,7 @@ describe('renderChartConfig', () => { connection: connection.id, }, metadata, + querySettings, ); expect(await queryData(query)).toMatchSnapshot(); }); @@ -434,6 +441,7 @@ describe('renderChartConfig', () => { connection: connection.id, }, metadata, + querySettings, ); expect(await queryData(query)).toMatchSnapshot(); }); @@ -459,6 +467,7 @@ describe('renderChartConfig', () => { connection: connection.id, }, metadata, + querySettings, ); expect(await queryData(query)).toMatchSnapshot(); }); @@ -485,6 +494,7 @@ describe('renderChartConfig', () => { connection: connection.id, }, metadata, + querySettings, ); expect(await queryData(query)).toMatchSnapshot(); }); @@ -657,6 +667,7 @@ describe('renderChartConfig', () => { connection: connection.id, }, metadata, + querySettings, ); expect(await queryData(query)).toMatchSnapshot(); }); @@ -681,6 +692,7 @@ describe('renderChartConfig', () => { connection: connection.id, }, metadata, + querySettings, ); expect(await queryData(query)).toMatchSnapshot(); }); @@ -705,6 +717,7 @@ describe('renderChartConfig', () => { connection: connection.id, }, metadata, + querySettings, ); expect(await queryData(query)).toMatchSnapshot(); }); @@ -749,6 +762,7 @@ describe('renderChartConfig', () => { connection: connection.id, }, metadata, + querySettings, ); expect(await queryData(minQuery)).toMatchSnapshot('minSum'); @@ -771,6 +785,7 @@ describe('renderChartConfig', () => { connection: connection.id, }, metadata, + querySettings, ); expect(await queryData(maxQuery)).toMatchSnapshot('maxSum'); }); @@ -1151,6 +1166,7 @@ describe('renderChartConfig', () => { connection: connection.id, }, metadata, + querySettings, ); const res = await queryData(query); expect(res).toMatchSnapshot(); @@ -1200,6 +1216,7 @@ describe('renderChartConfig', () => { connection: connection.id, }, metadata, + querySettings, ); const res = await queryData(query); expect(res).toMatchSnapshot(); @@ -1250,6 +1267,7 @@ describe('renderChartConfig', () => { connection: connection.id, }, metadata, + querySettings, ); const res = await queryData(query); expect(res).toMatchSnapshot(); @@ -1294,6 +1312,7 @@ describe('renderChartConfig', () => { connection: connection.id, }, metadata, + querySettings, ); const res = await queryData(query); expect(res).toMatchSnapshot(); @@ -1320,6 +1339,7 @@ describe('renderChartConfig', () => { connection: connection.id, }, metadata, + querySettings, ); const res = await queryData(query); expect(res).toMatchSnapshot(); @@ -1347,6 +1367,7 @@ describe('renderChartConfig', () => { connection: connection.id, }, metadata, + querySettings, ); const res = await queryData(query); expect(res).toMatchSnapshot(); @@ -1374,6 +1395,7 @@ describe('renderChartConfig', () => { connection: connection.id, }, metadata, + querySettings, ); const res = await queryData(query); expect(res).toMatchSnapshot(); @@ -1444,6 +1466,7 @@ describe('renderChartConfig', () => { connection: connection.id, }, metadata, + querySettings, ); const res = await queryData(query); @@ -1480,6 +1503,7 @@ describe('renderChartConfig', () => { connection: connection.id, }, metadata, + querySettings, ); const res = await queryData(query); @@ -1509,6 +1533,7 @@ describe('renderChartConfig', () => { connection: connection.id, }, metadata, + querySettings, ); const res = await queryData(query); @@ -1520,4 +1545,43 @@ describe('renderChartConfig', () => { expect(query.sql).not.toMatch(/MetricName IN /); }); }); + + describe('Query settings', () => { + it('handles the the query settings', async () => { + const now = new Date('2023-11-16T22:12:00.000Z'); + + await bulkInsertLogs([ + { + ServiceName: 'api', + Timestamp: now, + SeverityText: 'error', + Body: 'Oh no! Something went wrong!', + }, + { + ServiceName: 'api', + Timestamp: now, + SeverityText: 'info', + Body: 'This is a test message.', + }, + ]); + + const query = await renderChartConfig( + { + select: [{ valueExpression: 'Body' }], + from: logSource.from, + where: '', + timestampValueExpression: 'Timestamp', + connection: connection.id, + settings: chSql`max_result_rows = 1`, + }, + metadata, + [...querySettings, { setting: 'result_overflow_mode', value: 'break' }], + ); + + const res = await queryData(query); + // ensures `result_overflow_mode = break` is applied, otherwise query would error. + expect(res).toHaveLength(2); + expect(res).toMatchSnapshot(); + }); + }); }); diff --git a/packages/api/src/models/source.ts b/packages/api/src/models/source.ts index 68c6936a..dc91fb73 100644 --- a/packages/api/src/models/source.ts +++ b/packages/api/src/models/source.ts @@ -86,6 +86,17 @@ export const Source = mongoose.model( }, default: undefined, }, + + querySettings: { + type: [ + { + setting: { type: String, required: true }, + value: { type: String, required: true }, + }, + ], + default: undefined, + maxlength: 10, + }, }, { toJSON: { virtuals: true }, diff --git a/packages/api/src/routers/api/ai.ts b/packages/api/src/routers/api/ai.ts index 3f8fbdb5..de53866c 100644 --- a/packages/api/src/routers/api/ai.ts +++ b/packages/api/src/routers/api/ai.ts @@ -274,6 +274,7 @@ router.post( const keyValues = await metadata.getKeyValues({ chartConfig: cc, keys: keysToFetch.map(f => f.key), + source, }); const anthropic = createAnthropic({ diff --git a/packages/api/src/routers/external-api/v2/charts.ts b/packages/api/src/routers/external-api/v2/charts.ts index a560e2f6..80fd19d9 100644 --- a/packages/api/src/routers/external-api/v2/charts.ts +++ b/packages/api/src/routers/external-api/v2/charts.ts @@ -588,6 +588,7 @@ router.post( const result = await clickhouseClient.queryChartConfig({ config: chartConfig, metadata, + querySettings: source.querySettings, }); return { diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index f2882dac..93579135 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -448,6 +448,7 @@ export const processAlert = async ( const checksData = await clickhouseClient.queryChartConfig({ config: optimizedChartConfig, metadata, + querySettings: source.querySettings, }); logger.info( diff --git a/packages/api/src/tasks/checkAlerts/template.ts b/packages/api/src/tasks/checkAlerts/template.ts index 921233d2..3b9225dd 100644 --- a/packages/api/src/tasks/checkAlerts/template.ts +++ b/packages/api/src/tasks/checkAlerts/template.ts @@ -602,7 +602,11 @@ ${targetTemplate}`; let truncatedResults = ''; try { - const query = await renderChartConfig(chartConfig, metadata); + const query = await renderChartConfig( + chartConfig, + metadata, + source.querySettings, + ); const raw = await clickhouseClient .query<'CSV'>({ query: query.sql, diff --git a/packages/api/src/tasks/pingPongTask.ts b/packages/api/src/tasks/pingPongTask.ts index 841468c8..088aeff0 100644 --- a/packages/api/src/tasks/pingPongTask.ts +++ b/packages/api/src/tasks/pingPongTask.ts @@ -4,7 +4,6 @@ import logger from '@/utils/logger'; export default class PingPongTask implements HdxTask { constructor(private args: PingTaskArgs) {} - // eslint-disable-next-line @typescript-eslint/no-unused-vars async execute(): Promise { logger.info(` O . diff --git a/packages/app/src/TeamPage.tsx b/packages/app/src/TeamPage.tsx index 7627adc7..12faa904 100644 --- a/packages/app/src/TeamPage.tsx +++ b/packages/app/src/TeamPage.tsx @@ -1,9 +1,9 @@ import { useCallback, useState } from 'react'; import Head from 'next/head'; import { CopyToClipboard } from 'react-copy-to-clipboard'; -import { SubmitHandler, useForm, useWatch } from 'react-hook-form'; +import { SubmitHandler, useForm } from 'react-hook-form'; import { DEFAULT_METADATA_MAX_ROWS_TO_READ } from '@hyperdx/common-utils/dist/core/metadata'; -import { TeamClickHouseSettings } from '@hyperdx/common-utils/dist/types'; +import { type TeamClickHouseSettings } from '@hyperdx/common-utils/dist/types'; import { Box, Button, diff --git a/packages/app/src/__tests__/utils.test.ts b/packages/app/src/__tests__/utils.test.ts index adc58578..e130adca 100644 --- a/packages/app/src/__tests__/utils.test.ts +++ b/packages/app/src/__tests__/utils.test.ts @@ -8,6 +8,7 @@ import { formatAttributeClause, formatNumber, getMetricTableName, + mapKeyBy, orderByStringToSortingState, sortingStateToOrderByString, stripTrailingSlash, @@ -630,3 +631,25 @@ describe('orderByStringToSortingState', () => { expect(roundTripSort).toEqual(originalSort); }); }); + +describe('mapKeyBy', () => { + it('returns a map', () => { + const result = mapKeyBy([{ id: 'a' }, { id: 'b' }], 'id'); + expect(result).toBeInstanceOf(Map); + }); + + it('adds each item to the map, keyed by the provided `key` param', () => { + const data = [{ id: 'a' }, { id: 'b' }]; + const result = mapKeyBy(data, 'id'); + expect(result.size).toBe(2); + expect(result.get('a')).toBe(data.at(0)); + expect(result.get('b')).toBe(data.at(1)); + }); + + it('overwrites items with the same key', () => { + const data = [{ id: 'a' }, { id: 'a' }]; + const result = mapKeyBy(data, 'id'); + expect(result.size).toBe(1); + expect(result.get('a')).toBe(data.at(1)); + }); +}); diff --git a/packages/app/src/api.ts b/packages/app/src/api.ts index 59c5a6b2..539b4fd7 100644 --- a/packages/app/src/api.ts +++ b/packages/app/src/api.ts @@ -7,7 +7,7 @@ import type { PresetDashboardFilter, } from '@hyperdx/common-utils/dist/types'; import type { UseQueryOptions } from '@tanstack/react-query'; -import { useMutation, useQuery } from '@tanstack/react-query'; +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; import { IS_LOCAL_MODE } from './config'; import { Dashboard } from './dashboard'; diff --git a/packages/app/src/components/KubeComponents.tsx b/packages/app/src/components/KubeComponents.tsx index 39e53be1..40177bd7 100644 --- a/packages/app/src/components/KubeComponents.tsx +++ b/packages/app/src/components/KubeComponents.tsx @@ -99,6 +99,7 @@ export const useV2LogBatch = ( orderBy: `${logSource.timestampValueExpression} ${order}`, }, metadata, + logSource.querySettings, ); const json = await clickhouseClient diff --git a/packages/app/src/components/Sources/SourceForm.tsx b/packages/app/src/components/Sources/SourceForm.tsx index 15753ac5..1df60f27 100644 --- a/packages/app/src/components/Sources/SourceForm.tsx +++ b/packages/app/src/components/Sources/SourceForm.tsx @@ -1,4 +1,10 @@ -import React, { useCallback, useEffect, useRef, useState } from 'react'; +import React, { + Fragment, + useCallback, + useEffect, + useRef, + useState, +} from 'react'; import { Control, Controller, @@ -1582,6 +1588,7 @@ export function TableSourceForm({ databaseName: 'default', tableName: '', }, + querySettings: source?.querySettings, }, // TODO: HDX-1768 remove type assertion values: source as TSourceUnion, @@ -1937,6 +1944,12 @@ export function TableSourceForm({ defaultValue: source?.connection, }); + const { + fields: querySettingFields, + append: appendSetting, + remove: removeSetting, + } = useFieldArray({ control, name: 'querySettings' }); + return (
)} + + Query Settings + + } + helpText="Query-level Session Settings that will be added to each query for this source." + > + + {querySettingFields.map((field, index) => ( + + + + + + + + + + removeSetting(index)} + > + + + + + + ))} + + + diff --git a/packages/app/src/hdxMTViews.ts b/packages/app/src/hdxMTViews.ts index 71d1d09e..5b3d00f4 100644 --- a/packages/app/src/hdxMTViews.ts +++ b/packages/app/src/hdxMTViews.ts @@ -15,6 +15,7 @@ import { AggregateFunction, ChartConfigWithOptDateRange, DerivedColumn, + QuerySettings, SQLInterval, } from '@hyperdx/common-utils/dist/types'; @@ -97,6 +98,7 @@ const buildMTViewDDL = (name: string, table: string, query: ChSql) => { export const buildMTViewSelectQuery = async ( chartConfig: ChartConfigWithOptDateRange, metadata: Metadata, + querySettings: QuerySettings | undefined, customGranularity?: SQLInterval, ) => { const _config = { @@ -116,7 +118,7 @@ export const buildMTViewSelectQuery = async ( orderBy: undefined, limit: undefined, }; - const mtViewSQL = await renderChartConfig(_config, metadata); + const mtViewSQL = await renderChartConfig(_config, metadata, querySettings); const mtViewSQLHash = objectHash.sha1(mtViewSQL); const mtViewName = `${chartConfig.from.tableName}_mv_${mtViewSQLHash}`; const renderMTViewConfig = { @@ -148,7 +150,11 @@ export const buildMTViewSelectQuery = async ( ), renderMTViewConfig: async () => { try { - return await renderChartConfig(renderMTViewConfig, metadata); + return await renderChartConfig( + renderMTViewConfig, + metadata, + querySettings, + ); } catch (e) { console.error('Failed to render MTView config', e); return null; diff --git a/packages/app/src/hooks/__tests__/useDashboardFilterValues.test.tsx b/packages/app/src/hooks/__tests__/useDashboardFilterValues.test.tsx index 1cb24ef4..a815aab9 100644 --- a/packages/app/src/hooks/__tests__/useDashboardFilterValues.test.tsx +++ b/packages/app/src/hooks/__tests__/useDashboardFilterValues.test.tsx @@ -446,6 +446,16 @@ describe('useDashboardFilterKeyValues', () => { limit: 10000, disableRowLimit: true, signal: expect.any(AbortSignal), + source: { + connection: 'clickhouse-conn', + from: { + databaseName: 'telemetry', + tableName: 'logs', + }, + id: 'logs-source', + name: 'Logs', + timestampValueExpression: 'timestamp', + }, }); }); diff --git a/packages/app/src/hooks/useChartConfig.tsx b/packages/app/src/hooks/useChartConfig.tsx index b9a7f898..07bb2222 100644 --- a/packages/app/src/hooks/useChartConfig.tsx +++ b/packages/app/src/hooks/useChartConfig.tsx @@ -17,6 +17,7 @@ import { format } from '@hyperdx/common-utils/dist/sqlFormatter'; import { ChartConfigWithDateRange, ChartConfigWithOptDateRange, + QuerySettings, } from '@hyperdx/common-utils/dist/types'; import { useQuery, @@ -128,6 +129,7 @@ async function* fetchDataInChunks({ enableQueryChunking = false, enableParallelQueries = false, metadata, + querySettings, }: { config: ChartConfigWithOptDateRange; clickhouseClient: ClickhouseClient; @@ -135,6 +137,7 @@ async function* fetchDataInChunks({ enableQueryChunking?: boolean; enableParallelQueries?: boolean; metadata: Metadata; + querySettings: QuerySettings | undefined; }) { const windows = enableQueryChunking && shouldUseChunking(config) @@ -143,7 +146,7 @@ async function* fetchDataInChunks({ if (IS_MTVIEWS_ENABLED) { const { dataTableDDL, mtViewDDL, renderMTViewConfig } = - await buildMTViewSelectQuery(config, metadata); + await buildMTViewSelectQuery(config, metadata, querySettings); // TODO: show the DDLs in the UI so users can run commands manually // eslint-disable-next-line no-console console.log('dataTableDDL:', dataTableDDL); @@ -167,6 +170,7 @@ async function* fetchDataInChunks({ opts: { abort_signal: signal, }, + querySettings, }), }; }); @@ -208,6 +212,7 @@ async function* fetchDataInChunks({ opts: { abort_signal: signal, }, + querySettings, }); yield { chunk: result, isComplete: i === windows.length - 1 }; @@ -260,6 +265,10 @@ export function useQueriedChartConfig( placeholderData: undefined, }); + const { data: source, isLoading: isSourceLoading } = useSource({ + id: config.source, + }); + const query = useQuery({ // Include enableQueryChunking in the query key to ensure that queries with the // same config but different enableQueryChunking values do not share a query @@ -291,6 +300,7 @@ export function useQueriedChartConfig( enableQueryChunking: options?.enableQueryChunking, enableParallelQueries: options?.enableParallelQueries, metadata, + querySettings: source?.querySettings, }); let accumulatedChunks: TQueryFnData = emptyValue; @@ -322,7 +332,7 @@ export function useQueriedChartConfig( retry: 1, refetchOnWindowFocus: false, ...options, - enabled: enabled && !isLoadingMVOptimization, + enabled: enabled && !isLoadingMVOptimization && !isSourceLoading, }); if (query.isError && options?.onError) { @@ -348,15 +358,23 @@ export function useRenderedSqlChartConfig( placeholderData: undefined, }); + const { data: source, isLoading: isSourceLoading } = useSource({ + id: config.source, + }); + const query = useQuery({ queryKey: ['renderedSql', config], queryFn: async () => { const optimizedConfig = mvOptimizationData?.optimizedConfig ?? config; - const query = await renderChartConfig(optimizedConfig, metadata); + const query = await renderChartConfig( + optimizedConfig, + metadata, + source?.querySettings, + ); return format(parameterizedQueryToSql(query)); }, ...options, - enabled: enabled && !isLoadingMVOptimization, + enabled: enabled && !isLoadingMVOptimization && !isSourceLoading, }); return { @@ -379,6 +397,10 @@ export function useAliasMapFromChartConfig( const metadata = useMetadataWithSettings(); + const { data: source, isLoading: isSourceLoading } = useSource({ + id: config?.source, + }); + return useQuery>({ // Only include config properties that affect SELECT structure and aliases. // When adding new ChartConfig fields, check renderChartConfig.ts to see if they @@ -400,13 +422,17 @@ export function useAliasMapFromChartConfig( return {}; } - const query = await renderChartConfig(config, metadata); + const query = await renderChartConfig( + config, + metadata, + undefined, // no query settings for creating alias map + ); const aliasMap = chSqlToAliasMap(query); return aliasMap; }, - enabled: config != null, + enabled: config != null && !isSourceLoading, ...options, }); } diff --git a/packages/app/src/hooks/useDashboardFilterValues.tsx b/packages/app/src/hooks/useDashboardFilterValues.tsx index fd01725d..4b8cc0be 100644 --- a/packages/app/src/hooks/useDashboardFilterValues.tsx +++ b/packages/app/src/hooks/useDashboardFilterValues.tsx @@ -16,7 +16,7 @@ import { import { useClickhouseClient } from '@/clickhouse'; import { useSources } from '@/source'; -import { getMetricTableName } from '@/utils'; +import { getMetricTableName, mapKeyBy } from '@/utils'; import { useMetadataWithSettings } from './useMetadata'; @@ -129,6 +129,9 @@ export function useDashboardFilterKeyValues({ dateRange, }); + const { data: sources, isLoading: isSourcesLoading } = useSources(); + const sourcesLookup = useMemo(() => mapKeyBy(sources ?? [], 'id'), [sources]); + const queryClient = useQueryClient(); type TQueryData = { key: string; value: string[] }[]; @@ -140,6 +143,9 @@ export function useDashboardFilterKeyValues({ chartConfig.from, keys, ]; + + const source = sourcesLookup.get(chartConfig.source); + return { queryKey: [...queryKeyPrefix, chartConfig], placeholderData: () => { @@ -157,7 +163,7 @@ export function useDashboardFilterKeyValues({ }); return cached[0]?.data; }, - enabled: !isLoadingOptimizedCalls, + enabled: !isLoadingOptimizedCalls && !isSourcesLoading, staleTime: 1000 * 60 * 5, // Cache every 5 min queryFn: async ({ signal }) => metadata.getKeyValues({ @@ -166,6 +172,7 @@ export function useDashboardFilterKeyValues({ limit: 10000, disableRowLimit: true, signal, + source, }), }; }), diff --git a/packages/app/src/hooks/useExplainQuery.tsx b/packages/app/src/hooks/useExplainQuery.tsx index 36024691..81e0c181 100644 --- a/packages/app/src/hooks/useExplainQuery.tsx +++ b/packages/app/src/hooks/useExplainQuery.tsx @@ -3,6 +3,7 @@ import { ChartConfigWithOptDateRange } from '@hyperdx/common-utils/dist/types'; import { useQuery, UseQueryOptions } from '@tanstack/react-query'; import { useClickhouseClient } from '@/clickhouse'; +import { useSource } from '@/source'; import { useMetadataWithSettings } from './useMetadata'; @@ -15,12 +16,21 @@ export function useExplainQuery( with: undefined, }; const clickhouseClient = useClickhouseClient(); + const metadata = useMetadataWithSettings(); + const { data: source, isLoading: isSourceLoading } = useSource({ + id: config?.source, + }); + return useQuery({ queryKey: ['explain', config], queryFn: async ({ signal }) => { - const query = await renderChartConfig(config, metadata); + const query = await renderChartConfig( + config, + metadata, + source?.querySettings, + ); const response = await clickhouseClient.query<'JSONEachRow'>({ query: `EXPLAIN ESTIMATE ${query.sql}`, query_params: query.params, @@ -32,6 +42,7 @@ export function useExplainQuery( }, retry: false, staleTime: 1000 * 60, + enabled: !isSourceLoading, ...options, }); } diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx index bf174283..05e253a9 100644 --- a/packages/app/src/hooks/useMetadata.tsx +++ b/packages/app/src/hooks/useMetadata.tsx @@ -10,7 +10,10 @@ import { TableConnection, TableMetadata, } from '@hyperdx/common-utils/dist/core/metadata'; -import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import { + ChartConfigWithDateRange, + TSource, +} from '@hyperdx/common-utils/dist/types'; import { keepPreviousData, useQuery, @@ -22,7 +25,7 @@ import api from '@/api'; import { IS_LOCAL_MODE } from '@/config'; import { LOCAL_STORE_CONNECTIONS_KEY } from '@/connection'; import { getMetadata } from '@/metadata'; -import { useSources } from '@/source'; +import { useSource, useSources } from '@/source'; import { toArray } from '@/utils'; // Hook to get metadata with proper settings applied @@ -265,6 +268,10 @@ export function useGetValuesDistribution( options?: Omit, Error>, 'queryKey'>, ) { const metadata = useMetadataWithSettings(); + const { data: source, isLoading: isLoadingSource } = useSource({ + id: chartConfig.source, + }); + return useQuery>({ queryKey: ['useMetadata.useGetValuesDistribution', chartConfig, key], queryFn: async () => { @@ -272,10 +279,11 @@ export function useGetValuesDistribution( chartConfig, key, limit, + source, }); }, staleTime: Infinity, - enabled: !!key, + enabled: !!key && !isLoadingSource, placeholderData: keepPreviousData, retry: false, ...options, diff --git a/packages/app/src/hooks/useOffsetPaginatedQuery.tsx b/packages/app/src/hooks/useOffsetPaginatedQuery.tsx index 774d6136..217ad509 100644 --- a/packages/app/src/hooks/useOffsetPaginatedQuery.tsx +++ b/packages/app/src/hooks/useOffsetPaginatedQuery.tsx @@ -12,7 +12,10 @@ import { isFirstOrderByAscending, isTimestampExpressionInFirstOrderBy, } from '@hyperdx/common-utils/dist/core/utils'; -import { ChartConfigWithOptTimestamp } from '@hyperdx/common-utils/dist/types'; +import { + ChartConfigWithOptTimestamp, + TSource, +} from '@hyperdx/common-utils/dist/types'; import { QueryClient, QueryFunction, @@ -24,6 +27,7 @@ import api from '@/api'; import { getClickhouseClient } from '@/clickhouse'; import { useMetadataWithSettings } from '@/hooks/useMetadata'; import { useMVOptimizationExplanation } from '@/hooks/useMVOptimizationExplanation'; +import { useSource } from '@/source'; import { omit } from '@/utils'; import { generateTimeWindowsAscending, @@ -67,6 +71,7 @@ type QueryMeta = { hasPreviousQueries: boolean; metadata: Metadata; optimizedConfig?: ChartConfigWithOptTimestamp; + source: TSource | undefined; }; // Get time window from page param @@ -141,7 +146,7 @@ const queryFn: QueryFunction = async ({ throw new Error('Query missing client meta'); } - const { queryClient, metadata, hasPreviousQueries, optimizedConfig } = + const { queryClient, metadata, hasPreviousQueries, optimizedConfig, source } = meta as QueryMeta; // Only stream incrementally if this is a fresh query with no previous @@ -177,7 +182,11 @@ const queryFn: QueryFunction = async ({ }, }; - const query = await renderChartConfig(windowedConfig, metadata); + const query = await renderChartConfig( + windowedConfig, + metadata, + source?.querySettings, + ); // Create abort signal from timeout if provided const abortController = queryTimeout ? new AbortController() : undefined; @@ -399,6 +408,10 @@ export default function useOffsetPaginatedQuery( placeholderData: undefined, }); + const { data: source, isLoading: isSourceLoading } = useSource({ + id: config?.source, + }); + const { data, fetchNextPage, @@ -419,7 +432,8 @@ export default function useOffsetPaginatedQuery( // Only preserve previous query in live mode return isLive ? prev : undefined; }, - enabled: enabled && !isLoadingMe && !isLoadingMVOptimization, + enabled: + enabled && !isLoadingMe && !isLoadingMVOptimization && !isSourceLoading, initialPageParam: { windowIndex: 0, offset: 0 } as TPageParam, getNextPageParam: (lastPage, allPages) => { return getNextPageParam(lastPage, allPages, config); @@ -430,6 +444,7 @@ export default function useOffsetPaginatedQuery( hasPreviousQueries, metadata, optimizedConfig: mvOptimizationData?.optimizedConfig, + source, } satisfies QueryMeta, queryFn, gcTime: isLive ? ms('30s') : ms('5m'), // more aggressive gc for live data, since it can end up holding lots of data diff --git a/packages/app/src/hooks/useServiceMap.tsx b/packages/app/src/hooks/useServiceMap.tsx index d6a51b7d..34467742 100644 --- a/packages/app/src/hooks/useServiceMap.tsx +++ b/packages/app/src/hooks/useServiceMap.tsx @@ -94,6 +94,7 @@ async function getServiceMapQuery({ where: '', }, metadata, + source.querySettings, ), renderChartConfig( { @@ -108,6 +109,7 @@ async function getServiceMapQuery({ where: '', }, metadata, + source.querySettings, ), ]); diff --git a/packages/app/src/sessions.ts b/packages/app/src/sessions.ts index e162a947..d8ccaa9a 100644 --- a/packages/app/src/sessions.ts +++ b/packages/app/src/sessions.ts @@ -141,6 +141,7 @@ export function useSessions( groupBy: 'serviceName, sessionId', }, metadata, + traceSource.querySettings, ), renderChartConfig( { @@ -161,6 +162,7 @@ export function useSessions( connection: sessionSource.connection, }, metadata, + sessionSource.querySettings, ), renderChartConfig( { @@ -179,6 +181,7 @@ export function useSessions( connection: traceSource?.connection, }, metadata, + traceSource.querySettings, ), ]); @@ -372,6 +375,7 @@ export function useRRWebEventStream( }, }, metadata, + source.querySettings, ); const format = 'JSONEachRow'; diff --git a/packages/app/src/source.ts b/packages/app/src/source.ts index e87ef2e2..a8a15f89 100644 --- a/packages/app/src/source.ts +++ b/packages/app/src/source.ts @@ -20,7 +20,12 @@ import { TSource, TSourceUnion, } from '@hyperdx/common-utils/dist/types'; -import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; +import { + useMutation, + useQuery, + useQueryClient, + UseQueryOptions, +} from '@tanstack/react-query'; import { hdxServer } from '@/api'; import { HDX_LOCAL_DEFAULT_SOURCES } from '@/config'; diff --git a/packages/app/src/utils.ts b/packages/app/src/utils.ts index 32ef7649..6144c568 100644 --- a/packages/app/src/utils.ts +++ b/packages/app/src/utils.ts @@ -756,3 +756,13 @@ export const orderByStringToSortingState = ( }, ]; }; + +export const mapKeyBy = (array: T[], key: keyof T) => { + const map = new Map(); + + for (const item of array) { + map.set(item[key], item); + } + + return map; +}; diff --git a/packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap b/packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap index 8c82711f..15b2108e 100644 --- a/packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap +++ b/packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap @@ -1,30 +1,36 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`renderChartConfig Aggregate Merge Functions should generate SQL for an aggregate merge function 1`] = `"SELECT avgMerge(Duration),severity FROM default.logs WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY severity"`; +exports[`renderChartConfig Aggregate Merge Functions should generate SQL for an aggregate merge function 1`] = `"SELECT avgMerge(Duration),severity FROM default.logs WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY severity SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"`; -exports[`renderChartConfig Aggregate Merge Functions should generate SQL for an aggregate merge function with a condition 1`] = `"SELECT avgMergeIf(Duration, ((severity = 'ERROR')) AND toFloat64OrDefault(toString(Duration)) IS NOT NULL),severity FROM default.logs WHERE (((severity = 'ERROR'))) GROUP BY severity"`; +exports[`renderChartConfig Aggregate Merge Functions should generate SQL for an aggregate merge function with a condition 1`] = `"SELECT avgMergeIf(Duration, ((severity = 'ERROR')) AND toFloat64OrDefault(toString(Duration)) IS NOT NULL),severity FROM default.logs WHERE (((severity = 'ERROR'))) GROUP BY severity SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"`; -exports[`renderChartConfig Aggregate Merge Functions should generate SQL for an histogram merge function 1`] = `"SELECT histogramMerge(20)(Duration),severity FROM default.logs WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY severity"`; +exports[`renderChartConfig Aggregate Merge Functions should generate SQL for an histogram merge function 1`] = `"SELECT histogramMerge(20)(Duration),severity FROM default.logs WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY severity SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"`; -exports[`renderChartConfig Aggregate Merge Functions should generate SQL for an quantile merge function with a condition 1`] = `"SELECT quantileMergeIf(0.95)(Duration, ((severity = 'ERROR')) AND toFloat64OrDefault(toString(Duration)) IS NOT NULL),severity FROM default.logs WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) AND (((severity = 'ERROR'))) GROUP BY severity"`; +exports[`renderChartConfig Aggregate Merge Functions should generate SQL for an quantile merge function with a condition 1`] = `"SELECT quantileMergeIf(0.95)(Duration, ((severity = 'ERROR')) AND toFloat64OrDefault(toString(Duration)) IS NOT NULL),severity FROM default.logs WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) AND (((severity = 'ERROR'))) GROUP BY severity SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"`; -exports[`renderChartConfig HAVING clause should not render HAVING clause when having is empty string 1`] = `"SELECT count(),severity FROM default.logs WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY severity"`; +exports[`renderChartConfig HAVING clause should not render HAVING clause when having is empty string 1`] = `"SELECT count(),severity FROM default.logs WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY severity SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"`; -exports[`renderChartConfig HAVING clause should not render HAVING clause when not provided 1`] = `"SELECT count(),severity FROM default.logs WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY severity"`; +exports[`renderChartConfig HAVING clause should not render HAVING clause when not provided 1`] = `"SELECT count(),severity FROM default.logs WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY severity SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"`; -exports[`renderChartConfig HAVING clause should render HAVING clause with SQL language 1`] = `"SELECT count(),severity FROM default.logs WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY severity HAVING count(*) > 100"`; +exports[`renderChartConfig HAVING clause should render HAVING clause with SQL language 1`] = `"SELECT count(),severity FROM default.logs WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY severity HAVING count(*) > 100 SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"`; -exports[`renderChartConfig HAVING clause should render HAVING clause with granularity and groupBy 1`] = `"SELECT count(),event_type,toStartOfInterval(toDateTime(timestamp), INTERVAL 5 minute) AS \`__hdx_time_bucket\` FROM default.events WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY event_type,toStartOfInterval(toDateTime(timestamp), INTERVAL 5 minute) AS \`__hdx_time_bucket\` HAVING count(*) > 50 ORDER BY toStartOfInterval(toDateTime(timestamp), INTERVAL 5 minute) AS \`__hdx_time_bucket\`"`; +exports[`renderChartConfig HAVING clause should render HAVING clause with granularity and groupBy 1`] = `"SELECT count(),event_type,toStartOfInterval(toDateTime(timestamp), INTERVAL 5 minute) AS \`__hdx_time_bucket\` FROM default.events WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY event_type,toStartOfInterval(toDateTime(timestamp), INTERVAL 5 minute) AS \`__hdx_time_bucket\` HAVING count(*) > 50 ORDER BY toStartOfInterval(toDateTime(timestamp), INTERVAL 5 minute) AS \`__hdx_time_bucket\` SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"`; exports[`renderChartConfig HAVING clause should render HAVING clause with multiple conditions 1`] = ` "SELECT avg( toFloat64OrDefault(toString(response_time)) - ),count(),endpoint FROM default.metrics WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY endpoint HAVING avg(response_time) > 500 AND count(*) > 10" + ),count(),endpoint FROM default.metrics WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY endpoint HAVING avg(response_time) > 500 AND count(*) > 10 SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" `; -exports[`renderChartConfig containing CTE clauses should render a ChSql CTE configuration correctly 1`] = `"WITH TestCte AS (SELECT TimeUnix, Line FROM otel_logs) SELECT Line FROM TestCte"`; +exports[`renderChartConfig SETTINGS clause should apply the "chart config" settings to the query 1`] = `"SELECT histogramMerge(20)(Duration),severity FROM default.logs WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY severity SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"`; -exports[`renderChartConfig containing CTE clauses should render a chart config CTE configuration correctly 1`] = `"WITH Parts AS (SELECT _part, _part_offset FROM default.some_table WHERE ((FieldA = 'test')) ORDER BY rand() DESC LIMIT 1000) SELECT * FROM Parts WHERE ((FieldA = 'test') AND (indexHint((_part, _part_offset) IN (SELECT tuple(_part, _part_offset) FROM Parts)))) ORDER BY rand() DESC LIMIT 1000"`; +exports[`renderChartConfig SETTINGS clause should apply the "query settings" settings to the query 1`] = `"SELECT histogramMerge(20)(Duration),severity FROM default.logs WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY severity SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"`; + +exports[`renderChartConfig SETTINGS clause should concat the "chart config" and "query setting" settings and apply them to the query 1`] = `"SELECT histogramMerge(20)(Duration),severity FROM default.logs WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY severity SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"`; + +exports[`renderChartConfig containing CTE clauses should render a ChSql CTE configuration correctly 1`] = `"WITH TestCte AS (SELECT TimeUnix, Line FROM otel_logs) SELECT Line FROM TestCte SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"`; + +exports[`renderChartConfig containing CTE clauses should render a chart config CTE configuration correctly 1`] = `"WITH Parts AS (SELECT _part, _part_offset FROM default.some_table WHERE ((FieldA = 'test')) ORDER BY rand() DESC LIMIT 1000 SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000) SELECT * FROM Parts WHERE ((FieldA = 'test') AND (indexHint((_part, _part_offset) IN (SELECT tuple(_part, _part_offset) FROM Parts)))) ORDER BY rand() DESC LIMIT 1000 SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"`; exports[`renderChartConfig histogram metric queries count should generate a count query with grouping and time bucketing 1`] = ` "WITH source AS ( @@ -54,7 +60,7 @@ exports[`renderChartConfig histogram metric queries count should generate a coun sum(delta) AS \\"Value\\" FROM source GROUP BY group, \`__hdx_time_bucket\` - ) SELECT \`__hdx_time_bucket\`, group, \\"Value\\" FROM metrics WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable'" + ) SELECT \`__hdx_time_bucket\`, group, \\"Value\\" FROM metrics WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" `; exports[`renderChartConfig histogram metric queries count should generate a count query without grouping but time bucketing 1`] = ` @@ -85,7 +91,7 @@ exports[`renderChartConfig histogram metric queries count should generate a coun sum(delta) AS \\"Value\\" FROM source GROUP BY \`__hdx_time_bucket\` - ) SELECT \`__hdx_time_bucket\`, \\"Value\\" FROM metrics WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable'" + ) SELECT \`__hdx_time_bucket\`, \\"Value\\" FROM metrics WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" `; exports[`renderChartConfig histogram metric queries count should generate a count query without grouping or time bucketing 1`] = ` @@ -116,7 +122,7 @@ exports[`renderChartConfig histogram metric queries count should generate a coun sum(delta) AS \\"Value\\" FROM source GROUP BY \`__hdx_time_bucket\` - ) SELECT \`__hdx_time_bucket\`, \\"Value\\" FROM metrics WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable'" + ) SELECT \`__hdx_time_bucket\`, \\"Value\\" FROM metrics WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" `; exports[`renderChartConfig histogram metric queries quantile should generate a query with grouping and time bucketing 1`] = ` @@ -200,7 +206,7 @@ exports[`renderChartConfig histogram metric queries quantile should generate a q END AS \\"Value\\" FROM points WHERE length(point) > 1 AND total > 0 - ) SELECT \`__hdx_time_bucket\`, group, \\"Value\\" FROM metrics WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable'" + ) SELECT \`__hdx_time_bucket\`, group, \\"Value\\" FROM metrics WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" `; exports[`renderChartConfig histogram metric queries quantile should generate a query without grouping but time bucketing 1`] = ` @@ -284,7 +290,7 @@ exports[`renderChartConfig histogram metric queries quantile should generate a q END AS \\"Value\\" FROM points WHERE length(point) > 1 AND total > 0 - ) SELECT \`__hdx_time_bucket\`, \\"Value\\" FROM metrics WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable'" + ) SELECT \`__hdx_time_bucket\`, \\"Value\\" FROM metrics WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" `; exports[`renderChartConfig histogram metric queries quantile should generate a query without grouping or time bucketing 1`] = ` @@ -368,7 +374,7 @@ exports[`renderChartConfig histogram metric queries quantile should generate a q END AS \\"Value\\" FROM points WHERE length(point) > 1 AND total > 0 - ) SELECT \`__hdx_time_bucket\`, \\"Value\\" FROM metrics WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable'" + ) SELECT \`__hdx_time_bucket\`, \\"Value\\" FROM metrics WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" `; exports[`renderChartConfig k8s semantic convention migrations should generate SQL with metricNameSql for container.cpu.utilization histogram metric 1`] = ` @@ -452,7 +458,7 @@ exports[`renderChartConfig k8s semantic convention migrations should generate SQ END AS \\"Value\\" FROM points WHERE length(point) > 1 AND total > 0 - ) SELECT \`__hdx_time_bucket\`, \\"Value\\" FROM metrics WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable'" + ) SELECT \`__hdx_time_bucket\`, \\"Value\\" FROM metrics WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" `; exports[`renderChartConfig k8s semantic convention migrations should generate SQL with metricNameSql for histogram metric with groupBy 1`] = ` @@ -536,7 +542,7 @@ exports[`renderChartConfig k8s semantic convention migrations should generate SQ END AS \\"Value\\" FROM points WHERE length(point) > 1 AND total > 0 - ) SELECT \`__hdx_time_bucket\`, group, \\"Value\\" FROM metrics WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable'" + ) SELECT \`__hdx_time_bucket\`, group, \\"Value\\" FROM metrics WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" `; exports[`renderChartConfig k8s semantic convention migrations should generate SQL with metricNameSql for k8s.node.cpu.utilization sum metric 1`] = ` @@ -579,7 +585,7 @@ exports[`renderChartConfig k8s semantic convention migrations should generate SQ ORDER BY AttributesHash, \`__hdx_time_bucket2\` ) SELECT max( toFloat64OrDefault(toString(Rate)) - ) AS \\"Value\\",toStartOfInterval(toDateTime(\`__hdx_time_bucket2\`), INTERVAL 5 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (\`__hdx_time_bucket2\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket2\` <= fromUnixTimestamp64Milli(1765670400000)) GROUP BY toStartOfInterval(toDateTime(\`__hdx_time_bucket2\`), INTERVAL 5 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(\`__hdx_time_bucket2\`), INTERVAL 5 minute) AS \`__hdx_time_bucket\` LIMIT 10" + ) AS \\"Value\\",toStartOfInterval(toDateTime(\`__hdx_time_bucket2\`), INTERVAL 5 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (\`__hdx_time_bucket2\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket2\` <= fromUnixTimestamp64Milli(1765670400000)) GROUP BY toStartOfInterval(toDateTime(\`__hdx_time_bucket2\`), INTERVAL 5 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(\`__hdx_time_bucket2\`), INTERVAL 5 minute) AS \`__hdx_time_bucket\` LIMIT 10 SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" `; exports[`renderChartConfig k8s semantic convention migrations should generate SQL with metricNameSql for k8s.pod.cpu.utilization gauge metric 1`] = ` @@ -612,7 +618,7 @@ exports[`renderChartConfig k8s semantic convention migrations should generate SQ ORDER BY AttributesHash, __hdx_time_bucket2 ) SELECT avg( toFloat64OrDefault(toString(LastValue)) - ),toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (__hdx_time_bucket2 >= fromUnixTimestamp64Milli(1739318400000) AND __hdx_time_bucket2 <= fromUnixTimestamp64Milli(1765670400000)) GROUP BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable'" + ),toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (__hdx_time_bucket2 >= fromUnixTimestamp64Milli(1739318400000) AND __hdx_time_bucket2 <= fromUnixTimestamp64Milli(1765670400000)) GROUP BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" `; exports[`renderChartConfig k8s semantic convention migrations should handle metrics without metricNameSql (backward compatibility) 1`] = ` @@ -645,7 +651,7 @@ exports[`renderChartConfig k8s semantic convention migrations should handle metr ORDER BY AttributesHash, __hdx_time_bucket2 ) SELECT avg( toFloat64OrDefault(toString(LastValue)) - ),toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (__hdx_time_bucket2 >= fromUnixTimestamp64Milli(1739318400000) AND __hdx_time_bucket2 <= fromUnixTimestamp64Milli(1765670400000)) GROUP BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable'" + ),toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (__hdx_time_bucket2 >= fromUnixTimestamp64Milli(1739318400000) AND __hdx_time_bucket2 <= fromUnixTimestamp64Milli(1765670400000)) GROUP BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" `; exports[`renderChartConfig should generate sql for a single gauge metric 1`] = ` @@ -676,7 +682,7 @@ exports[`renderChartConfig should generate sql for a single gauge metric 1`] = ` FROM Source GROUP BY AttributesHash, __hdx_time_bucket2 ORDER BY AttributesHash, __hdx_time_bucket2 - ) SELECT quantile(0.95)(toFloat64OrDefault(toString(LastValue))),toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (__hdx_time_bucket2 >= fromUnixTimestamp64Milli(1739318400000) AND __hdx_time_bucket2 <= fromUnixTimestamp64Milli(1765670400000)) GROUP BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable'" + ) SELECT quantile(0.95)(toFloat64OrDefault(toString(LastValue))),toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (__hdx_time_bucket2 >= fromUnixTimestamp64Milli(1739318400000) AND __hdx_time_bucket2 <= fromUnixTimestamp64Milli(1765670400000)) GROUP BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" `; exports[`renderChartConfig should generate sql for a single gauge metric with a delta() function applied 1`] = ` @@ -709,7 +715,7 @@ exports[`renderChartConfig should generate sql for a single gauge metric with a ORDER BY AttributesHash, __hdx_time_bucket2 ) SELECT max( toFloat64OrDefault(toString(LastValue)) - ),toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (__hdx_time_bucket2 >= fromUnixTimestamp64Milli(1739318400000) AND __hdx_time_bucket2 <= fromUnixTimestamp64Milli(1765670400000)) GROUP BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable'" + ),toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (__hdx_time_bucket2 >= fromUnixTimestamp64Milli(1739318400000) AND __hdx_time_bucket2 <= fromUnixTimestamp64Milli(1765670400000)) GROUP BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" `; exports[`renderChartConfig should generate sql for a single sum metric 1`] = ` @@ -752,5 +758,5 @@ exports[`renderChartConfig should generate sql for a single sum metric 1`] = ` ORDER BY AttributesHash, \`__hdx_time_bucket2\` ) SELECT avg( toFloat64OrDefault(toString(Rate)) - ) AS \\"Value\\",toStartOfInterval(toDateTime(\`__hdx_time_bucket2\`), INTERVAL 5 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (\`__hdx_time_bucket2\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket2\` <= fromUnixTimestamp64Milli(1765670400000)) GROUP BY toStartOfInterval(toDateTime(\`__hdx_time_bucket2\`), INTERVAL 5 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(\`__hdx_time_bucket2\`), INTERVAL 5 minute) AS \`__hdx_time_bucket\` LIMIT 10" + ) AS \\"Value\\",toStartOfInterval(toDateTime(\`__hdx_time_bucket2\`), INTERVAL 5 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (\`__hdx_time_bucket2\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket2\` <= fromUnixTimestamp64Milli(1765670400000)) GROUP BY toStartOfInterval(toDateTime(\`__hdx_time_bucket2\`), INTERVAL 5 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(\`__hdx_time_bucket2\`), INTERVAL 5 minute) AS \`__hdx_time_bucket\` LIMIT 10 SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" `; diff --git a/packages/common-utils/src/__tests__/metadata.int.test.ts b/packages/common-utils/src/__tests__/metadata.int.test.ts index 09158833..3743d047 100644 --- a/packages/common-utils/src/__tests__/metadata.int.test.ts +++ b/packages/common-utils/src/__tests__/metadata.int.test.ts @@ -3,12 +3,19 @@ import { ClickHouseClient } from '@clickhouse/client-common'; import { ClickhouseClient as HdxClickhouseClient } from '@/clickhouse/node'; import { Metadata, MetadataCache } from '@/core/metadata'; -import { ChartConfigWithDateRange } from '@/types'; +import { ChartConfigWithDateRange, TSource } from '@/types'; describe('Metadata Integration Tests', () => { let client: ClickHouseClient; let hdxClient: HdxClickhouseClient; + const source = { + querySettings: [ + { setting: 'optimize_read_in_order', value: '0' }, + { setting: 'cast_keep_nullable', value: '0' }, + ], + } as TSource; + beforeAll(() => { const host = process.env.CLICKHOUSE_HOST || 'http://localhost:8123'; const username = process.env.CLICKHOUSE_USER || 'default'; @@ -85,6 +92,7 @@ describe('Metadata Integration Tests', () => { chartConfig, keys: ['SeverityText'], disableRowLimit, + source, }); expect(resultSeverityText).toHaveLength(1); @@ -98,6 +106,7 @@ describe('Metadata Integration Tests', () => { chartConfig, keys: ['TraceId'], disableRowLimit, + source, }); expect(resultTraceId).toHaveLength(1); @@ -115,6 +124,7 @@ describe('Metadata Integration Tests', () => { chartConfig, keys: ['TraceId', 'SeverityText'], disableRowLimit, + source, }); expect(resultBoth).toEqual([ @@ -138,6 +148,7 @@ describe('Metadata Integration Tests', () => { chartConfig, keys: ['__hdx_materialized_k8s.pod.name'], disableRowLimit, + source, }); expect(resultPodName).toHaveLength(1); @@ -153,6 +164,7 @@ describe('Metadata Integration Tests', () => { chartConfig, keys: ['LogAttributes.user'], disableRowLimit, + source, }); expect(resultLogAttributes).toHaveLength(1); @@ -167,6 +179,7 @@ describe('Metadata Integration Tests', () => { const resultEmpty = await metadata.getKeyValues({ chartConfig, keys: [], + source, }); expect(resultEmpty).toEqual([]); @@ -177,6 +190,7 @@ describe('Metadata Integration Tests', () => { chartConfig, keys: ['SeverityText'], limit: 2, + source, }); expect(resultLimited).toHaveLength(1); @@ -380,11 +394,12 @@ describe('Metadata Integration Tests', () => { ); }); - it('should work without source parameter (fall back to base table)', async () => { + it('should work with an undefined source parameter (fall back to base table)', async () => { const result = await metadata.getKeyValuesWithMVs({ chartConfig, keys: ['environment', 'service'], // No source parameter + source: undefined, }); expect(result).toHaveLength(2); diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index 4b1afbcb..0a8e2e72 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1,7 +1,7 @@ import { ClickhouseClient } from '../clickhouse/node'; import { Metadata, MetadataCache } from '../core/metadata'; import * as renderChartConfigModule from '../core/renderChartConfig'; -import { ChartConfigWithDateRange } from '../types'; +import { ChartConfigWithDateRange, TSource } from '../types'; // Mock ClickhouseClient const mockClickhouseClient = { @@ -20,6 +20,13 @@ jest.mock('../core/renderChartConfig', () => ({ .mockResolvedValue({ sql: 'SELECT 1', params: {} }), })); +const source = { + querySettings: [ + { setting: 'optimize_read_in_order', value: '0' }, + { setting: 'cast_keep_nullable', value: '0' }, + ], +} as TSource; + describe('MetadataCache', () => { let metadataCache: MetadataCache; @@ -259,6 +266,7 @@ describe('Metadata', () => { keys: ['column1', 'column2'], limit: 10, disableRowLimit: false, + source, }); expect(mockClickhouseClient.query).toHaveBeenCalledWith( @@ -278,6 +286,7 @@ describe('Metadata', () => { keys: ['column1', 'column2'], limit: 10, disableRowLimit: true, + source, }); expect(mockClickhouseClient.query).toHaveBeenCalledWith( @@ -292,6 +301,7 @@ describe('Metadata', () => { chartConfig: mockChartConfig, keys: ['column1', 'column2'], limit: 10, + source, }); expect(mockClickhouseClient.query).toHaveBeenCalledWith( @@ -310,6 +320,7 @@ describe('Metadata', () => { chartConfig: mockChartConfig, keys: ['column1', 'column2'], limit: 10, + source, }); expect(result).toEqual([ @@ -334,6 +345,7 @@ describe('Metadata', () => { chartConfig: mockChartConfig, keys: ['column1'], limit: 10, + source, }); expect(result).toEqual([{ key: 'column1', value: ['value1', 'value2'] }]); @@ -349,6 +361,7 @@ describe('Metadata', () => { chartConfig: mockChartConfig, keys: [], limit: 10, + source, }); expect(results).toEqual([]); @@ -400,6 +413,7 @@ describe('Metadata', () => { const result = await metadata.getValuesDistribution({ chartConfig: mockChartConfig, key: 'severity', + source, }); expect(result).toEqual( @@ -442,6 +456,7 @@ describe('Metadata', () => { await metadata.getValuesDistribution({ chartConfig: configWithAliases, key: 'severity', + source, }); const actualConfig = renderChartConfigSpy.mock.calls[0][0]; @@ -481,6 +496,7 @@ describe('Metadata', () => { await metadata.getValuesDistribution({ chartConfig: configWithFilters, key: 'severity', + source, }); const actualConfig = renderChartConfigSpy.mock.calls[0][0]; diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index 23547ca8..a9988a62 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -4,9 +4,14 @@ import { ChartConfigWithOptDateRange, DisplayType, MetricsDataType, + QuerySettings, } from '@/types'; -import { renderChartConfig, timeFilterExpr } from '../core/renderChartConfig'; +import { + ChartConfigWithOptDateRangeEx, + renderChartConfig, + timeFilterExpr, +} from '../core/renderChartConfig'; describe('renderChartConfig', () => { let mockMetadata: jest.Mocked; @@ -70,10 +75,19 @@ describe('renderChartConfig', () => { limit: { limit: 10 }, }; + const querySettings: QuerySettings = [ + { setting: 'optimize_read_in_order', value: '0' }, + { setting: 'cast_keep_nullable', value: '1' }, + { setting: 'additional_result_filter', value: 'x != 2' }, + { setting: 'count_distinct_implementation', value: 'uniqCombined64' }, + { setting: 'async_insert_busy_timeout_min_ms', value: '20000' }, + ]; + it('should generate sql for a single gauge metric', async () => { const generatedSql = await renderChartConfig( gaugeConfiguration, mockMetadata, + querySettings, ); const actual = parameterizedQueryToSql(generatedSql); expect(actual).toMatchSnapshot(); @@ -94,6 +108,7 @@ describe('renderChartConfig', () => { ], }, mockMetadata, + querySettings, ); const actual = parameterizedQueryToSql(generatedSql); expect(actual).toMatchSnapshot(); @@ -133,7 +148,11 @@ describe('renderChartConfig', () => { limit: { limit: 10 }, }; - const generatedSql = await renderChartConfig(config, mockMetadata); + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); const actual = parameterizedQueryToSql(generatedSql); expect(actual).toMatchSnapshot(); }); @@ -162,9 +181,9 @@ describe('renderChartConfig', () => { limit: { limit: 10 }, }; - await expect(renderChartConfig(config, mockMetadata)).rejects.toThrow( - 'multi select or string select on metrics not supported', - ); + await expect( + renderChartConfig(config, mockMetadata, querySettings), + ).rejects.toThrow('multi select or string select on metrics not supported'); }); describe('histogram metric queries', () => { @@ -200,7 +219,11 @@ describe('renderChartConfig', () => { limit: { limit: 10 }, }; - const generatedSql = await renderChartConfig(config, mockMetadata); + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); const actual = parameterizedQueryToSql(generatedSql); expect(actual).toMatchSnapshot(); }); @@ -237,7 +260,11 @@ describe('renderChartConfig', () => { limit: { limit: 10 }, }; - const generatedSql = await renderChartConfig(config, mockMetadata); + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); const actual = parameterizedQueryToSql(generatedSql); expect(actual).toMatchSnapshot(); }); @@ -275,7 +302,11 @@ describe('renderChartConfig', () => { limit: { limit: 10 }, }; - const generatedSql = await renderChartConfig(config, mockMetadata); + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); const actual = parameterizedQueryToSql(generatedSql); expect(actual).toMatchSnapshot(); }); @@ -312,7 +343,11 @@ describe('renderChartConfig', () => { limit: { limit: 10 }, }; - const generatedSql = await renderChartConfig(config, mockMetadata); + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); const actual = parameterizedQueryToSql(generatedSql); expect(actual).toMatchSnapshot(); }); @@ -348,7 +383,11 @@ describe('renderChartConfig', () => { limit: { limit: 10 }, }; - const generatedSql = await renderChartConfig(config, mockMetadata); + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); const actual = parameterizedQueryToSql(generatedSql); expect(actual).toMatchSnapshot(); }); @@ -385,7 +424,11 @@ describe('renderChartConfig', () => { limit: { limit: 10 }, }; - const generatedSql = await renderChartConfig(config, mockMetadata); + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); const actual = parameterizedQueryToSql(generatedSql); expect(actual).toMatchSnapshot(); }); @@ -408,7 +451,11 @@ describe('renderChartConfig', () => { whereLanguage: 'sql', }; - const generatedSql = await renderChartConfig(config, mockMetadata); + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); const actual = parameterizedQueryToSql(generatedSql); expect(actual).toMatchSnapshot(); }); @@ -458,7 +505,11 @@ describe('renderChartConfig', () => { limit: { limit: 1000 }, }; - const generatedSql = await renderChartConfig(config, mockMetadata); + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); const actual = parameterizedQueryToSql(generatedSql); expect(actual).toMatchSnapshot(); }); @@ -481,7 +532,9 @@ describe('renderChartConfig', () => { whereLanguage: 'sql', }; - await expect(renderChartConfig(config, mockMetadata)).rejects.toThrow( + await expect( + renderChartConfig(config, mockMetadata, querySettings), + ).rejects.toThrow( "must specify either 'sql' or 'chartConfig' in with clause", ); }); @@ -504,9 +557,9 @@ describe('renderChartConfig', () => { whereLanguage: 'sql', }; - await expect(renderChartConfig(config, mockMetadata)).rejects.toThrow( - 'non-conforming sql object in CTE', - ); + await expect( + renderChartConfig(config, mockMetadata, querySettings), + ).rejects.toThrow('non-conforming sql object in CTE'); }); it('should throw if the CTE chartConfig param is invalid', async () => { @@ -530,9 +583,9 @@ describe('renderChartConfig', () => { whereLanguage: 'sql', }; - await expect(renderChartConfig(config, mockMetadata)).rejects.toThrow( - 'non-conforming chartConfig object in CTE', - ); + await expect( + renderChartConfig(config, mockMetadata, querySettings), + ).rejects.toThrow('non-conforming chartConfig object in CTE'); }); }); @@ -572,7 +625,11 @@ describe('renderChartConfig', () => { limit: { limit: 10 }, }; - const generatedSql = await renderChartConfig(config, mockMetadata); + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); const actual = parameterizedQueryToSql(generatedSql); // Verify the SQL contains the IN-based metric name condition @@ -617,7 +674,11 @@ describe('renderChartConfig', () => { limit: { limit: 10 }, }; - const generatedSql = await renderChartConfig(config, mockMetadata); + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); const actual = parameterizedQueryToSql(generatedSql); expect(actual).toContain('k8s.node.cpu.utilization'); @@ -660,7 +721,11 @@ describe('renderChartConfig', () => { limit: { limit: 10 }, }; - const generatedSql = await renderChartConfig(config, mockMetadata); + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); const actual = parameterizedQueryToSql(generatedSql); expect(actual).toContain('container.cpu.utilization'); @@ -704,7 +769,11 @@ describe('renderChartConfig', () => { limit: { limit: 10 }, }; - const generatedSql = await renderChartConfig(config, mockMetadata); + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); const actual = parameterizedQueryToSql(generatedSql); expect(actual).toContain('k8s.pod.cpu.utilization'); @@ -747,7 +816,11 @@ describe('renderChartConfig', () => { limit: { limit: 10 }, }; - const generatedSql = await renderChartConfig(config, mockMetadata); + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); const actual = parameterizedQueryToSql(generatedSql); // Should use the simple string comparison for regular metrics (not IN-based) @@ -782,7 +855,11 @@ describe('renderChartConfig', () => { dateRange: [new Date('2025-02-12'), new Date('2025-02-14')], }; - const generatedSql = await renderChartConfig(config, mockMetadata); + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); const actual = parameterizedQueryToSql(generatedSql); expect(actual).toContain('HAVING'); expect(actual).toContain('count(*) > 100'); @@ -818,7 +895,11 @@ describe('renderChartConfig', () => { dateRange: [new Date('2025-02-12'), new Date('2025-02-14')], }; - const generatedSql = await renderChartConfig(config, mockMetadata); + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); const actual = parameterizedQueryToSql(generatedSql); expect(actual).toContain('HAVING'); expect(actual).toContain('avg(response_time) > 500 AND count(*) > 10'); @@ -847,7 +928,11 @@ describe('renderChartConfig', () => { dateRange: [new Date('2025-02-12'), new Date('2025-02-14')], }; - const generatedSql = await renderChartConfig(config, mockMetadata); + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); const actual = parameterizedQueryToSql(generatedSql); expect(actual).not.toContain('HAVING'); expect(actual).toMatchSnapshot(); @@ -878,7 +963,11 @@ describe('renderChartConfig', () => { granularity: '5 minute', }; - const generatedSql = await renderChartConfig(config, mockMetadata); + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); const actual = parameterizedQueryToSql(generatedSql); expect(actual).toContain('HAVING'); expect(actual).toContain('count(*) > 50'); @@ -910,7 +999,11 @@ describe('renderChartConfig', () => { dateRange: [new Date('2025-02-12'), new Date('2025-02-14')], }; - const generatedSql = await renderChartConfig(config, mockMetadata); + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); const actual = parameterizedQueryToSql(generatedSql); expect(actual).not.toContain('HAVING'); expect(actual).toMatchSnapshot(); @@ -1151,7 +1244,11 @@ describe('renderChartConfig', () => { dateRange: [new Date('2025-02-12'), new Date('2025-02-14')], }; - const generatedSql = await renderChartConfig(config, mockMetadata); + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); const actual = parameterizedQueryToSql(generatedSql); expect(actual).toContain('avgMerge(Duration)'); expect(actual).toMatchSnapshot(); @@ -1178,7 +1275,11 @@ describe('renderChartConfig', () => { groupBy: 'severity', }; - const generatedSql = await renderChartConfig(config, mockMetadata); + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); const actual = parameterizedQueryToSql(generatedSql); expect(actual).toContain( "avgMergeIf(Duration, ((severity = 'ERROR')) AND toFloat64OrDefault(toString(Duration)) IS NOT NULL)", @@ -1210,7 +1311,11 @@ describe('renderChartConfig', () => { dateRange: [new Date('2025-02-12'), new Date('2025-02-14')], }; - const generatedSql = await renderChartConfig(config, mockMetadata); + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); const actual = parameterizedQueryToSql(generatedSql); expect(actual).toContain( "quantileMergeIf(0.95)(Duration, ((severity = 'ERROR')) AND toFloat64OrDefault(toString(Duration)) IS NOT NULL)", @@ -1240,10 +1345,85 @@ describe('renderChartConfig', () => { dateRange: [new Date('2025-02-12'), new Date('2025-02-14')], }; - const generatedSql = await renderChartConfig(config, mockMetadata); + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); const actual = parameterizedQueryToSql(generatedSql); expect(actual).toContain('histogramMerge(20)(Duration)'); expect(actual).toMatchSnapshot(); }); }); + + describe('SETTINGS clause', () => { + const config: ChartConfigWithOptDateRangeEx = { + displayType: DisplayType.Table, + connection: 'test-connection', + from: { + databaseName: 'default', + tableName: 'logs', + }, + select: [ + { + aggFn: 'histogramMerge', + valueExpression: 'Duration', + level: 20, + }, + ], + where: '', + whereLanguage: 'sql', + groupBy: 'severity', + timestampValueExpression: 'timestamp', + dateRange: [new Date('2025-02-12'), new Date('2025-02-14')], + }; + + test('should apply the "query settings" settings to the query', async () => { + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); + + const actual = parameterizedQueryToSql(generatedSql); + expect(actual).toContain( + "SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000", + ); + expect(actual).toMatchSnapshot(); + }); + + test('should apply the "chart config" settings to the query', async () => { + const generatedSql = await renderChartConfig( + { + ...config, + settings: chSql`short_circuit_function_evaluation = 'force_enable'`, + }, + mockMetadata, + querySettings, + ); + + const actual = parameterizedQueryToSql(generatedSql); + expect(actual).toContain( + "SETTINGS short_circuit_function_evaluation = 'force_enable'", + ); + expect(actual).toMatchSnapshot(); + }); + + test('should concat the "chart config" and "query setting" settings and apply them to the query', async () => { + const generatedSql = await renderChartConfig( + { + ...config, + settings: chSql`short_circuit_function_evaluation = 'force_enable'`, + }, + mockMetadata, + querySettings, + ); + + const actual = parameterizedQueryToSql(generatedSql); + expect(actual).toContain( + "SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000", + ); + expect(actual).toMatchSnapshot(); + }); + }); }); diff --git a/packages/common-utils/src/__tests__/utils.test.ts b/packages/common-utils/src/__tests__/utils.test.ts index 96c2e619..b26f997e 100644 --- a/packages/common-utils/src/__tests__/utils.test.ts +++ b/packages/common-utils/src/__tests__/utils.test.ts @@ -10,6 +10,7 @@ import { import { convertToDashboardTemplate, + extractSettingsClauseFromEnd, findJsonExpressions, formatDate, getAlignedDateRange, @@ -17,7 +18,9 @@ import { isFirstOrderByAscending, isJsonExpression, isTimestampExpressionInFirstOrderBy, + joinQuerySettings, optimizeTimestampValueExpression, + parseToNumber, parseToStartOfFunction, replaceJsonExpressions, splitAndTrimCSV, @@ -1423,4 +1426,135 @@ describe('utils', () => { expect(alignedEnd.toISOString()).toBe('2025-11-26T13:00:00.000Z'); }); }); + + describe('extractSettingsClauseFromEnd', () => { + test.each([ + { + label: 'no settings clause', + sql: 'SELECT * FROM table', + withoutSettingsClause: 'SELECT * FROM table', + settingsClause: undefined, + }, + { + label: 'basic', + sql: 'SELECT * FROM table SETTINGS opt=1, cast=1', + withoutSettingsClause: 'SELECT * FROM table', + settingsClause: 'SETTINGS opt=1, cast=1', + }, + { + label: 'basic with semicolon', + sql: 'SELECT * FROM table SETTINGS opt = 1, cast = 1;', + withoutSettingsClause: 'SELECT * FROM table', + settingsClause: 'SETTINGS opt = 1, cast = 1', + }, + { + label: 'with WHERE clause', + sql: 'SELECT * FROM table WHERE col=Value SETTINGS opt = 1, cast = 1;', + withoutSettingsClause: 'SELECT * FROM table WHERE col=Value', + settingsClause: 'SETTINGS opt = 1, cast = 1', + }, + { + label: 'SETTINGS not at end', + sql: 'SELECT * FROM table WHERE col=Value SETTINGS opt = 1, cast = 1 FORMAT json;', + withoutSettingsClause: 'SELECT * FROM table WHERE col=Value', + // This test case illustrates that subsequent clauses will also be extracted. + settingsClause: 'SETTINGS opt = 1, cast = 1 FORMAT json', + }, + ])( + 'Extracts SETTINGS clause from: "$label" query', + ({ sql, settingsClause, withoutSettingsClause }) => { + const [remaining, extractedSettingsClause] = + extractSettingsClauseFromEnd(sql); + expect(remaining).toBe(withoutSettingsClause); + expect(extractedSettingsClause).toBe(settingsClause); + }, + ); + }); + + describe('parseToNumber', () => { + it('returns `undefined` for an empty string', () => { + expect(parseToNumber('')).toBe(undefined); + }); + + it('returns `undefined` for a whitespace string', () => { + expect(parseToNumber(' ')).toBe(undefined); + }); + + it('returns `undefined` for a non-numeric string', () => { + expect(parseToNumber(' . ? / ')).toBe(undefined); + expect(parseToNumber(' some string value ')).toBe(undefined); + expect(parseToNumber('5678abc')).toBe(undefined); + }); + + it('returns `undefined` for an infinite number', () => { + expect(parseToNumber('Infinity')).toBe(undefined); + expect(parseToNumber('-Infinity')).toBe(undefined); + }); + + it('returns the number value for a parseable number', () => { + expect(parseToNumber('123')).toBe(123); + expect(parseToNumber('0.123')).toBe(0.123); + expect(parseToNumber('1.123')).toBe(1.123); + expect(parseToNumber('10000000')).toBe(10000000); + }); + }); + + describe('joinQuerySettings', () => { + test('returns `undefined` if the querySettings are `undefined` or empty', () => { + expect(joinQuerySettings(undefined)).toBe(undefined); + expect(joinQuerySettings([])).toBe(undefined); + }); + + test('filters out items whose `setting` or `value` field is empty', () => { + expect( + joinQuerySettings([ + { setting: '', value: '1' }, + { setting: 'async_insert', value: '' }, + { setting: 'async_insert_busy_timeout_min_ms', value: '20000' }, + ]), + ).toEqual('async_insert_busy_timeout_min_ms = 20000'); + }); + + test('joins the values into key value pairs', () => { + const result = joinQuerySettings([ + { setting: 'additional_result_filter', value: 'x != 2' }, + { setting: 'async_insert', value: '0' }, + { setting: 'async_insert_busy_timeout_min_ms', value: '20000' }, + ]); + + expect(result).toContain("additional_result_filter = 'x != 2'"); + expect(result).toContain('async_insert = 0'); + expect(result).toContain('async_insert_busy_timeout_min_ms = 20000'); + }); + + test('joins the result into a comma separated string', () => { + expect( + joinQuerySettings([ + { setting: 'additional_result_filter', value: 'x != 2' }, + { setting: 'async_insert', value: '0' }, + { setting: 'async_insert_busy_timeout_min_ms', value: '20000' }, + ]), + ).toEqual( + "additional_result_filter = 'x != 2', async_insert = 0, async_insert_busy_timeout_min_ms = 20000", + ); + }); + + test('wraps non-numeric and infinite numeric values in quotes', () => { + expect( + joinQuerySettings([{ setting: 'setting_name', value: 'x != 2' }]), + ).toEqual("setting_name = 'x != 2'"); + + expect( + joinQuerySettings([{ setting: 'setting_name', value: 'string value' }]), + ).toEqual("setting_name = 'string value'"); + + expect( + joinQuerySettings([{ setting: 'setting_name', value: '1000' }]), + ).toEqual('setting_name = 1000'); + + expect( + joinQuerySettings([{ setting: 'setting_name', value: 'Infinity' }]), + ).toEqual("setting_name = 'Infinity'"); + }); + }); }); diff --git a/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts b/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts index 9c502fad..574b020e 100644 --- a/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts +++ b/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts @@ -9,6 +9,8 @@ import { Metadata } from '@/core/metadata'; import { ChartConfigWithOptDateRange, MaterializedViewConfiguration, + QuerySettings, + TSource, } from '@/types'; import { ColumnMeta } from '..'; @@ -81,7 +83,7 @@ describe('materializedViews', () => { const SOURCE = { from: { databaseName: 'default', tableName: 'otel_spans' }, materializedViews: [MV_CONFIG_METRIC_ROLLUP_1M], - }; + } as TSource; describe('tryConvertConfigToMaterializedViewSelect', () => { it('should return empty object if selecting a string instead of an array of aggregates', async () => { @@ -1092,7 +1094,7 @@ describe('materializedViews', () => { {} as any, { from: { databaseName: 'default', tableName: 'table_without_mv' }, - }, + } as TSource, ); expect(actual).toEqual(chartConfig); @@ -1507,7 +1509,7 @@ describe('materializedViews', () => { {} as any, { from: { databaseName: 'default', tableName: 'table_without_mv' }, - }, + } as TSource, ); expect(result).toEqual({ diff --git a/packages/common-utils/src/clickhouse/index.ts b/packages/common-utils/src/clickhouse/index.ts index 0bc35573..af812bf9 100644 --- a/packages/common-utils/src/clickhouse/index.ts +++ b/packages/common-utils/src/clickhouse/index.ts @@ -18,11 +18,12 @@ import { splitChartConfigs, } from '@/core/renderChartConfig'; import { + extractSettingsClauseFromEnd, hashCode, replaceJsonExpressions, splitAndTrimWithBracket, } from '@/core/utils'; -import { ChartConfigWithOptDateRange } from '@/types'; +import { ChartConfigWithOptDateRange, QuerySettings } from '@/types'; // export @clickhouse/client-common types export type { @@ -557,6 +558,7 @@ export abstract class BaseClickhouseClient { config, metadata, opts, + querySettings, }: { config: ChartConfigWithOptDateRange; metadata: Metadata; @@ -564,10 +566,13 @@ export abstract class BaseClickhouseClient { abort_signal?: AbortSignal; clickhouse_settings?: Record; }; + querySettings: QuerySettings | undefined; }): Promise>> { config = setChartSelectsAlias(config); const queries: ChSql[] = await Promise.all( - splitChartConfigs(config).map(c => renderChartConfig(c, metadata)), + splitChartConfigs(config).map(c => + renderChartConfig(c, metadata, querySettings), + ), ); const isTimeSeries = config.displayType === 'line'; @@ -654,6 +659,7 @@ export abstract class BaseClickhouseClient { config, metadata, opts, + querySettings, }: { config: ChartConfigWithOptDateRange; metadata: Metadata; @@ -661,9 +667,14 @@ export abstract class BaseClickhouseClient { abort_signal?: AbortSignal; clickhouse_settings?: Record; }; + querySettings: QuerySettings | undefined; }): Promise<{ isValid: boolean; rowEstimate?: number; error?: string }> { try { - const renderedConfig = await renderChartConfig(config, metadata); + const renderedConfig = await renderChartConfig( + config, + metadata, + querySettings, + ); const explainedQuery = chSql`EXPLAIN ESTIMATE ${renderedConfig}`; const result = await this.query<'JSON'>({ @@ -736,10 +747,12 @@ export function chSqlToAliasMap( try { const sql = parameterizedQueryToSql(chSql); + // Remove the SETTINGS clause because `SQLParser` doesn't understand it. + const [sqlWithoutSettingsClause] = extractSettingsClauseFromEnd(sql); + // Replace JSON expressions with replacement tokens so that node-sql-parser can parse the SQL const { sqlWithReplacements, replacements: jsonReplacementsToExpressions } = - replaceJsonExpressions(sql); - + replaceJsonExpressions(sqlWithoutSettingsClause); const parser = new SQLParser.Parser(); // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- astify returns union type const ast = parser.astify(sqlWithReplacements, { diff --git a/packages/common-utils/src/core/materializedViews.ts b/packages/common-utils/src/core/materializedViews.ts index 930bce7b..6f16fb40 100644 --- a/packages/common-utils/src/core/materializedViews.ts +++ b/packages/common-utils/src/core/materializedViews.ts @@ -383,7 +383,7 @@ async function tryOptimizeConfig( clickhouseClient: BaseClickhouseClient, signal: AbortSignal | undefined, mvConfig: MaterializedViewConfiguration, - sourceFrom: TSource['from'], + source: Omit, // for overlap with ISource type ) { const errors: string[] = []; // Attempt to optimize any CTEs that exist in the config @@ -393,8 +393,8 @@ async function tryOptimizeConfig( config.with.map(async cte => { if ( cte.chartConfig && - cte.chartConfig.from.databaseName === sourceFrom.databaseName && - cte.chartConfig.from.tableName === sourceFrom.tableName + cte.chartConfig.from.databaseName === source.from.databaseName && + cte.chartConfig.from.tableName === source.from.tableName ) { return tryConvertConfigToMaterializedViewSelect( cte.chartConfig, @@ -433,8 +433,8 @@ async function tryOptimizeConfig( // Attempt to optimize the main (outer) select if ( - config.from.databaseName === sourceFrom.databaseName && - config.from.tableName === sourceFrom.tableName + config.from.databaseName === source.from.databaseName && + config.from.tableName === source.from.tableName ) { const convertedOuterSelect = await tryConvertConfigToMaterializedViewSelect( optimizedConfig ?? config, @@ -460,6 +460,7 @@ async function tryOptimizeConfig( opts: { abort_signal: signal, }, + querySettings: source.querySettings, }); if (error) { @@ -486,7 +487,7 @@ export async function tryOptimizeConfigWithMaterializedViewWithExplanations< metadata: Metadata, clickhouseClient: BaseClickhouseClient, signal: AbortSignal | undefined, - source: Pick & Partial>, + source: Omit, // for overlap with ISource type ): Promise<{ optimizedConfig?: C; explanations: MVOptimizationExplanation[]; @@ -500,7 +501,7 @@ export async function tryOptimizeConfigWithMaterializedViewWithExplanations< clickhouseClient, signal, mvConfig, - source.from, + source, ).then(result => ({ ...result, mvConfig })), ), ); @@ -540,7 +541,7 @@ export async function tryOptimizeConfigWithMaterializedView< metadata: Metadata, clickhouseClient: BaseClickhouseClient, signal: AbortSignal | undefined, - source: Pick & Partial>, + source: Omit, // for overlap with ISource type ) { const { optimizedConfig } = await tryOptimizeConfigWithMaterializedViewWithExplanations( @@ -653,6 +654,7 @@ export async function optimizeGetKeyValuesCalls< config, metadata, opts: { abort_signal: signal }, + querySettings: source?.querySettings, }); return { id: toMvId({ diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index b6804377..fbc119ec 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -12,7 +12,12 @@ import { tableExpr, } from '@/clickhouse'; import { renderChartConfig } from '@/core/renderChartConfig'; -import type { ChartConfig, ChartConfigWithDateRange, TSource } from '@/types'; +import type { + ChartConfig, + ChartConfigWithDateRange, + QuerySettings, + TSource, +} from '@/types'; import { optimizeGetKeyValuesCalls } from './materializedViews'; import { objectHash } from './utils'; @@ -701,11 +706,13 @@ export class Metadata { key, samples = 100_000, limit = 100, + source, }: { chartConfig: ChartConfigWithDateRange; key: string; samples?: number; limit?: number; + source: TSource | undefined; }) { const cacheKeyConfig = pick(chartConfig, [ 'connection', @@ -746,7 +753,11 @@ export class Metadata { limit: { limit }, }; - const sql = await renderChartConfig(config, this); + const sql = await renderChartConfig( + config, + this, + source?.querySettings, + ); const json = await this.clickhouseClient .query<'JSON'>({ @@ -785,12 +796,16 @@ export class Metadata { limit = 20, disableRowLimit = false, signal, + source, }: { chartConfig: ChartConfigWithDateRange; keys: string[]; limit?: number; disableRowLimit?: boolean; signal?: AbortSignal; + source: + | Omit /* for overlap with ISource type */ + | undefined; }): Promise<{ key: string; value: string[] }[]> { const cacheKeyConfig = { ...pick(chartConfig, [ @@ -856,7 +871,11 @@ export class Metadata { }; })(); - const sql = await renderChartConfig(sqlConfig, this); + const sql = await renderChartConfig( + sqlConfig, + this, + source?.querySettings, + ); const json = await this.clickhouseClient .query<'JSON'>({ @@ -898,7 +917,7 @@ export class Metadata { }: { chartConfig: ChartConfigWithDateRange; keys: string[]; - source?: TSource; + source: TSource | undefined; limit?: number; disableRowLimit?: boolean; signal?: AbortSignal; @@ -940,6 +959,7 @@ export class Metadata { limit, disableRowLimit, signal, + source, }), ), ); diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts index f5ac41b8..15fdf454 100644 --- a/packages/common-utils/src/core/renderChartConfig.ts +++ b/packages/common-utils/src/core/renderChartConfig.ts @@ -8,8 +8,11 @@ import { Metadata } from '@/core/metadata'; import { convertDateRangeToGranularityString, convertGranularityToSeconds, + extractSettingsClauseFromEnd, getFirstTimestampValueExpression, + joinQuerySettings, optimizeTimestampValueExpression, + parseToNumber, parseToStartOfFunction, splitAndTrimWithBracket, } from '@/core/utils'; @@ -24,6 +27,7 @@ import { ChSqlSchema, CteChartConfig, MetricsDataType, + QuerySettings, SearchCondition, SearchConditionLanguage, SelectList, @@ -164,9 +168,12 @@ const fastifySQL = ({ }) => { // Parse the SQL AST try { + // Remove the SETTINGS clause because `SQLParser` doesn't understand it. + const [rawSqlWithoutSettingsClause] = extractSettingsClauseFromEnd(rawSQL); + const parser = new SQLParser.Parser(); // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- astify returns union type, we expect Select - const ast = parser.astify(rawSQL, { + const ast = parser.astify(rawSqlWithoutSettingsClause, { database: 'Postgresql', }) as SQLParser.Select; @@ -916,9 +923,21 @@ function renderLimit( return chSql`${{ Int32: chartConfig.limit.limit }}${offset}`; } +function renderSettings( + chartConfig: ChartConfigWithOptDateRangeEx, + querySettings: QuerySettings | undefined, +) { + const querySettingsJoined = joinQuerySettings(querySettings); + + return concatChSql(', ', [ + chSql`${chartConfig.settings ?? ''}`, + chSql`${querySettingsJoined ?? ''}`, + ]); +} + // includedDataInterval isn't exported at this time. It's only used internally // for metric SQL generation. -type ChartConfigWithOptDateRangeEx = ChartConfigWithOptDateRange & { +export type ChartConfigWithOptDateRangeEx = ChartConfigWithOptDateRange & { includedDataInterval?: string; settings?: ChSql; }; @@ -926,6 +945,7 @@ type ChartConfigWithOptDateRangeEx = ChartConfigWithOptDateRange & { async function renderWith( chartConfig: ChartConfigWithOptDateRangeEx, metadata: Metadata, + querySettings: QuerySettings | undefined, ): Promise { const { with: withClauses } = chartConfig; if (withClauses) { @@ -972,8 +992,12 @@ async function renderWith( // results in schema conformance. const resolvedSql = sql ? sql - : // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- intentional, see comment above - await renderChartConfig(chartConfig as ChartConfig, metadata); + : await renderChartConfig( + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- intentional, see comment above + chartConfig as ChartConfig, + metadata, + querySettings, + ); if (clause.isSubquery === false) { return chSql`(${resolvedSql}) AS ${{ Identifier: clause.name }}`; @@ -1339,8 +1363,9 @@ async function translateMetricChartConfig( } export async function renderChartConfig( - rawChartConfig: ChartConfigWithOptDateRange, + rawChartConfig: ChartConfigWithOptDateRangeEx, metadata: Metadata, + querySettings: QuerySettings | undefined, ): Promise { // metric types require more rewriting since we know more about the schema // but goes through the same generation process @@ -1348,7 +1373,7 @@ export async function renderChartConfig( ? await translateMetricChartConfig(rawChartConfig, metadata) : rawChartConfig; - const withClauses = await renderWith(chartConfig, metadata); + const withClauses = await renderWith(chartConfig, metadata, querySettings); const select = await renderSelect(chartConfig, metadata); const from = renderFrom(chartConfig); const where = await renderWhere(chartConfig, metadata); @@ -1357,6 +1382,7 @@ export async function renderChartConfig( const orderBy = renderOrderBy(chartConfig); //const fill = renderFill(chartConfig); //TODO: Fill breaks heatmaps and some charts const limit = renderLimit(chartConfig); + const settings = renderSettings(chartConfig, querySettings); return concatChSql(' ', [ chSql`${withClauses?.sql ? chSql`WITH ${withClauses}` : ''}`, @@ -1368,8 +1394,9 @@ export async function renderChartConfig( chSql`${orderBy?.sql ? chSql`ORDER BY ${orderBy}` : ''}`, //chSql`${fill?.sql ? chSql`WITH FILL ${fill}` : ''}`, chSql`${limit?.sql ? chSql`LIMIT ${limit}` : ''}`, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- settings type narrowing - chSql`${'settings' in chartConfig ? chSql`SETTINGS ${chartConfig.settings as ChSql}` : []}`, + + // SETTINGS must be last - see `extractSettingsClause` in "./utils.ts" + chSql`${settings.sql ? chSql`SETTINGS ${settings}` : []}`, ]); } diff --git a/packages/common-utils/src/core/utils.ts b/packages/common-utils/src/core/utils.ts index d14aedb5..e0f9e99c 100644 --- a/packages/common-utils/src/core/utils.ts +++ b/packages/common-utils/src/core/utils.ts @@ -13,6 +13,7 @@ import { DashboardSchema, DashboardTemplateSchema, DashboardWithoutId, + QuerySettings, SQLInterval, TileTemplateSchema, TSourceUnion, @@ -693,3 +694,56 @@ export function isDateRangeEqual(range1: [Date, Date], range2: [Date, Date]) { range1[1].getTime() === range2[1].getTime() ); } + +/* + This function extracts the SETTINGS clause from the end(!) of the sql string. +*/ +export function extractSettingsClauseFromEnd( + sqlInput: string, +): [string, string | undefined] { + const sql = sqlInput.trim().endsWith(';') + ? sqlInput.trim().slice(0, -1) + : sqlInput.trim(); + + const settingsIndex = sql.toUpperCase().indexOf('SETTINGS'); + + if (settingsIndex === -1) { + return [sql, undefined] as const; + } + + const settingsClause = sql.substring(settingsIndex).trim(); + const remaining = sql.substring(0, settingsIndex).trim(); + + return [remaining, settingsClause] as const; +} + +export function parseToNumber(input: string): number | undefined { + const trimmed = input.trim(); + + if (trimmed === '') { + return undefined; + } + + const num = Number(trimmed); + + return Number.isFinite(num) ? num : undefined; +} + +export function joinQuerySettings( + querySettings: QuerySettings | undefined, +): string | undefined { + if (!querySettings?.length) { + return undefined; + } + + const emptyFiltered = querySettings.filter( + ({ setting, value }) => setting.length && value.length, + ); + + const formattedPairs = emptyFiltered.map( + ({ setting, value }) => + `${setting} = ${parseToNumber(value) ?? `'${value}'`}`, + ); + + return formattedPairs.join(', '); +} diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 5e429b3f..891a407f 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -603,6 +603,12 @@ export enum SourceKind { // TABLE SOURCE FORM VALIDATION // -------------------------- +const QuerySettingsSchema = z.array( + z.object({ setting: z.string(), value: z.string() }), +); + +export type QuerySettings = z.infer; + // Base schema with fields common to all source types const SourceBaseSchema = z.object({ id: z.string(), @@ -613,6 +619,7 @@ const SourceBaseSchema = z.object({ databaseName: z.string().min(1, 'Database is required'), tableName: z.string().min(1, 'Table is required'), }), + querySettings: QuerySettingsSchema.optional(), }); const RequiredTimestampColumnSchema = z