feat: Add query params, sorting, and placeholders to Raw-SQL tables (#1857)

Closes HDX-3580
Closes HDX-3034

## Summary

This PR enhances the support for SQL-driven tables

1. The selected date range can now be referenced in the query with query params. The available query params are outlined in a new section above the SQL input.
2. The table no longer OOMs on large result sets (it is now truncated to the first 10k results), or crashes when selecting columns that are Map or JSON type
3. The table can now be sorted client-side for sql-driven tables
4. There is now placeholder / example SQL in the input

### Screenshots or video

https://github.com/user-attachments/assets/4f39fd0a-d33e-4f8c-9e91-84143d23e293

### How to test locally or on Vercel

This feature can be tested locally or on the preview environment without any special toggles.

### References



- Linear Issue: HDX-3580
- Related PRs:
This commit is contained in:
Drew Davis 2026-03-06 11:05:34 -05:00 committed by GitHub
parent 6e4660f834
commit fd9f290e2a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 735 additions and 97 deletions

View file

@ -0,0 +1,7 @@
---
"@hyperdx/common-utils": patch
"@hyperdx/api": patch
"@hyperdx/app": patch
---
feat: Add query params, sorting, and placeholders to Raw-SQL tables

View file

@ -6,10 +6,13 @@ import { IconDownload, IconTextWrap } from '@tabler/icons-react';
import {
flexRender,
getCoreRowModel,
getSortedRowModel,
Getter,
Row,
Row as TableRow,
SortingFnOption,
SortingState,
TableOptions,
useReactTable,
} from '@tanstack/react-table';
import { ColumnDef } from '@tanstack/react-table';
@ -25,12 +28,16 @@ import { formatNumber } from './utils';
export type TableVariant = 'default' | 'muted';
// Arbitrary limit to prevent OOM crashes for very large result sets. Most result sets should be paginated anyway.
export const MAX_TABLE_ROWS = 10_000;
export const Table = ({
data,
groupColumnName,
columns,
getRowSearchLink,
tableBottom,
enableClientSideSorting = false,
sorting,
onSortingChange,
variant = 'default',
@ -44,10 +51,12 @@ export const Table = ({
numberFormat?: NumberFormat;
columnWidthPercent?: number;
visible?: boolean;
sortingFn?: SortingFnOption<any>;
}[];
groupColumnName?: string;
getRowSearchLink?: (row: any) => string | null;
tableBottom?: React.ReactNode;
enableClientSideSorting?: boolean;
sorting: SortingState;
onSortingChange: (sorting: SortingState) => void;
variant?: TableVariant;
@ -57,6 +66,14 @@ export const Table = ({
//we need a reference to the scrolling element for logic down below
const tableContainerRef = useRef<HTMLDivElement>(null);
const truncatedData = useMemo(() => {
if (data.length > MAX_TABLE_ROWS) {
return data.slice(0, MAX_TABLE_ROWS);
}
return data;
}, [data]);
const isTruncated = truncatedData.length === MAX_TABLE_ROWS;
const tableWidth = tableContainerRef.current?.clientWidth;
const numColumns = columns.filter(c => c.visible !== false).length + 1;
const dataColumnsWidthPerc = columns
@ -92,95 +109,120 @@ export const Table = ({
.filter(c => c.visible !== false)
.map(
(
{ id, dataKey, displayName, numberFormat, columnWidthPercent },
{
id,
dataKey,
displayName,
numberFormat,
columnWidthPercent,
sortingFn,
},
i,
) => ({
id: id,
accessorKey: dataKey,
header: displayName,
accessorFn: (row: any) => row[dataKey],
cell: ({
getValue,
row,
}: {
getValue: Getter<number>;
row: Row<any>;
}) => {
const value = getValue();
let formattedValue: string | number | null = value ?? null;
if (numberFormat) {
formattedValue = formatNumber(value, numberFormat);
}
if (getRowSearchLink == null) {
return formattedValue;
}
const link = getRowSearchLink(row.original);
) =>
({
id: id,
accessorKey: dataKey,
header: displayName,
accessorFn: (row: any) => row[dataKey],
sortingFn,
cell: ({
getValue,
row,
}: {
getValue: Getter<number>;
row: Row<any>;
}) => {
const value = getValue();
let formattedValue: string | number | null = value ?? null;
// Table cannot accept values which are objects or arrays, so we need to stringify them
if (typeof value !== 'string' && typeof value !== 'number') {
formattedValue = JSON.stringify(value);
} else if (numberFormat) {
formattedValue = formatNumber(value, numberFormat);
}
if (getRowSearchLink == null) {
return formattedValue;
}
const link = getRowSearchLink(row.original);
if (!link) {
return (
<div
className={cx('align-top overflow-hidden py-1 pe-3', {
'text-break': wrapLinesEnabled,
'text-truncate': !wrapLinesEnabled,
})}
>
{formattedValue}
</div>
);
}
if (!link) {
return (
<div
<Link
href={link}
className={cx('align-top overflow-hidden py-1 pe-3', {
'text-break': wrapLinesEnabled,
'text-truncate': !wrapLinesEnabled,
})}
style={{
display: 'block',
color: 'inherit',
textDecoration: 'none',
}}
>
{formattedValue}
</div>
</Link>
);
}
return (
<Link
href={link}
className={cx('align-top overflow-hidden py-1 pe-3', {
'text-break': wrapLinesEnabled,
'text-truncate': !wrapLinesEnabled,
})}
style={{
display: 'block',
color: 'inherit',
textDecoration: 'none',
}}
>
{formattedValue}
</Link>
);
},
size:
i === numColumns - 2
? UNDEFINED_WIDTH
: tableWidth != null && columnWidthPercent != null
? Math.max(
tableWidth * (columnWidthPercent / 100),
MIN_COLUMN_WIDTH_PX,
)
: tableWidth != null
? tableWidth / numColumns
: 200,
enableResizing: i !== numColumns - 2,
}),
},
size:
i === numColumns - 2
? UNDEFINED_WIDTH
: tableWidth != null && columnWidthPercent != null
? Math.max(
tableWidth * (columnWidthPercent / 100),
MIN_COLUMN_WIDTH_PX,
)
: tableWidth != null
? tableWidth / numColumns
: 200,
enableResizing: i !== numColumns - 2,
}) satisfies ColumnDef<any>,
),
];
const sortingParams: Partial<TableOptions<any>> = useMemo(() => {
return enableClientSideSorting
? {
enableSorting: true,
getSortedRowModel: getSortedRowModel(),
}
: {
enableSorting: true,
manualSorting: true,
onSortingChange: v => {
if (typeof v === 'function') {
const newSortVal = v(sorting);
onSortingChange?.(newSortVal ?? null);
} else {
onSortingChange?.(v ?? null);
}
},
state: {
sorting,
},
};
}, [enableClientSideSorting, onSortingChange, sorting]);
const table = useReactTable({
data,
data: truncatedData,
columns: reactTableColumns,
getCoreRowModel: getCoreRowModel(),
enableColumnResizing: true,
columnResizeMode: 'onChange',
enableSorting: true,
manualSorting: true,
onSortingChange: v => {
if (typeof v === 'function') {
const newSortVal = v(sorting);
onSortingChange?.(newSortVal ?? null);
} else {
onSortingChange?.(v ?? null);
}
},
state: {
sorting,
},
...sortingParams,
});
const { rows } = table.getRowModel();
@ -209,7 +251,7 @@ export const Table = ({
const [wrapLinesEnabled, setWrapLinesEnabled] = useState(false);
const { csvData } = useCsvExport(
data,
truncatedData,
columns.map(col => ({
dataKey: col.dataKey,
displayName: col.displayName,
@ -307,6 +349,11 @@ export const Table = ({
)}
</tbody>
</table>
{isTruncated && (
<div className="p-2 text-center">
Showing the first {MAX_TABLE_ROWS} rows.
</div>
)}
{tableBottom}
</div>
);

View file

@ -1,27 +1,158 @@
import { Control } from 'react-hook-form';
import { Box, Button, Group, Stack, Text } from '@mantine/core';
import { useEffect } from 'react';
import { Control, UseFormSetValue, useWatch } from 'react-hook-form';
import { QUERY_PARAMS_BY_DISPLAY_TYPE } from '@hyperdx/common-utils/dist/rawSqlParams';
import { DisplayType } from '@hyperdx/common-utils/dist/types';
import {
ActionIcon,
Box,
Button,
Code,
Collapse,
Group,
List,
Paper,
Stack,
Text,
Tooltip,
} from '@mantine/core';
import { useClipboard, useDisclosure } from '@mantine/hooks';
import {
IconCheck,
IconChevronDown,
IconChevronRight,
IconCopy,
} from '@tabler/icons-react';
import useResizable from '@/hooks/useResizable';
import { useSources } from '@/source';
import { ConnectionSelectControlled } from '../ConnectionSelect';
import { SQLEditorControlled } from '../SQLEditor';
import { SQL_PLACEHOLDERS } from './constants';
import { ChartEditorFormState } from './types';
import resizeStyles from '@/../styles/ResizablePanel.module.scss';
function ParamSnippet({
value,
description,
}: {
value: string;
description: string;
}) {
const clipboard = useClipboard({ timeout: 1500 });
return (
<Group gap={4} display="inline-flex">
<Code fz="xs">{value}</Code>
<Tooltip label={clipboard.copied ? 'Copied!' : 'Copy'} withArrow>
<ActionIcon
variant="subtle"
size="xs"
color={clipboard.copied ? 'green' : 'gray'}
onClick={() => clipboard.copy(value)}
>
{clipboard.copied ? <IconCheck size={10} /> : <IconCopy size={10} />}
</ActionIcon>
</Tooltip>
<Text span size="xs">
&mdash; {description}
</Text>
</Group>
);
}
function AvailableParameters({ displayType }: { displayType: DisplayType }) {
const [helpOpened, { toggle: toggleHelp }] = useDisclosure(false);
const availableParams = QUERY_PARAMS_BY_DISPLAY_TYPE[displayType];
return (
<Paper
p="xs"
radius="sm"
style={{
background: 'var(--color-bg-muted)',
}}
>
<Stack gap={0}>
<Group
gap="xs"
align="center"
style={{ cursor: 'pointer', userSelect: 'none' }}
onClick={toggleHelp}
>
{helpOpened ? (
<IconChevronDown size={12} />
) : (
<IconChevronRight size={12} />
)}
<Text size="xs" mt={1}>
Query parameters
</Text>
</Group>
<Collapse in={helpOpened}>
<Stack gap={6} pl="xs" pt="md">
<Text size="xs">
The following parameters can be referenced in this chart's SQL:
</Text>
<List size="xs" withPadding spacing={3}>
{availableParams.map(({ name, type, description }) => (
<List.Item key={name}>
<ParamSnippet
value={`{${name}:${type}}`}
description={description}
/>
</List.Item>
))}
</List>
<Text size="xs">Example:</Text>
<Code fz="xs" block>
{
'WHERE Timestamp >= fromUnixTimestamp64Milli ({startDateMilliseconds:Int64})\n AND Timestamp <= fromUnixTimestamp64Milli ({endDateMilliseconds:Int64})'
}
</Code>
</Stack>
</Collapse>
</Stack>
</Paper>
);
}
export default function RawSqlChartEditor({
control,
setValue,
onOpenDisplaySettings,
}: {
control: Control<ChartEditorFormState>;
setValue: UseFormSetValue<ChartEditorFormState>;
onOpenDisplaySettings: () => void;
}) {
const { size, startResize } = useResizable(20, 'bottom');
const { data: sources } = useSources();
const displayType = useWatch({ control, name: 'displayType' });
const connection = useWatch({ control, name: 'connection' });
const source = useWatch({ control, name: 'source' });
// Set a default connection
useEffect(() => {
if (sources && !connection) {
const defaultConnection =
sources.find(s => s.id === source)?.connection ??
sources[0]?.connection;
if (defaultConnection && defaultConnection !== connection) {
setValue('connection', defaultConnection);
}
}
}, [connection, setValue, source, sources]);
const placeholderSQl = SQL_PLACEHOLDERS[displayType ?? DisplayType.Table];
return (
<Stack>
<Group mb="md" align="center">
<Group align="center">
<Text pe="md" size="sm">
Connection
</Text>
@ -31,12 +162,14 @@ export default function RawSqlChartEditor({
size="xs"
/>
</Group>
<AvailableParameters displayType={displayType ?? DisplayType.Table} />
<Box style={{ position: 'relative' }}>
<SQLEditorControlled
control={control}
name="sqlTemplate"
height={`${size}vh`}
enableLineWrapping
placeholder={placeholderSQl}
/>
<div className={resizeStyles.resizeYHandle} onMouseDown={startResize} />
</Box>

View file

@ -0,0 +1,20 @@
import { DisplayType } from '@hyperdx/common-utils/dist/types';
export const SQL_PLACEHOLDERS: Record<DisplayType, string> = {
[DisplayType.Line]: '',
[DisplayType.StackedBar]: '',
[DisplayType.Table]: `SELECT
count()
FROM
default.otel_logs
WHERE TimestampTime >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64})
AND TimestampTime <= fromUnixTimestamp64Milli({endDateMilliseconds:Int64})
LIMIT
200
`,
[DisplayType.Pie]: '',
[DisplayType.Number]: '',
[DisplayType.Search]: '',
[DisplayType.Heatmap]: '',
[DisplayType.Markdown]: '',
};

View file

@ -865,10 +865,10 @@ export default function EditTimeChartForm({
}, [granularity, onSubmit]);
useEffect(() => {
if (
displayType !== prevDisplayTypeRef.current ||
configType !== prevConfigTypeRef.current
) {
const displayTypeChanged = displayType !== prevDisplayTypeRef.current;
const configTypeChanged = configType !== prevConfigTypeRef.current;
if (displayTypeChanged || configTypeChanged) {
prevDisplayTypeRef.current = displayType;
prevConfigTypeRef.current = configType;
@ -876,6 +876,7 @@ export default function EditTimeChartForm({
setValue('select', '');
setValue('series', []);
}
if (displayType !== DisplayType.Search && !Array.isArray(select)) {
const defaultSeries: SavedChartConfigWithSelectArray['select'] = [
{
@ -889,7 +890,11 @@ export default function EditTimeChartForm({
setValue('select', defaultSeries);
setValue('series', defaultSeries);
}
onSubmit();
// Don't auto-submit when config type changes, to avoid clearing form state (like source)
if (displayTypeChanged) {
onSubmit();
}
}
}, [displayType, select, setValue, onSubmit, configType]);
@ -1136,6 +1141,7 @@ export default function EditTimeChartForm({
) : isRawSqlInput ? (
<RawSqlChartEditor
control={control}
setValue={setValue}
onOpenDisplaySettings={openDisplaySettings}
/>
) : (

View file

@ -0,0 +1,62 @@
import { numericRowSortingFn } from '@/components/DBTable/sorting';
function makeRow(value: unknown) {
return { getValue: (_key: string) => value } as any;
}
describe('numericRowSortingFn', () => {
const key = 'col';
it('sorts numbers in ascending order', () => {
expect(numericRowSortingFn(makeRow(1), makeRow(2), key)).toBeLessThan(0);
expect(numericRowSortingFn(makeRow(2), makeRow(1), key)).toBeGreaterThan(0);
expect(numericRowSortingFn(makeRow(5), makeRow(5), key)).toBe(0);
});
it('treats numeric strings as numbers', () => {
expect(
numericRowSortingFn(makeRow('10'), makeRow('9'), key),
).toBeGreaterThan(0);
expect(numericRowSortingFn(makeRow('3'), makeRow('20'), key)).toBeLessThan(
0,
);
});
it('sorts null as greater than any number (pushes to end)', () => {
expect(numericRowSortingFn(makeRow(null), makeRow(1), key)).toBeGreaterThan(
0,
);
expect(numericRowSortingFn(makeRow(1), makeRow(null), key)).toBeLessThan(0);
});
it('sorts NaN strings as greater than any number (pushes to end)', () => {
expect(
numericRowSortingFn(makeRow('abc'), makeRow(1), key),
).toBeGreaterThan(0);
expect(numericRowSortingFn(makeRow(1), makeRow('abc'), key)).toBeLessThan(
0,
);
});
it('handles negative numbers', () => {
expect(numericRowSortingFn(makeRow(-5), makeRow(0), key)).toBeLessThan(0);
expect(numericRowSortingFn(makeRow(0), makeRow(-5), key)).toBeGreaterThan(
0,
);
});
it('treats two null/NaN values as equal', () => {
expect(numericRowSortingFn(makeRow(null), makeRow(null), key)).toBe(0);
expect(numericRowSortingFn(makeRow('abc'), makeRow('xyz'), key)).toBe(0);
expect(numericRowSortingFn(makeRow(null), makeRow('abc'), key)).toBe(0);
});
it('handles floating point numbers', () => {
expect(numericRowSortingFn(makeRow(1.5), makeRow(1.6), key)).toBeLessThan(
0,
);
expect(
numericRowSortingFn(makeRow(1.6), makeRow(1.5), key),
).toBeGreaterThan(0);
});
});

View file

@ -0,0 +1,40 @@
import {
ColumnMetaType,
convertCHDataTypeToJSType,
} from '@hyperdx/common-utils/dist/clickhouse';
import { Row, SortingFnOption } from '@tanstack/react-table';
export const numericRowSortingFn = ((
a: Row<unknown>,
b: Row<unknown>,
columnId: string,
) => {
const aValue = a.getValue(columnId);
const bValue = b.getValue(columnId);
const aInvalid = aValue == null || isNaN(Number(aValue));
const bInvalid = bValue == null || isNaN(Number(bValue));
if (aInvalid && bInvalid) return 0;
if (aInvalid) return 1;
if (bInvalid) return -1;
return Number(aValue) - Number(bValue);
}) satisfies SortingFnOption<unknown>;
export const getClientSideSortingFn = (
meta: ColumnMetaType[] | undefined,
columnName: string,
): SortingFnOption<unknown> => {
const columnMeta = meta?.find(col => col.name === columnName);
const jsType = columnMeta
? convertCHDataTypeToJSType(columnMeta.type)
: undefined;
if (jsType === 'number') {
return numericRowSortingFn;
}
// Fallback to alphanumeric sorting for other types, including when metadata is unavailable, and when
// metadata indicates that the column type is date (in which case the value will be a string type in JS)
return 'alphanumeric';
};

View file

@ -19,6 +19,7 @@ import { useSource } from '@/source';
import { useIntersectionObserver } from '@/utils';
import ChartContainer from './charts/ChartContainer';
import { getClientSideSortingFn } from './DBTable/sorting';
import MVOptimizationIndicator from './MaterializedViews/MVOptimizationIndicator';
import { SQLPreview } from './ChartSQLPreview';
@ -143,6 +144,7 @@ export default function DBTableChart({
numberFormat: groupByKeys.includes(key)
? undefined
: config.numberFormat,
sortingFn: getClientSideSortingFn(data?.meta, key),
}));
}, [config.numberFormat, aliasMap, queriedConfig, data, hiddenColumns]);
@ -234,6 +236,7 @@ export default function DBTableChart({
columns={columns}
getRowSearchLink={getRowSearchLink}
sorting={effectiveSort}
enableClientSideSorting={isRawSqlChartConfig(config)}
onSortingChange={handleSortingChange}
variant={variant}
tableBottom={

View file

@ -31,6 +31,19 @@ jest.mock('@hyperdx/app/src/metadata', () => ({
getMetadata: jest.fn(),
}));
// Mock useMetadataWithSettings
jest.mock('@/hooks/useMetadata', () => ({
useMetadataWithSettings: jest.fn(),
}));
// Mock useSource
jest.mock('@/source', () => ({
useSource: jest.fn().mockReturnValue({
data: undefined,
isLoading: false,
}),
}));
// Mock the useMVOptimizationExplanation hook
jest.mock('@/hooks/useMVOptimizationExplanation', () => ({
useMVOptimizationExplanation: jest.fn().mockReturnValue({
@ -51,6 +64,7 @@ import { MVOptimizationExplanation } from '@hyperdx/common-utils/dist/core/mater
import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig';
import { isBuilderChartConfig } from '@hyperdx/common-utils/dist/guards';
import { useMetadataWithSettings } from '@/hooks/useMetadata';
import {
MVOptimizationExplanationResult,
useMVOptimizationExplanation,
@ -88,6 +102,7 @@ describe('useOffsetPaginatedQuery', () => {
let mockClickhouseClient: any;
let mockStream: any;
let mockReader: any;
let mockMetadata: { getSetting: jest.Mock };
beforeEach(() => {
// Reset mocks
@ -135,6 +150,12 @@ describe('useOffsetPaginatedQuery', () => {
sql: 'SELECT * FROM traces',
params: {},
});
// Mock metadata with getSetting returning null by default
mockMetadata = {
getSetting: jest.fn().mockResolvedValue(null),
};
jest.mocked(useMetadataWithSettings).mockReturnValue(mockMetadata as any);
});
describe('Time Window Generation', () => {
@ -815,6 +836,146 @@ describe('useOffsetPaginatedQuery', () => {
});
});
describe('ClickHouse Settings (readonly and max_result_rows)', () => {
const rawSqlConfig = {
configType: 'sql' as const,
sqlTemplate: 'SELECT status, count() FROM logs GROUP BY status',
connection: 'conn-1',
displayType: undefined,
dateRange: [
new Date('2024-01-01T00:00:00Z'),
new Date('2024-01-02T00:00:00Z'),
] as [Date, Date],
};
const mockRawSqlRead = () => {
mockReader.read
.mockResolvedValueOnce({
done: false,
value: [
{ json: () => ['status', 'count()'] },
{ json: () => ['String', 'UInt64'] },
{ json: () => ['error', 42] },
],
})
.mockResolvedValueOnce({ done: true });
};
it('should set readonly=2, result_overflow_mode=break, and max_result_rows=MAX_TABLE_ROWS when no existing setting', async () => {
mockMetadata.getSetting.mockResolvedValue(null);
mockRawSqlRead();
const { result } = renderHook(
() => useOffsetPaginatedQuery(rawSqlConfig),
{ wrapper },
);
await waitFor(() => expect(result.current.isLoading).toBe(false));
expect(mockClickhouseClient.query).toHaveBeenCalledTimes(1);
const clickhouseSettings =
mockClickhouseClient.query.mock.calls[0][0].clickhouse_settings;
expect(clickhouseSettings.readonly).toBe('2');
expect(clickhouseSettings.max_result_rows).toBe('10000');
expect(clickhouseSettings.result_overflow_mode).toBe('break');
});
it('should use MAX_TABLE_ROWS when existingMaxResultRowsSetting is 0', async () => {
mockMetadata.getSetting.mockResolvedValue('0');
mockRawSqlRead();
const { result } = renderHook(
() => useOffsetPaginatedQuery(rawSqlConfig),
{ wrapper },
);
await waitFor(() => expect(result.current.isLoading).toBe(false));
const clickhouseSettings =
mockClickhouseClient.query.mock.calls[0][0].clickhouse_settings;
expect(clickhouseSettings.max_result_rows).toBe('10000');
});
it('should use existingMaxResultRowsSetting when it is less than MAX_TABLE_ROWS', async () => {
mockMetadata.getSetting.mockResolvedValue('5000');
mockRawSqlRead();
const { result } = renderHook(
() => useOffsetPaginatedQuery(rawSqlConfig),
{ wrapper },
);
await waitFor(() => expect(result.current.isLoading).toBe(false));
const clickhouseSettings =
mockClickhouseClient.query.mock.calls[0][0].clickhouse_settings;
expect(clickhouseSettings.max_result_rows).toBe('5000');
});
it('should cap max_result_rows at MAX_TABLE_ROWS when existingMaxResultRowsSetting exceeds it', async () => {
mockMetadata.getSetting.mockResolvedValue('50000');
mockRawSqlRead();
const { result } = renderHook(
() => useOffsetPaginatedQuery(rawSqlConfig),
{ wrapper },
);
await waitFor(() => expect(result.current.isLoading).toBe(false));
const clickhouseSettings =
mockClickhouseClient.query.mock.calls[0][0].clickhouse_settings;
expect(clickhouseSettings.max_result_rows).toBe('10000');
});
it('should query getSetting with the correct settingName and connectionId', async () => {
mockRawSqlRead();
const { result } = renderHook(
() => useOffsetPaginatedQuery(rawSqlConfig),
{ wrapper },
);
await waitFor(() => expect(result.current.isLoading).toBe(false));
expect(mockMetadata.getSetting).toHaveBeenCalledWith({
settingName: 'max_result_rows',
connectionId: 'conn-1',
});
});
it('should not set readonly or max_result_rows for builder configs', async () => {
const config = createMockChartConfig();
mockReader.read
.mockResolvedValueOnce({
done: false,
value: [
{ json: () => ['timestamp', 'message'] },
{ json: () => ['DateTime', 'String'] },
{ json: () => ['2024-01-01T01:00:00Z', 'test log'] },
],
})
.mockResolvedValueOnce({ done: true });
const { result } = renderHook(() => useOffsetPaginatedQuery(config), {
wrapper,
});
await waitFor(() => expect(result.current.isLoading).toBe(false));
expect(mockClickhouseClient.query).toHaveBeenCalledTimes(1);
const clickhouseSettings =
mockClickhouseClient.query.mock.calls[0][0].clickhouse_settings;
expect(clickhouseSettings.readonly).toBeUndefined();
expect(clickhouseSettings.max_result_rows).toBeUndefined();
expect(clickhouseSettings.result_overflow_mode).toBeUndefined();
// getSetting should not be called for builder configs
expect(mockMetadata.getSetting).not.toHaveBeenCalled();
});
});
describe('MV Optimization Integration', () => {
it('should optimize queries using MVs when possible', async () => {
const config = createMockChartConfig({

View file

@ -1,6 +1,10 @@
import { useMemo } from 'react';
import ms from 'ms';
import type { ResponseJSON, Row } from '@hyperdx/common-utils/dist/clickhouse';
import type {
ClickHouseSettings,
ResponseJSON,
Row,
} from '@hyperdx/common-utils/dist/clickhouse';
import {
ChSql,
ClickHouseQueryError,
@ -29,6 +33,7 @@ import {
import api from '@/api';
import { getClickhouseClient } from '@/clickhouse';
import { MAX_TABLE_ROWS } from '@/HDXMultiSeriesTableChart';
import { useMetadataWithSettings } from '@/hooks/useMetadata';
import { useMVOptimizationExplanation } from '@/hooks/useMVOptimizationExplanation';
import { useSource } from '@/source';
@ -76,7 +81,6 @@ type QueryMeta = {
metadata: Metadata;
optimizedConfig?: ChartConfigWithOptTimestamp;
source: TSource | undefined;
readonly: boolean;
};
// Get time window from page param
@ -154,14 +158,8 @@ const queryFn: QueryFunction<TQueryFnData, TQueryKey, TPageParam> = async ({
throw new Error('Query missing client meta');
}
const {
queryClient,
metadata,
hasPreviousQueries,
optimizedConfig,
source,
readonly,
} = meta as QueryMeta;
const { queryClient, metadata, hasPreviousQueries, optimizedConfig, source } =
meta as QueryMeta;
// Only stream incrementally if this is a fresh query with no previous
// response or if it's a paginated query
@ -211,8 +209,28 @@ const queryFn: QueryFunction<TQueryFnData, TQueryKey, TPageParam> = async ({
setTimeout(() => abortController.abort(), queryTimeout * 1000);
}
// Readonly = 2 means the query is readonly but can still specify query settings.
const clickHouseSettings = readonly ? { readonly: '2' } : {};
const clickHouseSettings: ClickHouseSettings = {};
if (isRawSqlChartConfig(config)) {
// Readonly = 2 means the query is readonly but can still specify query settings.
clickHouseSettings.readonly = '2';
const existingMaxResultRowsSetting = await metadata.getSetting({
settingName: 'max_result_rows',
connectionId: config.connection,
});
const maxResultRows =
existingMaxResultRowsSetting != null &&
Number(existingMaxResultRowsSetting) > 0
? Math.min(Number(existingMaxResultRowsSetting), MAX_TABLE_ROWS)
: MAX_TABLE_ROWS;
// result_overflow_mode=break will prevent an error when the result set exceeds max_result_rows,
// and instead just return the first max_result_rows rows.
clickHouseSettings.max_result_rows = String(maxResultRows);
clickHouseSettings.result_overflow_mode = 'break';
}
const resultSet =
await clickhouseClient.query<'JSONCompactEachRowWithNamesAndTypes'>({
query: query.sql,
@ -466,8 +484,6 @@ export default function useOffsetPaginatedQuery(
metadata,
optimizedConfig: mvOptimizationData?.optimizedConfig,
source,
// Additional readonly protection when the user is running a raw SQL query
readonly: isRawSqlChartConfig(config),
} satisfies QueryMeta,
queryFn,
gcTime: isLive ? ms('30s') : ms('5m'), // more aggressive gc for live data, since it can end up holding lots of data

View file

@ -0,0 +1,56 @@
import { DisplayType } from '@/types';
import { renderRawSqlChartConfig } from '../rawSqlParams';
describe('renderRawSqlChartConfig', () => {
describe('DisplayType.Table', () => {
it('returns the sqlTemplate with no params when no dateRange provided', () => {
const result = renderRawSqlChartConfig({
configType: 'sql',
sqlTemplate: 'SELECT count() FROM logs',
connection: 'conn-1',
displayType: DisplayType.Table,
});
expect(result.sql).toBe('SELECT count() FROM logs');
expect(result.params).toEqual({
startDateMilliseconds: undefined,
endDateMilliseconds: undefined,
});
});
it('injects startDateMilliseconds and endDateMilliseconds when dateRange provided', () => {
const start = new Date('2024-01-01T00:00:00.000Z');
const end = new Date('2024-01-02T00:00:00.000Z');
const result = renderRawSqlChartConfig({
configType: 'sql',
sqlTemplate:
'SELECT count() FROM logs WHERE ts BETWEEN {startDateMilliseconds:Int64} AND {endDateMilliseconds:Int64}',
connection: 'conn-1',
displayType: DisplayType.Table,
dateRange: [start, end],
});
expect(result.sql).toBe(
'SELECT count() FROM logs WHERE ts BETWEEN {startDateMilliseconds:Int64} AND {endDateMilliseconds:Int64}',
);
expect(result.params).toEqual({
startDateMilliseconds: start.getTime(),
endDateMilliseconds: end.getTime(),
});
});
it('defaults to Table display type when displayType is not specified', () => {
const start = new Date('2024-06-15T12:00:00.000Z');
const end = new Date('2024-06-15T13:00:00.000Z');
const result = renderRawSqlChartConfig({
configType: 'sql',
sqlTemplate: 'SELECT * FROM events',
connection: 'conn-1',
dateRange: [start, end],
});
expect(result.params).toEqual({
startDateMilliseconds: start.getTime(),
endDateMilliseconds: end.getTime(),
});
});
});
});

View file

@ -1442,6 +1442,33 @@ describe('renderChartConfig', () => {
expect(result.sql).toBe(
'SELECT count() FROM logs WHERE level = {level:String}',
);
expect(result.params).toEqual({});
expect(result.params).toEqual({
startDateMilliseconds: undefined,
endDateMilliseconds: undefined,
});
});
it('injects startDateMilliseconds and endDateMilliseconds params for raw sql config with dateRange', async () => {
const start = new Date('2024-01-01T00:00:00.000Z');
const end = new Date('2024-01-02T00:00:00.000Z');
const rawSqlConfig: ChartConfigWithOptDateRangeEx = {
configType: 'sql',
sqlTemplate:
'SELECT count() FROM logs WHERE ts BETWEEN {startDateMilliseconds:Int64} AND {endDateMilliseconds:Int64}',
connection: 'conn-1',
dateRange: [start, end],
};
const result = await renderChartConfig(
rawSqlConfig,
mockMetadata,
undefined,
);
expect(result.sql).toBe(
'SELECT count() FROM logs WHERE ts BETWEEN {startDateMilliseconds:Int64} AND {endDateMilliseconds:Int64}',
);
expect(result.params).toEqual({
startDateMilliseconds: start.getTime(),
endDateMilliseconds: end.getTime(),
});
});
});

View file

@ -12,12 +12,12 @@ import {
getFirstTimestampValueExpression,
joinQuerySettings,
optimizeTimestampValueExpression,
parseToNumber,
parseToStartOfFunction,
splitAndTrimWithBracket,
} from '@/core/utils';
import { isBuilderChartConfig, isRawSqlChartConfig } from '@/guards';
import { CustomSchemaSQLSerializerV2, SearchQueryBuilder } from '@/queryParser';
import { renderRawSqlChartConfig } from '@/rawSqlParams';
import {
AggregateFunction,
AggregateFunctionWithCombinators,
@ -1408,7 +1408,7 @@ export async function renderChartConfig(
querySettings: QuerySettings | undefined,
): Promise<ChSql> {
if (isRawSqlChartConfig(rawChartConfig)) {
return chSql`${{ UNSAFE_RAW_SQL: rawChartConfig.sqlTemplate ?? '' }}`;
return renderRawSqlChartConfig(rawChartConfig);
}
// metric types require more rewriting since we know more about the schema

View file

@ -0,0 +1,60 @@
import { ChSql } from './clickhouse';
import { DateRange, DisplayType, RawSqlChartConfig } from './types';
type QueryParamDefinition = {
name: string;
type: string;
description: string;
get: (config: RawSqlChartConfig & Partial<DateRange>) => any;
};
export const QUERY_PARAMS: Record<string, QueryParamDefinition> = {
startDateMilliseconds: {
name: 'startDateMilliseconds',
type: 'Int64',
description:
'start of the dashboard date range, in milliseconds since epoch',
get: (config: RawSqlChartConfig & Partial<DateRange>) =>
config.dateRange ? config.dateRange[0].getTime() : undefined,
},
endDateMilliseconds: {
name: 'endDateMilliseconds',
type: 'Int64',
description: 'end of the dashboard date range, in milliseconds since epoch',
get: (config: RawSqlChartConfig & Partial<DateRange>) =>
config.dateRange ? config.dateRange[1].getTime() : undefined,
},
};
export const QUERY_PARAMS_BY_DISPLAY_TYPE: Record<
DisplayType,
QueryParamDefinition[]
> = {
[DisplayType.Line]: [],
[DisplayType.StackedBar]: [],
[DisplayType.Table]: [
QUERY_PARAMS.startDateMilliseconds,
QUERY_PARAMS.endDateMilliseconds,
],
[DisplayType.Pie]: [],
[DisplayType.Number]: [],
[DisplayType.Search]: [],
[DisplayType.Heatmap]: [],
[DisplayType.Markdown]: [],
};
export function renderRawSqlChartConfig(
chartConfig: RawSqlChartConfig & Partial<DateRange>,
): ChSql {
const displayType = chartConfig.displayType ?? DisplayType.Table;
// eslint-disable-next-line security/detect-object-injection
const queryParams = QUERY_PARAMS_BY_DISPLAY_TYPE[displayType];
return {
sql: chartConfig.sqlTemplate ?? '',
params: Object.fromEntries(
queryParams.map(param => [param.name, param.get(chartConfig)]),
),
};
}