feat: Add support for dashboard filters on Raw SQL Charts (#1924)

## Summary

This PR updates Raw SQL charts with support for dashboard filters, using the $__filters macro.

Lucene global filters require a Source to be included in the ChartConfig, for schema introspection and the `implicitColumnExpression` value. To support Lucene filters, this PR also updates the RawSqlChartConfig type to include optional `source`, `implicitColumnExpression`, and `from` properties. Only `source` is saved in the database. The external API has been updated to accept the `source` field for raw SQL charts as well.

Dashboard import/export has also been updated to support source mapping for raw sql charts with sources.

### Screenshots or video

Both the global filter and the drop-down filters are applied:

<img width="683" height="574" alt="Screenshot 2026-03-17 at 10 57 36 AM" src="https://github.com/user-attachments/assets/280ba0b5-55f7-4107-a55c-eeb1497ac7de" />

To render Lucene conditions, we require Source information (the `from` and `implicitColumnExpression` fields). When a source is not set, filters are therefore not applied:

<img width="782" height="618" alt="Screenshot 2026-03-17 at 10 54 41 AM" src="https://github.com/user-attachments/assets/3ad19ea7-12ee-4334-abe2-8985a0be952c" />

Similarly, if the `$__filters` macro is not used, then the filters are not applied

<img width="704" height="292" alt="Screenshot 2026-03-17 at 10 56 33 AM" src="https://github.com/user-attachments/assets/e1169e4a-2f64-4cd2-bc05-f699fecef8c1" />

Import:

<img width="993" height="669" alt="Screenshot 2026-03-17 at 3 35 16 PM" src="https://github.com/user-attachments/assets/6ebe20c0-19e2-4e90-95d0-8b02c2af0612" />

### How to test locally or on Vercel

This can be tested in the preview environment.
1. Create a saved dashboard and add some filters.
2. Create a raw sql tile, with or without the $__filters macro
3. Try selecting filters, deselecting filters, and applying global SQL and Lucene filters
4. Inspect the generated queries in the console.

### References



- Linear Issue: Closes HDX-3694
- Related PRs:
This commit is contained in:
Drew Davis 2026-03-18 08:03:27 -04:00 committed by GitHub
parent 031ca831bf
commit 1d83bebb54
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
29 changed files with 826 additions and 218 deletions

View file

@ -0,0 +1,6 @@
---
"@hyperdx/common-utils": patch
"@hyperdx/app": patch
---
feat: Add support for dashboard filters on Raw SQL Charts

View file

@ -1389,6 +1389,11 @@
"description": "SQL query template to execute. Supports HyperDX template variables.",
"example": "SELECT count() FROM otel_logs WHERE timestamp > now() - INTERVAL 1 HOUR"
},
"sourceId": {
"type": "string",
"description": "Optional ID of the data source associated with this Raw SQL chart. Used for applying dashboard filters.",
"example": "65f5e4a3b9e77c001a567890"
},
"numberFormat": {
"$ref": "#/components/schemas/NumberFormat",
"description": "Number formatting options for displayed values."

View file

@ -2311,6 +2311,7 @@ describe('External API v2 Dashboards - new format', () => {
it('can round-trip all raw SQL chart config types', async () => {
const connectionId = connection._id.toString();
const sourceId = traceSource._id.toString();
const sqlTemplate = 'SELECT count() FROM otel_logs WHERE {timeFilter}';
const lineRawSql: ExternalDashboardTile = {
@ -2324,6 +2325,7 @@ describe('External API v2 Dashboards - new format', () => {
displayType: 'line',
connectionId,
sqlTemplate,
sourceId,
compareToPreviousPeriod: true,
fillNulls: true,
alignDateRangeToGranularity: true,
@ -2342,6 +2344,7 @@ describe('External API v2 Dashboards - new format', () => {
displayType: 'stacked_bar',
connectionId,
sqlTemplate,
sourceId,
fillNulls: false,
alignDateRangeToGranularity: false,
numberFormat: { output: 'byte', decimalBytes: true },
@ -2359,6 +2362,7 @@ describe('External API v2 Dashboards - new format', () => {
displayType: 'table',
connectionId,
sqlTemplate,
sourceId,
numberFormat: { output: 'percent', mantissa: 1 },
},
};
@ -2374,6 +2378,7 @@ describe('External API v2 Dashboards - new format', () => {
displayType: 'number',
connectionId,
sqlTemplate,
sourceId,
numberFormat: { output: 'currency', currencySymbol: '$' },
},
};
@ -2389,6 +2394,7 @@ describe('External API v2 Dashboards - new format', () => {
displayType: 'pie',
connectionId,
sqlTemplate,
sourceId,
},
};
@ -2457,6 +2463,43 @@ describe('External API v2 Dashboards - new format', () => {
});
});
it('should return 400 when source connection does not match tile connection', async () => {
const otherConnection = await Connection.create({
team: team._id,
name: 'Other Connection',
host: config.CLICKHOUSE_HOST,
username: config.CLICKHOUSE_USER,
password: config.CLICKHOUSE_PASSWORD,
});
const response = await authRequest('post', BASE_URL)
.send({
name: 'Dashboard with Mismatched Source Connection',
tiles: [
{
name: 'Raw SQL Tile',
x: 0,
y: 0,
w: 6,
h: 3,
config: {
configType: 'sql',
displayType: 'table',
connectionId: otherConnection._id.toString(),
sourceId: traceSource._id.toString(),
sqlTemplate: 'SELECT count() FROM otel_logs',
},
},
],
tags: [],
})
.expect(400);
expect(response.body).toEqual({
message: `The following source IDs do not match the specified connections: ${traceSource._id.toString()}`,
});
});
it('should create a dashboard with filters', async () => {
const dashboardPayload = {
name: 'Dashboard with Filters',
@ -3100,6 +3143,7 @@ describe('External API v2 Dashboards - new format', () => {
it('can round-trip all raw SQL chart config types', async () => {
const connectionId = connection._id.toString();
const sourceId = traceSource._id.toString();
const sqlTemplate = 'SELECT count() FROM otel_logs WHERE {timeFilter}';
const lineRawSql: ExternalDashboardTileWithId = {
@ -3114,6 +3158,7 @@ describe('External API v2 Dashboards - new format', () => {
displayType: 'line',
connectionId,
sqlTemplate,
sourceId,
compareToPreviousPeriod: true,
fillNulls: true,
alignDateRangeToGranularity: true,
@ -3133,6 +3178,7 @@ describe('External API v2 Dashboards - new format', () => {
displayType: 'stacked_bar',
connectionId,
sqlTemplate,
sourceId,
fillNulls: false,
alignDateRangeToGranularity: false,
numberFormat: { output: 'byte', decimalBytes: true },
@ -3151,6 +3197,7 @@ describe('External API v2 Dashboards - new format', () => {
displayType: 'table',
connectionId,
sqlTemplate,
sourceId,
numberFormat: { output: 'percent', mantissa: 1 },
},
};
@ -3167,6 +3214,7 @@ describe('External API v2 Dashboards - new format', () => {
displayType: 'number',
connectionId,
sqlTemplate,
sourceId,
numberFormat: { output: 'currency', currencySymbol: '$' },
},
};
@ -3183,6 +3231,7 @@ describe('External API v2 Dashboards - new format', () => {
displayType: 'pie',
connectionId,
sqlTemplate,
sourceId,
},
};
@ -3271,6 +3320,45 @@ describe('External API v2 Dashboards - new format', () => {
});
});
it('should return 400 when source connection does not match tile connection', async () => {
const dashboard = await createTestDashboard();
const otherConnection = await Connection.create({
team: team._id,
name: 'Other Connection',
host: config.CLICKHOUSE_HOST,
username: config.CLICKHOUSE_USER,
password: config.CLICKHOUSE_PASSWORD,
});
const response = await authRequest('put', `${BASE_URL}/${dashboard._id}`)
.send({
name: 'Updated Dashboard with Mismatched Source Connection',
tiles: [
{
id: new ObjectId().toString(),
name: 'Raw SQL Tile',
x: 0,
y: 0,
w: 6,
h: 3,
config: {
configType: 'sql',
displayType: 'table',
connectionId: otherConnection._id.toString(),
sourceId: traceSource._id.toString(),
sqlTemplate: 'SELECT count() FROM otel_logs',
},
},
],
tags: [],
})
.expect(400);
expect(response.body).toEqual({
message: `The following source IDs do not match the specified connections: ${traceSource._id.toString()}`,
});
});
it('should delete alert when tile is updated from builder to raw SQL config', async () => {
const tileId = new ObjectId().toString();
const dashboard = await createTestDashboard({

View file

@ -53,7 +53,7 @@ async function getMissingSources(
}
}
} else if (isConfigTile(tile)) {
if ('sourceId' in tile.config) {
if ('sourceId' in tile.config && tile.config.sourceId) {
sourceIds.add(tile.config.sourceId);
}
}
@ -99,6 +99,30 @@ async function getMissingConnections(
);
}
async function getSourceConnectionMismatches(
team: string | mongoose.Types.ObjectId,
tiles: ExternalDashboardTileWithId[],
): Promise<string[]> {
const existingSources = await getSources(team.toString());
const sourceById = new Map(existingSources.map(s => [s._id.toString(), s]));
const sourcesWithInvalidConnections: string[] = [];
for (const tile of tiles) {
if (
isConfigTile(tile) &&
isRawSqlExternalTileConfig(tile.config) &&
tile.config.sourceId
) {
const source = sourceById.get(tile.config.sourceId);
if (source && source.connection.toString() !== tile.config.connectionId) {
sourcesWithInvalidConnections.push(tile.config.sourceId);
}
}
}
return sourcesWithInvalidConnections;
}
type SavedQueryLanguage = z.infer<typeof whereLanguageSchema>;
function resolveSavedQueryLanguage(params: {
@ -823,6 +847,10 @@ const updateDashboardBodySchema = buildDashboardBodySchema(
* maxLength: 100000
* description: SQL query template to execute. Supports HyperDX template variables.
* example: "SELECT count() FROM otel_logs WHERE timestamp > now() - INTERVAL 1 HOUR"
* sourceId:
* type: string
* description: Optional ID of the data source associated with this Raw SQL chart. Used for applying dashboard filters.
* example: "65f5e4a3b9e77c001a567890"
* numberFormat:
* $ref: '#/components/schemas/NumberFormat'
* description: Number formatting options for displayed values.
@ -1669,10 +1697,12 @@ router.post(
savedFilterValues,
} = req.body;
const [missingSources, missingConnections] = await Promise.all([
getMissingSources(teamId, tiles, filters),
getMissingConnections(teamId, tiles),
]);
const [missingSources, missingConnections, sourceConnectionMismatches] =
await Promise.all([
getMissingSources(teamId, tiles, filters),
getMissingConnections(teamId, tiles),
getSourceConnectionMismatches(teamId, tiles),
]);
if (missingSources.length > 0) {
return res.status(400).json({
message: `Could not find the following source IDs: ${missingSources.join(
@ -1687,6 +1717,13 @@ router.post(
)}`,
});
}
if (sourceConnectionMismatches.length > 0) {
return res.status(400).json({
message: `The following source IDs do not match the specified connections: ${sourceConnectionMismatches.join(
', ',
)}`,
});
}
const internalTiles = tiles.map(tile => {
const tileId = new ObjectId().toString();
@ -1902,10 +1939,12 @@ router.put(
savedFilterValues,
} = req.body ?? {};
const [missingSources, missingConnections] = await Promise.all([
getMissingSources(teamId, tiles, filters),
getMissingConnections(teamId, tiles),
]);
const [missingSources, missingConnections, sourceConnectionMismatches] =
await Promise.all([
getMissingSources(teamId, tiles, filters),
getMissingConnections(teamId, tiles),
getSourceConnectionMismatches(teamId, tiles),
]);
if (missingSources.length > 0) {
return res.status(400).json({
message: `Could not find the following source IDs: ${missingSources.join(
@ -1920,6 +1959,13 @@ router.put(
)}`,
});
}
if (sourceConnectionMismatches.length > 0) {
return res.status(400).json({
message: `The following source IDs do not match the specified connections: ${sourceConnectionMismatches.join(
', ',
)}`,
});
}
const existingDashboard = await Dashboard.findOne(
{ _id: dashboardId, team: teamId },

View file

@ -102,6 +102,7 @@ const convertToExternalTileChartConfig = (
displayType: DisplayType.Line,
connectionId: config.connection,
sqlTemplate: config.sqlTemplate,
sourceId: config.source,
alignDateRangeToGranularity: config.alignDateRangeToGranularity,
fillNulls: config.fillNulls !== false,
numberFormat: config.numberFormat,
@ -113,6 +114,7 @@ const convertToExternalTileChartConfig = (
displayType: config.displayType,
connectionId: config.connection,
sqlTemplate: config.sqlTemplate,
sourceId: config.source,
alignDateRangeToGranularity: config.alignDateRangeToGranularity,
fillNulls: config.fillNulls !== false,
numberFormat: config.numberFormat,
@ -123,6 +125,7 @@ const convertToExternalTileChartConfig = (
displayType: DisplayType.Table,
connectionId: config.connection,
sqlTemplate: config.sqlTemplate,
sourceId: config.source,
numberFormat: config.numberFormat,
};
case DisplayType.Number:
@ -131,6 +134,7 @@ const convertToExternalTileChartConfig = (
displayType: DisplayType.Number,
connectionId: config.connection,
sqlTemplate: config.sqlTemplate,
sourceId: config.source,
numberFormat: config.numberFormat,
};
case DisplayType.Pie:
@ -139,6 +143,7 @@ const convertToExternalTileChartConfig = (
displayType: DisplayType.Pie,
connectionId: config.connection,
sqlTemplate: config.sqlTemplate,
sourceId: config.source,
numberFormat: config.numberFormat,
};
case DisplayType.Search:
@ -342,6 +347,7 @@ export function convertToInternalTileConfig(
name,
connection: externalConfig.connectionId,
sqlTemplate: externalConfig.sqlTemplate,
source: externalConfig.sourceId,
} satisfies RawSqlSavedChartConfig;
break;
case 'table':
@ -358,6 +364,7 @@ export function convertToInternalTileConfig(
name,
connection: externalConfig.connectionId,
sqlTemplate: externalConfig.sqlTemplate,
source: externalConfig.sourceId,
numberFormat: externalConfig.numberFormat,
} satisfies RawSqlSavedChartConfig;
break;

View file

@ -211,6 +211,7 @@ const externalDashboardRawSqlChartConfigBaseSchema = z.object({
configType: z.literal('sql'),
connectionId: objectIdSchema,
sqlTemplate: z.string().max(100000),
sourceId: objectIdSchema.optional(),
numberFormat: NumberFormatSchema.optional(),
});

View file

@ -214,9 +214,9 @@ function Mapping({ input }: { input: Input }) {
const sourceMappings = input.tiles.map(tile => {
const config = tile.config as SavedChartConfig;
if (isRawSqlSavedChartConfig(config)) return '';
if (!config.source) return '';
const match = sources.find(
source => source.name.toLowerCase() === config.source.toLowerCase(),
source => source.name.toLowerCase() === config.source!.toLowerCase(),
);
return match?.id || '';
});
@ -253,7 +253,6 @@ function Mapping({ input }: { input: Input }) {
if (isUpdatingRef.current) return;
if (!sourceMappings || !input.tiles) return;
// Find which mapping changed
const changedIdx = sourceMappings.findIndex(
(mapping, idx) => mapping !== prevSourceMappingsRef.current?.[idx],
);
@ -263,21 +262,14 @@ function Mapping({ input }: { input: Input }) {
const inputTile = input.tiles[changedIdx];
const inputTileConfig = inputTile?.config;
if (!inputTileConfig || isRawSqlSavedChartConfig(inputTileConfig)) return;
if (!inputTileConfig || !inputTileConfig.source) return;
const sourceId = sourceMappings[changedIdx] ?? '';
const inputTileSource = inputTileConfig.source;
const keysForTilesWithMatchingSource = input.tiles
.map((tile, index) => ({
config: tile.config,
index,
}))
.filter(
tile =>
!isRawSqlSavedChartConfig(tile.config) &&
tile.config.source === inputTileSource,
)
.map((tile, index) => ({ config: tile.config, index }))
.filter(tile => tile.config.source === inputTileSource)
.map(({ index }) => `sourceMappings.${index}` as const);
const keysForFiltersWithMatchingSource =
@ -345,6 +337,10 @@ function Mapping({ input }: { input: Input }) {
try {
// Zip the source/connection mappings with the input tiles
const zippedTiles = input.tiles.map((tile, idx) => {
const source = sources?.find(
source => source.id === data.sourceMappings[idx],
);
if (isRawSqlSavedChartConfig(tile.config)) {
const connection = connections?.find(
conn => conn.id === data.connectionMappings[idx],
@ -354,12 +350,10 @@ function Mapping({ input }: { input: Input }) {
config: {
...tile.config,
connection: connection!.id,
...(source ? { source: source.id } : {}),
},
};
}
const source = sources?.find(
source => source.id === data.sourceMappings[idx],
);
return {
...tile,
config: {
@ -430,21 +424,35 @@ function Mapping({ input }: { input: Input }) {
<Table.Thead>
<Table.Tr>
<Table.Th>Name</Table.Th>
<Table.Th>Input Source Name</Table.Th>
<Table.Th>Mapped Source Name</Table.Th>
<Table.Th>Input Source</Table.Th>
<Table.Th>Mapped Source</Table.Th>
<Table.Th>Input Connection</Table.Th>
<Table.Th>Mapped Connection</Table.Th>
</Table.Tr>
</Table.Thead>
<Table.Tbody>
{input.tiles.map((tile, i) => {
const config = tile.config;
const isRawSql = isRawSqlSavedChartConfig(config);
const inputSourceName = isRawSql
? `${config.connection} (Connection)`
: `${config.source} (Source)`;
return (
<Table.Tr key={tile.id}>
<Table.Td>{tile.config.name}</Table.Td>
<Table.Td>{inputSourceName}</Table.Td>
<Table.Td>{config.source ?? ''}</Table.Td>
<Table.Td>
{config.source != null && (
<SelectControlled
control={control}
name={`sourceMappings.${i}`}
data={sources?.map(source => ({
value: source.id,
label: source.name,
}))}
placeholder="Select a source"
/>
)}
</Table.Td>
<Table.Td>{isRawSql ? config.connection : ''}</Table.Td>
<Table.Td>
{isRawSql ? (
<SelectControlled
@ -456,17 +464,7 @@ function Mapping({ input }: { input: Input }) {
}))}
placeholder="Select a connection"
/>
) : (
<SelectControlled
control={control}
name={`sourceMappings.${i}`}
data={sources?.map(source => ({
value: source.id,
label: source.name,
}))}
placeholder="Select a source"
/>
)}
) : null}
</Table.Td>
</Table.Tr>
);
@ -486,6 +484,8 @@ function Mapping({ input }: { input: Input }) {
placeholder="Select a source"
/>
</Table.Td>
<Table.Td />
<Table.Td />
</Table.Tr>
))}
</Table.Tbody>

View file

@ -12,7 +12,8 @@ import Head from 'next/head';
import { useRouter } from 'next/router';
import { formatRelative } from 'date-fns';
import produce from 'immer';
import { parseAsJson, parseAsString, useQueryState } from 'nuqs';
import { pick } from 'lodash';
import { parseAsString, useQueryState } from 'nuqs';
import { ErrorBoundary } from 'react-error-boundary';
import RGL, { WidthProvider } from 'react-grid-layout';
import { useForm, useWatch } from 'react-hook-form';
@ -21,6 +22,7 @@ import { convertToDashboardTemplate } from '@hyperdx/common-utils/dist/core/util
import {
isBuilderChartConfig,
isBuilderSavedChartConfig,
isRawSqlChartConfig,
isRawSqlSavedChartConfig,
} from '@hyperdx/common-utils/dist/guards';
import {
@ -67,6 +69,7 @@ import {
IconTrash,
IconUpload,
IconX,
IconZoomExclamation,
} from '@tabler/icons-react';
import { ContactSupportText } from '@/components/ContactSupportText';
@ -202,14 +205,30 @@ const Tile = forwardRef(
>(undefined);
const { data: source } = useSource({
id: isBuilderSavedChartConfig(chart.config)
? chart.config.source
: undefined,
id: chart.config.source,
});
useEffect(() => {
if (isRawSqlSavedChartConfig(chart.config)) {
setQueriedConfig({ ...chart.config, dateRange, granularity });
// Some raw SQL charts don't have a source
if (!chart.config.source) {
setQueriedConfig({
...chart.config,
dateRange,
granularity,
filters,
});
} else if (source != null) {
setQueriedConfig({
...chart.config,
// Populate these two columns from the source to support Lucene-based filters
...pick(source, ['implicitColumnExpression', 'from']),
dateRange,
granularity,
filters,
});
}
return;
}
@ -272,6 +291,35 @@ const Tile = forwardRef(
return tooltip;
}, [alert]);
const filterWarning = useMemo(() => {
const doFiltersExist = !!filters?.filter(
f => (f.type === 'lucene' || f.type === 'sql') && f.condition.trim(),
)?.length;
if (
!doFiltersExist ||
!queriedConfig ||
!isRawSqlChartConfig(queriedConfig)
)
return null;
const isMissingSourceForFiltering = !queriedConfig.source;
const isMissingFiltersMacro =
!queriedConfig.sqlTemplate.includes('$__filters');
if (!isMissingSourceForFiltering && !isMissingFiltersMacro) return null;
const message = isMissingFiltersMacro
? 'Filters are not applied because the SQL does not include the required $__filters macro'
: 'Filters are not applied because no Source is set for this chart';
return (
<Tooltip multiline maw={500} label={message} key="filter-warning">
<IconZoomExclamation size={16} color="var(--color-text-danger)" />
</Tooltip>
);
}, [filters, queriedConfig]);
const hoverToolbar = useMemo(() => {
return (
<Flex
@ -369,7 +417,9 @@ const Tile = forwardRef(
// Render chart content (used in both tile and fullscreen views)
const renderChartContent = useCallback(
(hideToolbar: boolean = false, isFullscreenView: boolean = false) => {
const toolbar = hideToolbar ? [] : [hoverToolbar];
const toolbar = hideToolbar
? [filterWarning]
: [hoverToolbar, filterWarning];
const keyPrefix = isFullscreenView ? 'fullscreen' : 'tile';
// Markdown charts may not have queriedConfig, if config.source is not set
@ -390,11 +440,7 @@ const Tile = forwardRef(
key={`${keyPrefix}-${chart.id}`}
title={title}
toolbarPrefix={toolbar}
sourceId={
isBuilderSavedChartConfig(chart.config)
? chart.config.source
: undefined
}
sourceId={chart.config.source}
showDisplaySwitcher={true}
config={queriedConfig}
onTimeRangeSelect={onTimeRangeSelect}
@ -504,6 +550,7 @@ const Tile = forwardRef(
onUpdateChart,
source,
dateRange,
filterWarning,
],
);
@ -627,6 +674,7 @@ const EditTileModal = ({
}}
onClose={handleClose}
onDirtyChange={setHasUnsavedChanges}
isDashboardForm
/>
</ZIndexContext.Provider>
)}

View file

@ -7,15 +7,17 @@ import {
import { MACRO_SUGGESTIONS } from '@hyperdx/common-utils/dist/macros';
import { QUERY_PARAMS_BY_DISPLAY_TYPE } from '@hyperdx/common-utils/dist/rawSqlParams';
import { DisplayType, SourceKind } from '@hyperdx/common-utils/dist/types';
import { Box, Button, Group, Stack, Text } from '@mantine/core';
import { Box, Button, Group, Stack, Text, Tooltip } from '@mantine/core';
import { IconHelpCircle } from '@tabler/icons-react';
import { SQLEditorControlled } from '@/components/SQLEditor/SQLEditor';
import { type SQLCompletion } from '@/components/SQLEditor/utils';
import useResizable from '@/hooks/useResizable';
import { useSources } from '@/source';
import { getAllMetricTables } from '@/utils';
import { getAllMetricTables, usePrevious } from '@/utils';
import { ConnectionSelectControlled } from '../ConnectionSelect';
import { SourceSelectControlled } from '../SourceSelect';
import { SQL_PLACEHOLDERS } from './constants';
import { RawSqlChartInstructions } from './RawSqlChartInstructions';
@ -27,10 +29,12 @@ export default function RawSqlChartEditor({
control,
setValue,
onOpenDisplaySettings,
isDashboardForm,
}: {
control: Control<ChartEditorFormState>;
setValue: UseFormSetValue<ChartEditorFormState>;
onOpenDisplaySettings: () => void;
isDashboardForm: boolean;
}) {
const { size, startResize } = useResizable(20, 'bottom');
@ -40,17 +44,29 @@ export default function RawSqlChartEditor({
const connection = useWatch({ control, name: 'connection' });
const source = useWatch({ control, name: 'source' });
// Set a default connection
const prevSource = usePrevious(source);
const prevConnection = usePrevious(connection);
useEffect(() => {
if (sources && !connection) {
const defaultConnection =
sources.find(s => s.id === source)?.connection ??
sources[0]?.connection;
if (defaultConnection && defaultConnection !== connection) {
if (!sources) return;
// When the source changes, sync the connection to match.
if (source !== prevSource) {
const sourceConnection = sources.find(s => s.id === source)?.connection;
if (sourceConnection && sourceConnection !== connection) {
setValue('connection', sourceConnection);
}
} else if (!connection) {
// Set a default connection
const defaultConnection = sources[0]?.connection;
if (defaultConnection) {
setValue('connection', defaultConnection);
}
} else if (connection !== prevConnection && prevConnection !== undefined) {
// When the connection changes, clear the source
setValue('source', '');
}
}, [connection, setValue, source, sources]);
}, [connection, prevConnection, prevSource, setValue, source, sources]);
const placeholderSQl = SQL_PLACEHOLDERS[displayType ?? DisplayType.Table];
@ -104,7 +120,7 @@ export default function RawSqlChartEditor({
return (
<Stack>
<Group align="center">
<Group align="center" gap={0}>
<Text pe="md" size="sm">
Connection
</Text>
@ -113,8 +129,32 @@ export default function RawSqlChartEditor({
name="connection"
size="xs"
/>
<Group align="center" gap={8} mx="md">
<Text size="sm" ps="md">
Source
</Text>
{isDashboardForm && (
<Tooltip
label="Optional. Required to apply dashboard filters to this chart."
pe="md"
>
<IconHelpCircle size={14} className="cursor-pointer" />
</Tooltip>
)}
</Group>
<SourceSelectControlled
control={control}
name="source"
connectionId={connection}
size="xs"
clearable
placeholder="None"
/>
</Group>
<RawSqlChartInstructions displayType={displayType ?? DisplayType.Table} />
<RawSqlChartInstructions
displayType={displayType ?? DisplayType.Table}
isDashboardForm={isDashboardForm}
/>
<Box style={{ position: 'relative' }}>
<SQLEditorControlled
control={control}

View file

@ -59,8 +59,10 @@ function ParamSnippet({
export function RawSqlChartInstructions({
displayType,
isDashboardForm,
}: {
displayType: DisplayType;
isDashboardForm: boolean;
}) {
const [helpOpened, setHelpOpened] = useAtom(helpOpenedAtom);
const toggleHelp = () => setHelpOpened(v => !v);
@ -96,7 +98,7 @@ export function RawSqlChartInstructions({
{DISPLAY_TYPE_INSTRUCTIONS[displayType]}
<Text size="xs" fw="bold">
The following parameters can be referenced in this chart's SQL:
The following parameters and macros can be used in this chart:
</Text>
<List size="xs" withPadding spacing={3}>
{availableParams.map(({ name, type, description }) => (
@ -107,6 +109,23 @@ export function RawSqlChartInstructions({
/>
</List.Item>
))}
<List.Item>
<ParamSnippet
value={`$__filters`}
description="Applies the selected dashboard filter conditions to the chart (Source must be selected)"
/>
</List.Item>
<List.Item>
<Text size="xs">
Macros from the{' '}
<Anchor
href="https://github.com/grafana/clickhouse-datasource?tab=readme-ov-file#macros"
target="_blank"
>
ClickHouse Datasource Grafana Plugin
</Anchor>
</Text>
</List.Item>
</List>
<Text size="xs" fw="bold">
@ -141,16 +160,6 @@ export function RawSqlChartInstructions({
<Code fz="xs" block>
{QUERY_PARAM_EXAMPLES[displayType]}
</Code>
<Text size="xs" mt="xs">
Macros from the{' '}
<Anchor
href="https://github.com/grafana/clickhouse-datasource?tab=readme-ov-file#macros"
target="_blank"
>
ClickHouse Datasource Grafana Plugin
</Anchor>{' '}
may also be used.
</Text>
</div>
</Stack>
</Collapse>

View file

@ -74,6 +74,7 @@ export function convertFormStateToSavedChartConfig(
]),
sqlTemplate: form.sqlTemplate ?? '',
connection: form.connection ?? '',
source: form.source || undefined,
};
return rawSqlConfig;
}
@ -116,6 +117,7 @@ export function convertFormStateToChartConfig(
]),
sqlTemplate: form.sqlTemplate ?? '',
connection: form.connection ?? '',
source: form.source || undefined,
};
return { ...rawSqlConfig, dateRange };

View file

@ -542,6 +542,7 @@ export default function EditTimeChartForm({
onDirtyChange,
'data-testid': dataTestId,
submitRef,
isDashboardForm = false,
}: {
dashboardId?: string;
chartConfig: SavedChartConfig;
@ -557,6 +558,7 @@ export default function EditTimeChartForm({
onTimeRangeSelect?: (start: Date, end: Date) => void;
'data-testid'?: string;
submitRef?: React.MutableRefObject<(() => void) | undefined>;
isDashboardForm?: boolean;
}) {
const formValue: ChartEditorFormState = useMemo(
() => convertSavedChartConfigToFormState(chartConfig),
@ -731,63 +733,71 @@ export default function EditTimeChartForm({
const [saveToDashboardModalOpen, setSaveToDashboardModalOpen] =
useState(false);
const onSubmit = useCallback(() => {
handleSubmit(form => {
const isRawSqlChart =
form.configType === 'sql' && isRawSqlDisplayType(form.displayType);
const onSubmit = useCallback(
(suppressErrorNotification: boolean = false) => {
handleSubmit(form => {
const isRawSqlChart =
form.configType === 'sql' && isRawSqlDisplayType(form.displayType);
const errors = validateChartForm(form, tableSource, setError);
if (errors.length > 0) {
notifications.show({
id: 'chart-error',
title: 'Invalid Chart',
message: <ErrorNotificationMessage errors={errors} />,
color: 'red',
});
return;
}
const errors = validateChartForm(form, tableSource, setError);
if (errors.length > 0) {
if (!suppressErrorNotification) {
notifications.show({
id: 'chart-error',
title: 'Invalid Chart',
message: <ErrorNotificationMessage errors={errors} />,
color: 'red',
});
}
return;
}
const savedConfig = convertFormStateToSavedChartConfig(form, tableSource);
const queriedConfig = convertFormStateToChartConfig(
form,
dateRange,
tableSource,
);
if (savedConfig && queriedConfig) {
const normalizedSavedConfig = isRawSqlSavedChartConfig(savedConfig)
? savedConfig
: {
...savedConfig,
alert: normalizeNoOpAlertScheduleFields(
savedConfig.alert,
chartConfigAlert,
{
preserveExplicitScheduleOffsetMinutes:
dirtyFields.alert?.scheduleOffsetMinutes === true,
preserveExplicitScheduleStartAt:
dirtyFields.alert?.scheduleStartAt === true,
},
),
};
setChartConfig?.(normalizedSavedConfig);
setQueriedConfigAndSource(
queriedConfig,
isRawSqlChart ? undefined : tableSource,
const savedConfig = convertFormStateToSavedChartConfig(
form,
tableSource,
);
}
})();
}, [
chartConfigAlert,
dirtyFields.alert?.scheduleOffsetMinutes,
dirtyFields.alert?.scheduleStartAt,
handleSubmit,
setChartConfig,
setQueriedConfigAndSource,
tableSource,
dateRange,
setError,
]);
const queriedConfig = convertFormStateToChartConfig(
form,
dateRange,
tableSource,
);
if (savedConfig && queriedConfig) {
const normalizedSavedConfig = isRawSqlSavedChartConfig(savedConfig)
? savedConfig
: {
...savedConfig,
alert: normalizeNoOpAlertScheduleFields(
savedConfig.alert,
chartConfigAlert,
{
preserveExplicitScheduleOffsetMinutes:
dirtyFields.alert?.scheduleOffsetMinutes === true,
preserveExplicitScheduleStartAt:
dirtyFields.alert?.scheduleStartAt === true,
},
),
};
setChartConfig?.(normalizedSavedConfig);
setQueriedConfigAndSource(
queriedConfig,
isRawSqlChart ? undefined : tableSource,
);
}
})();
},
[
chartConfigAlert,
dirtyFields.alert?.scheduleOffsetMinutes,
dirtyFields.alert?.scheduleStartAt,
handleSubmit,
setChartConfig,
setQueriedConfigAndSource,
tableSource,
dateRange,
setError,
],
);
const onTableSortingChange = useCallback(
(sortState: SortingState | null) => {
@ -904,7 +914,8 @@ export default function EditTimeChartForm({
// Don't auto-submit when config type changes, to avoid clearing form state (like source)
if (displayTypeChanged) {
onSubmit();
// true = Suppress error notification (because we're auto-submitting)
onSubmit(true);
}
}
}, [displayType, select, setValue, onSubmit, configType]);
@ -1155,6 +1166,7 @@ export default function EditTimeChartForm({
control={control}
setValue={setValue}
onOpenDisplaySettings={openDisplaySettings}
isDashboardForm={isDashboardForm}
/>
) : (
<>
@ -1512,7 +1524,7 @@ export default function EditTimeChartForm({
data-testid="chart-run-query-button"
variant="primary"
type="submit"
onClick={onSubmit}
onClick={() => onSubmit()}
leftSection={<IconPlayerPlay size={16} />}
style={{ flexShrink: 0 }}
>

View file

@ -87,7 +87,7 @@ export default function DBNumberChart({
const formattedValue = formatNumber(value as number, config.numberFormat);
const { data: source } = useSource({
id: isBuilderChartConfig(config) ? config.source : undefined,
id: config.source,
});
const toolbarItemsMemo = useMemo(() => {

View file

@ -71,7 +71,7 @@ export const DBPieChart = ({
errorVariant?: ChartErrorStateVariant;
}) => {
const { data: source } = useSource({
id: isBuilderChartConfig(config) ? config.source : undefined,
id: config.source,
});
const queriedConfig = useMemo(() => {

View file

@ -54,7 +54,7 @@ export default function DBTableChart({
const [sort, setSort] = useState<SortingState>([]);
const { data: source } = useSource({
id: isBuilderChartConfig(config) ? config.source : undefined,
id: config.source,
});
const effectiveSort = useMemo(

View file

@ -362,7 +362,7 @@ function DBTimeChartComponent({
isPlaceholderData;
const { data: source } = useSource({
id: sourceId || (isBuilderChartConfig(config) ? config.source : undefined),
id: sourceId || config.source,
});
const {

View file

@ -47,6 +47,7 @@ function SourceSelectControlledComponent({
size,
onCreate,
allowedSourceKinds,
connectionId,
comboboxProps,
sourceSchemaPreview,
...props
@ -54,6 +55,7 @@ function SourceSelectControlledComponent({
size?: string;
onCreate?: () => void;
allowedSourceKinds?: SourceKind[];
connectionId?: string;
sourceSchemaPreview?: React.ReactNode;
} & UseControllerProps<any> &
SelectProps) {
@ -66,7 +68,9 @@ function SourceSelectControlledComponent({
data
?.filter(
source =>
!allowedSourceKinds || allowedSourceKinds.includes(source.kind),
(!allowedSourceKinds ||
allowedSourceKinds.includes(source.kind)) &&
(!connectionId || source.connection === connectionId),
)
.map(d => ({
value: d.id,
@ -82,7 +86,7 @@ function SourceSelectControlledComponent({
]
: []),
],
[data, onCreate, allowedSourceKinds, hasLocalDefaultSources],
[data, onCreate, allowedSourceKinds, connectionId, hasLocalDefaultSources],
);
const rightSectionProps = SourceSelectRightSection({ sourceSchemaPreview });
@ -91,7 +95,6 @@ function SourceSelectControlledComponent({
<SelectControlled
{...props}
data={values}
// disabled={isDatabasesLoading}
comboboxProps={{ withinPortal: false, ...comboboxProps }}
searchable
placeholder="Data Source"

View file

@ -280,7 +280,7 @@ export function useQueriedChartConfig(
});
const { data: source, isLoading: isSourceLoading } = useSource({
id: builderConfig?.source,
id: config.source,
});
const query = useQuery<TQueryFnData, ClickHouseQueryError | Error>({
@ -374,7 +374,7 @@ export function useRenderedSqlChartConfig(
});
const { data: source, isLoading: isSourceLoading } = useSource({
id: builderConfig?.source,
id: config.source,
});
const query = useQuery({

View file

@ -21,7 +21,7 @@ export function useExplainQuery(
const metadata = useMetadataWithSettings();
const { data: source, isLoading: isSourceLoading } = useSource({
id: isBuilderChartConfig(config) ? config.source : undefined,
id: config.source,
});
return useQuery({

View file

@ -448,7 +448,7 @@ export default function useOffsetPaginatedQuery(
});
const { data: source, isLoading: isSourceLoading } = useSource({
id: builderConfig?.source,
id: config.source,
});
const {

View file

@ -81,9 +81,9 @@ describe('replaceMacros', () => {
});
it('should replace multiple macros in one query', () => {
const sql =
'SELECT $__timeInterval(ts), count() FROM t WHERE $__timeFilter(ts) GROUP BY 1';
const result = replaceMacros(sql);
const result = replaceMacros(
'SELECT $__timeInterval(ts), count() FROM t WHERE $__timeFilter(ts) GROUP BY 1',
);
expect(result).toContain('toStartOfInterval');
expect(result).toContain(
'ts >= toDateTime(fromUnixTimestamp64Milli({startDateMilliseconds:Int64}))',
@ -101,4 +101,24 @@ describe('replaceMacros', () => {
'Failed to parse macro arguments',
);
});
it('should replace $__filters with provided filtersSQL', () => {
const result = replaceMacros(
'WHERE $__filters',
"(col = 'val') AND (x > 1)",
);
expect(result).toBe("WHERE (col = 'val') AND (x > 1)");
});
it('should replace $__filters with fallback when no filtersSQL provided', () => {
expect(replaceMacros('WHERE $__filters')).toBe(
'WHERE (1=1 /** no filters applied */)',
);
});
it('should replace $__filters with fallback when filtersSQL is empty', () => {
expect(replaceMacros('WHERE $__filters', '')).toBe(
'WHERE (1=1 /** no filters applied */)',
);
});
});

View file

@ -1,16 +1,29 @@
import { Metadata } from '@/core/metadata';
import { DisplayType } from '@/types';
import { renderRawSqlChartConfig } from '../core/renderChartConfig';
const mockMetadata = {
getColumns: jest.fn().mockResolvedValue([]),
getMaterializedColumnsLookupTable: jest.fn().mockResolvedValue(null),
getColumn: jest.fn().mockResolvedValue(undefined),
getTableMetadata: jest.fn().mockResolvedValue({ primary_key: '' }),
getSkipIndices: jest.fn().mockResolvedValue([]),
getSetting: jest.fn().mockResolvedValue(undefined),
} as unknown as Metadata;
describe('renderRawSqlChartConfig', () => {
describe('DisplayType.Table', () => {
it('returns the sqlTemplate with no params when no dateRange provided', () => {
const result = renderRawSqlChartConfig({
configType: 'sql',
sqlTemplate: 'SELECT count() FROM logs',
connection: 'conn-1',
displayType: DisplayType.Table,
});
it('returns the sqlTemplate with no params when no dateRange provided', async () => {
const result = await renderRawSqlChartConfig(
{
configType: 'sql',
sqlTemplate: 'SELECT count() FROM logs',
connection: 'conn-1',
displayType: DisplayType.Table,
},
mockMetadata,
);
expect(result.sql).toBe('SELECT count() FROM logs');
expect(result.params).toEqual({
startDateMilliseconds: undefined,
@ -18,17 +31,20 @@ describe('renderRawSqlChartConfig', () => {
});
});
it('injects startDateMilliseconds and endDateMilliseconds when dateRange provided', () => {
it('injects startDateMilliseconds and endDateMilliseconds when dateRange provided', async () => {
const start = new Date('2024-01-01T00:00:00.000Z');
const end = new Date('2024-01-02T00:00:00.000Z');
const result = renderRawSqlChartConfig({
configType: 'sql',
sqlTemplate:
'SELECT count() FROM logs WHERE ts BETWEEN {startDateMilliseconds:Int64} AND {endDateMilliseconds:Int64}',
connection: 'conn-1',
displayType: DisplayType.Table,
dateRange: [start, end],
});
const result = await renderRawSqlChartConfig(
{
configType: 'sql',
sqlTemplate:
'SELECT count() FROM logs WHERE ts BETWEEN {startDateMilliseconds:Int64} AND {endDateMilliseconds:Int64}',
connection: 'conn-1',
displayType: DisplayType.Table,
dateRange: [start, end],
},
mockMetadata,
);
expect(result.sql).toBe(
'SELECT count() FROM logs WHERE ts BETWEEN {startDateMilliseconds:Int64} AND {endDateMilliseconds:Int64}',
);
@ -39,13 +55,16 @@ describe('renderRawSqlChartConfig', () => {
});
describe('DisplayType.Line', () => {
it('returns undefined params when no dateRange is provided', () => {
const result = renderRawSqlChartConfig({
configType: 'sql',
sqlTemplate: 'SELECT ts, count() FROM logs GROUP BY ts',
connection: 'conn-1',
displayType: DisplayType.Line,
});
it('returns undefined params when no dateRange is provided', async () => {
const result = await renderRawSqlChartConfig(
{
configType: 'sql',
sqlTemplate: 'SELECT ts, count() FROM logs GROUP BY ts',
connection: 'conn-1',
displayType: DisplayType.Line,
},
mockMetadata,
);
expect(result.params).toEqual({
startDateMilliseconds: undefined,
endDateMilliseconds: undefined,
@ -54,17 +73,20 @@ describe('renderRawSqlChartConfig', () => {
});
});
it('injects all four params when dateRange is provided', () => {
it('injects all four params when dateRange is provided', async () => {
const start = new Date('2024-01-01T00:00:00.000Z');
const end = new Date('2024-01-02T00:00:00.000Z');
const result = renderRawSqlChartConfig({
configType: 'sql',
sqlTemplate:
'SELECT toStartOfInterval(ts, INTERVAL {intervalSeconds:Int64} SECOND) AS ts, count() FROM logs WHERE ts >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) AND ts <= fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) GROUP BY ts ORDER BY ts ASC',
connection: 'conn-1',
displayType: DisplayType.Line,
dateRange: [start, end],
});
const result = await renderRawSqlChartConfig(
{
configType: 'sql',
sqlTemplate:
'SELECT toStartOfInterval(ts, INTERVAL {intervalSeconds:Int64} SECOND) AS ts, count() FROM logs WHERE ts >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) AND ts <= fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) GROUP BY ts ORDER BY ts ASC',
connection: 'conn-1',
displayType: DisplayType.Line,
dateRange: [start, end],
},
mockMetadata,
);
expect(result.params.startDateMilliseconds).toBe(start.getTime());
expect(result.params.endDateMilliseconds).toBe(end.getTime());
expect(typeof result.params.intervalSeconds).toBe('number');
@ -74,49 +96,58 @@ describe('renderRawSqlChartConfig', () => {
);
});
it('returns the granularity from the config when available', () => {
it('returns the granularity from the config when available', async () => {
// 1-hour range: auto-granularity should be 1 minute (60s) for 60 max buckets
const start = new Date('2024-01-01T00:00:00.000Z');
const end = new Date('2024-01-01T01:00:00.000Z');
const result = renderRawSqlChartConfig({
configType: 'sql',
sqlTemplate: 'SELECT ts, count() FROM logs GROUP BY ts',
connection: 'conn-1',
displayType: DisplayType.Line,
dateRange: [start, end],
granularity: '5 minute',
});
const result = await renderRawSqlChartConfig(
{
configType: 'sql',
sqlTemplate: 'SELECT ts, count() FROM logs GROUP BY ts',
connection: 'conn-1',
displayType: DisplayType.Line,
dateRange: [start, end],
granularity: '5 minute',
},
mockMetadata,
);
expect(result.params.intervalSeconds).toBe(300); // 5 minutes
expect(result.params.intervalMilliseconds).toBe(300000);
});
it('computes intervalSeconds based on the date range duration when granularity is auto', () => {
it('computes intervalSeconds based on the date range duration when granularity is auto', async () => {
// 1-hour range: auto-granularity should be 1 minute (60s) for 60 max buckets
const start = new Date('2024-01-01T00:00:00.000Z');
const end = new Date('2024-01-01T01:00:00.000Z');
const result = renderRawSqlChartConfig({
configType: 'sql',
sqlTemplate: 'SELECT ts, count() FROM logs GROUP BY ts',
connection: 'conn-1',
granularity: 'auto',
displayType: DisplayType.Line,
dateRange: [start, end],
});
const result = await renderRawSqlChartConfig(
{
configType: 'sql',
sqlTemplate: 'SELECT ts, count() FROM logs GROUP BY ts',
connection: 'conn-1',
granularity: 'auto',
displayType: DisplayType.Line,
dateRange: [start, end],
},
mockMetadata,
);
// 1-hour range / 60 buckets = 60s per bucket → "1 minute" interval → 60 seconds
expect(result.params.intervalSeconds).toBe(60);
expect(result.params.intervalMilliseconds).toBe(60000);
});
});
it('defaults to Table display type when displayType is not specified', () => {
it('defaults to Table display type when displayType is not specified', async () => {
const start = new Date('2024-06-15T12:00:00.000Z');
const end = new Date('2024-06-15T13:00:00.000Z');
const result = renderRawSqlChartConfig({
configType: 'sql',
sqlTemplate: 'SELECT * FROM events',
connection: 'conn-1',
dateRange: [start, end],
});
const result = await renderRawSqlChartConfig(
{
configType: 'sql',
sqlTemplate: 'SELECT * FROM events',
connection: 'conn-1',
dateRange: [start, end],
},
mockMetadata,
);
expect(result.params).toEqual({
startDateMilliseconds: start.getTime(),
endDateMilliseconds: end.getTime(),

View file

@ -1678,5 +1678,132 @@ describe('renderChartConfig', () => {
);
expect(result.sql).toBe(sql);
});
it('replaces $__filters macro with rendered filter conditions', async () => {
const result = await renderChartConfig(
{
configType: 'sql',
sqlTemplate:
'SELECT * FROM logs WHERE $__timeFilter(ts) AND $__filters',
connection: 'conn-1',
dateRange: [start, end],
source: 'source-1',
from: { databaseName: 'default', tableName: 'logs' },
filters: [
{ type: 'sql', condition: "ServiceName = 'api'" },
{ type: 'sql_ast', operator: '>', left: 'duration', right: '100' },
],
},
mockMetadata,
undefined,
);
expect(result.sql).toContain(
"AND ((ServiceName = 'api') AND (duration > 100))",
);
});
it('replaces $__filters with 1 = 1 when no filters provided', async () => {
const result = await renderChartConfig(
{
configType: 'sql',
sqlTemplate: 'SELECT * FROM logs WHERE $__filters',
connection: 'conn-1',
dateRange: [start, end],
},
mockMetadata,
undefined,
);
expect(result.sql).toBe(
'SELECT * FROM logs WHERE (1=1 /** no filters applied */)',
);
});
it('replaces $__filters with 1 = 1 when source and from are defined but filters is empty', async () => {
const result = await renderChartConfig(
{
configType: 'sql',
sqlTemplate: 'SELECT * FROM logs WHERE $__filters',
connection: 'conn-1',
dateRange: [start, end],
source: 'source-1',
from: { databaseName: 'default', tableName: 'logs' },
filters: [],
},
mockMetadata,
undefined,
);
expect(result.sql).toBe(
'SELECT * FROM logs WHERE (1=1 /** no filters applied */)',
);
});
it('renders lucene filters to SQL in $__filters when source is specified', async () => {
mockMetadata.getMaterializedColumnsLookupTable = jest
.fn()
.mockResolvedValue(new Map());
const result = await renderChartConfig(
{
configType: 'sql',
sqlTemplate: 'SELECT * FROM logs WHERE $__filters',
connection: 'conn-1',
dateRange: [start, end],
source: 'source-1',
from: { databaseName: 'default', tableName: 'logs' },
implicitColumnExpression: 'Body',
filters: [{ type: 'lucene', condition: 'ServiceName:api' }],
},
mockMetadata,
undefined,
);
expect(result.sql).toBe(
"SELECT * FROM logs WHERE (((ServiceName ILIKE '%api%')))",
);
});
it('renders mixed lucene and sql filters in $__filters', async () => {
mockMetadata.getMaterializedColumnsLookupTable = jest
.fn()
.mockResolvedValue(new Map());
const result = await renderChartConfig(
{
configType: 'sql',
sqlTemplate: 'SELECT * FROM logs WHERE $__filters',
connection: 'conn-1',
dateRange: [start, end],
source: 'source-1',
from: { databaseName: 'default', tableName: 'logs' },
implicitColumnExpression: 'Body',
filters: [
{ type: 'lucene', condition: 'ServiceName:api' },
{ type: 'sql', condition: 'duration > 100' },
],
},
mockMetadata,
undefined,
);
expect(result.sql).toBe(
"SELECT * FROM logs WHERE (((ServiceName ILIKE '%api%')) AND (duration > 100))",
);
});
it('skips filters without source metadata (no from)', async () => {
const result = await renderChartConfig(
{
configType: 'sql',
sqlTemplate: 'SELECT * FROM logs WHERE $__filters',
connection: 'conn-1',
dateRange: [start, end],
filters: [
{ type: 'lucene', condition: 'ServiceName:api' },
{ type: 'sql', condition: 'duration > 100' },
],
},
mockMetadata,
undefined,
);
expect(result.sql).toBe(
'SELECT * FROM logs WHERE (1=1 /** no filters applied */)',
);
});
});
});

View file

@ -815,6 +815,83 @@ describe('utils', () => {
});
});
it('should convert source IDs to names for RawSQL tiles with a source', () => {
const dashboard: z.infer<typeof DashboardSchema> = {
id: 'dashboard1',
name: 'SQL Dashboard',
tags: [],
tiles: [
{
id: 'tile1',
config: {
name: 'SQL Tile With Source',
configType: 'sql',
sqlTemplate: 'SELECT 1',
connection: 'conn1',
source: 'source1',
},
x: 0,
y: 0,
w: 6,
h: 6,
},
{
id: 'tile2',
config: {
name: 'SQL Tile Without Source',
configType: 'sql',
sqlTemplate: 'SELECT 2',
connection: 'conn1',
},
x: 6,
y: 0,
w: 6,
h: 6,
},
],
};
const sources: TSourceUnion[] = [
{
id: 'source1',
kind: SourceKind.Log,
name: 'My Logs',
from: { databaseName: 'default', tableName: 'otel_logs' },
timestampValueExpression: 'Timestamp',
defaultTableSelectExpression: '',
connection: 'conn1',
},
];
const connections: Connection[] = [
{
id: 'conn1',
name: 'Production DB',
host: 'http://localhost:8123',
username: 'default',
},
];
const template = convertToDashboardTemplate(
dashboard,
sources,
connections,
);
expect(template.tiles[0].config).toMatchObject({
configType: 'sql',
connection: 'Production DB',
source: 'My Logs',
});
// Tile without source should not have source set
expect(template.tiles[1].config).toMatchObject({
configType: 'sql',
connection: 'Production DB',
});
expect(
(template.tiles[1].config as { source?: string }).source,
).toBeUndefined();
});
it('should fall back to empty string for unknown connection IDs in RawSQL tiles', () => {
const dashboard: z.infer<typeof DashboardSchema> = {
id: 'dashboard1',

View file

@ -31,6 +31,7 @@ import {
CteChartConfig,
DateRange,
DisplayType,
Filter,
MetricsDataType,
QuerySettings,
RawSqlChartConfig,
@ -698,7 +699,7 @@ function renderFrom({
);
}
async function renderWhereExpression({
async function renderWhereExpressionStr({
condition,
language,
metadata,
@ -714,7 +715,7 @@ async function renderWhereExpression({
implicitColumnExpression?: string;
connectionId: string;
with?: BuilderChartConfigWithDateRange['with'];
}): Promise<ChSql> {
}): Promise<string> {
let _condition = condition;
if (language === 'lucene') {
const serializer = new CustomSchemaSQLSerializerV2({
@ -757,6 +758,14 @@ async function renderWhereExpression({
'',
);
}
return _condition;
}
async function renderWhereExpression(
args: Parameters<typeof renderWhereExpressionStr>[0],
): Promise<ChSql> {
const _condition = await renderWhereExpressionStr(args);
return chSql`${{ UNSAFE_RAW_SQL: _condition }}`;
}
@ -1404,12 +1413,58 @@ async function translateMetricChartConfig(
throw new Error(`no query support for metric type=${metricType}`);
}
export function renderRawSqlChartConfig(
/** Renders the config's filters into a SQL condition string */
async function renderFiltersToSql(
chartConfig: RawSqlChartConfig,
metadata: Metadata,
): Promise<string | undefined> {
if (
!chartConfig.filters?.length ||
!chartConfig.source ||
!chartConfig.from
) {
return undefined;
}
const conditions = (
await Promise.all(
chartConfig.filters.map(async filter => {
if (filter.type === 'sql_ast') {
return `(${filter.left} ${filter.operator} ${filter.right})`;
} else if (
(filter.type === 'lucene' || filter.type === 'sql') &&
filter.condition.trim() &&
chartConfig.from &&
chartConfig.source
) {
const condition = await renderWhereExpressionStr({
condition: filter.condition,
from: chartConfig.from,
language: filter.type,
implicitColumnExpression: chartConfig.implicitColumnExpression,
metadata,
connectionId: chartConfig.connection,
});
return condition ? `(${condition})` : undefined;
}
}),
)
).filter(condition => condition !== undefined);
return conditions.length > 0 ? `(${conditions.join(' AND ')})` : undefined;
}
export async function renderRawSqlChartConfig(
chartConfig: RawSqlChartConfig & Partial<DateRange>,
): ChSql {
metadata: Metadata,
): Promise<ChSql> {
const displayType = chartConfig.displayType ?? DisplayType.Table;
const sqlWithMacrosReplaced = replaceMacros(chartConfig.sqlTemplate);
const filtersSQL = await renderFiltersToSql(chartConfig, metadata);
const sqlWithMacrosReplaced = replaceMacros(
chartConfig.sqlTemplate,
filtersSQL,
);
// eslint-disable-next-line security/detect-object-injection
const queryParams = QUERY_PARAMS_BY_DISPLAY_TYPE[displayType];
@ -1428,7 +1483,7 @@ export async function renderChartConfig(
querySettings: QuerySettings | undefined,
): Promise<ChSql> {
if (isRawSqlChartConfig(rawChartConfig)) {
return renderRawSqlChartConfig(rawChartConfig);
return renderRawSqlChartConfig(rawChartConfig, metadata);
}
// metric types require more rewriting since we know more about the schema

View file

@ -487,6 +487,10 @@ export function convertToDashboardTemplate(
name: '',
}
).name;
if (tileConfig.source) {
tileConfig.source =
sources.find(source => source.id === tileConfig.source)?.name ?? '';
}
}
return tile;
};

View file

@ -124,10 +124,10 @@ const MACROS: Macro[] = [
];
/** Macro metadata for autocomplete suggestions */
export const MACRO_SUGGESTIONS = MACROS.map(({ name, argCount }) => ({
name,
argCount,
}));
export const MACRO_SUGGESTIONS = [
...MACROS.map(({ name, argCount }) => ({ name, argCount })),
{ name: 'filters', argCount: 0 },
];
type MacroMatch = {
full: string;
@ -184,11 +184,26 @@ function findMacros(input: string, name: string): MacroMatch[] {
return matches;
}
export function replaceMacros(sql: string): string {
const sortedMacros = [...MACROS].sort(
const NO_FILTERS = '(1=1 /** no filters applied */)';
export function replaceMacros(
sqlTemplate: string,
filtersSQL?: string,
): string {
const allMacros: Macro[] = [
...MACROS,
{
name: 'filters',
argCount: 0,
replace: () => filtersSQL || NO_FILTERS,
},
];
const sortedMacros = allMacros.sort(
(m1, m2) => m2.name.length - m1.name.length,
);
let sql = sqlTemplate;
for (const macro of sortedMacros) {
const matches = findMacros(sql, macro.name);

View file

@ -94,10 +94,12 @@ const TIME_CHART_EXAMPLE_SQL = `SELECT
FROM otel_logs
WHERE TimestampTime >= fromUnixTimestamp64Milli ({startDateMilliseconds:Int64})
AND TimestampTime < fromUnixTimestamp64Milli ({endDateMilliseconds:Int64})
AND $__filters
GROUP BY ServiceName, ts`;
export const DATE_RANGE_WHERE_EXAMPLE_SQL = `WHERE TimestampTime >= fromUnixTimestamp64Milli ({startDateMilliseconds:Int64})
AND TimestampTime <= fromUnixTimestamp64Milli ({endDateMilliseconds:Int64})`;
AND TimestampTime <= fromUnixTimestamp64Milli ({endDateMilliseconds:Int64})
AND $__filters`;
export const QUERY_PARAM_EXAMPLES: Record<DisplayType, string> = {
[DisplayType.Line]: TIME_CHART_EXAMPLE_SQL,

View file

@ -574,11 +574,21 @@ const BuilderChartConfigSchema = z.intersection(
export type BuilderChartConfig = z.infer<typeof BuilderChartConfigSchema>;
/** Schema describing Raw SQL chart configs */
const RawSqlChartConfigSchema = SharedChartDisplaySettingsSchema.extend({
/** Base schema for Raw SQL chart configs */
const RawSqlBaseChartConfigSchema = SharedChartDisplaySettingsSchema.extend({
configType: z.literal('sql'),
sqlTemplate: z.string(),
connection: z.string(),
source: z.string().optional(),
});
/** Schema describing Raw SQL chart configs with runtime-only fields */
const RawSqlChartConfigSchema = RawSqlBaseChartConfigSchema.extend({
filters: z.array(FilterSchema).optional(),
from: z
.object({ databaseName: z.string(), tableName: z.string() })
.optional(),
implicitColumnExpression: z.string().optional(),
});
export type RawSqlChartConfig = z.infer<typeof RawSqlChartConfigSchema>;
@ -655,7 +665,7 @@ export type BuilderSavedChartConfig = z.infer<
typeof BuilderSavedChartConfigSchema
>;
const RawSqlSavedChartConfigSchema = RawSqlChartConfigSchema.extend({
const RawSqlSavedChartConfigSchema = RawSqlBaseChartConfigSchema.extend({
name: z.string().optional(),
});