mirror of
https://github.com/hyperdxio/hyperdx
synced 2026-04-21 13:37:15 +00:00
feat: Add $__sourceTable macro (#2018)
## Summary This PR adds a $__sourceTable macro which is replaced (in Raw SQL-based charts) with the selected source (if there is one). This allows for easier import/export when using raw SQL charts, since sources mapped during import will automatically be replaced with the correct source table name when queried. `$__sourceTable` also supports arguments for the metric tables, eg. `$__sourceTable(sum)`. ### Screenshots or video <img width="1439" height="1151" alt="Screenshot 2026-03-31 at 3 07 43 PM" src="https://github.com/user-attachments/assets/bdbaa7fb-0570-46bc-90c5-032e3d64fd34" /> ### How to test locally or on Vercel This can all be tested in the preview environment. ### References - Linear Issue: Closes HDX-3834 - Related PRs:
This commit is contained in:
parent
c4dcfd75e2
commit
308da30bb7
12 changed files with 346 additions and 69 deletions
6
.changeset/nice-suns-melt.md
Normal file
6
.changeset/nice-suns-melt.md
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
---
|
||||
"@hyperdx/common-utils": patch
|
||||
"@hyperdx/app": patch
|
||||
---
|
||||
|
||||
feat: Add $\_\_sourceTable macro
|
||||
|
|
@ -235,8 +235,12 @@ const Tile = forwardRef(
|
|||
} else if (source != null) {
|
||||
setQueriedConfig({
|
||||
...chart.config,
|
||||
// Populate these two columns from the source to support Lucene-based filters
|
||||
...pick(source, ['implicitColumnExpression', 'from']),
|
||||
// Populate these columns from the source to support Lucene-based filters and metric table macros
|
||||
...pick(source, [
|
||||
'implicitColumnExpression',
|
||||
'from',
|
||||
'metricTables',
|
||||
]),
|
||||
sampleWeightExpression: getSampleWeightExpression(source),
|
||||
dateRange,
|
||||
granularity,
|
||||
|
|
@ -315,6 +319,9 @@ const Tile = forwardRef(
|
|||
const doFiltersExist = !!filters?.filter(
|
||||
f => (f.type === 'lucene' || f.type === 'sql') && f.condition.trim(),
|
||||
)?.length;
|
||||
const doLuceneFiltersExist = !!filters?.filter(
|
||||
f => f.type === 'lucene' && f.condition.trim(),
|
||||
)?.length;
|
||||
|
||||
if (
|
||||
!doFiltersExist ||
|
||||
|
|
@ -326,19 +333,28 @@ const Tile = forwardRef(
|
|||
const isMissingSourceForFiltering = !queriedConfig.source;
|
||||
const isMissingFiltersMacro =
|
||||
!queriedConfig.sqlTemplate.includes('$__filters');
|
||||
const isMetricsSourceWithLuceneFilter =
|
||||
source?.kind === SourceKind.Metric && doLuceneFiltersExist;
|
||||
|
||||
if (!isMissingSourceForFiltering && !isMissingFiltersMacro) return null;
|
||||
if (
|
||||
!isMissingSourceForFiltering &&
|
||||
!isMissingFiltersMacro &&
|
||||
!isMetricsSourceWithLuceneFilter
|
||||
)
|
||||
return null;
|
||||
|
||||
const message = isMissingFiltersMacro
|
||||
? 'Filters are not applied because the SQL does not include the required $__filters macro'
|
||||
: 'Filters are not applied because no Source is set for this chart';
|
||||
: isMetricsSourceWithLuceneFilter
|
||||
? 'Lucene filters are not applied because they are not supported for metrics sources.'
|
||||
: 'Filters are not applied because no Source is set for this chart';
|
||||
|
||||
return (
|
||||
<Tooltip multiline maw={500} label={message} key="filter-warning">
|
||||
<IconZoomExclamation size={16} color="var(--color-text-danger)" />
|
||||
</Tooltip>
|
||||
);
|
||||
}, [filters, queriedConfig]);
|
||||
}, [filters, queriedConfig, source]);
|
||||
|
||||
const hoverToolbar = useMemo(() => {
|
||||
return (
|
||||
|
|
|
|||
|
|
@ -87,9 +87,9 @@ export default function RawSqlChartEditor({
|
|||
}));
|
||||
|
||||
const macroCompletions: SQLCompletion[] = MACRO_SUGGESTIONS.map(
|
||||
({ name, argCount }) => ({
|
||||
({ name, minArgs }) => ({
|
||||
label: `$__${name}`,
|
||||
apply: argCount > 0 ? `$__${name}(` : `$__${name}`,
|
||||
apply: minArgs > 0 ? `$__${name}(` : `$__${name}`,
|
||||
detail: 'macro',
|
||||
type: 'function',
|
||||
}),
|
||||
|
|
|
|||
|
|
@ -107,6 +107,12 @@ export function RawSqlChartInstructions({
|
|||
/>
|
||||
</List.Item>
|
||||
))}
|
||||
<List.Item>
|
||||
<ParamSnippet
|
||||
value={`$__sourceTable([metricType])`}
|
||||
description="Resolves to selected source table (Source must be selected)"
|
||||
/>
|
||||
</List.Item>
|
||||
<List.Item>
|
||||
<ParamSnippet
|
||||
value={`$__filters`}
|
||||
|
|
|
|||
|
|
@ -7,9 +7,10 @@ const TIMESERIES_PLACEHOLDER_SQL = `SELECT
|
|||
SeverityText,
|
||||
count() AS count
|
||||
FROM
|
||||
default.otel_logs
|
||||
$__sourceTable
|
||||
WHERE TimestampTime >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64})
|
||||
AND TimestampTime < fromUnixTimestamp64Milli({endDateMilliseconds:Int64})
|
||||
AND $__filters
|
||||
GROUP BY ts, SeverityText
|
||||
ORDER BY ts ASC;`;
|
||||
|
||||
|
|
@ -19,9 +20,10 @@ export const SQL_PLACEHOLDERS: Record<DisplayType, string> = {
|
|||
[DisplayType.Table]: `SELECT
|
||||
count()
|
||||
FROM
|
||||
default.otel_logs
|
||||
$__sourceTable
|
||||
WHERE TimestampTime >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64})
|
||||
AND TimestampTime <= fromUnixTimestamp64Milli({endDateMilliseconds:Int64})
|
||||
AND $__filters
|
||||
LIMIT
|
||||
200
|
||||
`,
|
||||
|
|
@ -29,16 +31,18 @@ LIMIT
|
|||
ServiceName,
|
||||
count()
|
||||
FROM
|
||||
default.otel_logs
|
||||
$__sourceTable
|
||||
WHERE TimestampTime >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64})
|
||||
AND TimestampTime < fromUnixTimestamp64Milli({endDateMilliseconds:Int64})
|
||||
AND $__filters
|
||||
GROUP BY ServiceName;`,
|
||||
[DisplayType.Number]: `SELECT
|
||||
count()
|
||||
FROM
|
||||
default.otel_logs
|
||||
$__sourceTable
|
||||
WHERE TimestampTime >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64})
|
||||
AND TimestampTime < fromUnixTimestamp64Milli({endDateMilliseconds:Int64});`,
|
||||
AND TimestampTime < fromUnixTimestamp64Milli({endDateMilliseconds:Int64})
|
||||
AND $__filters;`,
|
||||
[DisplayType.Search]: '',
|
||||
[DisplayType.Heatmap]: '',
|
||||
[DisplayType.Markdown]: '',
|
||||
|
|
|
|||
|
|
@ -122,6 +122,13 @@ export function convertFormStateToChartConfig(
|
|||
sqlTemplate: form.sqlTemplate ?? '',
|
||||
connection: form.connection ?? '',
|
||||
source: form.source || undefined,
|
||||
from: source?.from,
|
||||
implicitColumnExpression:
|
||||
source && (isLogSource(source) || isTraceSource(source))
|
||||
? source.implicitColumnExpression
|
||||
: undefined,
|
||||
metricTables:
|
||||
source && isMetricSource(source) ? source.metricTables : undefined,
|
||||
};
|
||||
|
||||
return { ...rawSqlConfig, dateRange };
|
||||
|
|
|
|||
|
|
@ -1,89 +1,113 @@
|
|||
import { replaceMacros } from '../macros';
|
||||
import type { MetricTable } from '../types';
|
||||
|
||||
const ALL_METRIC_TABLES: MetricTable = {
|
||||
gauge: 'otel_metrics_gauge',
|
||||
histogram: 'otel_metrics_histogram',
|
||||
sum: 'otel_metrics_sum',
|
||||
summary: 'otel_metrics_summary',
|
||||
'exponential histogram': 'otel_metrics_exponential_histogram',
|
||||
};
|
||||
|
||||
describe('replaceMacros', () => {
|
||||
it('should replace $__fromTime with seconds-precision DateTime', () => {
|
||||
expect(replaceMacros('SELECT $__fromTime')).toBe(
|
||||
expect(replaceMacros({ sqlTemplate: 'SELECT $__fromTime' })).toBe(
|
||||
'SELECT toDateTime(fromUnixTimestamp64Milli({startDateMilliseconds:Int64}))',
|
||||
);
|
||||
});
|
||||
|
||||
it('should replace $__toTime with seconds-precision DateTime', () => {
|
||||
expect(replaceMacros('SELECT $__toTime')).toBe(
|
||||
expect(replaceMacros({ sqlTemplate: 'SELECT $__toTime' })).toBe(
|
||||
'SELECT toDateTime(fromUnixTimestamp64Milli({endDateMilliseconds:Int64}))',
|
||||
);
|
||||
});
|
||||
|
||||
it('should replace $__fromTime_ms with millisecond-precision DateTime64', () => {
|
||||
expect(replaceMacros('SELECT $__fromTime_ms')).toBe(
|
||||
expect(replaceMacros({ sqlTemplate: 'SELECT $__fromTime_ms' })).toBe(
|
||||
'SELECT fromUnixTimestamp64Milli({startDateMilliseconds:Int64})',
|
||||
);
|
||||
});
|
||||
|
||||
it('should replace $__toTime_ms with millisecond-precision DateTime64', () => {
|
||||
expect(replaceMacros('SELECT $__toTime_ms')).toBe(
|
||||
expect(replaceMacros({ sqlTemplate: 'SELECT $__toTime_ms' })).toBe(
|
||||
'SELECT fromUnixTimestamp64Milli({endDateMilliseconds:Int64})',
|
||||
);
|
||||
});
|
||||
|
||||
it('should replace $__timeFilter with seconds-precision range filter', () => {
|
||||
const result = replaceMacros('WHERE $__timeFilter(ts)');
|
||||
const result = replaceMacros({
|
||||
sqlTemplate: 'WHERE $__timeFilter(ts)',
|
||||
});
|
||||
expect(result).toBe(
|
||||
'WHERE ts >= toDateTime(fromUnixTimestamp64Milli({startDateMilliseconds:Int64})) AND ts <= toDateTime(fromUnixTimestamp64Milli({endDateMilliseconds:Int64}))',
|
||||
);
|
||||
});
|
||||
|
||||
it('should replace $__timeFilter_ms with millisecond-precision range filter', () => {
|
||||
const result = replaceMacros('WHERE $__timeFilter_ms(ts)');
|
||||
const result = replaceMacros({
|
||||
sqlTemplate: 'WHERE $__timeFilter_ms(ts)',
|
||||
});
|
||||
expect(result).toBe(
|
||||
'WHERE ts >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) AND ts <= fromUnixTimestamp64Milli({endDateMilliseconds:Int64})',
|
||||
);
|
||||
});
|
||||
|
||||
it('should replace $__dateFilter with date-only range filter', () => {
|
||||
const result = replaceMacros('WHERE $__dateFilter(d)');
|
||||
const result = replaceMacros({
|
||||
sqlTemplate: 'WHERE $__dateFilter(d)',
|
||||
});
|
||||
expect(result).toBe(
|
||||
'WHERE d >= toDate(fromUnixTimestamp64Milli({startDateMilliseconds:Int64})) AND d <= toDate(fromUnixTimestamp64Milli({endDateMilliseconds:Int64}))',
|
||||
);
|
||||
});
|
||||
|
||||
it('should replace $__dateTimeFilter with combined date and time filter', () => {
|
||||
const result = replaceMacros('WHERE $__dateTimeFilter(d, ts)');
|
||||
const result = replaceMacros({
|
||||
sqlTemplate: 'WHERE $__dateTimeFilter(d, ts)',
|
||||
});
|
||||
expect(result).toBe(
|
||||
'WHERE (d >= toDate(fromUnixTimestamp64Milli({startDateMilliseconds:Int64})) AND d <= toDate(fromUnixTimestamp64Milli({endDateMilliseconds:Int64}))) AND (ts >= toDateTime(fromUnixTimestamp64Milli({startDateMilliseconds:Int64})) AND ts <= toDateTime(fromUnixTimestamp64Milli({endDateMilliseconds:Int64})))',
|
||||
);
|
||||
});
|
||||
|
||||
it('should replace $__dt as an alias for dateTimeFilter', () => {
|
||||
const result = replaceMacros('WHERE $__dt(d, ts)');
|
||||
const result = replaceMacros({
|
||||
sqlTemplate: 'WHERE $__dt(d, ts)',
|
||||
});
|
||||
expect(result).toBe(
|
||||
'WHERE (d >= toDate(fromUnixTimestamp64Milli({startDateMilliseconds:Int64})) AND d <= toDate(fromUnixTimestamp64Milli({endDateMilliseconds:Int64}))) AND (ts >= toDateTime(fromUnixTimestamp64Milli({startDateMilliseconds:Int64})) AND ts <= toDateTime(fromUnixTimestamp64Milli({endDateMilliseconds:Int64})))',
|
||||
);
|
||||
});
|
||||
|
||||
it('should replace $__timeInterval with interval bucketing expression', () => {
|
||||
const result = replaceMacros('SELECT $__timeInterval(ts)');
|
||||
const result = replaceMacros({
|
||||
sqlTemplate: 'SELECT $__timeInterval(ts)',
|
||||
});
|
||||
expect(result).toBe(
|
||||
'SELECT toStartOfInterval(toDateTime(ts), INTERVAL {intervalSeconds:Int64} second)',
|
||||
);
|
||||
});
|
||||
|
||||
it('should replace $__timeInterval_ms with millisecond interval bucketing', () => {
|
||||
const result = replaceMacros('SELECT $__timeInterval_ms(ts)');
|
||||
const result = replaceMacros({
|
||||
sqlTemplate: 'SELECT $__timeInterval_ms(ts)',
|
||||
});
|
||||
expect(result).toBe(
|
||||
'SELECT toStartOfInterval(toDateTime64(ts, 3), INTERVAL {intervalMilliseconds:Int64} millisecond)',
|
||||
);
|
||||
});
|
||||
|
||||
it('should replace $__interval_s with interval seconds param', () => {
|
||||
expect(replaceMacros('INTERVAL $__interval_s second')).toBe(
|
||||
'INTERVAL {intervalSeconds:Int64} second',
|
||||
);
|
||||
expect(
|
||||
replaceMacros({ sqlTemplate: 'INTERVAL $__interval_s second' }),
|
||||
).toBe('INTERVAL {intervalSeconds:Int64} second');
|
||||
});
|
||||
|
||||
it('should replace multiple macros in one query', () => {
|
||||
const result = replaceMacros(
|
||||
'SELECT $__timeInterval(ts), count() FROM t WHERE $__timeFilter(ts) GROUP BY 1',
|
||||
);
|
||||
const result = replaceMacros({
|
||||
sqlTemplate:
|
||||
'SELECT $__timeInterval(ts), count() FROM t WHERE $__timeFilter(ts) GROUP BY 1',
|
||||
});
|
||||
expect(result).toContain('toStartOfInterval');
|
||||
expect(result).toContain(
|
||||
'ts >= toDateTime(fromUnixTimestamp64Milli({startDateMilliseconds:Int64}))',
|
||||
|
|
@ -91,34 +115,135 @@ describe('replaceMacros', () => {
|
|||
});
|
||||
|
||||
it('should throw on wrong argument count', () => {
|
||||
expect(() => replaceMacros('$__timeFilter(a, b)')).toThrow(
|
||||
expect(() => replaceMacros({ sqlTemplate: '$__timeFilter(a, b)' })).toThrow(
|
||||
'expects 1 argument(s), but got 2',
|
||||
);
|
||||
});
|
||||
|
||||
it('should throw on missing close bracket', () => {
|
||||
expect(() => replaceMacros('$__timeFilter(col')).toThrow(
|
||||
expect(() => replaceMacros({ sqlTemplate: '$__timeFilter(col' })).toThrow(
|
||||
'Failed to parse macro arguments',
|
||||
);
|
||||
});
|
||||
|
||||
it('should replace $__filters with provided filtersSQL', () => {
|
||||
const result = replaceMacros(
|
||||
'WHERE $__filters',
|
||||
{ sqlTemplate: 'WHERE $__filters' },
|
||||
"(col = 'val') AND (x > 1)",
|
||||
);
|
||||
expect(result).toBe("WHERE (col = 'val') AND (x > 1)");
|
||||
});
|
||||
|
||||
it('should replace $__filters with fallback when no filtersSQL provided', () => {
|
||||
expect(replaceMacros('WHERE $__filters')).toBe(
|
||||
expect(replaceMacros({ sqlTemplate: 'WHERE $__filters' })).toBe(
|
||||
'WHERE (1=1 /** no filters applied */)',
|
||||
);
|
||||
});
|
||||
|
||||
it('should replace $__filters with fallback when filtersSQL is empty', () => {
|
||||
expect(replaceMacros('WHERE $__filters', '')).toBe(
|
||||
expect(replaceMacros({ sqlTemplate: 'WHERE $__filters' }, '')).toBe(
|
||||
'WHERE (1=1 /** no filters applied */)',
|
||||
);
|
||||
});
|
||||
|
||||
it('should replace $__sourceTable with databaseName.tableName', () => {
|
||||
const result = replaceMacros({
|
||||
sqlTemplate: 'SELECT * FROM $__sourceTable',
|
||||
from: { databaseName: 'otel', tableName: 'otel_logs' },
|
||||
});
|
||||
expect(result).toBe('SELECT * FROM `otel`.`otel_logs`');
|
||||
});
|
||||
|
||||
it('should replace $__sourceTable in a complex query', () => {
|
||||
const result = replaceMacros({
|
||||
sqlTemplate: 'SELECT count() FROM $__sourceTable WHERE $__timeFilter(ts)',
|
||||
from: { databaseName: 'default', tableName: 'my_table' },
|
||||
});
|
||||
expect(result).toContain('FROM `default`.`my_table`');
|
||||
expect(result).toContain('ts >=');
|
||||
});
|
||||
|
||||
it('should throw when $__sourceTable is used without a source', () => {
|
||||
expect(() =>
|
||||
replaceMacros({ sqlTemplate: 'SELECT * FROM $__sourceTable' }),
|
||||
).toThrow("Macro '$__sourceTable' requires a source to be selected");
|
||||
});
|
||||
|
||||
it('should replace $__sourceTable(gauge) with the gauge metric table', () => {
|
||||
const result = replaceMacros({
|
||||
sqlTemplate: 'SELECT * FROM $__sourceTable(gauge)',
|
||||
from: { databaseName: 'otel', tableName: 'otel_logs' },
|
||||
metricTables: ALL_METRIC_TABLES,
|
||||
});
|
||||
expect(result).toBe('SELECT * FROM `otel`.`otel_metrics_gauge`');
|
||||
});
|
||||
|
||||
it('should replace $__sourceTable(sum) with the sum metric table', () => {
|
||||
const result = replaceMacros({
|
||||
sqlTemplate: 'SELECT * FROM $__sourceTable(sum)',
|
||||
from: { databaseName: 'otel', tableName: 'otel_logs' },
|
||||
metricTables: ALL_METRIC_TABLES,
|
||||
});
|
||||
expect(result).toBe('SELECT * FROM `otel`.`otel_metrics_sum`');
|
||||
});
|
||||
|
||||
it('should replace $__sourceTable(histogram) with the histogram metric table', () => {
|
||||
const result = replaceMacros({
|
||||
sqlTemplate: 'SELECT * FROM $__sourceTable(histogram)',
|
||||
from: { databaseName: 'otel', tableName: 'otel_logs' },
|
||||
metricTables: ALL_METRIC_TABLES,
|
||||
});
|
||||
expect(result).toBe('SELECT * FROM `otel`.`otel_metrics_histogram`');
|
||||
});
|
||||
|
||||
it('should throw when $__sourceTable is called with an invalid metric type', () => {
|
||||
expect(() =>
|
||||
replaceMacros({
|
||||
sqlTemplate: 'SELECT * FROM $__sourceTable(invalid)',
|
||||
from: { databaseName: 'otel', tableName: 'otel_logs' },
|
||||
metricTables: ALL_METRIC_TABLES,
|
||||
}),
|
||||
).toThrow('Expected a valid metrics data type');
|
||||
});
|
||||
|
||||
it('should throw when $__sourceTable is called with a metric type that has no table', () => {
|
||||
expect(() =>
|
||||
replaceMacros({
|
||||
sqlTemplate: 'SELECT * FROM $__sourceTable(gauge)',
|
||||
from: { databaseName: 'otel', tableName: 'otel_logs' },
|
||||
metricTables: {} as MetricTable,
|
||||
}),
|
||||
).toThrow("No table configured for metric type 'gauge'");
|
||||
});
|
||||
|
||||
it('should throw when $__sourceTable is called with a metric type but no metricTables', () => {
|
||||
expect(() =>
|
||||
replaceMacros({
|
||||
sqlTemplate: 'SELECT * FROM $__sourceTable(gauge)',
|
||||
from: { databaseName: 'otel', tableName: 'otel_logs' },
|
||||
}),
|
||||
).toThrow(
|
||||
'with a metric type argument requires a metrics source to be selected',
|
||||
);
|
||||
});
|
||||
|
||||
it('should throw when $__sourceTable is used without a metricType but metricTables is set', () => {
|
||||
expect(() =>
|
||||
replaceMacros({
|
||||
sqlTemplate: 'SELECT * FROM $__sourceTable',
|
||||
from: { databaseName: 'otel', tableName: 'otel_logs' },
|
||||
metricTables: ALL_METRIC_TABLES,
|
||||
}),
|
||||
).toThrow('requires a metricType when a metrics source is selected');
|
||||
});
|
||||
|
||||
it('should throw when $__sourceTable is called with too many arguments', () => {
|
||||
expect(() =>
|
||||
replaceMacros({
|
||||
sqlTemplate: 'SELECT * FROM $__sourceTable(gauge, sum)',
|
||||
from: { databaseName: 'otel', tableName: 'otel_logs' },
|
||||
metricTables: ALL_METRIC_TABLES,
|
||||
}),
|
||||
).toThrow('expects 0-1 argument(s), but got 2');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1891,6 +1891,28 @@ describe('renderChartConfig', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('renders sql filters raw when source has no tableName (metric source)', async () => {
|
||||
const result = await renderChartConfig(
|
||||
{
|
||||
configType: 'sql',
|
||||
sqlTemplate: 'SELECT * FROM logs WHERE $__filters',
|
||||
connection: 'conn-1',
|
||||
dateRange: [start, end],
|
||||
source: 'source-1',
|
||||
from: { databaseName: 'default', tableName: '' },
|
||||
filters: [
|
||||
{ type: 'sql', condition: 'duration > 100' },
|
||||
{ type: 'sql_ast', operator: '=', left: 'status', right: "'ok'" },
|
||||
],
|
||||
},
|
||||
mockMetadata,
|
||||
undefined,
|
||||
);
|
||||
expect(result.sql).toBe(
|
||||
"SELECT * FROM logs WHERE ((duration > 100) AND (status = 'ok'))",
|
||||
);
|
||||
});
|
||||
|
||||
it('skips filters without source metadata (no from)', async () => {
|
||||
const result = await renderChartConfig(
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1523,17 +1523,23 @@ async function renderFiltersToSql(
|
|||
const conditions = (
|
||||
await Promise.all(
|
||||
chartConfig.filters.map(async filter => {
|
||||
const hasSourceTable =
|
||||
chartConfig.from &&
|
||||
chartConfig.from.tableName && // tableName is falsy for metric sources
|
||||
chartConfig.source;
|
||||
|
||||
if (filter.type === 'sql_ast') {
|
||||
return `(${filter.left} ${filter.operator} ${filter.right})`;
|
||||
} else if (filter.type === 'sql' && !hasSourceTable) {
|
||||
return `(${filter.condition})`; // Don't pass to renderWhereExpressionStr since it requires source table metadata
|
||||
} else if (
|
||||
(filter.type === 'lucene' || filter.type === 'sql') &&
|
||||
filter.condition.trim() &&
|
||||
chartConfig.from &&
|
||||
chartConfig.source
|
||||
hasSourceTable
|
||||
) {
|
||||
const condition = await renderWhereExpressionStr({
|
||||
condition: filter.condition,
|
||||
from: chartConfig.from,
|
||||
from: chartConfig.from!,
|
||||
language: filter.type,
|
||||
implicitColumnExpression: chartConfig.implicitColumnExpression,
|
||||
metadata,
|
||||
|
|
@ -1555,10 +1561,7 @@ export async function renderRawSqlChartConfig(
|
|||
const displayType = chartConfig.displayType ?? DisplayType.Table;
|
||||
|
||||
const filtersSQL = await renderFiltersToSql(chartConfig, metadata);
|
||||
const sqlWithMacrosReplaced = replaceMacros(
|
||||
chartConfig.sqlTemplate,
|
||||
filtersSQL,
|
||||
);
|
||||
const sqlWithMacrosReplaced = replaceMacros(chartConfig, filtersSQL);
|
||||
|
||||
// eslint-disable-next-line security/detect-object-injection
|
||||
const queryParams = QUERY_PARAMS_BY_DISPLAY_TYPE[displayType];
|
||||
|
|
|
|||
|
|
@ -1,8 +1,20 @@
|
|||
import { splitAndTrimWithBracket } from './core/utils';
|
||||
import { renderQueryParam } from './rawSqlParams';
|
||||
import {
|
||||
MetricsDataType,
|
||||
MetricsDataTypeSchema,
|
||||
RawSqlChartConfig,
|
||||
} from './types';
|
||||
|
||||
function expectArgs(macroName: string, args: string[], expected: number) {
|
||||
if (args.length !== expected) {
|
||||
function expectArgs(
|
||||
macroName: string,
|
||||
args: string[],
|
||||
minArgs: number,
|
||||
maxArgs: number,
|
||||
) {
|
||||
if (args.length < minArgs || args.length > maxArgs) {
|
||||
const expected =
|
||||
minArgs === maxArgs ? `${minArgs}` : `${minArgs}-${maxArgs}`;
|
||||
throw new Error(
|
||||
`Macro '${macroName}' expects ${expected} argument(s), but got ${args.length}`,
|
||||
);
|
||||
|
|
@ -24,63 +36,72 @@ const timeToDateTime64 = (msParam: string) =>
|
|||
|
||||
type Macro = {
|
||||
name: string;
|
||||
argCount: number;
|
||||
minArgs: number;
|
||||
maxArgs: number;
|
||||
replace: (args: string[]) => string;
|
||||
};
|
||||
|
||||
const MACROS: Macro[] = [
|
||||
{
|
||||
name: 'fromTime',
|
||||
argCount: 0,
|
||||
minArgs: 0,
|
||||
maxArgs: 0,
|
||||
replace: () => timeToDateTime(startMs()),
|
||||
},
|
||||
{
|
||||
name: 'toTime',
|
||||
argCount: 0,
|
||||
minArgs: 0,
|
||||
maxArgs: 0,
|
||||
replace: () => timeToDateTime(endMs()),
|
||||
},
|
||||
{
|
||||
name: 'fromTime_ms',
|
||||
argCount: 0,
|
||||
minArgs: 0,
|
||||
maxArgs: 0,
|
||||
replace: () => timeToDateTime64(startMs()),
|
||||
},
|
||||
{
|
||||
name: 'toTime_ms',
|
||||
argCount: 0,
|
||||
minArgs: 0,
|
||||
maxArgs: 0,
|
||||
replace: () => timeToDateTime64(endMs()),
|
||||
},
|
||||
{
|
||||
name: 'timeFilter',
|
||||
argCount: 1,
|
||||
minArgs: 1,
|
||||
maxArgs: 1,
|
||||
replace: (args: string[]) => {
|
||||
expectArgs('timeFilter', args, 1);
|
||||
expectArgs('timeFilter', args, 1, 1);
|
||||
const [col] = args;
|
||||
return `${col} >= ${timeToDateTime(startMs())} AND ${col} <= ${timeToDateTime(endMs())}`;
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'timeFilter_ms',
|
||||
argCount: 1,
|
||||
minArgs: 1,
|
||||
maxArgs: 1,
|
||||
replace: (args: string[]) => {
|
||||
expectArgs('timeFilter_ms', args, 1);
|
||||
expectArgs('timeFilter_ms', args, 1, 1);
|
||||
const [col] = args;
|
||||
return `${col} >= ${timeToDateTime64(startMs())} AND ${col} <= ${timeToDateTime64(endMs())}`;
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'dateFilter',
|
||||
argCount: 1,
|
||||
minArgs: 1,
|
||||
maxArgs: 1,
|
||||
replace: (args: string[]) => {
|
||||
expectArgs('dateFilter', args, 1);
|
||||
expectArgs('dateFilter', args, 1, 1);
|
||||
const [col] = args;
|
||||
return `${col} >= ${timeToDate(startMs())} AND ${col} <= ${timeToDate(endMs())}`;
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'dateTimeFilter',
|
||||
argCount: 2,
|
||||
minArgs: 2,
|
||||
maxArgs: 2,
|
||||
replace: (args: string[]) => {
|
||||
expectArgs('dateTimeFilter', args, 2);
|
||||
expectArgs('dateTimeFilter', args, 2, 2);
|
||||
const [dateCol, timeCol] = args;
|
||||
const dateFilter = `(${dateCol} >= ${timeToDate(startMs())} AND ${dateCol} <= ${timeToDate(endMs())})`;
|
||||
const timeFilter = `(${timeCol} >= ${timeToDateTime(startMs())} AND ${timeCol} <= ${timeToDateTime(endMs())})`;
|
||||
|
|
@ -89,9 +110,10 @@ const MACROS: Macro[] = [
|
|||
},
|
||||
{
|
||||
name: 'dt',
|
||||
argCount: 2,
|
||||
minArgs: 2,
|
||||
maxArgs: 2,
|
||||
replace: (args: string[]) => {
|
||||
expectArgs('dt', args, 2);
|
||||
expectArgs('dt', args, 2, 2);
|
||||
const [dateCol, timeCol] = args;
|
||||
const dateFilter = `(${dateCol} >= ${timeToDate(startMs())} AND ${dateCol} <= ${timeToDate(endMs())})`;
|
||||
const timeFilter = `(${timeCol} >= ${timeToDateTime(startMs())} AND ${timeCol} <= ${timeToDateTime(endMs())})`;
|
||||
|
|
@ -100,33 +122,42 @@ const MACROS: Macro[] = [
|
|||
},
|
||||
{
|
||||
name: 'timeInterval',
|
||||
argCount: 1,
|
||||
minArgs: 1,
|
||||
maxArgs: 1,
|
||||
replace: (args: string[]) => {
|
||||
expectArgs('timeInterval', args, 1);
|
||||
expectArgs('timeInterval', args, 1, 1);
|
||||
const [col] = args;
|
||||
return `toStartOfInterval(toDateTime(${col}), INTERVAL ${intervalS()} second)`;
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'timeInterval_ms',
|
||||
argCount: 1,
|
||||
minArgs: 1,
|
||||
maxArgs: 1,
|
||||
replace: (args: string[]) => {
|
||||
expectArgs('timeInterval_ms', args, 1);
|
||||
expectArgs('timeInterval_ms', args, 1, 1);
|
||||
const [col] = args;
|
||||
return `toStartOfInterval(toDateTime64(${col}, 3), INTERVAL ${intervalMs()} millisecond)`;
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'interval_s',
|
||||
argCount: 0,
|
||||
minArgs: 0,
|
||||
maxArgs: 0,
|
||||
replace: () => intervalS(),
|
||||
},
|
||||
];
|
||||
|
||||
/** Macro metadata for autocomplete suggestions */
|
||||
export const MACRO_SUGGESTIONS = [
|
||||
...MACROS.map(({ name, argCount }) => ({ name, argCount })),
|
||||
{ name: 'filters', argCount: 0 },
|
||||
...MACROS.map(({ name, minArgs, maxArgs }) => ({ name, minArgs, maxArgs })),
|
||||
{ name: 'filters', minArgs: 0, maxArgs: 0 },
|
||||
{ name: 'sourceTable', minArgs: 0, maxArgs: 1 },
|
||||
...Object.values(MetricsDataType).map(type => ({
|
||||
name: `sourceTable(${type})`,
|
||||
minArgs: 0,
|
||||
maxArgs: 0,
|
||||
})),
|
||||
];
|
||||
|
||||
type MacroMatch = {
|
||||
|
|
@ -187,23 +218,77 @@ function findMacros(input: string, name: string): MacroMatch[] {
|
|||
const NO_FILTERS = '(1=1 /** no filters applied */)';
|
||||
|
||||
export function replaceMacros(
|
||||
sqlTemplate: string,
|
||||
chartConfig: Pick<RawSqlChartConfig, 'sqlTemplate' | 'from' | 'metricTables'>,
|
||||
filtersSQL?: string,
|
||||
): string {
|
||||
const { from, metricTables } = chartConfig;
|
||||
|
||||
const allMacros: Macro[] = [
|
||||
...MACROS,
|
||||
{
|
||||
name: 'filters',
|
||||
argCount: 0,
|
||||
minArgs: 0,
|
||||
maxArgs: 0,
|
||||
replace: () => filtersSQL || NO_FILTERS,
|
||||
},
|
||||
{
|
||||
name: 'sourceTable',
|
||||
minArgs: 0,
|
||||
maxArgs: 1,
|
||||
replace: (args: string[]) => {
|
||||
expectArgs('sourceTable', args, 0, 1);
|
||||
if (!from) {
|
||||
throw new Error(
|
||||
"Macro '$__sourceTable' requires a source to be selected",
|
||||
);
|
||||
}
|
||||
|
||||
if (args.length === 0 && metricTables) {
|
||||
throw new Error(
|
||||
"Macro '$__sourceTable(metricType)' requires a metricType when a metrics source is selected",
|
||||
);
|
||||
}
|
||||
|
||||
if (args.length === 0 && !from.tableName) {
|
||||
throw new Error(
|
||||
"Macro '$__sourceTable' requires a source with a table to be selected when no arguments are provided",
|
||||
);
|
||||
}
|
||||
|
||||
if (args.length === 0) {
|
||||
return `\`${from.databaseName}\`.\`${from.tableName}\``;
|
||||
}
|
||||
|
||||
if (!metricTables) {
|
||||
throw new Error(
|
||||
"Macro '$__sourceTable(metricType)' with a metric type argument requires a metrics source to be selected",
|
||||
);
|
||||
}
|
||||
|
||||
const metricsTypeParseResult = MetricsDataTypeSchema.safeParse(args[0]);
|
||||
if (!metricsTypeParseResult.success) {
|
||||
throw new Error(
|
||||
`Macro '$__sourceTable(metricType)' invalid argument '${args[0]}'. Expected a valid metrics data type (${Object.values(MetricsDataType).join(', ')}).`,
|
||||
);
|
||||
}
|
||||
|
||||
const metricType = metricsTypeParseResult.data;
|
||||
const table = metricTables[metricType];
|
||||
if (!table) {
|
||||
throw new Error(
|
||||
`Macro '$__sourceTable(metricType)': No table configured for metric type '${metricType}'.`,
|
||||
);
|
||||
}
|
||||
return `\`${from.databaseName}\`.\`${table}\``;
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const sortedMacros = allMacros.sort(
|
||||
(m1, m2) => m2.name.length - m1.name.length,
|
||||
);
|
||||
|
||||
let sql = sqlTemplate;
|
||||
let sql = chartConfig.sqlTemplate;
|
||||
for (const macro of sortedMacros) {
|
||||
const matches = findMacros(sql, macro.name);
|
||||
|
||||
|
|
|
|||
|
|
@ -91,7 +91,7 @@ const TIME_CHART_EXAMPLE_SQL = `SELECT
|
|||
toStartOfInterval(TimestampTime, INTERVAL {intervalSeconds:Int64} second) AS ts, -- (Timestamp column)
|
||||
ServiceName, -- (Group name column)
|
||||
count() -- (Series value column)
|
||||
FROM otel_logs
|
||||
FROM $__sourceTable
|
||||
WHERE TimestampTime >= fromUnixTimestamp64Milli ({startDateMilliseconds:Int64})
|
||||
AND TimestampTime < fromUnixTimestamp64Milli ({endDateMilliseconds:Int64})
|
||||
AND $__filters
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ export enum MetricsDataType {
|
|||
ExponentialHistogram = 'exponential histogram',
|
||||
}
|
||||
|
||||
export const MetricsDataTypeSchema = z.nativeEnum(MetricsDataType);
|
||||
|
||||
// --------------------------
|
||||
// UI
|
||||
// --------------------------
|
||||
|
|
@ -590,6 +592,7 @@ const RawSqlChartConfigSchema = RawSqlBaseChartConfigSchema.extend({
|
|||
.object({ databaseName: z.string(), tableName: z.string() })
|
||||
.optional(),
|
||||
implicitColumnExpression: z.string().optional(),
|
||||
metricTables: MetricTableSchema.optional(),
|
||||
});
|
||||
|
||||
export type RawSqlChartConfig = z.infer<typeof RawSqlChartConfigSchema>;
|
||||
|
|
|
|||
Loading…
Reference in a new issue