diff --git a/.changeset/calm-dragons-lie.md b/.changeset/calm-dragons-lie.md new file mode 100644 index 00000000..5f8df9f4 --- /dev/null +++ b/.changeset/calm-dragons-lie.md @@ -0,0 +1,6 @@ +--- +"@hyperdx/common-utils": patch +"@hyperdx/app": patch +--- + +feat: Add a minimum date to MV configuration diff --git a/packages/app/src/components/MaterializedViews/MVConfigSummary.tsx b/packages/app/src/components/MaterializedViews/MVConfigSummary.tsx index a67b04da..ef8a34fa 100644 --- a/packages/app/src/components/MaterializedViews/MVConfigSummary.tsx +++ b/packages/app/src/components/MaterializedViews/MVConfigSummary.tsx @@ -1,7 +1,9 @@ import { useMemo } from 'react'; import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils'; import { MaterializedViewConfiguration } from '@hyperdx/common-utils/dist/types'; -import { Group, Pill, Stack, Table, Text } from '@mantine/core'; +import { Grid, Group, Pill, Stack, Table, Text } from '@mantine/core'; + +import { FormatTime } from '@/useFormatTime'; export default function MVConfigSummary({ config, @@ -29,12 +31,24 @@ export default function MVConfigSummary({ return ( -
- - Minimum Granularity - - {config.minGranularity} -
+ + + + Granularity + + {config.minGranularity} + + {config.minDate && ( + + + Minimum Date + + + + + + )} +
diff --git a/packages/app/src/components/SourceForm.tsx b/packages/app/src/components/SourceForm.tsx index d785e7f7..dd15f308 100644 --- a/packages/app/src/components/SourceForm.tsx +++ b/packages/app/src/components/SourceForm.tsx @@ -33,6 +33,7 @@ import { Text, Tooltip, } from '@mantine/core'; +import { DateInput } from '@mantine/dates'; import { notifications } from '@mantine/notifications'; import { IconCirclePlus, @@ -402,7 +403,7 @@ function MaterializedViewFormSection({ - + Timestamp Column @@ -445,6 +446,38 @@ function MaterializedViewFormSection({ )} /> + + + + Minimum Date + + + + + ( + + field.onChange(dateStr ? dateStr.toISOString() : null) + } + clearable + highlightToday + placeholder="YYYY-MM-DD HH:mm:ss" + valueFormat="YYYY-MM-DD HH:mm:ss" + /> + )} + /> + diff --git a/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts b/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts index 2dbc4a13..b9a683e2 100644 --- a/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts +++ b/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts @@ -781,6 +781,122 @@ describe('materializedViews', () => { 'Custom count() expressions are not supported with materialized views.', ]); }); + + it("should not use the materialized view when the chart config references a date range prior to the MV's date range", async () => { + const chartConfig: ChartConfigWithOptDateRange = { + from: { + databaseName: 'default', + tableName: 'otel_spans', + }, + select: [ + { + valueExpression: '', + aggFn: 'count', + }, + ], + where: '', + connection: 'test-connection', + dateRange: [ + new Date('2022-12-01T00:00:00Z'), + new Date('2022-12-31T23:59:59Z'), + ], + }; + + const result = await tryConvertConfigToMaterializedViewSelect( + chartConfig, + { + ...MV_CONFIG_METRIC_ROLLUP_1M, + minDate: '2023-01-01T00:00:00Z', + }, + metadata, + ); + + expect(result.optimizedConfig).toBeUndefined(); + expect(result.errors).toEqual([ + 'The selected date range includes dates for which this view does not contain data.', + ]); + }); + + it('should not use the materialized view when the chart config has no date range and the MV has a minimum date', async () => { + const chartConfig: ChartConfigWithOptDateRange = { + from: { + databaseName: 'default', + tableName: 'otel_spans', + }, + select: [ + { + valueExpression: '', + aggFn: 'count', + }, + ], + where: '', + connection: 'test-connection', + }; + + const result = await tryConvertConfigToMaterializedViewSelect( + chartConfig, + { + ...MV_CONFIG_METRIC_ROLLUP_1M, + minDate: '2023-01-01T00:00:00Z', + }, + metadata, + ); + + expect(result.optimizedConfig).toBeUndefined(); + expect(result.errors).toEqual([ + 'The selected date range includes dates for which this view does not contain data.', + ]); + }); + + it("should use the materialized view when the chart config's date range is after the MV's minimum date", async () => { + const chartConfig: ChartConfigWithOptDateRange = { + from: { + databaseName: 'default', + tableName: 'otel_spans', + }, + select: [ + { + valueExpression: '', + aggFn: 'count', + }, + ], + where: '', + connection: 'test-connection', + dateRange: [ + new Date('2023-12-01T00:00:00Z'), + new Date('2023-12-31T23:59:59Z'), + ], + }; + + const result = await tryConvertConfigToMaterializedViewSelect( + chartConfig, + { + ...MV_CONFIG_METRIC_ROLLUP_1M, + minDate: '2023-01-01T00:00:00Z', + }, + metadata, + ); + + expect(result.optimizedConfig).toEqual({ + from: { + databaseName: 'default', + tableName: 'metrics_rollup_1m', + }, + dateRange: [ + new Date('2023-12-01T00:00:00Z'), + new Date('2023-12-31T23:59:59Z'), + ], + select: [ + { + valueExpression: 'count', + aggFn: 'sum', + }, + ], + where: '', + connection: 'test-connection', + }); + expect(result.errors).toBeUndefined(); + }); }); describe('isUnsupportedCountFunction', () => { diff --git a/packages/common-utils/src/core/materializedViews.ts b/packages/common-utils/src/core/materializedViews.ts index fc66b1a8..43c3cc56 100644 --- a/packages/common-utils/src/core/materializedViews.ts +++ b/packages/common-utils/src/core/materializedViews.ts @@ -153,6 +153,24 @@ function mvConfigSupportsGranularity( return chartGranularitySeconds >= mvGranularitySeconds; } +function mvConfigSupportsDateRange( + mvConfig: MaterializedViewConfiguration, + chartConfig: ChartConfigWithOptDateRange, +) { + if (mvConfig.minDate && !chartConfig.dateRange) { + return false; + } + + if (!mvConfig.minDate || !chartConfig.dateRange) { + return true; + } + + const [startDate] = chartConfig.dateRange; + const minDate = new Date(mvConfig.minDate); + + return startDate >= minDate; +} + const COUNT_FUNCTION_PATTERN = /\bcount(If)?\s*\(/i; export function isUnsupportedCountFunction(selectItem: SelectItem): boolean { return COUNT_FUNCTION_PATTERN.test(selectItem.valueExpression); @@ -268,6 +286,14 @@ export async function tryConvertConfigToMaterializedViewSelect< }; } + if (mvConfig.minDate && !mvConfigSupportsDateRange(mvConfig, chartConfig)) { + return { + errors: [ + 'The selected date range includes dates for which this view does not contain data.', + ], + }; + } + if (!mvConfigSupportsGranularity(mvConfig, chartConfig)) { const error = chartConfig.granularity ? `Granularity must be at least ${mvConfig.minGranularity}.` diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 9419a938..64ae0307 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -646,6 +646,7 @@ export const MaterializedViewConfigurationSchema = z.object({ tableName: z.string().min(1, 'Materialized View Table is required'), dimensionColumns: z.string(), minGranularity: SQLIntervalSchema, + minDate: z.string().datetime().nullish(), timestampColumn: z .string() .min(1, 'Materialized View Timestamp column is required'),