From ab7645deb69a6e847fb3965666dcfcbe5ccaab29 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Mon, 5 Jan 2026 12:59:39 -0500 Subject: [PATCH] feat: Add a minimum date to MV configuration (#1549) Closes HDX-3129 # Summary This PR adds a "minimum date" setting to Materialized view configurations. The minimum date is optional. When provided, the materialized view will not be used for any query that includes date ranges prior to the minimum date. ## Demo https://github.com/user-attachments/assets/de0efb45-786e-4b45-b053-f1dc7c322f2e --- .changeset/calm-dragons-lie.md | 6 + .../MaterializedViews/MVConfigSummary.tsx | 28 +++-- packages/app/src/components/SourceForm.tsx | 35 +++++- .../__tests__/materializedViews.test.ts | 116 ++++++++++++++++++ .../src/core/materializedViews.ts | 26 ++++ packages/common-utils/src/types.ts | 1 + 6 files changed, 204 insertions(+), 8 deletions(-) create mode 100644 .changeset/calm-dragons-lie.md diff --git a/.changeset/calm-dragons-lie.md b/.changeset/calm-dragons-lie.md new file mode 100644 index 00000000..5f8df9f4 --- /dev/null +++ b/.changeset/calm-dragons-lie.md @@ -0,0 +1,6 @@ +--- +"@hyperdx/common-utils": patch +"@hyperdx/app": patch +--- + +feat: Add a minimum date to MV configuration diff --git a/packages/app/src/components/MaterializedViews/MVConfigSummary.tsx b/packages/app/src/components/MaterializedViews/MVConfigSummary.tsx index a67b04da..ef8a34fa 100644 --- a/packages/app/src/components/MaterializedViews/MVConfigSummary.tsx +++ b/packages/app/src/components/MaterializedViews/MVConfigSummary.tsx @@ -1,7 +1,9 @@ import { useMemo } from 'react'; import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils'; import { MaterializedViewConfiguration } from '@hyperdx/common-utils/dist/types'; -import { Group, Pill, Stack, Table, Text } from '@mantine/core'; +import { Grid, Group, Pill, Stack, Table, Text } from '@mantine/core'; + +import { FormatTime } from '@/useFormatTime'; export default function MVConfigSummary({ config, @@ -29,12 +31,24 @@ export default function MVConfigSummary({ return ( -
- - Minimum Granularity - - {config.minGranularity} -
+ + + + Granularity + + {config.minGranularity} + + {config.minDate && ( + + + Minimum Date + + + + + + )} +
diff --git a/packages/app/src/components/SourceForm.tsx b/packages/app/src/components/SourceForm.tsx index d785e7f7..dd15f308 100644 --- a/packages/app/src/components/SourceForm.tsx +++ b/packages/app/src/components/SourceForm.tsx @@ -33,6 +33,7 @@ import { Text, Tooltip, } from '@mantine/core'; +import { DateInput } from '@mantine/dates'; import { notifications } from '@mantine/notifications'; import { IconCirclePlus, @@ -402,7 +403,7 @@ function MaterializedViewFormSection({ - + Timestamp Column @@ -445,6 +446,38 @@ function MaterializedViewFormSection({ )} /> + + + + Minimum Date + + + + + ( + + field.onChange(dateStr ? dateStr.toISOString() : null) + } + clearable + highlightToday + placeholder="YYYY-MM-DD HH:mm:ss" + valueFormat="YYYY-MM-DD HH:mm:ss" + /> + )} + /> + diff --git a/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts b/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts index 2dbc4a13..b9a683e2 100644 --- a/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts +++ b/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts @@ -781,6 +781,122 @@ describe('materializedViews', () => { 'Custom count() expressions are not supported with materialized views.', ]); }); + + it("should not use the materialized view when the chart config references a date range prior to the MV's date range", async () => { + const chartConfig: ChartConfigWithOptDateRange = { + from: { + databaseName: 'default', + tableName: 'otel_spans', + }, + select: [ + { + valueExpression: '', + aggFn: 'count', + }, + ], + where: '', + connection: 'test-connection', + dateRange: [ + new Date('2022-12-01T00:00:00Z'), + new Date('2022-12-31T23:59:59Z'), + ], + }; + + const result = await tryConvertConfigToMaterializedViewSelect( + chartConfig, + { + ...MV_CONFIG_METRIC_ROLLUP_1M, + minDate: '2023-01-01T00:00:00Z', + }, + metadata, + ); + + expect(result.optimizedConfig).toBeUndefined(); + expect(result.errors).toEqual([ + 'The selected date range includes dates for which this view does not contain data.', + ]); + }); + + it('should not use the materialized view when the chart config has no date range and the MV has a minimum date', async () => { + const chartConfig: ChartConfigWithOptDateRange = { + from: { + databaseName: 'default', + tableName: 'otel_spans', + }, + select: [ + { + valueExpression: '', + aggFn: 'count', + }, + ], + where: '', + connection: 'test-connection', + }; + + const result = await tryConvertConfigToMaterializedViewSelect( + chartConfig, + { + ...MV_CONFIG_METRIC_ROLLUP_1M, + minDate: '2023-01-01T00:00:00Z', + }, + metadata, + ); + + expect(result.optimizedConfig).toBeUndefined(); + expect(result.errors).toEqual([ + 'The selected date range includes dates for which this view does not contain data.', + ]); + }); + + it("should use the materialized view when the chart config's date range is after the MV's minimum date", async () => { + const chartConfig: ChartConfigWithOptDateRange = { + from: { + databaseName: 'default', + tableName: 'otel_spans', + }, + select: [ + { + valueExpression: '', + aggFn: 'count', + }, + ], + where: '', + connection: 'test-connection', + dateRange: [ + new Date('2023-12-01T00:00:00Z'), + new Date('2023-12-31T23:59:59Z'), + ], + }; + + const result = await tryConvertConfigToMaterializedViewSelect( + chartConfig, + { + ...MV_CONFIG_METRIC_ROLLUP_1M, + minDate: '2023-01-01T00:00:00Z', + }, + metadata, + ); + + expect(result.optimizedConfig).toEqual({ + from: { + databaseName: 'default', + tableName: 'metrics_rollup_1m', + }, + dateRange: [ + new Date('2023-12-01T00:00:00Z'), + new Date('2023-12-31T23:59:59Z'), + ], + select: [ + { + valueExpression: 'count', + aggFn: 'sum', + }, + ], + where: '', + connection: 'test-connection', + }); + expect(result.errors).toBeUndefined(); + }); }); describe('isUnsupportedCountFunction', () => { diff --git a/packages/common-utils/src/core/materializedViews.ts b/packages/common-utils/src/core/materializedViews.ts index fc66b1a8..43c3cc56 100644 --- a/packages/common-utils/src/core/materializedViews.ts +++ b/packages/common-utils/src/core/materializedViews.ts @@ -153,6 +153,24 @@ function mvConfigSupportsGranularity( return chartGranularitySeconds >= mvGranularitySeconds; } +function mvConfigSupportsDateRange( + mvConfig: MaterializedViewConfiguration, + chartConfig: ChartConfigWithOptDateRange, +) { + if (mvConfig.minDate && !chartConfig.dateRange) { + return false; + } + + if (!mvConfig.minDate || !chartConfig.dateRange) { + return true; + } + + const [startDate] = chartConfig.dateRange; + const minDate = new Date(mvConfig.minDate); + + return startDate >= minDate; +} + const COUNT_FUNCTION_PATTERN = /\bcount(If)?\s*\(/i; export function isUnsupportedCountFunction(selectItem: SelectItem): boolean { return COUNT_FUNCTION_PATTERN.test(selectItem.valueExpression); @@ -268,6 +286,14 @@ export async function tryConvertConfigToMaterializedViewSelect< }; } + if (mvConfig.minDate && !mvConfigSupportsDateRange(mvConfig, chartConfig)) { + return { + errors: [ + 'The selected date range includes dates for which this view does not contain data.', + ], + }; + } + if (!mvConfigSupportsGranularity(mvConfig, chartConfig)) { const error = chartConfig.granularity ? `Granularity must be at least ${mvConfig.minGranularity}.` diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 9419a938..64ae0307 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -646,6 +646,7 @@ export const MaterializedViewConfigurationSchema = z.object({ tableName: z.string().min(1, 'Materialized View Table is required'), dimensionColumns: z.string(), minGranularity: SQLIntervalSchema, + minDate: z.string().datetime().nullish(), timestampColumn: z .string() .min(1, 'Materialized View Timestamp column is required'),