mirror of
https://github.com/hyperdxio/hyperdx
synced 2026-04-21 13:37:15 +00:00
feat: Align date ranges to chart and MV granularity (Line/Bar charts) (#1533)
Closes HDX-3067 Closes #1331 Closes #1212 Closes #1468 # Summary This PR makes a number of improvements around the way we handle date ranges and granularities, in an effort to minimize discrepancies between aggregate values queried from original data and aggregate values queried from materialized views. 1. Date ranges for Line and Bar chart queries are now (by default) auto-aligned to the chart's granularity. **This is not limited to materialized view queries.** Since the chart granularity is a multiple of the MV granularity, this ensures that the date range is aligned to the MV granularity as well. This also address a number of related issues that point out 0-values or low-values in the first or last data points. This PR also includes an option to disable this behavior for charts in Chart Explorer or Dashboard Tiles. 2. All materialized view queries and all time chart queries are now end-exclusive, to avoid selecting the entirety of the next "time bucket" from the materialized view when the date range is aligned with the materialized view granularity 3. Materialized views are only used for a query with a granularity if the chart query granularity is a multiple of the MV granularity. Previously, we'd use the MV as long as the chart query granularity was at least as large as the MV granularity, but this could cause unequal distributions of data across time buckets. Nearly all available granularities are multiples of all smaller available granularities - so this should only impact queries with granularity 15 minutes with MVs with granularity 10 minutes. 10m granularity support is being removed in #1551 ## Demo <details> <summary>Show Complete Intervals Option</summary> https://github.com/user-attachments/assets/4b903adb-4edf-4481-93d6-2a0c42589a37 </details>
This commit is contained in:
Drew Davis
GitHub
parent
fd81c4cb4f
commit
725dbc2f3b
11 changed files with 323 additions and 54 deletions
7
.changeset/itchy-zebras-train.md
Normal file
7
.changeset/itchy-zebras-train.md
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
---
|
||||
"@hyperdx/common-utils": patch
|
||||
"@hyperdx/api": patch
|
||||
"@hyperdx/app": patch
|
||||
---
|
||||
|
||||
feat: Align line/bar chart date ranges to chart granularity
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import { useMemo } from 'react';
|
||||
import { add } from 'date-fns';
|
||||
import { add, differenceInSeconds } from 'date-fns';
|
||||
import { omit } from 'lodash';
|
||||
import SqlString from 'sqlstring';
|
||||
import { z } from 'zod';
|
||||
|
|
@ -135,20 +135,45 @@ export const DEFAULT_CHART_CONFIG: Omit<
|
|||
whereLanguage: 'lucene',
|
||||
displayType: DisplayType.Line,
|
||||
granularity: 'auto',
|
||||
alignDateRangeToGranularity: true,
|
||||
};
|
||||
|
||||
export const isGranularity = (value: string): value is Granularity => {
|
||||
return Object.values(Granularity).includes(value as Granularity);
|
||||
};
|
||||
|
||||
export function getAlignedDateRange(
|
||||
[originalStart, originalEnd]: [Date, Date],
|
||||
granularity: SQLInterval,
|
||||
): [Date, Date] {
|
||||
// Round the start time down to the previous interval boundary
|
||||
const alignedStart = toStartOfInterval(originalStart, granularity);
|
||||
|
||||
// Round the end time up to the next interval boundary
|
||||
let alignedEnd = toStartOfInterval(originalEnd, granularity);
|
||||
if (alignedEnd.getTime() < originalEnd.getTime()) {
|
||||
const intervalSeconds = convertGranularityToSeconds(granularity);
|
||||
alignedEnd = add(alignedEnd, { seconds: intervalSeconds });
|
||||
}
|
||||
|
||||
return [alignedStart, alignedEnd];
|
||||
}
|
||||
|
||||
export function convertToTimeChartConfig(config: ChartConfigWithDateRange) {
|
||||
const granularity =
|
||||
config.granularity === 'auto' || config.granularity == null
|
||||
? convertDateRangeToGranularityString(config.dateRange, 80)
|
||||
: config.granularity;
|
||||
|
||||
const dateRange =
|
||||
config.alignDateRangeToGranularity === false
|
||||
? config.dateRange
|
||||
: getAlignedDateRange(config.dateRange, granularity);
|
||||
|
||||
return {
|
||||
...config,
|
||||
dateRange,
|
||||
dateRangeEndInclusive: false,
|
||||
granularity,
|
||||
limit: { limit: 100000 },
|
||||
};
|
||||
|
|
@ -549,16 +574,9 @@ function inferGroupColumns(meta: Array<{ name: string; type: string }>) {
|
|||
]);
|
||||
}
|
||||
|
||||
export function getPreviousPeriodOffsetSeconds(
|
||||
dateRange: [Date, Date],
|
||||
): number {
|
||||
const [start, end] = dateRange;
|
||||
return Math.round((end.getTime() - start.getTime()) / 1000);
|
||||
}
|
||||
|
||||
export function getPreviousDateRange(currentRange: [Date, Date]): [Date, Date] {
|
||||
const [start, end] = currentRange;
|
||||
const offsetSeconds = getPreviousPeriodOffsetSeconds(currentRange);
|
||||
const offsetSeconds = differenceInSeconds(end, start);
|
||||
return [
|
||||
new Date(start.getTime() - offsetSeconds * 1000),
|
||||
new Date(end.getTime() - offsetSeconds * 1000),
|
||||
|
|
@ -622,7 +640,7 @@ function addResponseToFormattedData({
|
|||
lineDataMap,
|
||||
tsBucketMap,
|
||||
source,
|
||||
currentPeriodDateRange,
|
||||
previousPeriodOffsetSeconds,
|
||||
isPreviousPeriod,
|
||||
hiddenSeries = [],
|
||||
}: {
|
||||
|
|
@ -631,7 +649,7 @@ function addResponseToFormattedData({
|
|||
response: ResponseJSON<Record<string, any>>;
|
||||
source?: TSource;
|
||||
isPreviousPeriod: boolean;
|
||||
currentPeriodDateRange: [Date, Date];
|
||||
previousPeriodOffsetSeconds: number;
|
||||
hiddenSeries?: string[];
|
||||
}) {
|
||||
const { meta, data } = response;
|
||||
|
|
@ -655,9 +673,7 @@ function addResponseToFormattedData({
|
|||
const date = new Date(row[timestampColumn.name]);
|
||||
|
||||
// Previous period data needs to be shifted forward to align with current period
|
||||
const offsetSeconds = isPreviousPeriod
|
||||
? getPreviousPeriodOffsetSeconds(currentPeriodDateRange)
|
||||
: 0;
|
||||
const offsetSeconds = isPreviousPeriod ? previousPeriodOffsetSeconds : 0;
|
||||
const ts = Math.round(date.getTime() / 1000 + offsetSeconds);
|
||||
|
||||
for (const valueColumn of valueColumns) {
|
||||
|
|
@ -712,6 +728,7 @@ export function formatResponseForTimeChart({
|
|||
generateEmptyBuckets = true,
|
||||
source,
|
||||
hiddenSeries = [],
|
||||
previousPeriodOffsetSeconds = 0,
|
||||
}: {
|
||||
dateRange: [Date, Date];
|
||||
granularity?: SQLInterval;
|
||||
|
|
@ -720,6 +737,7 @@ export function formatResponseForTimeChart({
|
|||
generateEmptyBuckets?: boolean;
|
||||
source?: TSource;
|
||||
hiddenSeries?: string[];
|
||||
previousPeriodOffsetSeconds?: number;
|
||||
}) {
|
||||
const meta = currentPeriodResponse.meta;
|
||||
|
||||
|
|
@ -750,7 +768,7 @@ export function formatResponseForTimeChart({
|
|||
tsBucketMap,
|
||||
source,
|
||||
isPreviousPeriod: false,
|
||||
currentPeriodDateRange: dateRange,
|
||||
previousPeriodOffsetSeconds,
|
||||
hiddenSeries,
|
||||
});
|
||||
|
||||
|
|
@ -761,7 +779,7 @@ export function formatResponseForTimeChart({
|
|||
tsBucketMap,
|
||||
source,
|
||||
isPreviousPeriod: true,
|
||||
currentPeriodDateRange: dateRange,
|
||||
previousPeriodOffsetSeconds,
|
||||
hiddenSeries,
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1367,6 +1367,9 @@ function DBSearchPage() {
|
|||
with: aliasWith,
|
||||
// Preserve the original table select string for "View Events" links
|
||||
eventTableSelect: searchedConfig.select,
|
||||
// In live mode, when the end date is aligned to the granularity, the end date does
|
||||
// not change on every query, resulting in cached data being re-used.
|
||||
alignDateRangeToGranularity: !isLive,
|
||||
...variableConfig,
|
||||
};
|
||||
}, [
|
||||
|
|
@ -1375,6 +1378,7 @@ function DBSearchPage() {
|
|||
aliasWith,
|
||||
searchedTimeRange,
|
||||
searchedConfig.select,
|
||||
isLive,
|
||||
]);
|
||||
|
||||
const onFormSubmit = useCallback<FormEventHandler<HTMLFormElement>>(
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import {
|
|||
convertToTableChartConfig,
|
||||
convertToTimeChartConfig,
|
||||
formatResponseForTimeChart,
|
||||
getAlignedDateRange,
|
||||
} from '@/ChartUtils';
|
||||
|
||||
if (!globalThis.structuredClone) {
|
||||
|
|
@ -503,6 +504,7 @@ describe('ChartUtils', () => {
|
|||
],
|
||||
granularity: '1 minute',
|
||||
generateEmptyBuckets: false,
|
||||
previousPeriodOffsetSeconds: 120,
|
||||
});
|
||||
|
||||
expect(actual.graphResults).toEqual([
|
||||
|
|
@ -652,4 +654,111 @@ describe('ChartUtils', () => {
|
|||
expect(convertedConfig.limit).toEqual({ limit: 200 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('getAlignedDateRange', () => {
|
||||
it('should align start time down to the previous minute boundary', () => {
|
||||
const dateRange: [Date, Date] = [
|
||||
new Date('2025-11-26T12:23:37Z'), // 37 seconds
|
||||
new Date('2025-11-26T12:25:00Z'),
|
||||
];
|
||||
|
||||
const [alignedStart, alignedEnd] = getAlignedDateRange(
|
||||
dateRange,
|
||||
'1 minute',
|
||||
);
|
||||
|
||||
expect(alignedStart.toISOString()).toBe('2025-11-26T12:23:00.000Z');
|
||||
expect(alignedEnd.toISOString()).toBe('2025-11-26T12:25:00.000Z');
|
||||
});
|
||||
|
||||
it('should align end time up to the next minute boundary', () => {
|
||||
const dateRange: [Date, Date] = [
|
||||
new Date('2025-11-26T12:23:00Z'),
|
||||
new Date('2025-11-26T12:25:42Z'), // 42 seconds
|
||||
];
|
||||
|
||||
const [alignedStart, alignedEnd] = getAlignedDateRange(
|
||||
dateRange,
|
||||
'1 minute',
|
||||
);
|
||||
|
||||
expect(alignedStart.toISOString()).toBe('2025-11-26T12:23:00.000Z');
|
||||
expect(alignedEnd.toISOString()).toBe('2025-11-26T12:26:00.000Z');
|
||||
});
|
||||
|
||||
it('should align both start and end times with 5 minute granularity', () => {
|
||||
const dateRange: [Date, Date] = [
|
||||
new Date('2025-11-26T12:23:17Z'), // Should round down to 12:20:00
|
||||
new Date('2025-11-26T12:27:42Z'), // Should round up to 12:30:00
|
||||
];
|
||||
|
||||
const [alignedStart, alignedEnd] = getAlignedDateRange(
|
||||
dateRange,
|
||||
'5 minute',
|
||||
);
|
||||
|
||||
expect(alignedStart.toISOString()).toBe('2025-11-26T12:20:00.000Z');
|
||||
expect(alignedEnd.toISOString()).toBe('2025-11-26T12:30:00.000Z');
|
||||
});
|
||||
|
||||
it('should align with 30 second granularity', () => {
|
||||
const dateRange: [Date, Date] = [
|
||||
new Date('2025-11-26T12:23:17Z'), // Should round down to 12:23:00
|
||||
new Date('2025-11-26T12:25:42Z'), // Should round up to 12:26:00
|
||||
];
|
||||
|
||||
const [alignedStart, alignedEnd] = getAlignedDateRange(
|
||||
dateRange,
|
||||
'30 second',
|
||||
);
|
||||
|
||||
expect(alignedStart.toISOString()).toBe('2025-11-26T12:23:00.000Z');
|
||||
expect(alignedEnd.toISOString()).toBe('2025-11-26T12:26:00.000Z');
|
||||
});
|
||||
|
||||
it('should align with 1 day granularity', () => {
|
||||
const dateRange: [Date, Date] = [
|
||||
new Date('2025-11-26T12:23:17Z'), // Should round down to start of day
|
||||
new Date('2025-11-28T08:15:00Z'), // Should round up to start of next day
|
||||
];
|
||||
|
||||
const [alignedStart, alignedEnd] = getAlignedDateRange(
|
||||
dateRange,
|
||||
'1 day',
|
||||
);
|
||||
|
||||
expect(alignedStart.toISOString()).toBe('2025-11-26T00:00:00.000Z');
|
||||
expect(alignedEnd.toISOString()).toBe('2025-11-29T00:00:00.000Z');
|
||||
});
|
||||
|
||||
it('should not change range when already aligned to the interval', () => {
|
||||
const dateRange: [Date, Date] = [
|
||||
new Date('2025-11-26T12:23:00Z'), // Already aligned
|
||||
new Date('2025-11-26T12:25:00Z'), // Already aligned
|
||||
];
|
||||
|
||||
const [alignedStart, alignedEnd] = getAlignedDateRange(
|
||||
dateRange,
|
||||
'1 minute',
|
||||
);
|
||||
|
||||
expect(alignedStart.toISOString()).toBe('2025-11-26T12:23:00.000Z');
|
||||
expect(alignedEnd.toISOString()).toBe('2025-11-26T12:25:00.000Z');
|
||||
});
|
||||
|
||||
it('should align with 15 minute granularity', () => {
|
||||
const dateRange: [Date, Date] = [
|
||||
new Date('2025-11-26T12:23:17Z'), // Should round down to 12:15:00
|
||||
new Date('2025-11-26T12:47:42Z'), // Should round up to 13:00:00
|
||||
];
|
||||
|
||||
const [alignedStart, alignedEnd] = getAlignedDateRange(
|
||||
dateRange,
|
||||
'15 minute',
|
||||
);
|
||||
|
||||
expect(alignedStart.toISOString()).toBe('2025-11-26T12:15:00.000Z');
|
||||
expect(alignedEnd.toISOString()).toBe('2025-11-26T13:00:00.000Z');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ jest.mock('@/ChartUtils', () => ({
|
|||
new Date('2023-12-31'),
|
||||
new Date('2024-01-01'),
|
||||
],
|
||||
getPreviousPeriodOffsetSeconds: () => 86400,
|
||||
getAlignedDateRange: (dateRange: [Date, Date]) => dateRange,
|
||||
convertToTimeChartConfig:
|
||||
jest.requireActual('@/ChartUtils').convertToTimeChartConfig,
|
||||
}));
|
||||
|
|
|
|||
|
|
@ -485,6 +485,10 @@ export default function EditTimeChartForm({
|
|||
control,
|
||||
name: 'compareToPreviousPeriod',
|
||||
});
|
||||
const alignDateRangeToGranularity = useWatch({
|
||||
control,
|
||||
name: 'alignDateRangeToGranularity',
|
||||
});
|
||||
const groupBy = useWatch({ control, name: 'groupBy' });
|
||||
const displayType =
|
||||
useWatch({ control, name: 'displayType' }) ?? DisplayType.Line;
|
||||
|
|
@ -496,9 +500,6 @@ export default function EditTimeChartForm({
|
|||
const databaseName = tableSource?.from.databaseName;
|
||||
const tableName = tableSource?.from.tableName;
|
||||
|
||||
// const tableSource = tableSourceWatch();
|
||||
// const databaseName = tableSourceWatch('from.databaseName');
|
||||
// const tableName = tableSourceWatch('from.tableName');
|
||||
const activeTab = useMemo(() => {
|
||||
switch (displayType) {
|
||||
case DisplayType.Search:
|
||||
|
|
@ -523,19 +524,6 @@ export default function EditTimeChartForm({
|
|||
const showGeneratedSql = ['table', 'time', 'number'].includes(activeTab); // Whether to show the generated SQL preview
|
||||
const showSampleEvents = tableSource?.kind !== SourceKind.Metric;
|
||||
|
||||
// const queriedConfig: ChartConfigWithDateRange | undefined = useMemo(() => {
|
||||
// if (queriedTableSource == null) {
|
||||
// return undefined;
|
||||
// }
|
||||
|
||||
// return {
|
||||
// ...chartConfig,
|
||||
// from: queriedTableSource.from,
|
||||
// timestampValueExpression: queriedTableSource?.timestampValueExpression,
|
||||
// dateRange,
|
||||
// };
|
||||
// }, [dateRange, chartConfig, queriedTableSource]);
|
||||
|
||||
// Only update this on submit, otherwise we'll have issues
|
||||
// with using the source value from the last submit
|
||||
// (ex. ignoring local custom source updates)
|
||||
|
|
@ -707,6 +695,20 @@ export default function EditTimeChartForm({
|
|||
});
|
||||
}, [dateRange]);
|
||||
|
||||
// Trigger a search when "Show Complete Intervals" changes
|
||||
useEffect(() => {
|
||||
setQueriedConfig((config: ChartConfigWithDateRange | undefined) => {
|
||||
if (config == null) {
|
||||
return config;
|
||||
}
|
||||
|
||||
return {
|
||||
...config,
|
||||
alignDateRangeToGranularity,
|
||||
};
|
||||
});
|
||||
}, [alignDateRangeToGranularity]);
|
||||
|
||||
// Trigger a search when "compare to previous period" changes
|
||||
useEffect(() => {
|
||||
setQueriedConfig((config: ChartConfigWithDateRange | undefined) => {
|
||||
|
|
@ -1213,6 +1215,11 @@ export default function EditTimeChartForm({
|
|||
</Flex>
|
||||
{activeTab === 'time' && (
|
||||
<Group justify="end" mb="xs">
|
||||
<SwitchControlled
|
||||
control={control}
|
||||
name="alignDateRangeToGranularity"
|
||||
label="Show Complete Intervals"
|
||||
/>
|
||||
<SwitchControlled
|
||||
control={control}
|
||||
name="compareToPreviousPeriod"
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { memo, useCallback, useEffect, useMemo, useState } from 'react';
|
||||
import Link from 'next/link';
|
||||
import cx from 'classnames';
|
||||
import { add } from 'date-fns';
|
||||
import { add, differenceInSeconds } from 'date-fns';
|
||||
import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse';
|
||||
import {
|
||||
ChartConfigWithDateRange,
|
||||
|
|
@ -35,8 +35,8 @@ import {
|
|||
convertGranularityToSeconds,
|
||||
convertToTimeChartConfig,
|
||||
formatResponseForTimeChart,
|
||||
getAlignedDateRange,
|
||||
getPreviousDateRange,
|
||||
getPreviousPeriodOffsetSeconds,
|
||||
PreviousPeriodSuffix,
|
||||
useTimeChartSettings,
|
||||
} from '@/ChartUtils';
|
||||
|
|
@ -233,6 +233,8 @@ function DBTimeChartComponent({
|
|||
hiddenSeries,
|
||||
}: DBTimeChartComponentProps) {
|
||||
const [isErrorExpanded, errorExpansion] = useDisclosure(false);
|
||||
|
||||
const originalDateRange = config.dateRange;
|
||||
const {
|
||||
displayType: displayTypeProp,
|
||||
dateRange,
|
||||
|
|
@ -266,17 +268,32 @@ function DBTimeChartComponent({
|
|||
});
|
||||
|
||||
const previousPeriodChartConfig: ChartConfigWithDateRange = useMemo(() => {
|
||||
const previousPeriodDateRange =
|
||||
queriedConfig.alignDateRangeToGranularity === false
|
||||
? getPreviousDateRange(originalDateRange)
|
||||
: getAlignedDateRange(
|
||||
getPreviousDateRange(originalDateRange),
|
||||
queriedConfig.granularity,
|
||||
);
|
||||
|
||||
return {
|
||||
...queriedConfig,
|
||||
dateRange: getPreviousDateRange(dateRange),
|
||||
dateRange: previousPeriodDateRange,
|
||||
};
|
||||
}, [queriedConfig, dateRange]);
|
||||
}, [queriedConfig, originalDateRange]);
|
||||
|
||||
const previousPeriodOffsetSeconds = useMemo(() => {
|
||||
return config.compareToPreviousPeriod
|
||||
? getPreviousPeriodOffsetSeconds(dateRange)
|
||||
? differenceInSeconds(
|
||||
dateRange[0],
|
||||
previousPeriodChartConfig.dateRange[0],
|
||||
)
|
||||
: undefined;
|
||||
}, [dateRange, config.compareToPreviousPeriod]);
|
||||
}, [
|
||||
config.compareToPreviousPeriod,
|
||||
dateRange,
|
||||
previousPeriodChartConfig.dateRange,
|
||||
]);
|
||||
|
||||
const { data: previousPeriodData, isLoading: isPreviousPeriodLoading } =
|
||||
useQueriedChartConfig(previousPeriodChartConfig, {
|
||||
|
|
@ -332,6 +349,7 @@ function DBTimeChartComponent({
|
|||
generateEmptyBuckets: fillNulls !== false,
|
||||
source,
|
||||
hiddenSeries,
|
||||
previousPeriodOffsetSeconds,
|
||||
});
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
|
|
@ -347,6 +365,7 @@ function DBTimeChartComponent({
|
|||
config.compareToPreviousPeriod,
|
||||
previousPeriodData,
|
||||
hiddenSeries,
|
||||
previousPeriodOffsetSeconds,
|
||||
]);
|
||||
|
||||
// To enable backward compatibility, allow non-controlled usage of displayType
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import { useMemo } from 'react';
|
||||
import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils';
|
||||
import { MaterializedViewConfiguration } from '@hyperdx/common-utils/dist/types';
|
||||
import { Grid, Group, Pill, Stack, Table, Text } from '@mantine/core';
|
||||
import { Grid, Group, Pill, Stack, Table, Text, Tooltip } from '@mantine/core';
|
||||
import { IconInfoCircle } from '@tabler/icons-react';
|
||||
|
||||
import { FormatTime } from '@/useFormatTime';
|
||||
|
||||
|
|
@ -33,9 +34,23 @@ export default function MVConfigSummary({
|
|||
<Stack gap="md">
|
||||
<Grid columns={2}>
|
||||
<Grid.Col span={1}>
|
||||
<Text size="sm" fw={500} mb="xs">
|
||||
Granularity
|
||||
</Text>
|
||||
<Group align="center" mb="xs" gap="xs">
|
||||
<Text size="sm" fw={500}>
|
||||
Granularity
|
||||
</Text>
|
||||
<Tooltip
|
||||
multiline
|
||||
maw={400}
|
||||
label={`
|
||||
The size of the time buckets into which data is pre-aggregated.
|
||||
Aggregate values can vary slightly between materialized views and the base source
|
||||
table when the selected time range does not align with the view's granularity.
|
||||
`}
|
||||
>
|
||||
<IconInfoCircle size={16} />
|
||||
</Tooltip>
|
||||
</Group>
|
||||
|
||||
<Pill>{config.minGranularity}</Pill>
|
||||
</Grid.Col>
|
||||
{config.minDate && (
|
||||
|
|
|
|||
|
|
@ -105,7 +105,7 @@ describe('materializedViews', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('should return mvConfig and errors if selecting a column which is not in the materialized view', async () => {
|
||||
it('should return errors if selecting a column which is not in the materialized view', async () => {
|
||||
const chartConfig: ChartConfigWithOptDateRange = {
|
||||
from: {
|
||||
databaseName: 'default',
|
||||
|
|
@ -133,7 +133,7 @@ describe('materializedViews', () => {
|
|||
]);
|
||||
});
|
||||
|
||||
it('should return mvConfig and errors if selecting an aggregation which is not supported for the specified column', async () => {
|
||||
it('should return errors if selecting an aggregation which is not supported for the specified column', async () => {
|
||||
const chartConfig: ChartConfigWithOptDateRange = {
|
||||
from: {
|
||||
databaseName: 'default',
|
||||
|
|
@ -627,7 +627,7 @@ describe('materializedViews', () => {
|
|||
expect(result.errors).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should return mvConfig and errors if the granularity of the query is less than the materialized view granularity', async () => {
|
||||
it('should return errors if the granularity of the query is less than the materialized view granularity', async () => {
|
||||
const chartConfig: ChartConfigWithOptDateRange = {
|
||||
from: {
|
||||
databaseName: 'default',
|
||||
|
|
@ -641,7 +641,7 @@ describe('materializedViews', () => {
|
|||
],
|
||||
where: '',
|
||||
connection: 'test-connection',
|
||||
granularity: '30 seconds',
|
||||
granularity: '30 second',
|
||||
};
|
||||
|
||||
const result = await tryConvertConfigToMaterializedViewSelect(
|
||||
|
|
@ -651,10 +651,41 @@ describe('materializedViews', () => {
|
|||
);
|
||||
|
||||
expect(result.optimizedConfig).toBeUndefined();
|
||||
expect(result.errors).toEqual(['Granularity must be at least 1 minute.']);
|
||||
expect(result.errors).toEqual([
|
||||
"Granularity must be a multiple of the view's granularity (1 minute).",
|
||||
]);
|
||||
});
|
||||
|
||||
it('should return mvConfig and errors if no granularity is specified but the date range is too short for the MV granularity', async () => {
|
||||
it('should return errors if the granularity of the query is greater than but not a multiple of the materialized view granularity', async () => {
|
||||
const chartConfig: ChartConfigWithOptDateRange = {
|
||||
from: {
|
||||
databaseName: 'default',
|
||||
tableName: 'otel_spans',
|
||||
},
|
||||
select: [
|
||||
{
|
||||
valueExpression: '',
|
||||
aggFn: 'count',
|
||||
},
|
||||
],
|
||||
where: '',
|
||||
connection: 'test-connection',
|
||||
granularity: '90 second',
|
||||
};
|
||||
|
||||
const result = await tryConvertConfigToMaterializedViewSelect(
|
||||
chartConfig,
|
||||
MV_CONFIG_METRIC_ROLLUP_1M,
|
||||
metadata,
|
||||
);
|
||||
|
||||
expect(result.optimizedConfig).toBeUndefined();
|
||||
expect(result.errors).toEqual([
|
||||
"Granularity must be a multiple of the view's granularity (1 minute).",
|
||||
]);
|
||||
});
|
||||
|
||||
it('should return errors if no granularity is specified but the date range is too short for the MV granularity', async () => {
|
||||
const chartConfig: ChartConfigWithOptDateRange = {
|
||||
from: {
|
||||
databaseName: 'default',
|
||||
|
|
@ -894,6 +925,54 @@ describe('materializedViews', () => {
|
|||
],
|
||||
where: '',
|
||||
connection: 'test-connection',
|
||||
dateRangeEndInclusive: false,
|
||||
});
|
||||
expect(result.errors).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should set dateRangeEndInclusive to false when optimizing a config with dateRange', async () => {
|
||||
const chartConfig: ChartConfigWithOptDateRange = {
|
||||
from: {
|
||||
databaseName: 'default',
|
||||
tableName: 'otel_spans',
|
||||
},
|
||||
select: [
|
||||
{
|
||||
valueExpression: '',
|
||||
aggFn: 'count',
|
||||
},
|
||||
],
|
||||
where: '',
|
||||
connection: 'test-connection',
|
||||
granularity: '1 minute',
|
||||
dateRange: [
|
||||
new Date('2023-01-01T00:00:00Z'),
|
||||
new Date('2023-01-02T01:00:00Z'),
|
||||
],
|
||||
};
|
||||
|
||||
const result = await tryConvertConfigToMaterializedViewSelect(
|
||||
chartConfig,
|
||||
MV_CONFIG_METRIC_ROLLUP_1M,
|
||||
metadata,
|
||||
);
|
||||
|
||||
expect(result.optimizedConfig).toEqual({
|
||||
from: {
|
||||
databaseName: 'default',
|
||||
tableName: 'metrics_rollup_1m',
|
||||
},
|
||||
select: [
|
||||
{
|
||||
valueExpression: 'count',
|
||||
aggFn: 'sum',
|
||||
},
|
||||
],
|
||||
where: '',
|
||||
connection: 'test-connection',
|
||||
granularity: '1 minute',
|
||||
dateRange: chartConfig.dateRange,
|
||||
dateRangeEndInclusive: false,
|
||||
});
|
||||
expect(result.errors).toBeUndefined();
|
||||
});
|
||||
|
|
@ -1371,7 +1450,7 @@ describe('materializedViews', () => {
|
|||
).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should return mvConfig and errors if the generated MV config is not valid', async () => {
|
||||
it('should return errors if the generated MV config is not valid', async () => {
|
||||
const chartConfig: ChartConfigWithOptDateRange = {
|
||||
from: {
|
||||
databaseName: 'default',
|
||||
|
|
@ -1539,7 +1618,9 @@ describe('materializedViews', () => {
|
|||
expect(result.explanations).toEqual([
|
||||
{
|
||||
mvConfig: MV_CONFIG_METRIC_ROLLUP_1M,
|
||||
errors: ['Granularity must be at least 1 minute.'],
|
||||
errors: [
|
||||
"Granularity must be a multiple of the view's granularity (1 minute).",
|
||||
],
|
||||
success: false,
|
||||
},
|
||||
{
|
||||
|
|
|
|||
|
|
@ -150,7 +150,13 @@ function mvConfigSupportsGranularity(
|
|||
mvConfig.minGranularity,
|
||||
);
|
||||
|
||||
return chartGranularitySeconds >= mvGranularitySeconds;
|
||||
// The chart granularity must be a multiple of the MV granularity,
|
||||
// to avoid unequal distribution of data across chart time buckets
|
||||
// which don't align with the MV time buckets.
|
||||
return (
|
||||
chartGranularitySeconds >= mvGranularitySeconds &&
|
||||
chartGranularitySeconds % mvGranularitySeconds === 0
|
||||
);
|
||||
}
|
||||
|
||||
function mvConfigSupportsDateRange(
|
||||
|
|
@ -296,7 +302,7 @@ export async function tryConvertConfigToMaterializedViewSelect<
|
|||
|
||||
if (!mvConfigSupportsGranularity(mvConfig, chartConfig)) {
|
||||
const error = chartConfig.granularity
|
||||
? `Granularity must be at least ${mvConfig.minGranularity}.`
|
||||
? `Granularity must be a multiple of the view's granularity (${mvConfig.minGranularity}).`
|
||||
: 'The selected date range is too short for the granularity of this materialized view.';
|
||||
return { errors: [error] };
|
||||
}
|
||||
|
|
@ -334,13 +340,15 @@ export async function tryConvertConfigToMaterializedViewSelect<
|
|||
};
|
||||
}
|
||||
|
||||
const clonedConfig = {
|
||||
const clonedConfig: C = {
|
||||
...structuredClone(chartConfig),
|
||||
select,
|
||||
from: {
|
||||
databaseName: mvConfig.databaseName,
|
||||
tableName: mvConfig.tableName,
|
||||
},
|
||||
// Make the date range end exclusive to avoid selecting the entire next time bucket from the MV
|
||||
...('dateRange' in chartConfig ? { dateRangeEndInclusive: false } : {}),
|
||||
};
|
||||
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -422,6 +422,7 @@ export const _ChartConfigSchema = z.object({
|
|||
eventTableSelect: z.string().optional(),
|
||||
compareToPreviousPeriod: z.boolean().optional(),
|
||||
source: z.string().optional(),
|
||||
alignDateRangeToGranularity: z.boolean().optional(),
|
||||
});
|
||||
|
||||
// This is a ChartConfig type without the `with` CTE clause included.
|
||||
|
|
|
|||
Loading…
Reference in a new issue