Elgato_dark/hyperdx
1
0
Fork 0
mirror of https://github.com/hyperdxio/hyperdx synced 2026-04-21 13:37:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

perf: Query filter values from MVs (#1591)

Browse source 
Closes HDX-3066

# Summary

This PR improves the performance of Search and Dashboard filters by querying available filter values from materialized views, when possible. The existing `useMultipleGetKeyValues` has been updated to make use of `getKeyValuesWithMVs`, which works as follows:

1. Identify which materialized views support each of the requested keys. Keys must be `dimensionColumns` in the materialized view, the materialized view must support the provided date range, and the materialized view must support the provided filters (determined by running an EXPLAIN query).
2. Split the keys into groups based on which Materialized view can provide their values. Query values for each group using the existing `getKeyValues` function. Sampling is disabled because it is assumed that MVs are small enough to be queried without sampling.
3. Query any keys which are not supported by any materialized view from the base table.

To reduce the number of EXPLAIN queries required to support this, and to generally decrease the number of concurrent requests for filters, Dashboard filter value queries are now batched by source. Values for each batch are then queried using `getKeyValuesWithMVs` (described above).

Other fixes:
1. I've also updated the various filter functions and hooks to support abort signals, so that filter queries are canceled when a query value is no longer needed.
2. The getKeyValues cache key now includes `where` and `filters`, so that the filter values correctly update when new filters or where conditions are added on the search page.
This commit is contained in:
Drew Davis 2026-01-14 13:05:11 -05:00 committed by GitHub
parent 6752b3f862
commit f98fc51946
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 2397 additions and 80 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.changeset/rotten-oranges-agree.md Normal file
View file

@ -0,0 +1,6 @@
---
"@hyperdx/common-utils": patch
"@hyperdx/app": patch
---
perf: Query filter values from MVs

95
packages/app/src/DashboardFilters.tsx
View file

@ -1,68 +1,29 @@
import { DashboardFilter } from '@hyperdx/common-utils/dist/types';
import { Group, Select } from '@mantine/core';
import { IconRefresh } from '@tabler/icons-react';
import { useGetKeyValues } from './hooks/useMetadata';
import { useDashboardFilterKeyValues } from './hooks/useDashboardFilterValues';
import { FilterState } from './searchFilters';
import { useSource } from './source';
import { getMetricTableName } from './utils';
interface DashboardFilterSelectProps {
filter: DashboardFilter;
dateRange: [Date, Date];
onChange: (value: string | null) => void;
value?: string | null;
values?: string[];
isLoading?: boolean;
}
const DashboardFilterSelect = ({
filter,
dateRange,
onChange,
value,
values,
isLoading,
}: DashboardFilterSelectProps) => {
const { data: source, isLoading: isSourceLoading } = useSource({
id: filter.source,
});
const { timestampValueExpression, connection } = source || {};
const databaseName = source?.from.databaseName;
const tableName =
source && getMetricTableName(source, filter.sourceMetricType);
const { data: keys, isLoading: isKeyValuesLoading } = useGetKeyValues(
{
chartConfig: {
dateRange,
timestampValueExpression: timestampValueExpression!,
connection: connection!,
from: {
databaseName: databaseName!,
tableName: tableName!,
}!,
where: '',
whereLanguage: 'sql',
select: '',
},
keys: [filter.expression],
disableRowLimit: true,
limit: 10000,
},
{
enabled:
!!timestampValueExpression &&
!!connection &&
!!tableName &&
!!databaseName,
},
);
const selectValues = keys?.[0]?.value
.map(value => String(value))
.sort()
.map(value => ({
value,
label: value,
}));
const selectValues = values?.toSorted().map(value => ({
value,
label: value,
}));
return (
<Select
@ -74,7 +35,7 @@ const DashboardFilterSelect = ({
allowDeselect
size="xs"
maxDropdownHeight={280}
disabled={isSourceLoading || isKeyValuesLoading}
disabled={isLoading}
variant="filled"
w={200}
limit={20}
@ -96,20 +57,30 @@ const DashboardFilters = ({
filterValues,
onSetFilterValue,
}: DashboardFilterProps) => {
const { data: filterValuesBySource, isFetching } =
useDashboardFilterKeyValues({ filters, dateRange });
return (
<Group mt="sm">
{Object.values(filters).map(filter => (
<DashboardFilterSelect
key={filter.id}
filter={filter}
dateRange={dateRange}
onChange={value => onSetFilterValue(filter.expression, value)}
value={filterValues[filter.expression]?.included
.values()
.next()
.value?.toString()}
/>
))}
{Object.values(filters).map(filter => {
const queriedFilterValues = filterValuesBySource?.get(
filter.expression,
);
return (
<DashboardFilterSelect
key={filter.id}
filter={filter}
isLoading={!queriedFilterValues}
onChange={value => onSetFilterValue(filter.expression, value)}
values={queriedFilterValues?.values}
value={filterValues[filter.expression]?.included
.values()
.next()
.value?.toString()}
/>
);
})}
{isFetching && <IconRefresh className="spin-animate" size={12} />}
</Group>
);
};

1
packages/app/src/NamespaceDetailsSidePanel.tsx
View file

@ -276,6 +276,7 @@ export default function NamespaceDetailsSidePanel({
const { data: logServiceNames } = useGetKeyValues(
{
chartConfig: {
source: logSource.id,
from: logSource.from,
where: `${logSource?.resourceAttributesExpression}.k8s.namespace.name:"${namespaceName}"`,
whereLanguage: 'lucene',

1
packages/app/src/NodeDetailsSidePanel.tsx
View file

@ -289,6 +289,7 @@ export default function NodeDetailsSidePanel({
const { data: logServiceNames } = useGetKeyValues(
{
chartConfig: {
source: logSource.id,
from: logSource.from,
where: `${logSource?.resourceAttributesExpression}.k8s.node.name:"${nodeName}"`,
whereLanguage: 'lucene',

1
packages/app/src/PodDetailsSidePanel.tsx
View file

@ -282,6 +282,7 @@ export default function PodDetailsSidePanel({
const { data: logServiceNames } = useGetKeyValues(
{
chartConfig: {
source: logSource.id,
from: logSource.from,
where: `${logSource?.resourceAttributesExpression}.k8s.pod.name:"${podName}"`,
whereLanguage: 'lucene',

5
packages/app/src/components/DBSearchPageFilters.tsx
View file

@ -997,7 +997,7 @@ const DBSearchPageFiltersComponent = ({
async (key: string) => {
setLoadMoreLoadingKeys(prev => new Set(prev).add(key));
try {
const newKeyVals = await metadata.getKeyValues({
const newKeyVals = await metadata.getKeyValuesWithMVs({
chartConfig: {
...chartConfig,
dateRange,
@ -1005,6 +1005,7 @@ const DBSearchPageFiltersComponent = ({
keys: [key],
limit: LOAD_MORE_LOAD_LIMIT,
disableRowLimit: true,
source,
});
const newValues = newKeyVals[0].value;
if (newValues.length > 0) {
@ -1023,7 +1024,7 @@ const DBSearchPageFiltersComponent = ({
});
}
},
[chartConfig, setExtraFacets, dateRange, metadata],
[chartConfig, setExtraFacets, dateRange, metadata, source],
);
const shownFacets = useMemo(() => {

980
packages/app/src/hooks/__tests__/useDashboardFilterValues.test.tsx Normal file
View file

@ -0,0 +1,980 @@
/* eslint-disable @typescript-eslint/no-unsafe-type-assertion */
import React from 'react';
import { optimizeGetKeyValuesCalls } from '@hyperdx/common-utils/dist/core/materializedViews';
import { Metadata } from '@hyperdx/common-utils/dist/core/metadata';
import { DashboardFilter } from '@hyperdx/common-utils/dist/types';
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
import { renderHook, waitFor } from '@testing-library/react';
import * as sourceModule from '@/source';
import { useDashboardFilterKeyValues } from '../useDashboardFilterValues';
import * as useMetadataModule from '../useMetadata';
// Mock modules
jest.mock('@/source');
jest.mock('../useMetadata');
jest.mock('@hyperdx/common-utils/dist/core/materializedViews', () => ({
optimizeGetKeyValuesCalls: jest
.fn()
.mockImplementation(async ({ keys, chartConfig }) => [
{ keys, chartConfig },
]),
}));
describe('useDashboardFilterKeyValues', () => {
let queryClient: QueryClient;
let wrapper: React.ComponentType<{ children: any }>;
let mockMetadata: jest.Mocked<Metadata>;
const mockSources = [
{
id: 'logs-source',
name: 'Logs',
timestampValueExpression: 'timestamp',
connection: 'clickhouse-conn',
from: {
databaseName: 'telemetry',
tableName: 'logs',
},
},
{
id: 'traces-source',
name: 'Traces',
timestampValueExpression: 'timestamp',
connection: 'clickhouse-conn',
from: {
databaseName: 'telemetry',
tableName: 'traces',
},
},
];
const mockFilters: DashboardFilter[] = [
{
id: 'filter1',
type: 'QUERY_EXPRESSION',
name: 'Environment',
expression: 'environment',
source: 'logs-source',
},
{
id: 'filter2',
type: 'QUERY_EXPRESSION',
name: 'Service',
expression: 'service.name',
source: 'traces-source',
},
];
const mockDateRange: [Date, Date] = [
new Date('2024-01-01'),
new Date('2024-01-02'),
];
beforeEach(() => {
jest.clearAllMocks();
// Mock metadata with getKeyValues
mockMetadata = {
getKeyValues: jest.fn(),
} as unknown as jest.Mocked<Metadata>;
// Mock useMetadataWithSettings
jest
.spyOn(useMetadataModule, 'useMetadataWithSettings')
.mockReturnValue(mockMetadata);
// Create a new QueryClient for each test
queryClient = new QueryClient({
defaultOptions: {
queries: {
retry: false,
},
},
});
// Create a wrapper component with QueryClientProvider
wrapper = ({ children }) => (
<QueryClientProvider client={queryClient}>{children}</QueryClientProvider>
);
});
afterEach(() => {
jest.restoreAllMocks();
});
it('should fetch key values for filters grouped by source', async () => {
// Arrange
jest.spyOn(sourceModule, 'useSources').mockReturnValue({
data: mockSources,
isLoading: false,
} as any);
mockMetadata.getKeyValues.mockImplementation(({ keys }) => {
return Promise.resolve(
keys.map(key => ({
key,
value:
key === 'environment'
? ['production', 'staging', 'development']
: ['frontend', 'backend', 'database'],
})),
);
});
// Act
const { result } = renderHook(
() =>
useDashboardFilterKeyValues({
filters: mockFilters,
dateRange: mockDateRange,
}),
{ wrapper },
);
// Assert
await waitFor(() => expect(result.current.isFetching).toBe(false));
expect(result.current.data).toEqual(
new Map([
[
'environment',
{
values: ['production', 'staging', 'development'],
isLoading: false,
},
],
[
'service.name',
{
values: ['frontend', 'backend', 'database'],
isLoading: false,
},
],
]),
);
expect(optimizeGetKeyValuesCalls).toHaveBeenCalledTimes(2);
expect(optimizeGetKeyValuesCalls).toHaveBeenCalledWith(
expect.objectContaining({
chartConfig: expect.objectContaining({
from: { databaseName: 'telemetry', tableName: 'logs' },
source: 'logs-source',
dateRange: mockDateRange,
}),
keys: ['environment'],
}),
);
expect(optimizeGetKeyValuesCalls).toHaveBeenCalledWith(
expect.objectContaining({
chartConfig: expect.objectContaining({
from: { databaseName: 'telemetry', tableName: 'traces' },
source: 'traces-source',
dateRange: mockDateRange,
}),
keys: ['service.name'],
}),
);
expect(mockMetadata.getKeyValues).toHaveBeenCalledTimes(2);
expect(mockMetadata.getKeyValues).toHaveBeenCalledWith(
expect.objectContaining({
chartConfig: expect.objectContaining({
from: { databaseName: 'telemetry', tableName: 'logs' },
source: 'logs-source',
dateRange: mockDateRange,
}),
keys: ['environment'],
}),
);
expect(mockMetadata.getKeyValues).toHaveBeenCalledWith(
expect.objectContaining({
chartConfig: expect.objectContaining({
from: { databaseName: 'telemetry', tableName: 'traces' },
source: 'traces-source',
dateRange: mockDateRange,
}),
keys: ['service.name'],
}),
);
});
it('should group multiple filters from the same source', async () => {
// Arrange
const sameSourceFilters: DashboardFilter[] = [
{
id: 'filter1',
type: 'QUERY_EXPRESSION',
name: 'Environment',
expression: 'environment',
source: 'logs-source',
},
{
id: 'filter2',
type: 'QUERY_EXPRESSION',
name: 'Status',
expression: 'status',
source: 'logs-source',
},
];
jest.spyOn(sourceModule, 'useSources').mockReturnValue({
data: mockSources,
isLoading: false,
} as any);
mockMetadata.getKeyValues.mockResolvedValueOnce([
{
key: 'environment',
value: ['production', 'staging'],
},
{
key: 'status',
value: ['200', '404', '500'],
},
]);
// Act
const { result } = renderHook(
() =>
useDashboardFilterKeyValues({
filters: sameSourceFilters,
dateRange: mockDateRange,
}),
{ wrapper },
);
// Assert
await waitFor(() => expect(result.current.isFetching).toBe(false));
expect(mockMetadata.getKeyValues).toHaveBeenCalledTimes(1);
expect(mockMetadata.getKeyValues).toHaveBeenCalledWith(
expect.objectContaining({
keys: ['environment', 'status'],
}),
);
});
it('should not fetch when filters array is empty', () => {
// Arrange
jest.spyOn(sourceModule, 'useSources').mockReturnValue({
data: mockSources,
isLoading: false,
} as any);
// Act
const { result } = renderHook(
() =>
useDashboardFilterKeyValues({
filters: [],
dateRange: mockDateRange,
}),
{ wrapper },
);
// Assert
expect(result.current.data).toEqual(new Map());
expect(mockMetadata.getKeyValues).not.toHaveBeenCalled();
});
it('should not fetch when sources are still loading', () => {
// Arrange
jest.spyOn(sourceModule, 'useSources').mockReturnValue({
data: undefined,
isLoading: true,
} as any);
// Act
renderHook(
() =>
useDashboardFilterKeyValues({
filters: mockFilters,
dateRange: mockDateRange,
}),
{ wrapper },
);
// Assert
expect(mockMetadata.getKeyValues).not.toHaveBeenCalled();
});
it('should filter out filters for sources that do not exist', async () => {
// Arrange
const filtersWithInvalidSource: DashboardFilter[] = [
...mockFilters,
{
id: 'filter3',
type: 'QUERY_EXPRESSION',
name: 'Invalid',
expression: 'invalid.field',
source: 'nonexistent-source',
},
];
jest.spyOn(sourceModule, 'useSources').mockReturnValue({
data: mockSources,
isLoading: false,
} as any);
mockMetadata.getKeyValues
.mockResolvedValueOnce([
{
key: 'environment',
value: ['production'],
},
])
.mockResolvedValueOnce([
{
key: 'service.name',
value: ['backend'],
},
]);
// Act
const { result } = renderHook(
() =>
useDashboardFilterKeyValues({
filters: filtersWithInvalidSource,
dateRange: mockDateRange,
}),
{ wrapper },
);
// Assert
await waitFor(() => expect(result.current.isFetching).toBe(false));
// Should only call getKeyValues for valid sources
expect(mockMetadata.getKeyValues).toHaveBeenCalledTimes(2);
});
it('should handle errors when fetching key values', async () => {
// Arrange
jest.spyOn(sourceModule, 'useSources').mockReturnValue({
data: mockSources,
isLoading: false,
} as any);
mockMetadata.getKeyValues.mockRejectedValue(
new Error('Failed to fetch key values'),
);
// Act
const { result } = renderHook(
() =>
useDashboardFilterKeyValues({
filters: mockFilters,
dateRange: mockDateRange,
}),
{ wrapper },
);
// Assert
await waitFor(() => expect(result.current.isError).toBe(true));
expect(result.current.isError).toBeTruthy();
});
it('should pass correct parameters to getKeyValues', async () => {
// Arrange
jest.spyOn(sourceModule, 'useSources').mockReturnValue({
data: mockSources,
isLoading: false,
} as any);
mockMetadata.getKeyValues.mockResolvedValue([
{
key: 'environment',
value: ['production'],
},
]);
// Act
const { result } = renderHook(
() =>
useDashboardFilterKeyValues({
filters: [mockFilters[0]], // Only first filter (logs-source)
dateRange: mockDateRange,
}),
{ wrapper },
);
// Assert
await waitFor(() => expect(result.current.isFetching).toBe(false));
expect(mockMetadata.getKeyValues).toHaveBeenCalledWith({
chartConfig: {
timestampValueExpression: 'timestamp',
connection: 'clickhouse-conn',
from: {
databaseName: 'telemetry',
tableName: 'logs',
},
source: 'logs-source',
dateRange: mockDateRange,
where: '',
whereLanguage: 'sql',
select: '',
},
keys: ['environment'],
limit: 10000,
disableRowLimit: true,
signal: expect.any(AbortSignal),
});
});
it('should use placeholderData to keep previous data', async () => {
// Arrange
jest.spyOn(sourceModule, 'useSources').mockReturnValue({
data: mockSources,
isLoading: false,
} as any);
mockMetadata.getKeyValues.mockResolvedValue([
{
key: 'environment',
value: ['production'],
},
]);
// Act
const { result, rerender } = renderHook(
({ filters, dateRange }) =>
useDashboardFilterKeyValues({
filters,
dateRange,
}),
{
wrapper,
initialProps: {
filters: [mockFilters[0]],
dateRange: mockDateRange,
},
},
);
// Assert - Wait for first fetch to complete
await waitFor(() => expect(result.current.isFetching).toBe(false));
// Update with new date range (should trigger refetch)
const newDateRange: [Date, Date] = [
new Date('2024-01-02'),
new Date('2024-01-03'),
];
mockMetadata.getKeyValues.mockResolvedValue([
{
key: 'environment',
value: ['staging'],
},
]);
rerender({
filters: [mockFilters[0]],
dateRange: newDateRange,
});
// During fetching, previous data should still be available
// Note: This tests the keepPreviousData behavior
expect(result.current.data).toBeDefined();
});
it('should flatten results from multiple sources into a single Map', async () => {
// Arrange
jest.spyOn(sourceModule, 'useSources').mockReturnValue({
data: mockSources,
isLoading: false,
} as any);
mockMetadata.getKeyValues
.mockResolvedValueOnce([
{
key: 'environment',
value: ['production', 'staging'],
},
{
key: 'log_level',
value: ['info', 'error'],
},
])
.mockResolvedValueOnce([
{
key: 'service.name',
value: ['frontend', 'backend'],
},
]);
const multiFilters: DashboardFilter[] = [
{
id: 'filter1',
type: 'QUERY_EXPRESSION',
name: 'Environment',
expression: 'environment',
source: 'logs-source',
},
{
id: 'filter2',
type: 'QUERY_EXPRESSION',
name: 'Log Level',
expression: 'log_level',
source: 'logs-source',
},
{
id: 'filter3',
type: 'QUERY_EXPRESSION',
name: 'Service',
expression: 'service.name',
source: 'traces-source',
},
];
// Act
const { result } = renderHook(
() =>
useDashboardFilterKeyValues({
filters: multiFilters,
dateRange: mockDateRange,
}),
{ wrapper },
);
// Assert
await waitFor(() => expect(result.current.isFetching).toBe(false));
expect(result.current.data).toEqual(
new Map([
[
'environment',
{ values: ['production', 'staging'], isLoading: false },
],
['log_level', { values: ['info', 'error'], isLoading: false }],
['service.name', { values: ['frontend', 'backend'], isLoading: false }],
]),
);
// Should have size of 3 (all keys)
expect(result.current.data?.size).toBe(3);
});
it('should query keys from materialized views when optimizeGetKeyValuesCalls indicates MVs are available', async () => {
// Arrange
jest.spyOn(sourceModule, 'useSources').mockReturnValue({
data: mockSources,
isLoading: false,
} as any);
// Mock optimizeGetKeyValuesCalls to return multiple calls (one for MV, one for original source)
jest.mocked(optimizeGetKeyValuesCalls).mockResolvedValue([
{
chartConfig: {
from: { databaseName: 'telemetry', tableName: 'logs_rollup_1m' },
dateRange: mockDateRange,
connection: 'clickhouse-conn',
timestampValueExpression: 'timestamp',
source: 'logs-source',
where: '',
whereLanguage: 'sql',
select: '',
},
keys: ['environment'],
},
{
chartConfig: {
from: { databaseName: 'telemetry', tableName: 'logs' },
dateRange: mockDateRange,
connection: 'clickhouse-conn',
timestampValueExpression: 'timestamp',
source: 'logs-source',
where: '',
whereLanguage: 'sql',
select: '',
},
keys: ['service.name'],
},
]);
mockMetadata.getKeyValues
.mockResolvedValueOnce([
{
key: 'environment',
value: ['production', 'staging'],
},
])
.mockResolvedValueOnce([
{
key: 'service.name',
value: ['frontend', 'backend'],
},
]);
const filtersForSameSource: DashboardFilter[] = [
{
id: 'filter1',
type: 'QUERY_EXPRESSION',
name: 'Environment',
expression: 'environment',
source: 'logs-source',
},
{
id: 'filter2',
type: 'QUERY_EXPRESSION',
name: 'Service',
expression: 'service.name',
source: 'logs-source',
},
];
// Act
const { result } = renderHook(
() =>
useDashboardFilterKeyValues({
filters: filtersForSameSource,
dateRange: mockDateRange,
}),
{ wrapper },
);
// Assert
await waitFor(() => expect(result.current.isFetching).toBe(false));
// Should call getKeyValues twice (once for MV, once for original source)
expect(mockMetadata.getKeyValues).toHaveBeenCalledTimes(2);
expect(mockMetadata.getKeyValues).toHaveBeenCalledWith(
expect.objectContaining({
chartConfig: expect.objectContaining({
from: { databaseName: 'telemetry', tableName: 'logs_rollup_1m' },
}),
keys: ['environment'],
}),
);
expect(mockMetadata.getKeyValues).toHaveBeenCalledWith(
expect.objectContaining({
chartConfig: expect.objectContaining({
from: { databaseName: 'telemetry', tableName: 'logs' },
}),
keys: ['service.name'],
}),
);
// Should return combined results
expect(result.current.data).toEqual(
new Map([
[
'environment',
{ values: ['production', 'staging'], isLoading: false },
],
['service.name', { values: ['frontend', 'backend'], isLoading: false }],
]),
);
});
it('should provide partial results when some keys have loaded and others have not', async () => {
// Arrange
jest.spyOn(sourceModule, 'useSources').mockReturnValue({
data: mockSources,
isLoading: false,
} as any);
// Mock optimizeGetKeyValuesCalls to return two separate calls
jest
.mocked(optimizeGetKeyValuesCalls)
.mockResolvedValueOnce([
{
chartConfig: {
from: { databaseName: 'telemetry', tableName: 'logs' },
dateRange: mockDateRange,
connection: 'clickhouse-conn',
timestampValueExpression: 'timestamp',
source: 'logs-source',
where: '',
whereLanguage: 'sql',
select: '',
},
keys: ['environment'],
},
])
.mockResolvedValueOnce([
{
chartConfig: {
from: { databaseName: 'telemetry', tableName: 'traces' },
dateRange: mockDateRange,
connection: 'clickhouse-conn',
timestampValueExpression: 'timestamp',
source: 'traces-source',
where: '',
whereLanguage: 'sql',
select: '',
},
keys: ['service.name'],
},
]);
// First query resolves quickly, second query takes longer
let resolveQuery;
mockMetadata.getKeyValues
.mockImplementationOnce(async () => {
return [
{
key: 'environment',
value: ['production'],
},
];
})
.mockImplementationOnce(
async () =>
new Promise(
resolve => {
resolveQuery = resolve;
},
// setTimeout(
// () =>
// resolve([
// {
// key: 'service.name',
// value: ['backend'],
// },
// ]),
// 100,
// ),
),
);
// Act
const { result } = renderHook(
() =>
useDashboardFilterKeyValues({
filters: mockFilters,
dateRange: mockDateRange,
}),
{ wrapper },
);
// Assert - Wait for first query to complete
await waitFor(() => expect(result.current.isLoading).toBe(false));
// At this point, environment should be loaded but service.name should still be loading
expect(result.current.data?.get('environment')).toEqual({
values: ['production'],
isLoading: false,
});
// Wait for all queries to complete
resolveQuery!([
{
key: 'service.name',
value: ['backend'],
},
]);
await waitFor(() => expect(result.current.isFetching).toBe(false));
// Now both should be loaded
expect(result.current.data).toEqual(
new Map([
['environment', { values: ['production'], isLoading: false }],
['service.name', { values: ['backend'], isLoading: false }],
]),
);
});
it('should provide partial results when some keys have failed and others have not', async () => {
// Arrange
jest.spyOn(sourceModule, 'useSources').mockReturnValue({
data: mockSources,
isLoading: false,
} as any);
// Mock optimizeGetKeyValuesCalls to return two separate calls
(optimizeGetKeyValuesCalls as jest.Mock)
.mockResolvedValueOnce([
{
chartConfig: {
from: { databaseName: 'telemetry', tableName: 'logs' },
dateRange: mockDateRange,
connection: 'clickhouse-conn',
timestampValueExpression: 'timestamp',
source: 'logs-source',
where: '',
whereLanguage: 'sql',
select: '',
},
keys: ['environment'],
},
])
.mockResolvedValueOnce([
{
chartConfig: {
from: { databaseName: 'telemetry', tableName: 'traces' },
dateRange: mockDateRange,
connection: 'clickhouse-conn',
timestampValueExpression: 'timestamp',
source: 'traces-source',
where: '',
whereLanguage: 'sql',
select: '',
},
keys: ['service.name'],
},
]);
// First query succeeds, second query fails
mockMetadata.getKeyValues
.mockResolvedValueOnce([
{
key: 'environment',
value: ['production', 'staging'],
},
])
.mockRejectedValueOnce(new Error('Failed to fetch service.name'));
// Act
const { result } = renderHook(
() =>
useDashboardFilterKeyValues({
filters: mockFilters,
dateRange: mockDateRange,
}),
{ wrapper },
);
// Assert - Wait for queries to settle
await waitFor(() => expect(result.current.isLoading).toBe(false));
await waitFor(() => expect(result.current.isFetching).toBe(false));
// Should have partial results - environment loaded successfully
expect(result.current.data?.get('environment')).toEqual({
values: ['production', 'staging'],
isLoading: false,
});
// service.name should not be in the map because the query failed
expect(result.current.data?.has('service.name')).toBe(false);
// Overall error state should be true
expect(result.current.isError).toBe(true);
// Should have called getKeyValues twice
expect(mockMetadata.getKeyValues).toHaveBeenCalledTimes(2);
});
it('should keep previous data while fetching new data (placeholderData behavior)', async () => {
// Arrange
jest.spyOn(sourceModule, 'useSources').mockReturnValue({
data: mockSources,
isLoading: false,
} as any);
const initialDateRange: [Date, Date] = [
new Date('2024-01-01'),
new Date('2024-01-02'),
];
const updatedDateRange: [Date, Date] = [
new Date('2024-01-03'),
new Date('2024-01-04'),
];
// Mock optimizeGetKeyValuesCalls for both date ranges
jest
.mocked(optimizeGetKeyValuesCalls)
.mockImplementation(async ({ chartConfig }) => [
{
chartConfig,
keys: ['environment'],
},
]);
// Initial data
mockMetadata.getKeyValues.mockResolvedValueOnce([
{
key: 'environment',
value: ['production', 'staging'],
},
]);
// Act - Initial render
const { result, rerender } = renderHook(
({ filters, dateRange }) =>
useDashboardFilterKeyValues({
filters,
dateRange,
}),
{
wrapper,
initialProps: {
filters: [mockFilters[0]], // Only environment filter from logs-source
dateRange: initialDateRange,
},
},
);
// Wait for initial data to load
await waitFor(() => expect(result.current.isFetching).toBe(false));
const initialData = result.current.data;
expect(initialData?.get('environment')).toEqual({
values: ['production', 'staging'],
isLoading: false,
});
// Setup a Promise we can control for the next query
let resolveNextQuery: (value: { key: string; value: string[] }[]) => void;
const nextQueryPromise = new Promise<{ key: string; value: string[] }[]>(
resolve => {
resolveNextQuery = resolve;
},
);
mockMetadata.getKeyValues.mockImplementationOnce(async () => {
return nextQueryPromise;
});
// Rerender with new date range
rerender({
filters: [mockFilters[0]],
dateRange: updatedDateRange,
});
// Wait for refetch to start
await waitFor(() => expect(result.current.isFetching).toBe(true));
// Verify that previous data is still available during fetch (placeholderData behavior)
expect(result.current.data?.get('environment')).toEqual({
values: ['production', 'staging'],
isLoading: false,
});
expect(result.current.isFetching).toBe(true);
// Resolve the new query with updated data
resolveNextQuery!([
{
key: 'environment',
value: ['development', 'testing'],
},
]);
// Wait for new data to load
await waitFor(() => expect(result.current.isFetching).toBe(false));
// Verify that new data has replaced the old data
expect(result.current.data?.get('environment')).toEqual({
values: ['development', 'testing'],
isLoading: false,
});
// Verify optimizeGetKeyValuesCalls was called twice (once for each date range)
expect(optimizeGetKeyValuesCalls).toHaveBeenCalledTimes(2);
expect(optimizeGetKeyValuesCalls).toHaveBeenCalledWith(
expect.objectContaining({
chartConfig: expect.objectContaining({
dateRange: initialDateRange,
}),
}),
);
expect(optimizeGetKeyValuesCalls).toHaveBeenCalledWith(
expect.objectContaining({
chartConfig: expect.objectContaining({
dateRange: updatedDateRange,
}),
}),
);
});
});

57
packages/app/src/hooks/__tests__/useMetadata.test.tsx
View file

@ -9,6 +9,8 @@ import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
import { renderHook, waitFor } from '@testing-library/react';
import { useSources } from '@/source';
import {
deduplicate2dArray,
useGetKeyValues,
@ -29,6 +31,13 @@ const createMockChartConfig = (
...overrides,
}) as ChartConfigWithDateRange;
jest.mock('@/source', () => ({
useSources: jest.fn().mockReturnValue({
data: [{ id: 'source1' }, { id: 'source2' }],
isLoading: false,
}),
}));
describe('useGetKeyValues', () => {
let queryClient: QueryClient;
let wrapper: React.ComponentType<{ children: any }>;
@ -70,7 +79,9 @@ describe('useGetKeyValues', () => {
},
];
jest.spyOn(mockMetadata, 'getKeyValues').mockResolvedValue(mockKeyValues);
jest
.spyOn(mockMetadata, 'getKeyValuesWithMVs')
.mockResolvedValue(mockKeyValues);
// Act
const { result } = renderHook(
@ -108,7 +119,7 @@ describe('useGetKeyValues', () => {
];
jest
.spyOn(mockMetadata, 'getKeyValues')
.spyOn(mockMetadata, 'getKeyValuesWithMVs')
.mockResolvedValueOnce([
{
key: "ResourceAttributes['service.name']",
@ -145,7 +156,9 @@ describe('useGetKeyValues', () => {
value: ['production', 'staging'],
},
]);
expect(jest.spyOn(mockMetadata, 'getKeyValues')).toHaveBeenCalledTimes(2);
expect(
jest.spyOn(mockMetadata, 'getKeyValuesWithMVs'),
).toHaveBeenCalledTimes(2);
});
// Test case: Handling empty keys
@ -165,7 +178,9 @@ describe('useGetKeyValues', () => {
// Assert
expect(result.current.isFetched).toBe(false);
expect(jest.spyOn(mockMetadata, 'getKeyValues')).not.toHaveBeenCalled();
expect(
jest.spyOn(mockMetadata, 'getKeyValuesWithMVs'),
).not.toHaveBeenCalled();
});
// Test case: Custom limit and disableRowLimit
@ -181,7 +196,9 @@ describe('useGetKeyValues', () => {
},
];
jest.spyOn(mockMetadata, 'getKeyValues').mockResolvedValue(mockKeyValues);
jest
.spyOn(mockMetadata, 'getKeyValuesWithMVs')
.mockResolvedValue(mockKeyValues);
// Act
const { result } = renderHook(
@ -206,7 +223,7 @@ describe('useGetKeyValues', () => {
const mockKeys = ['ResourceAttributes.service.name'];
jest
.spyOn(mockMetadata, 'getKeyValues')
.spyOn(mockMetadata, 'getKeyValuesWithMVs')
.mockRejectedValue(new Error('Fetch failed'));
// Act
@ -225,6 +242,34 @@ describe('useGetKeyValues', () => {
expect(result.current.error).toEqual(expect.any(Error));
expect(result.current.error!.message).toBe('Fetch failed');
});
it('should be in a loading state while fetching sources', async () => {
jest.mocked(useSources).mockReturnValue({
data: undefined,
isLoading: true,
isFetching: true,
} as any);
// Arrange
const mockChartConfig = createMockChartConfig();
const mockKeys = ['ResourceAttributes.service.name'];
// Act
const { result } = renderHook(
() =>
useGetKeyValues({
chartConfig: mockChartConfig,
keys: mockKeys,
}),
{ wrapper },
);
// Assert
expect(
jest.spyOn(mockMetadata, 'getKeyValuesWithMVs'),
).not.toHaveBeenCalled();
await waitFor(() => expect(result.current.isLoading).toBe(true));
});
});
describe('deduplicate2dArray', () => {

171
packages/app/src/hooks/useDashboardFilterValues.tsx Normal file
View file

@ -0,0 +1,171 @@
import { useMemo } from 'react';
import { omit, pick } from 'lodash';
import {
GetKeyValueCall,
optimizeGetKeyValuesCalls,
} from '@hyperdx/common-utils/dist/core/materializedViews';
import {
ChartConfigWithDateRange,
DashboardFilter,
} from '@hyperdx/common-utils/dist/types';
import {
useQueries,
useQueryClient,
UseQueryResult,
} from '@tanstack/react-query';
import { useClickhouseClient } from '@/clickhouse';
import { useSources } from '@/source';
import { useMetadataWithSettings } from './useMetadata';
function useOptimizedKeyValuesCalls({
filters,
dateRange,
}: {
filters: DashboardFilter[];
dateRange: [Date, Date];
}) {
const clickhouseClient = useClickhouseClient();
const metadata = useMetadataWithSettings();
const { data: sources, isLoading: isLoadingSources } = useSources();
const filtersBySourceId = useMemo(() => {
const filtersBySourceId = new Map<string, DashboardFilter[]>();
for (const filter of filters) {
if (!filtersBySourceId.has(filter.source)) {
filtersBySourceId.set(filter.source, [filter]);
} else {
filtersBySourceId.get(filter.source)!.push(filter);
}
}
return filtersBySourceId;
}, [filters]);
const results: UseQueryResult<GetKeyValueCall<ChartConfigWithDateRange>[]>[] =
useQueries({
queries: Array.from(filtersBySourceId.entries())
.filter(([sourceId]) => sources?.some(s => s.id === sourceId))
.map(([sourceId, filters]) => {
const source = sources!.find(s => s.id === sourceId)!;
const keys = filters.map(f => f.expression);
const chartConfig: ChartConfigWithDateRange = {
...pick(source, ['timestampValueExpression', 'connection', 'from']),
dateRange,
source: source.id,
where: '',
whereLanguage: 'sql',
select: '',
};
return {
queryKey: [
'dashboard-filters-key-value-calls',
sourceId,
dateRange,
keys,
],
enabled: !isLoadingSources,
staleTime: 1000 * 60 * 5, // Cache every 5 min
queryFn: async ({ signal }) =>
await optimizeGetKeyValuesCalls({
chartConfig,
source,
clickhouseClient,
metadata,
keys,
signal,
}),
};
}),
});
return {
data: results.map(r => r.data ?? []).flat(),
isFetching: isLoadingSources || results.some(r => r.isFetching),
isLoading: isLoadingSources || results.every(r => r.isLoading),
};
}
export function useDashboardFilterKeyValues({
filters,
dateRange,
}: {
filters: DashboardFilter[];
dateRange: [Date, Date];
}) {
const metadata = useMetadataWithSettings();
const {
data: calls,
isFetching: isFetchingOptimizedCalls,
isLoading: isLoadingOptimizedCalls,
} = useOptimizedKeyValuesCalls({
filters,
dateRange,
});
const queryClient = useQueryClient();
type TQueryData = { key: string; value: string[] }[];
const results: UseQueryResult<TQueryData>[] = useQueries({
queries: calls.map(({ chartConfig, keys }) => {
// Construct a query key prefix which will allow us to use placeholder data from the previous query for the same keys
const queryKeyPrefix = [
'dashboard-filter-key-values',
chartConfig.from,
keys,
];
return {
queryKey: [...queryKeyPrefix, chartConfig],
placeholderData: () => {
// Use placeholder data from the most recently cached query with the same key prefix
const cached = queryClient
.getQueriesData<TQueryData>({ queryKey: queryKeyPrefix })
.map(([key, data]) => ({ key, data }))
.filter(({ data }) => !!data)
.toSorted((a, b) => {
const aTime =
queryClient.getQueryState(a.key)?.dataUpdatedAt ?? 0;
const bTime =
queryClient.getQueryState(b.key)?.dataUpdatedAt ?? 0;
return bTime - aTime;
});
return cached[0]?.data;
},
enabled: !isLoadingOptimizedCalls,
staleTime: 1000 * 60 * 5, // Cache every 5 min
queryFn: async ({ signal }) =>
metadata.getKeyValues({
chartConfig,
keys,
limit: 10000,
disableRowLimit: true,
signal,
}),
};
}),
});
const flattenedData = useMemo(
() =>
new Map(
results.flatMap(({ isLoading, data = [] }) => {
return data.map(({ key, value }) => [
key,
{
values: value,
isLoading,
},
]);
}),
),
[results],
);
return {
data: flattenedData,
isLoading: isLoadingOptimizedCalls || results.every(r => r.isLoading),
isFetching: isFetchingOptimizedCalls || results.some(r => r.isFetching),
isError: results.some(r => r.isError),
};
}

29
packages/app/src/hooks/useMetadata.tsx
View file

@ -22,6 +22,7 @@ import api from '@/api';
import { IS_LOCAL_MODE } from '@/config';
import { LOCAL_STORE_CONNECTIONS_KEY } from '@/connection';
import { getMetadata } from '@/metadata';
import { useSources } from '@/source';
import { toArray } from '@/utils';
// Hook to get metadata with proper settings applied
@ -209,32 +210,46 @@ export function useMultipleGetKeyValues(
) {
const metadata = useMetadataWithSettings();
const chartConfigsArr = toArray(chartConfigs);
return useQuery<{ key: string; value: string[] }[]>({
const { enabled = true } = options || {};
const { data: sources, isLoading: isLoadingSources } = useSources();
const query = useQuery<{ key: string; value: string[] }[]>({
queryKey: [
'useMetadata.useGetKeyValues',
...chartConfigsArr.map(cc => ({ ...cc })),
...keys,
disableRowLimit,
],
queryFn: async () => {
queryFn: async ({ signal }) => {
return (
await Promise.all(
chartConfigsArr.map(chartConfig =>
metadata.getKeyValues({
chartConfigsArr.map(chartConfig => {
const source = chartConfig.source
? sources?.find(s => s.id === chartConfig.source)
: undefined;
return metadata.getKeyValuesWithMVs({
chartConfig,
keys: keys.slice(0, 20), // Limit to 20 keys for now, otherwise request fails (max header size)
limit,
disableRowLimit,
}),
),
source,
signal,
});
}),
)
).flatMap(v => v);
},
staleTime: 1000 * 60 * 5, // Cache every 5 min
enabled: !!keys.length,
placeholderData: keepPreviousData,
...options,
enabled: !!enabled && !!keys.length && !isLoadingSources,
});
return {
...query,
isLoading: query.isLoading || isLoadingSources,
};
}
export function useGetValuesDistribution(

317
packages/common-utils/src/__tests__/metadata.int.test.ts
View file

@ -190,4 +190,321 @@ describe('Metadata Integration Tests', () => {
});
});
});
describe('getKeyValuesWithMVs', () => {
let metadata: Metadata;
const baseTableName = 'test_logs_base';
const mvTableName = 'test_logs_mv_1m';
const chartConfig: ChartConfigWithDateRange = {
connection: 'test_connection',
from: {
databaseName: 'default',
tableName: baseTableName,
},
dateRange: [new Date('2024-01-01'), new Date('2024-01-31')],
select: '',
timestampValueExpression: 'Timestamp',
where: '',
};
beforeAll(async () => {
// Create base table
await client.command({
query: `CREATE OR REPLACE TABLE default.${baseTableName} (
Timestamp DateTime64(9) CODEC(Delta(8), ZSTD(1)),
environment LowCardinality(String) CODEC(ZSTD(1)),
service LowCardinality(String) CODEC(ZSTD(1)),
status_code LowCardinality(String) CODEC(ZSTD(1)),
region LowCardinality(String) CODEC(ZSTD(1)),
message String CODEC(ZSTD(1))
)
ENGINE = MergeTree()
ORDER BY (Timestamp)
`,
});
// Create materialized view
await client.command({
query: `CREATE MATERIALIZED VIEW IF NOT EXISTS default.${mvTableName}
ENGINE = SummingMergeTree()
ORDER BY (environment, service, status_code, Timestamp)
AS SELECT
toStartOfMinute(Timestamp) as Timestamp,
environment,
service,
status_code,
count() as count
FROM default.${baseTableName}
GROUP BY Timestamp, environment, service, status_code
`,
});
// Insert data again to populate the MV (MVs don't get historical data)
await client.command({
query: `INSERT INTO default.${baseTableName}
(Timestamp, environment, service, status_code, region, message) VALUES
('2024-01-10 12:00:00', 'production', 'api', '200', 'us-east', 'Success'),
('2024-01-10 13:00:00', 'production', 'web', '200', 'us-west', 'Success'),
('2024-01-10 14:00:00', 'staging', 'api', '500', 'us-east', 'Error'),
('2024-01-10 15:00:00', 'staging', 'worker', '200', 'eu-west', 'Success'),
('2024-01-10 16:00:00', 'production', 'api', '404', 'us-east', 'Not found')
`,
});
});
beforeEach(async () => {
metadata = new Metadata(hdxClient, new MetadataCache());
});
afterAll(async () => {
await client.command({
query: `DROP VIEW IF EXISTS default.${mvTableName}`,
});
await client.command({
query: `DROP TABLE IF EXISTS default.${baseTableName}`,
});
});
it('should fetch key values using materialized views when available', async () => {
const source = {
id: 'test-source',
name: 'Test Logs',
kind: 'otel-logs',
from: { databaseName: 'default', tableName: baseTableName },
timestampValueExpression: 'Timestamp',
connection: 'test_connection',
materializedViews: [
{
databaseName: 'default',
tableName: mvTableName,
dimensionColumns: 'environment, service, status_code',
minGranularity: '1 minute',
timestampColumn: 'Timestamp',
aggregatedColumns: [{ aggFn: 'count', mvColumn: 'count' }],
},
],
};
const result = await metadata.getKeyValuesWithMVs({
chartConfig,
keys: ['environment', 'service', 'status_code'],
source: source as any,
});
expect(result).toHaveLength(3);
const environmentResult = result.find(r => r.key === 'environment');
expect(environmentResult?.value).toEqual(
expect.arrayContaining(['production', 'staging']),
);
const serviceResult = result.find(r => r.key === 'service');
expect(serviceResult?.value).toEqual(
expect.arrayContaining(['api', 'web', 'worker']),
);
const statusCodeResult = result.find(r => r.key === 'status_code');
expect(statusCodeResult?.value).toEqual(
expect.arrayContaining(['200', '404', '500']),
);
});
it('should fall back to base table for keys not in materialized view', async () => {
const source = {
id: 'test-source',
name: 'Test Logs',
kind: 'otel-logs',
from: { databaseName: 'default', tableName: baseTableName },
timestampValueExpression: 'Timestamp',
connection: 'test_connection',
materializedViews: [
{
databaseName: 'default',
tableName: mvTableName,
dimensionColumns: 'environment, service, status_code',
minGranularity: '1 minute',
timestampColumn: 'Timestamp',
aggregatedColumns: [{ aggFn: 'count', mvColumn: 'count' }],
},
],
};
// Query for keys both in and not in the MV
const result = await metadata.getKeyValuesWithMVs({
chartConfig,
keys: ['environment', 'region'], // 'region' is NOT in the MV
source: source as any,
});
expect(result).toHaveLength(2);
const environmentResult = result.find(r => r.key === 'environment');
expect(environmentResult?.value).toEqual(
expect.arrayContaining(['production', 'staging']),
);
const regionResult = result.find(r => r.key === 'region');
expect(regionResult?.value).toEqual(
expect.arrayContaining(['us-east', 'us-west', 'eu-west']),
);
});
it('should work without materialized views (fall back to base table)', async () => {
const source = {
id: 'test-source',
name: 'Test Logs',
kind: 'otel-logs',
from: { databaseName: 'default', tableName: baseTableName },
timestampValueExpression: 'Timestamp',
connection: 'test_connection',
materializedViews: [], // No MVs
};
const result = await metadata.getKeyValuesWithMVs({
chartConfig,
keys: ['environment', 'service'],
source: source as any,
});
expect(result).toHaveLength(2);
const environmentResult = result.find(r => r.key === 'environment');
expect(environmentResult?.value).toEqual(
expect.arrayContaining(['production', 'staging']),
);
const serviceResult = result.find(r => r.key === 'service');
expect(serviceResult?.value).toEqual(
expect.arrayContaining(['api', 'web', 'worker']),
);
});
it('should work without source parameter (fall back to base table)', async () => {
const result = await metadata.getKeyValuesWithMVs({
chartConfig,
keys: ['environment', 'service'],
// No source parameter
});
expect(result).toHaveLength(2);
const environmentResult = result.find(r => r.key === 'environment');
expect(environmentResult?.value).toEqual(
expect.arrayContaining(['production', 'staging']),
);
const serviceResult = result.find(r => r.key === 'service');
expect(serviceResult?.value).toEqual(
expect.arrayContaining(['api', 'web', 'worker']),
);
});
it('should return empty array for empty keys', async () => {
const source = {
id: 'test-source',
name: 'Test Logs',
kind: 'otel-logs',
from: { databaseName: 'default', tableName: baseTableName },
timestampValueExpression: 'Timestamp',
connection: 'test_connection',
materializedViews: [
{
databaseName: 'default',
tableName: mvTableName,
dimensionColumns: 'environment, service, status_code',
minGranularity: '1 minute',
timestampColumn: 'Timestamp',
aggregatedColumns: [{ aggFn: 'count', mvColumn: 'count' }],
},
],
};
const result = await metadata.getKeyValuesWithMVs({
chartConfig,
keys: [],
source: source as any,
});
expect(result).toEqual([]);
});
it('should respect limit parameter', async () => {
const source = {
id: 'test-source',
name: 'Test Logs',
kind: 'otel-logs',
from: { databaseName: 'default', tableName: baseTableName },
timestampValueExpression: 'Timestamp',
connection: 'test_connection',
materializedViews: [
{
databaseName: 'default',
tableName: mvTableName,
dimensionColumns: 'environment, service, status_code',
minGranularity: '1 minute',
timestampColumn: 'Timestamp',
aggregatedColumns: [{ aggFn: 'count', mvColumn: 'count' }],
},
],
};
const result = await metadata.getKeyValuesWithMVs({
chartConfig,
keys: ['service'],
source: source as any,
limit: 2,
});
expect(result).toHaveLength(1);
expect(result[0].key).toBe('service');
expect(result[0].value.length).toBeLessThanOrEqual(2);
});
it('should work with disableRowLimit: true', async () => {
const source = {
id: 'test-source',
name: 'Test Logs',
kind: 'otel-logs',
from: { databaseName: 'default', tableName: baseTableName },
timestampValueExpression: 'Timestamp',
connection: 'test_connection',
materializedViews: [
{
databaseName: 'default',
tableName: mvTableName,
dimensionColumns: 'environment, service, status_code',
minGranularity: '1 minute',
timestampColumn: 'Timestamp',
aggregatedColumns: [{ aggFn: 'count', mvColumn: 'count' }],
},
],
};
// Should work with disableRowLimit: true (no row limits applied)
const result = await metadata.getKeyValuesWithMVs({
chartConfig,
keys: ['environment', 'service', 'status_code'],
source: source as any,
disableRowLimit: true,
});
expect(result).toHaveLength(3);
const environmentResult = result.find(r => r.key === 'environment');
expect(environmentResult?.value).toEqual(
expect.arrayContaining(['production', 'staging']),
);
const serviceResult = result.find(r => r.key === 'service');
expect(serviceResult?.value).toEqual(
expect.arrayContaining(['api', 'web', 'worker']),
);
const statusCodeResult = result.find(r => r.key === 'status_code');
expect(statusCodeResult?.value).toEqual(
expect.arrayContaining(['200', '404', '500']),
);
});
});
});

577
packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts
View file

@ -1,5 +1,6 @@
import {
isUnsupportedCountFunction,
optimizeGetKeyValuesCalls,
tryConvertConfigToMaterializedViewSelect,
tryOptimizeConfigWithMaterializedView,
tryOptimizeConfigWithMaterializedViewWithExplanations,
@ -188,6 +189,7 @@ describe('materializedViews', () => {
databaseName: 'default',
tableName: 'metrics_rollup_1m',
},
timestampValueExpression: 'Timestamp',
select: [
{
valueExpression: 'sum__Duration',
@ -233,6 +235,7 @@ describe('materializedViews', () => {
aggFn: 'avgMerge',
},
],
timestampValueExpression: 'Timestamp',
where: '',
connection: 'test-connection',
});
@ -274,6 +277,7 @@ describe('materializedViews', () => {
level: 0.95,
},
],
timestampValueExpression: 'Timestamp',
where: '',
connection: 'test-connection',
});
@ -315,6 +319,7 @@ describe('materializedViews', () => {
level: 20,
},
],
timestampValueExpression: 'Timestamp',
where: '',
connection: 'test-connection',
});
@ -355,6 +360,7 @@ describe('materializedViews', () => {
aggFn: 'sum',
},
],
timestampValueExpression: 'Timestamp',
where: "StatusCode = '200'",
groupBy: 'StatusCode',
connection: 'test-connection',
@ -399,6 +405,7 @@ describe('materializedViews', () => {
],
where: '',
connection: 'test-connection',
timestampValueExpression: 'Timestamp',
});
expect(result.errors).toBeUndefined();
});
@ -458,6 +465,7 @@ describe('materializedViews', () => {
],
where: '',
connection: 'test-connection',
timestampValueExpression: 'Timestamp',
});
expect(result.errors).toBeUndefined();
});
@ -501,6 +509,7 @@ describe('materializedViews', () => {
],
where: '',
connection: 'test-connection',
timestampValueExpression: 'Timestamp',
});
expect(result.errors).toBeUndefined();
});
@ -540,6 +549,7 @@ describe('materializedViews', () => {
],
where: '',
connection: 'test-connection',
timestampValueExpression: 'Timestamp',
});
expect(result.errors).toBeUndefined();
});
@ -579,6 +589,7 @@ describe('materializedViews', () => {
],
where: '',
connection: 'test-connection',
timestampValueExpression: 'Timestamp',
});
expect(result.errors).toBeUndefined();
});
@ -623,6 +634,7 @@ describe('materializedViews', () => {
],
where: '',
connection: 'test-connection',
timestampValueExpression: 'Timestamp',
});
expect(result.errors).toBeUndefined();
});
@ -754,6 +766,7 @@ describe('materializedViews', () => {
where: '',
connection: 'test-connection',
granularity: '1 minute',
timestampValueExpression: 'Timestamp',
});
expect(result.errors).toBeUndefined();
});
@ -926,6 +939,7 @@ describe('materializedViews', () => {
where: '',
connection: 'test-connection',
dateRangeEndInclusive: false,
timestampValueExpression: 'Timestamp',
});
expect(result.errors).toBeUndefined();
});
@ -973,6 +987,7 @@ describe('materializedViews', () => {
granularity: '1 minute',
dateRange: chartConfig.dateRange,
dateRangeEndInclusive: false,
timestampValueExpression: 'Timestamp',
});
expect(result.errors).toBeUndefined();
});
@ -1021,6 +1036,7 @@ describe('materializedViews', () => {
new Date('2023-01-02T01:01:00Z'),
],
dateRangeEndInclusive: false,
timestampValueExpression: 'Timestamp',
});
});
});
@ -1130,6 +1146,7 @@ describe('materializedViews', () => {
],
where: '',
connection: 'test-connection',
timestampValueExpression: 'Timestamp',
},
}),
);
@ -1164,6 +1181,7 @@ describe('materializedViews', () => {
],
where: '',
connection: 'test-connection',
timestampValueExpression: 'Timestamp',
};
const actual = await tryOptimizeConfigWithMaterializedView(
@ -1235,6 +1253,7 @@ describe('materializedViews', () => {
],
where: '',
connection: 'test-connection',
timestampValueExpression: 'Timestamp',
};
const actual = await tryOptimizeConfigWithMaterializedView(
@ -1300,6 +1319,7 @@ describe('materializedViews', () => {
],
where: '',
connection: 'test-connection',
timestampValueExpression: 'Timestamp',
},
},
],
@ -1396,6 +1416,7 @@ describe('materializedViews', () => {
],
where: '',
connection: 'test-connection',
timestampValueExpression: 'Timestamp',
},
},
{
@ -1413,6 +1434,7 @@ describe('materializedViews', () => {
],
where: '',
connection: 'test-connection',
timestampValueExpression: 'Timestamp',
},
},
],
@ -1550,6 +1572,7 @@ describe('materializedViews', () => {
],
where: '',
connection: 'test-connection',
timestampValueExpression: 'Timestamp',
},
}),
);
@ -1584,6 +1607,7 @@ describe('materializedViews', () => {
],
where: '',
connection: 'test-connection',
timestampValueExpression: 'Timestamp',
};
const result =
@ -1804,6 +1828,7 @@ describe('materializedViews', () => {
],
where: '',
connection: 'test-connection',
timestampValueExpression: 'Timestamp',
};
const result =
@ -1844,4 +1869,556 @@ describe('materializedViews', () => {
);
});
});
describe('optimizeGetKeyValuesCalls', () => {
const mockClickHouseClient = {
testChartConfigValidity: jest.fn(),
} as unknown as jest.Mocked<ClickhouseClient>;
const MV_CONFIG_LOGS_1M: MaterializedViewConfiguration = {
databaseName: 'default',
tableName: 'logs_rollup_1m',
dimensionColumns: 'environment, service, status_code',
minGranularity: '1 minute',
timestampColumn: 'Timestamp',
aggregatedColumns: [{ aggFn: 'count', mvColumn: 'count' }],
};
const MV_CONFIG_LOGS_1H: MaterializedViewConfiguration = {
databaseName: 'default',
tableName: 'logs_rollup_1h',
dimensionColumns: 'environment, region',
minGranularity: '1 hour',
timestampColumn: 'Timestamp',
aggregatedColumns: [{ aggFn: 'count', mvColumn: 'count' }],
};
beforeEach(() => {
jest.clearAllMocks();
});
it('should return MVs for all keys when single MV covers all keys', async () => {
const chartConfig: ChartConfigWithOptDateRange = {
from: {
databaseName: 'default',
tableName: 'logs',
},
where: '',
connection: 'test-connection',
dateRange: [new Date('2024-01-01'), new Date('2024-01-02')],
select: '',
};
const keys = ['environment', 'service', 'status_code'];
const source = {
from: { databaseName: 'default', tableName: 'logs' },
materializedViews: [MV_CONFIG_LOGS_1M],
};
mockClickHouseClient.testChartConfigValidity.mockResolvedValue({
isValid: true,
rowEstimate: 1000,
});
const result = await optimizeGetKeyValuesCalls({
chartConfig,
keys,
source: source as any,
clickhouseClient: mockClickHouseClient,
metadata,
});
expect(result).toHaveLength(1);
expect(result[0].keys).toEqual(['environment', 'service', 'status_code']);
expect(result[0].chartConfig.from).toEqual({
databaseName: 'default',
tableName: 'logs_rollup_1m',
});
expect(
mockClickHouseClient.testChartConfigValidity,
).toHaveBeenCalledTimes(1);
});
it('should distribute keys across multiple MVs', async () => {
const chartConfig: ChartConfigWithOptDateRange = {
from: {
databaseName: 'default',
tableName: 'logs',
},
where: '',
connection: 'test-connection',
dateRange: [new Date('2024-01-01'), new Date('2024-01-02')],
select: '',
};
const keys = ['environment', 'service', 'region'];
const source = {
from: { databaseName: 'default', tableName: 'logs' },
materializedViews: [MV_CONFIG_LOGS_1M, MV_CONFIG_LOGS_1H],
};
mockClickHouseClient.testChartConfigValidity.mockImplementation(
({ config }) =>
Promise.resolve({
isValid: true,
rowEstimate:
config.from.tableName === 'logs_rollup_1h' ? 500 : 1000,
}),
);
const result = await optimizeGetKeyValuesCalls({
chartConfig,
keys,
source: source as any,
clickhouseClient: mockClickHouseClient,
metadata,
});
// Should prefer logs_rollup_1h (500 rows) over logs_rollup_1m (1000 rows) for shared keys
expect(result).toHaveLength(2);
expect(result).toEqual(
expect.arrayContaining([
expect.objectContaining({
chartConfig: expect.objectContaining({
from: {
databaseName: 'default',
tableName: 'logs_rollup_1h',
},
}),
keys: expect.arrayContaining(['environment', 'region']),
}),
expect.objectContaining({
chartConfig: expect.objectContaining({
from: {
databaseName: 'default',
tableName: 'logs_rollup_1m',
},
}),
keys: ['service'],
}),
]),
);
});
it('should return uncovered keys when no MV supports them', async () => {
const chartConfig: ChartConfigWithOptDateRange = {
from: {
databaseName: 'default',
tableName: 'logs',
},
where: '',
connection: 'test-connection',
dateRange: [new Date('2024-01-01'), new Date('2024-01-02')],
select: '',
};
const keys = ['environment', 'unsupported_key'];
const source = {
from: { databaseName: 'default', tableName: 'logs' },
materializedViews: [MV_CONFIG_LOGS_1M],
};
mockClickHouseClient.testChartConfigValidity.mockResolvedValue({
isValid: true,
rowEstimate: 1000,
});
const result = await optimizeGetKeyValuesCalls({
chartConfig,
keys,
source: source as any,
clickhouseClient: mockClickHouseClient,
metadata,
});
expect(result).toHaveLength(2);
expect(result).toEqual(
expect.arrayContaining([
expect.objectContaining({
chartConfig: expect.objectContaining({
from: {
databaseName: 'default',
tableName: 'logs_rollup_1m',
},
}),
keys: ['environment'],
}),
expect.objectContaining({
chartConfig: expect.objectContaining({
from: {
databaseName: 'default',
tableName: 'logs',
},
}),
keys: ['unsupported_key'],
}),
]),
);
});
it('should skip invalid MVs and return uncovered keys', async () => {
const chartConfig: ChartConfigWithOptDateRange = {
from: {
databaseName: 'default',
tableName: 'logs',
},
where: '',
connection: 'test-connection',
dateRange: [new Date('2024-01-01'), new Date('2024-01-02')],
select: '',
};
const keys = ['environment', 'service'];
const source = {
from: { databaseName: 'default', tableName: 'logs' },
materializedViews: [MV_CONFIG_LOGS_1M],
};
mockClickHouseClient.testChartConfigValidity.mockResolvedValue({
isValid: false,
error: 'Invalid query',
});
const result = await optimizeGetKeyValuesCalls({
chartConfig,
keys,
source: source as any,
clickhouseClient: mockClickHouseClient,
metadata,
});
expect(result).toHaveLength(1);
expect(result[0]).toEqual({
chartConfig: expect.objectContaining({
from: {
databaseName: 'default',
tableName: 'logs',
},
}),
keys: ['environment', 'service'],
});
});
it('should prefer MVs with lower row estimates', async () => {
const chartConfig: ChartConfigWithOptDateRange = {
from: {
databaseName: 'default',
tableName: 'logs',
},
where: '',
connection: 'test-connection',
dateRange: [new Date('2024-01-01'), new Date('2024-01-02')],
select: '',
};
const keys = ['environment'];
const source = {
from: { databaseName: 'default', tableName: 'logs' },
materializedViews: [MV_CONFIG_LOGS_1M, MV_CONFIG_LOGS_1H],
};
mockClickHouseClient.testChartConfigValidity.mockImplementation(
({ config }) =>
Promise.resolve({
isValid: true,
rowEstimate:
config.from.tableName === 'logs_rollup_1h' ? 500 : 1000,
}),
);
const result = await optimizeGetKeyValuesCalls({
chartConfig,
keys,
source: source as any,
clickhouseClient: mockClickHouseClient,
metadata,
});
expect(result).toHaveLength(1);
expect(result[0]).toEqual({
chartConfig: expect.objectContaining({
from: {
databaseName: 'default',
tableName: 'logs_rollup_1h',
},
}),
keys: ['environment'],
});
});
it('should handle empty keys array', async () => {
const chartConfig: ChartConfigWithOptDateRange = {
from: {
databaseName: 'default',
tableName: 'logs',
},
where: '',
connection: 'test-connection',
dateRange: [new Date('2024-01-01'), new Date('2024-01-02')],
select: '',
};
const keys: string[] = [];
const source = {
from: { databaseName: 'default', tableName: 'logs' },
materializedViews: [MV_CONFIG_LOGS_1M],
};
const result = await optimizeGetKeyValuesCalls({
chartConfig,
keys,
source: source as any,
clickhouseClient: mockClickHouseClient,
metadata,
});
expect(result).toEqual([]);
expect(
mockClickHouseClient.testChartConfigValidity,
).not.toHaveBeenCalled();
});
it('should handle source with no materialized views', async () => {
const chartConfig: ChartConfigWithOptDateRange = {
from: {
databaseName: 'default',
tableName: 'logs',
},
where: '',
connection: 'test-connection',
dateRange: [new Date('2024-01-01'), new Date('2024-01-02')],
select: '',
};
const keys = ['environment', 'service'];
const source = {
from: { databaseName: 'default', tableName: 'logs' },
materializedViews: [],
};
const result = await optimizeGetKeyValuesCalls({
chartConfig,
keys,
source: source as any,
clickhouseClient: mockClickHouseClient,
metadata,
});
expect(result).toHaveLength(1);
expect(result[0]).toEqual({
chartConfig: expect.objectContaining({
from: {
databaseName: 'default',
tableName: 'logs',
},
}),
keys: ['environment', 'service'],
});
expect(
mockClickHouseClient.testChartConfigValidity,
).not.toHaveBeenCalled();
});
it('should filter out MVs that do not support the date range', async () => {
const MV_CONFIG_WITH_MIN_DATE: MaterializedViewConfiguration = {
databaseName: 'default',
tableName: 'logs_rollup_recent',
dimensionColumns: 'environment, service',
minGranularity: '1 minute',
timestampColumn: 'Timestamp',
aggregatedColumns: [{ aggFn: 'count', mvColumn: 'count' }],
minDate: '2024-01-15',
};
const chartConfig: ChartConfigWithOptDateRange = {
from: {
databaseName: 'default',
tableName: 'logs',
},
where: '',
connection: 'test-connection',
dateRange: [new Date('2024-01-01'), new Date('2024-01-02')],
select: '',
};
const keys = ['environment', 'service'];
const source = {
from: { databaseName: 'default', tableName: 'logs' },
materializedViews: [MV_CONFIG_WITH_MIN_DATE],
};
const result = await optimizeGetKeyValuesCalls({
chartConfig,
keys,
source: source as any,
clickhouseClient: mockClickHouseClient,
metadata,
});
expect(result).toHaveLength(1);
expect(result[0]).toEqual({
chartConfig: expect.objectContaining({
from: {
databaseName: 'default',
tableName: 'logs',
},
}),
keys: ['environment', 'service'],
});
expect(
mockClickHouseClient.testChartConfigValidity,
).not.toHaveBeenCalled();
});
it('should filter out MVs which have a granularity too large to support the date range', async () => {
const MV_CONFIG_WITH_MIN_DATE: MaterializedViewConfiguration = {
databaseName: 'default',
tableName: 'logs_rollup_recent',
dimensionColumns: 'environment, service',
minGranularity: '1 day',
timestampColumn: 'Timestamp',
aggregatedColumns: [{ aggFn: 'count', mvColumn: 'count' }],
};
const chartConfig: ChartConfigWithOptDateRange = {
from: {
databaseName: 'default',
tableName: 'logs',
},
where: '',
connection: 'test-connection',
dateRange: [new Date('2024-01-01'), new Date('2024-01-02')], // only contains 1 MV interval
select: '',
};
const keys = ['environment', 'service'];
const source = {
from: { databaseName: 'default', tableName: 'logs' },
materializedViews: [MV_CONFIG_WITH_MIN_DATE],
};
const result = await optimizeGetKeyValuesCalls({
chartConfig,
keys,
source: source as any,
clickhouseClient: mockClickHouseClient,
metadata,
});
expect(result).toHaveLength(1);
expect(result[0]).toEqual({
chartConfig: expect.objectContaining({
from: {
databaseName: 'default',
tableName: 'logs',
},
}),
keys: ['environment', 'service'],
});
expect(
mockClickHouseClient.testChartConfigValidity,
).not.toHaveBeenCalled();
});
it('should generate correct select statement with multiple keys', async () => {
const chartConfig: ChartConfigWithOptDateRange = {
from: {
databaseName: 'default',
tableName: 'logs',
},
where: '',
connection: 'test-connection',
dateRange: [new Date('2024-01-01'), new Date('2024-01-02')],
select: '',
};
const keys = ['environment', 'service', 'status_code'];
const source = {
from: { databaseName: 'default', tableName: 'logs' },
materializedViews: [MV_CONFIG_LOGS_1M],
};
mockClickHouseClient.testChartConfigValidity.mockResolvedValue({
isValid: true,
rowEstimate: 1000,
});
await optimizeGetKeyValuesCalls({
chartConfig,
keys,
source: source as any,
clickhouseClient: mockClickHouseClient,
metadata,
});
expect(mockClickHouseClient.testChartConfigValidity).toHaveBeenCalledWith(
expect.objectContaining({
config: expect.objectContaining({
select:
'groupUniqArray(1)(environment) AS param0, groupUniqArray(1)(service) AS param1, groupUniqArray(1)(status_code) AS param2',
}),
metadata,
}),
);
});
it('should use the MV timestamp column when it differs from the source table', async () => {
const MV_CONFIG_WITH_DIFFERENT_TIMESTAMP: MaterializedViewConfiguration =
{
databaseName: 'default',
tableName: 'logs_rollup_1h',
dimensionColumns: 'environment, service',
minGranularity: '1 hour',
timestampColumn: 'mv_timestamp', // Different from source
aggregatedColumns: [{ aggFn: 'count', mvColumn: 'count' }],
};
const chartConfig: ChartConfigWithOptDateRange = {
from: {
databaseName: 'default',
tableName: 'logs',
},
where: '',
connection: 'test-connection',
dateRange: [new Date('2024-01-01'), new Date('2024-01-02')],
select: '',
timestampValueExpression: 'source_timestamp', // Source uses this
};
const keys = ['environment', 'service'];
const source = {
from: { databaseName: 'default', tableName: 'logs' },
materializedViews: [MV_CONFIG_WITH_DIFFERENT_TIMESTAMP],
timestampValueExpression: 'source_timestamp',
};
mockClickHouseClient.testChartConfigValidity.mockResolvedValue({
isValid: true,
rowEstimate: 500,
});
const result = await optimizeGetKeyValuesCalls({
chartConfig,
keys,
source: source as any,
clickhouseClient: mockClickHouseClient,
metadata,
});
// Should return one call using the MV
expect(result).toHaveLength(1);
expect(result[0].chartConfig.from).toEqual({
databaseName: 'default',
tableName: 'logs_rollup_1h',
});
// Should use the MV's timestamp column
expect(result[0].chartConfig.timestampValueExpression).toBe(
'mv_timestamp',
);
// Keys should be from the MV
expect(result[0].keys).toEqual(['environment', 'service']);
});
});
});

152
packages/common-utils/src/core/materializedViews.ts
View file

@ -1,3 +1,5 @@
import { differenceInSeconds } from 'date-fns';
import { BaseClickhouseClient } from '@/clickhouse';
import {
ChartConfigWithOptDateRange,
@ -13,6 +15,7 @@ import {
convertDateRangeToGranularityString,
convertGranularityToSeconds,
getAlignedDateRange,
splitAndTrimWithBracket,
} from './utils';
type SelectItem = Exclude<
@ -156,6 +159,16 @@ function mvConfigSupportsGranularity(
);
}
function countIntervalsInDateRange(
dateRange: [Date, Date],
granularity: string,
) {
const [startDate, endDate] = dateRange;
const granularitySeconds = convertGranularityToSeconds(granularity);
const diffSeconds = differenceInSeconds(endDate, startDate);
return Math.floor(diffSeconds / granularitySeconds);
}
function mvConfigSupportsDateRange(
mvConfig: MaterializedViewConfiguration,
chartConfig: ChartConfigWithOptDateRange,
@ -340,6 +353,7 @@ export async function tryConvertConfigToMaterializedViewSelect<
const clonedConfig: C = {
...structuredClone(chartConfig),
select,
timestampValueExpression: mvConfig.timestampColumn,
from: {
databaseName: mvConfig.databaseName,
tableName: mvConfig.tableName,
@ -558,3 +572,141 @@ function formatAggregateFunction(aggFn: string, level: number | undefined) {
return aggFn;
}
}
function toMvId(
mv: Pick<MaterializedViewConfiguration, 'databaseName' | 'tableName'>,
) {
return `${mv.databaseName}.${mv.tableName}`;
}
export interface GetKeyValueCall<C extends ChartConfigWithOptDateRange> {
chartConfig: C;
keys: string[];
}
export async function optimizeGetKeyValuesCalls<
C extends ChartConfigWithOptDateRange,
>({
chartConfig,
keys,
source,
clickhouseClient,
metadata,
signal,
}: {
chartConfig: C;
keys: string[];
source: TSource;
clickhouseClient: BaseClickhouseClient;
metadata: Metadata;
signal?: AbortSignal;
}): Promise<GetKeyValueCall<C>[]> {
// Get the MVs from the source
const mvs = source?.materializedViews || [];
const mvsById = new Map(mvs.map(mv => [toMvId(mv), mv]));
// Identify keys which can be queried from a materialized view
const supportedKeysByMv = new Map<string, string[]>();
for (const [mvId, mv] of mvsById.entries()) {
const mvIntervalsInDateRange = chartConfig.dateRange
? countIntervalsInDateRange(chartConfig.dateRange, mv.minGranularity)
: Infinity;
if (
// Ensures that the MV contains data for the selected date range
mvConfigSupportsDateRange(mv, chartConfig) &&
// Ensures that the MV's granularity is small enough that the selected date
// range will include multiple MV time buckets. (3 is an arbitrary cutoff)
mvIntervalsInDateRange >= 3
) {
const dimensionColumns = splitAndTrimWithBracket(mv.dimensionColumns);
const keysInMV = keys.filter(k => dimensionColumns.includes(k));
if (keysInMV.length > 0) {
supportedKeysByMv.set(mvId, keysInMV);
}
}
}
// Build the configs which would be used to query each MV for all of the keys it supports
const configsToExplain = [...supportedKeysByMv.entries()].map(
([mvId, mvKeys]) => {
const { databaseName, tableName, timestampColumn } = mvsById.get(mvId)!;
return {
...structuredClone(chartConfig),
timestampValueExpression: timestampColumn,
from: {
databaseName,
tableName,
},
// These are dimension columns so we don't need to add any -Merge combinators
select: mvKeys
.map((k, i) => `groupUniqArray(1)(${k}) AS param${i}`)
.join(', '),
};
},
);
// Figure out which of those configs are valid by running EXPLAIN queries
const explainResults = await Promise.all(
configsToExplain.map(async config => {
const { isValid, rowEstimate = Number.POSITIVE_INFINITY } =
await clickhouseClient.testChartConfigValidity({
config,
metadata,
opts: { abort_signal: signal },
});
return {
id: toMvId({
databaseName: config.from.databaseName,
tableName: config.from.tableName,
}),
isValid,
rowEstimate,
};
}),
);
// For each key, find the best MV that can provide it while reading the fewest rows
const finalKeysByMv = new Map<string, string[]>();
const uncoveredKeys = new Set<string>(keys);
const sortedValidConfigs = explainResults
.filter(r => r.isValid)
.sort((a, b) => a.rowEstimate - b.rowEstimate);
for (const config of sortedValidConfigs) {
const mvKeys = supportedKeysByMv.get(config.id) ?? [];
// Only include keys which have not already been covered by a previous MV
const keysNotAlreadyCovered = mvKeys.filter(k => uncoveredKeys.has(k));
if (keysNotAlreadyCovered.length) {
finalKeysByMv.set(config.id, keysNotAlreadyCovered);
for (const key of keysNotAlreadyCovered) {
uncoveredKeys.delete(key);
}
}
}
// Build the final list of optimized calls
const calls = [...finalKeysByMv.entries()].map(([mvId, mvKeys]) => {
const { databaseName, tableName, timestampColumn } = mvsById.get(mvId)!;
const optimizedConfig: C = {
...structuredClone(chartConfig),
timestampValueExpression: timestampColumn,
from: {
databaseName,
tableName,
},
};
return {
chartConfig: optimizedConfig,
keys: mvKeys,
};
});
if (uncoveredKeys.size) {
calls.push({
chartConfig: structuredClone(chartConfig),
keys: [...uncoveredKeys],
});
}
return calls;
}

85
packages/common-utils/src/core/metadata.ts
View file

@ -14,6 +14,9 @@ import {
import { renderChartConfig } from '@/core/renderChartConfig';
import type { ChartConfig, ChartConfigWithDateRange, TSource } from '@/types';
import { optimizeGetKeyValuesCalls } from './materializedViews';
import { objectHash } from './utils';
// If filters initially are taking too long to load, decrease this number.
// Between 1e6 - 5e6 is a good range.
export const DEFAULT_METADATA_MAX_ROWS_TO_READ = 3e6;
@ -621,7 +624,7 @@ export class Metadata {
'with',
]);
return this.cache.getOrFetch(
`${JSON.stringify(cacheKeyConfig)}.${key}.valuesDistribution`,
`${objectHash(cacheKeyConfig)}.${key}.valuesDistribution`,
async () => {
const config: ChartConfigWithDateRange = {
...chartConfig,
@ -689,14 +692,28 @@ export class Metadata {
keys,
limit = 20,
disableRowLimit = false,
signal,
}: {
chartConfig: ChartConfigWithDateRange;
keys: string[];
limit?: number;
disableRowLimit?: boolean;
}) {
signal?: AbortSignal;
}): Promise<{ key: string; value: string[] }[]> {
const cacheKeyConfig = {
...pick(chartConfig, [
'connection',
'from',
'dateRange',
'where',
'with',
'filters',
]),
keys,
disableRowLimit,
};
return this.cache.getOrFetch(
`${chartConfig.connection}.${chartConfig.from.databaseName}.${chartConfig.from.tableName}.${keys.join(',')}.${chartConfig.dateRange.toString()}.${disableRowLimit}.values`,
`${objectHash(cacheKeyConfig)}.getKeyValues`,
async () => {
if (keys.length === 0) return [];
@ -764,6 +781,7 @@ export class Metadata {
max_rows_to_read: '0',
}
: undefined,
abort_signal: signal,
})
.then(res => res.json<any>());
@ -777,6 +795,67 @@ export class Metadata {
},
);
}
async getKeyValuesWithMVs({
chartConfig,
keys,
source,
limit = 20,
disableRowLimit,
signal,
}: {
chartConfig: ChartConfigWithDateRange;
keys: string[];
source?: TSource;
limit?: number;
disableRowLimit?: boolean;
signal?: AbortSignal;
}): Promise<{ key: string; value: string[] }[]> {
const cacheKeyConfig = {
...pick(chartConfig, [
'connection',
'from',
'dateRange',
'where',
'with',
'filters',
]),
keys,
disableRowLimit,
};
return this.cache.getOrFetch(
`${objectHash(cacheKeyConfig)}.getKeyValuesWithMVs`,
async () => {
if (keys.length === 0) return [];
const defaultKeyValueCall = { chartConfig, keys };
const getKeyValueCalls = source
? await optimizeGetKeyValuesCalls({
chartConfig,
keys,
source,
clickhouseClient: this.clickhouseClient,
metadata: this,
signal,
})
: [defaultKeyValueCall];
const allResults = await Promise.all(
getKeyValueCalls.map(async ({ chartConfig, keys }) =>
this.getKeyValues({
chartConfig,
keys,
limit,
disableRowLimit,
signal,
}),
),
);
return allResults.flat();
},
);
}
}
export type Field = {