Elgato_dark/hyperdx
1
0
Fork 0
mirror of https://github.com/hyperdxio/hyperdx synced 2026-04-21 13:37:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

feat: Add RawSqlChartConfig types for SQL-based Table (#1846)

Browse source 
## Summary



This PR is the first step towards raw SQL-driven charts. 
- It introduces updated ChartConfig types, which are now unions of `BuilderChartConfig` (which is unchanged from the current `ChartConfig` types` and `RawSqlChartConfig` types which represent sql-driven charts. 
- It adds _very basic_ support for SQL-driven tables in the Chart Explorer and Dashboard pages. This is currently behind a feature toggle and enabled only in preview environments and for local development.

The changes in most of the files in this PR are either type updates or the addition of type guards to handle the new ChartConfig union type. 

The DBEditTimeChartForm has been updated significantly to (a) add the Raw SQL option to the table chart editor and (b) handle conversion from internal form state (which can now include properties from either branch of the ChartConfig union) to valid SavedChartConfigs (which may only include properties from one branch).

Significant changes are in:
- packages/app/src/components/ChartEditor/types.ts
- packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx
- packages/app/src/components/ChartEditor/utils.ts
- packages/app/src/components/DBEditTimeChartForm.tsx
- packages/app/src/components/DBTableChart.tsx
- packages/app/src/components/SQLEditor.tsx
- packages/app/src/hooks/useOffsetPaginatedQuery.tsx

Future PRs will add templating to the Raw SQL driven charts for date range and granularity injection; support for other chart types driven by SQL; improved placeholder, validation, and error states; and improved support in the external API and import/export.

### Screenshots or video

https://github.com/user-attachments/assets/008579cc-ef3c-496e-9899-88bbb21eaa5e

### How to test locally or on Vercel

The SQL-driven table can be tested in the preview environment or locally. 

### References



- Linear Issue: HDX-3580
- Related PRs:
This commit is contained in:
Drew Davis 2026-03-05 15:30:58 -05:00 committed by GitHub
parent 32d45f738a
commit 32f1189a7d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
69 changed files with 1690 additions and 798 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.changeset/fast-swans-brake.md Normal file
View file

@ -0,0 +1,7 @@
---
"@hyperdx/common-utils": patch
"@hyperdx/api": patch
"@hyperdx/app": patch
---
feat: Add RawSqlChartConfig types for SQL-based Table

17
packages/api/src/controllers/dashboard.ts
View file

@ -1,4 +1,6 @@
import { isBuilderSavedChartConfig } from '@hyperdx/common-utils/dist/guards';
import {
BuilderSavedChartConfig,
DashboardWithoutIdSchema,
Tile,
} from '@hyperdx/common-utils/dist/types';
@ -17,7 +19,7 @@ import Dashboard from '@/models/dashboard';
function pickAlertsByTile(tiles: Tile[]) {
return tiles.reduce((acc, tile) => {
if (tile.config.alert) {
if (isBuilderSavedChartConfig(tile.config) && tile.config.alert) {
acc[tile.id] = tile.config.alert;
}
return acc;
@ -25,7 +27,9 @@ function pickAlertsByTile(tiles: Tile[]) {
}
type TileForAlertSync = Pick<Tile, 'id'> & {
config?: Pick<Tile['config'], 'alert'> | { alert?: IAlert | AlertDocument };
config?:
| Pick<BuilderSavedChartConfig, 'alert'>
| { alert?: IAlert | AlertDocument };
};
function extractTileAlertData(tiles: TileForAlertSync[]): {
@ -48,8 +52,15 @@ async function syncDashboardAlerts(
): Promise<void> {
const { tileIds: oldTileIds, tileIdsWithAlerts: oldTileIdsWithAlerts } =
extractTileAlertData(oldTiles);
const newTilesForAlertSync: TileForAlertSync[] = newTiles.map(t => ({
id: t.id,
config: isBuilderSavedChartConfig(t.config)
? { alert: t.config.alert }
: {},
}));
const { tileIds: newTileIds, tileIdsWithAlerts: newTileIdsWithAlerts } =
extractTileAlertData(newTiles);
extractTileAlertData(newTilesForAlertSync);
// 1. Create/update alerts for tiles that have alerts
const alertsByTile = pickAlertsByTile(newTiles);

9
packages/api/src/fixtures.ts
View file

@ -1,5 +1,6 @@
import { createNativeClient } from '@hyperdx/common-utils/dist/clickhouse/node';
import {
BuilderSavedChartConfig,
DisplayType,
SavedChartConfig,
Tile,
@ -421,7 +422,8 @@ export const randomMongoId = () =>
export const makeTile = (opts?: {
id?: string;
alert?: SavedChartConfig['alert'];
alert?: BuilderSavedChartConfig['alert'];
sourceId?: string;
}): Tile => ({
id: opts?.id ?? randomMongoId(),
x: 1,
@ -433,10 +435,11 @@ export const makeTile = (opts?: {
export const makeChartConfig = (opts?: {
id?: string;
alert?: SavedChartConfig['alert'];
alert?: BuilderSavedChartConfig['alert'];
sourceId?: string;
}): SavedChartConfig => ({
name: 'Test Chart',
source: 'test-source',
source: opts?.sourceId ?? 'test-source',
displayType: DisplayType.Line,
select: [
{

18
packages/api/src/routers/external-api/v2/utils/dashboards.ts
View file

@ -1,5 +1,7 @@
import { isRawSqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards';
import {
AggregateFunctionSchema,
BuilderSavedChartConfig,
DisplayType,
SavedChartConfig,
} from '@hyperdx/common-utils/dist/types';
@ -62,7 +64,7 @@ const DEFAULT_SELECT_ITEM: ExternalDashboardSelectItem = {
};
const convertToExternalSelectItem = (
item: Exclude<SavedChartConfig['select'][number], string>,
item: Exclude<BuilderSavedChartConfig['select'][number], string>,
): ExternalDashboardSelectItem => {
const parsedAggFn = AggregateFunctionSchema.safeParse(item.aggFn);
const aggFn = parsedAggFn.success ? parsedAggFn.data : 'none';
@ -84,6 +86,9 @@ const convertToExternalSelectItem = (
const convertToExternalTileChartConfig = (
config: SavedChartConfig,
): ExternalDashboardTileConfig | undefined => {
// HDX-3582: Implement this for Raw SQL charts
if (isRawSqlSavedChartConfig(config)) return undefined;
const sourceId = config.source?.toString() ?? '';
const stringValueOrDefault = <D>(
@ -175,7 +180,10 @@ const convertToExternalTileChartConfig = (
function convertTileToExternalChart(
tile: DashboardDocument['tiles'][number],
): ExternalDashboardTileWithId {
): ExternalDashboardTileWithId | undefined {
// HDX-3582: Implement this for Raw SQL charts
if (isRawSqlSavedChartConfig(tile.config)) return undefined;
// Returned in case of a failure converting the saved chart config
const defaultTileConfig: ExternalDashboardTileConfig = {
displayType: 'line',
@ -196,7 +204,9 @@ export function convertToExternalDashboard(
return {
id: dashboard._id.toString(),
name: dashboard.name,
tiles: dashboard.tiles.map(convertTileToExternalChart),
tiles: dashboard.tiles
.map(convertTileToExternalChart)
.filter(t => t !== undefined),
tags: dashboard.tags || [],
filters: dashboard.filters?.map(translateFilterToExternalFilter) || [],
savedQuery: dashboard.savedQuery ?? null,
@ -211,7 +221,7 @@ export function convertToExternalDashboard(
const convertToInternalSelectItem = (
item: ExternalDashboardSelectItem,
): Exclude<SavedChartConfig['select'][number], string> => {
): Exclude<BuilderSavedChartConfig['select'][number], string> => {
return {
...pick(item, ['alias', 'metricType', 'metricName', 'aggFn', 'level']),
aggCondition: item.where,

17
packages/api/src/tasks/checkAlerts/index.ts
View file

@ -17,7 +17,11 @@ import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartCo
import { aliasMapToWithClauses } from '@hyperdx/common-utils/dist/core/utils';
import { timeBucketByGranularity } from '@hyperdx/common-utils/dist/core/utils';
import {
ChartConfigWithOptDateRange,
isBuilderSavedChartConfig,
isRawSqlSavedChartConfig,
} from '@hyperdx/common-utils/dist/guards';
import {
BuilderChartConfigWithOptDateRange,
DisplayType,
} from '@hyperdx/common-utils/dist/types';
import * as fns from 'date-fns';
@ -67,6 +71,7 @@ export const alertHasGroupBy = (details: AlertDetails): boolean => {
}
if (
details.taskType === AlertTaskType.TILE &&
isBuilderSavedChartConfig(details.tile.config) &&
details.tile.config.groupBy &&
details.tile.config.groupBy.length > 0
) {
@ -84,10 +89,10 @@ export async function computeAliasWithClauses(
savedSearch: Pick<ISavedSearch, 'select' | 'where' | 'whereLanguage'>,
source: ISource,
metadata: Metadata,
): Promise<ChartConfigWithOptDateRange['with']> {
): Promise<BuilderChartConfigWithOptDateRange['with']> {
const resolvedSelect =
savedSearch.select || source.defaultTableSelectExpression || '';
const config: ChartConfigWithOptDateRange = {
const config: BuilderChartConfigWithOptDateRange = {
connection: '',
displayType: DisplayType.Search,
from: source.from,
@ -317,7 +322,7 @@ const getChartConfigFromAlert = (
connection: string,
dateRange: [Date, Date],
windowSizeInMins: number,
): ChartConfigWithOptDateRange | undefined => {
): BuilderChartConfigWithOptDateRange | undefined => {
const { alert, source } = details;
if (details.taskType === AlertTaskType.SAVED_SEARCH) {
const savedSearch = details.savedSearch;
@ -344,6 +349,10 @@ const getChartConfigFromAlert = (
};
} else if (details.taskType === AlertTaskType.TILE) {
const tile = details.tile;
// Alerts are not supported for raw sql based charts
if (isRawSqlSavedChartConfig(tile.config)) return undefined;
// Doesn't work for metric alerts yet
if (
tile.config.displayType === DisplayType.Line ||

24
packages/api/src/tasks/checkAlerts/providers/__tests__/default.test.ts
View file

@ -148,8 +148,10 @@ describe('DefaultAlertProvider', () => {
});
// Create tile with source
const tile = makeTile({ id: 'test-tile-123' });
tile.config.source = source._id.toString();
const tile = makeTile({
id: 'test-tile-123',
sourceId: source._id.toString(),
});
// Create dashboard
const dashboard = await Dashboard.create({
@ -301,8 +303,10 @@ describe('DefaultAlertProvider', () => {
);
// Create tile and alert
const tile = makeTile({ id: 'test-tile-123' });
tile.config.source = source._id.toString();
const tile = makeTile({
id: 'test-tile-123',
sourceId: source._id.toString(),
});
const dashboard = await Dashboard.create({
team: team._id,
@ -503,8 +507,10 @@ describe('DefaultAlertProvider', () => {
it('should skip alerts with missing source', async () => {
const team = await createTeam({ name: 'Test Team' });
const tile = makeTile({ id: 'test-tile' });
tile.config.source = new mongoose.Types.ObjectId().toString(); // Non-existent source
const tile = makeTile({
id: 'test-tile',
sourceId: new mongoose.Types.ObjectId().toString(), // Non-existent source
});
const dashboard = await Dashboard.create({
team: team._id,
@ -684,8 +690,10 @@ describe('DefaultAlertProvider', () => {
});
// Create tile with source
const tile = makeTile({ id: 'test-tile-123' });
tile.config.source = source._id.toString();
const tile = makeTile({
id: 'test-tile-123',
sourceId: source._id.toString(),
});
// Create dashboard
const dashboard = await Dashboard.create({

11
packages/api/src/tasks/checkAlerts/providers/default.ts
View file

@ -1,4 +1,5 @@
import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node';
import { isRawSqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards';
import { Tile } from '@hyperdx/common-utils/dist/types';
import mongoose from 'mongoose';
import ms from 'ms';
@ -106,6 +107,16 @@ async function getTileDetails(
return [];
}
if (isRawSqlSavedChartConfig(tile.config)) {
logger.warn({
tileId,
dashboardId: dashboard._id,
alertId: alert.id,
message: 'skipping alert with raw sql chart config, not supported',
});
return [];
}
const source = await Source.findOne({
_id: tile.config.source,
team: alert.team,

194
packages/api/src/utils/externalApi.ts
View file

@ -1,10 +1,8 @@
import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils';
import {
AggregateFunctionSchema,
BuilderSavedChartConfig,
DashboardFilter,
DisplayType,
SavedChartConfig,
SelectList,
} from '@hyperdx/common-utils/dist/types';
import { omit } from 'lodash';
import { FlattenMaps, LeanDocument } from 'mongoose';
@ -18,42 +16,7 @@ import {
} from '@/models/alert';
import type { DashboardDocument } from '@/models/dashboard';
import { SeriesTile } from '@/routers/external-api/v2/utils/dashboards';
import {
ChartSeries,
ExternalDashboardFilterWithId,
MarkdownChartSeries,
NumberChartSeries,
SearchChartSeries,
TableChartSeries,
TimeChartSeries,
} from '@/utils/zod';
import logger from './logger';
type NonStringSelectItem = Exclude<SelectList[number], string>;
type NonStringSelectWithLevel = NonStringSelectItem & { level: number };
function hasLevel(
series: NonStringSelectItem,
): series is NonStringSelectWithLevel {
return 'level' in series && typeof series.level === 'number';
}
function isSortOrderDesc(config: SavedChartConfig): boolean {
if (!config.orderBy) {
return false;
}
if (typeof config.orderBy === 'string') {
return config.orderBy.toLowerCase().endsWith(' desc');
}
if (Array.isArray(config.orderBy) && config.orderBy.length === 0) {
return false;
}
return Array.isArray(config.orderBy) && config.orderBy[0].ordering === 'DESC';
}
import { ExternalDashboardFilterWithId } from '@/utils/zod';
/** Returns a new object containing only the truthy, requested keys from the original object */
const pickIfTruthy = <T, K extends keyof T>(obj: T, keys: K[]): Partial<T> => {
@ -66,143 +29,6 @@ const pickIfTruthy = <T, K extends keyof T>(obj: T, keys: K[]): Partial<T> => {
return result;
};
const convertChartConfigToExternalChartSeries = (
config: SavedChartConfig,
): ChartSeries[] => {
const {
displayType,
source: sourceId,
select,
groupBy,
numberFormat,
} = config;
const isSelectArray = Array.isArray(select);
const convertedGroupBy = Array.isArray(groupBy)
? groupBy.map(g => g.valueExpression)
: splitAndTrimWithBracket(groupBy ?? '');
switch (displayType) {
case 'line':
case 'stacked_bar':
if (!isSelectArray) {
logger.error(`Expected array select for displayType ${displayType}`);
return [];
}
return select.map(s => {
const aggFnSanitized = AggregateFunctionSchema.safeParse(
s.aggFn ?? 'none',
);
return {
aggFn: aggFnSanitized.success ? aggFnSanitized.data : 'none',
alias: s.alias ?? undefined,
type: 'time',
sourceId,
displayType,
level: hasLevel(s) ? s.level : undefined,
field: s.valueExpression,
where: s.aggCondition ?? '',
whereLanguage: s.aggConditionLanguage ?? 'lucene',
groupBy: convertedGroupBy,
metricName: s.metricName ?? undefined,
metricDataType: s.metricType ?? undefined,
numberFormat: numberFormat ?? undefined,
} satisfies TimeChartSeries;
});
case 'table':
if (!isSelectArray) {
logger.error(`Expected array select for displayType ${displayType}`);
return [];
}
return select.map(s => {
const aggFnSanitized = AggregateFunctionSchema.safeParse(
s.aggFn ?? 'none',
);
return {
aggFn: aggFnSanitized.success ? aggFnSanitized.data : 'none',
alias: s.alias ?? undefined,
type: 'table',
sourceId,
level: hasLevel(s) ? s.level : undefined,
field: s.valueExpression,
where: s.aggCondition ?? '',
whereLanguage: s.aggConditionLanguage ?? 'lucene',
groupBy: convertedGroupBy,
metricName: s.metricName ?? undefined,
metricDataType: s.metricType ?? undefined,
sortOrder: isSortOrderDesc(config) ? 'desc' : 'asc',
numberFormat: numberFormat ?? undefined,
} satisfies TableChartSeries;
});
case 'number': {
if (!isSelectArray || select.length === 0) {
logger.error(
`Expected non-empty array select for displayType ${displayType}`,
);
return [];
}
const firstSelect = select[0];
const aggFnSanitized = AggregateFunctionSchema.safeParse(
firstSelect.aggFn ?? 'none',
);
return [
{
alias: firstSelect.alias ?? undefined,
aggFn: aggFnSanitized.success ? aggFnSanitized.data : 'none',
type: 'number',
sourceId,
level: hasLevel(firstSelect) ? firstSelect.level : undefined,
field: firstSelect.valueExpression,
where: firstSelect.aggCondition ?? '',
whereLanguage: firstSelect.aggConditionLanguage ?? 'lucene',
metricName: firstSelect.metricName ?? undefined,
metricDataType: firstSelect.metricType ?? undefined,
numberFormat: numberFormat ?? undefined,
},
] satisfies [NumberChartSeries];
}
case 'search': {
if (isSelectArray) {
logger.error(
`Expected non-array select for displayType ${displayType}`,
);
return [];
}
return [
{
type: 'search',
sourceId,
fields: splitAndTrimWithBracket(select ?? ''),
where: config.where ?? '',
whereLanguage: config.whereLanguage ?? 'lucene',
},
] satisfies [SearchChartSeries];
}
case 'markdown':
return [
{
type: 'markdown',
content: config.markdown || '',
},
] satisfies [MarkdownChartSeries];
case 'heatmap': // Heatmap is not supported in external API, and should not be present in dashboards
default:
logger.error(
`DisplayType ${displayType} is not supported in external API`,
);
return [];
}
};
export function translateExternalChartToTileConfig(
chart: SeriesTile,
): DashboardDocument['tiles'][number] {
@ -218,14 +44,14 @@ export function translateExternalChartToTileConfig(
// Determine the sourceId and displayType based on series type
let sourceId: string =
firstSeries.type === 'markdown' ? '' : firstSeries.sourceId;
let select: SavedChartConfig['select'] = '';
let displayType: SavedChartConfig['displayType'];
let groupBy: SavedChartConfig['groupBy'] = '';
let where: SavedChartConfig['where'] = '';
let whereLanguage: SavedChartConfig['whereLanguage'] = 'lucene';
let orderBy: SavedChartConfig['orderBy'] = '';
let markdown: SavedChartConfig['markdown'] = '';
let numberFormat: SavedChartConfig['numberFormat'] = undefined;
let select: BuilderSavedChartConfig['select'] = '';
let displayType: BuilderSavedChartConfig['displayType'];
let groupBy: BuilderSavedChartConfig['groupBy'] = '';
let where: BuilderSavedChartConfig['where'] = '';
let whereLanguage: BuilderSavedChartConfig['whereLanguage'] = 'lucene';
let orderBy: BuilderSavedChartConfig['orderBy'] = '';
let markdown: BuilderSavedChartConfig['markdown'] = '';
let numberFormat: BuilderSavedChartConfig['numberFormat'] = undefined;
switch (firstSeries.type) {
case 'time': {

3
packages/app/.env.development
View file

@ -7,4 +7,5 @@ OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4318"
OTEL_SERVICE_NAME="hdx-oss-dev-app"
PORT=${HYPERDX_APP_PORT}
NODE_OPTIONS="--max-http-header-size=131072"
NEXT_PUBLIC_HYPERDX_BASE_PATH=
NEXT_PUBLIC_HYPERDX_BASE_PATH=
NEXT_PUBLIC_IS_SQL_CHARTS_ENABLED=true

36
packages/app/src/ChartUtils.tsx
View file

@ -18,13 +18,13 @@ import {
} from '@hyperdx/common-utils/dist/core/utils';
import {
AggregateFunction as AggFnV2,
ChartConfigWithDateRange,
BuilderChartConfigWithDateRange,
BuilderChartConfigWithOptTimestamp,
BuilderSavedChartConfig,
ChartConfigWithOptDateRange,
ChartConfigWithOptTimestamp,
DisplayType,
Filter,
MetricsDataType as MetricsDataTypeV2,
SavedChartConfig,
SourceKind,
SQLInterval,
TSource,
@ -109,7 +109,7 @@ export const getMetricAggFns = (
};
export const DEFAULT_CHART_CONFIG: Omit<
SavedChartConfig,
BuilderSavedChartConfig,
'source' | 'connection'
> = {
name: '',
@ -132,7 +132,9 @@ export const isGranularity = (value: string): value is Granularity => {
return Object.values(Granularity).includes(value as Granularity);
};
export function convertToTimeChartConfig(config: ChartConfigWithDateRange) {
export function convertToTimeChartConfig(
config: BuilderChartConfigWithDateRange,
) {
const granularity =
config.granularity === 'auto' || config.granularity == null
? convertDateRangeToGranularityString(config.dateRange, 80)
@ -152,7 +154,9 @@ export function convertToTimeChartConfig(config: ChartConfigWithDateRange) {
};
}
export function useTimeChartSettings(chartConfig: ChartConfigWithDateRange) {
export function useTimeChartSettings(
chartConfig: BuilderChartConfigWithDateRange,
) {
return useMemo(() => {
const convertedConfig = convertToTimeChartConfig(chartConfig);
@ -884,7 +888,7 @@ export const convertV1ChartConfigToV2 = (
metric?: TSource;
trace?: TSource;
},
): ChartConfigWithDateRange => {
): BuilderChartConfigWithDateRange => {
const {
series,
granularity,
@ -959,7 +963,7 @@ export function buildEventsSearchUrl({
valueRangeFilter,
}: {
source: TSource;
config: ChartConfigWithDateRange;
config: BuilderChartConfigWithDateRange;
dateRange: [Date, Date];
groupFilters?: Array<{ column: string; value: any }>;
valueRangeFilter?: { expression: string; value: number; threshold?: number };
@ -1056,7 +1060,7 @@ export function buildEventsSearchUrl({
* Handles both string format ("col1, col2") and array format ([{ valueExpression: "col1" }, ...])
*/
function extractGroupColumns(
groupBy: ChartConfigWithDateRange['groupBy'],
groupBy: BuilderChartConfigWithDateRange['groupBy'],
): string[] {
if (!groupBy) return [];
@ -1081,7 +1085,7 @@ export function buildTableRowSearchUrl({
}: {
row: Record<string, any>;
source: TSource | undefined;
config: ChartConfigWithDateRange;
config: BuilderChartConfigWithDateRange;
dateRange: [Date, Date];
}): string | null {
if (!source?.id) {
@ -1143,20 +1147,20 @@ export function buildTableRowSearchUrl({
}
export function convertToNumberChartConfig(
config: ChartConfigWithDateRange,
): ChartConfigWithOptTimestamp {
config: BuilderChartConfigWithDateRange,
): BuilderChartConfigWithOptTimestamp {
return omit(config, ['granularity', 'groupBy']);
}
export function convertToPieChartConfig(
config: ChartConfigWithOptTimestamp,
): ChartConfigWithOptTimestamp {
config: BuilderChartConfigWithOptTimestamp,
): BuilderChartConfigWithOptTimestamp {
return omit(config, ['granularity']);
}
export function convertToTableChartConfig(
config: ChartConfigWithOptTimestamp,
): ChartConfigWithOptTimestamp {
config: BuilderChartConfigWithOptTimestamp,
): BuilderChartConfigWithOptTimestamp {
const convertedConfig = structuredClone(omit(config, ['granularity']));
// Set a default limit if not already set

1
packages/app/src/DBChartPage.tsx
View file

@ -226,6 +226,7 @@ function DBChartExplorerPage() {
parseAsJson<SavedChartConfig>().withDefault({
...DEFAULT_CHART_CONFIG,
source: sources?.[0]?.id ?? '',
connection: sources?.[0]?.connection,
}),
);

25
packages/app/src/DBDashboardImportPage.tsx
View file

@ -8,6 +8,7 @@ import { StringParam, useQueryParam } from 'use-query-params';
import { z } from 'zod';
import { zodResolver } from '@hookform/resolvers/zod';
import { convertToDashboardDocument } from '@hyperdx/common-utils/dist/core/utils';
import { isRawSqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards';
import { DashboardTemplateSchema } from '@hyperdx/common-utils/dist/types';
import {
Button,
@ -207,9 +208,13 @@ function Mapping({ input }: { input: Input }) {
const sourceMappings = input.tiles.map(tile => {
// find matching source name
const configSource = !isRawSqlSavedChartConfig(tile.config)
? tile.config.source
: undefined;
const match = sources.find(
source =>
source.name.toLowerCase() === tile.config.source.toLowerCase(),
configSource &&
source.name.toLowerCase() === configSource.toLowerCase(),
);
return match?.id || '';
});
@ -230,6 +235,7 @@ function Mapping({ input }: { input: Input }) {
const sourceMappings = useWatch({ control, name: 'sourceMappings' });
const prevSourceMappingsRef = useRef(sourceMappings);
// HDX-3583: Extend this to support connection matching for Raw SQL-based charts.
useEffect(() => {
if (isUpdatingRef.current) return;
if (!sourceMappings || !input.tiles) return;
@ -245,15 +251,22 @@ function Mapping({ input }: { input: Input }) {
const inputTile = input.tiles[changedIdx];
if (!inputTile) return;
const sourceId = sourceMappings[changedIdx] ?? '';
const inputTileSource = !isRawSqlSavedChartConfig(inputTile.config)
? inputTile.config.source
: undefined;
const keysForTilesWithMatchingSource = input.tiles
.map((tile, index) => ({ ...tile, index }))
.filter(tile => tile.config.source === inputTile.config.source)
.filter(
tile =>
!isRawSqlSavedChartConfig(tile.config) &&
tile.config.source === inputTileSource,
)
.map(({ index }) => `sourceMappings.${index}` as const);
const keysForFiltersWithMatchingSource =
input.filters
?.map((filter, index) => ({ ...filter, index }))
.filter(f => f.source === inputTile.config.source)
.filter(f => f.source === inputTileSource)
.map(({ index }) => `filterSourceMappings.${index}` as const) ?? [];
isUpdatingRef.current = true;
@ -362,7 +375,11 @@ function Mapping({ input }: { input: Input }) {
{input.tiles.map((tile, i) => (
<Table.Tr key={tile.id}>
<Table.Td>{tile.config.name}</Table.Td>
<Table.Td>{tile.config.source}</Table.Td>
<Table.Td>
{!isRawSqlSavedChartConfig(tile.config)
? tile.config.source
: ''}
</Table.Td>
<Table.Td>
<SelectControlled
control={control}

212
packages/app/src/DBDashboardPage.tsx
View file

@ -19,18 +19,22 @@ import { useForm, useWatch } from 'react-hook-form';
import { TableConnection } from '@hyperdx/common-utils/dist/core/metadata';
import { convertToDashboardTemplate } from '@hyperdx/common-utils/dist/core/utils';
import {
AlertState,
DashboardFilter,
SourceKind,
TSourceUnion,
} from '@hyperdx/common-utils/dist/types';
isBuilderChartConfig,
isBuilderSavedChartConfig,
isRawSqlChartConfig,
isRawSqlSavedChartConfig,
} from '@hyperdx/common-utils/dist/guards';
import {
AlertState,
ChartConfigWithDateRange,
DashboardFilter,
DisplayType,
Filter,
SearchCondition,
SearchConditionLanguage,
SourceKind,
SQLInterval,
TSourceUnion,
} from '@hyperdx/common-utils/dist/types';
import {
ActionIcon,
@ -188,11 +192,18 @@ const Tile = forwardRef(
>(undefined);
const { data: source } = useSource({
id: chart.config.source,
id: isBuilderSavedChartConfig(chart.config)
? chart.config.source
: undefined,
});
useEffect(() => {
if (source != null) {
if (isRawSqlSavedChartConfig(chart.config)) {
setQueriedConfig({ ...chart.config, dateRange, granularity });
return;
}
if (source != null && isBuilderSavedChartConfig(chart.config)) {
const isMetricSource = source.kind === SourceKind.Metric;
// TODO: will need to update this when we allow for multiple metrics per chart
@ -223,7 +234,9 @@ const Tile = forwardRef(
const [hovered, setHovered] = useState(false);
const alert = chart.config.alert;
const alert = isBuilderSavedChartConfig(chart.config)
? chart.config.alert
: undefined;
const alertIndicatorColor = useMemo(() => {
if (!alert) {
return 'transparent';
@ -349,6 +362,9 @@ const Tile = forwardRef(
const toolbar = hideToolbar ? [] : [hoverToolbar];
const keyPrefix = isFullscreenView ? 'fullscreen' : 'tile';
// Markdown charts may not have queriedConfig, if config.source is not set
const effectiveMarkdownConfig = queriedConfig ?? chart.config;
return (
<ErrorBoundary
onError={console.error}
@ -359,26 +375,28 @@ const Tile = forwardRef(
}
>
{(queriedConfig?.displayType === DisplayType.Line ||
queriedConfig?.displayType === DisplayType.StackedBar) && (
<DBTimeChart
key={`${keyPrefix}-${chart.id}`}
title={title}
toolbarPrefix={toolbar}
sourceId={chart.config.source}
showDisplaySwitcher={true}
config={queriedConfig}
onTimeRangeSelect={onTimeRangeSelect}
setDisplayType={displayType => {
onUpdateChart?.({
...chart,
config: {
...chart.config,
displayType,
},
});
}}
/>
)}
queriedConfig?.displayType === DisplayType.StackedBar) &&
isBuilderChartConfig(queriedConfig) &&
isBuilderSavedChartConfig(chart.config) && (
<DBTimeChart
key={`${keyPrefix}-${chart.id}`}
title={title}
toolbarPrefix={toolbar}
sourceId={chart.config.source}
showDisplaySwitcher={true}
config={queriedConfig}
onTimeRangeSelect={onTimeRangeSelect}
setDisplayType={displayType => {
onUpdateChart?.({
...chart,
config: {
...chart.config,
displayType,
},
});
}}
/>
)}
{queriedConfig?.displayType === DisplayType.Table && (
<Box p="xs" h="100%">
<DBTableChart
@ -387,78 +405,83 @@ const Tile = forwardRef(
toolbarPrefix={toolbar}
config={queriedConfig}
variant="muted"
getRowSearchLink={row =>
buildTableRowSearchUrl({
row,
source,
config: queriedConfig,
dateRange: dateRange,
})
getRowSearchLink={
isBuilderChartConfig(queriedConfig)
? row =>
buildTableRowSearchUrl({
row,
source,
config: queriedConfig,
dateRange: dateRange,
})
: undefined
}
/>
</Box>
)}
{queriedConfig?.displayType === DisplayType.Number && (
<DBNumberChart
key={`${keyPrefix}-${chart.id}`}
title={title}
toolbarPrefix={toolbar}
config={queriedConfig}
/>
)}
{queriedConfig?.displayType === DisplayType.Pie && (
<DBPieChart
key={`${keyPrefix}-${chart.id}`}
title={title}
toolbarPrefix={toolbar}
config={queriedConfig}
/>
)}
{/* Markdown charts may not have queriedConfig, if source is not set */}
{(queriedConfig?.displayType === DisplayType.Markdown ||
(!queriedConfig &&
chart.config.displayType === DisplayType.Markdown)) && (
<HDXMarkdownChart
key={`${keyPrefix}-${chart.id}`}
title={title}
toolbarItems={toolbar}
config={queriedConfig ?? chart.config}
/>
)}
{queriedConfig?.displayType === DisplayType.Search && (
<ChartContainer
title={title}
toolbarItems={toolbar}
disableReactiveContainer
>
<DBSqlRowTableWithSideBar
{queriedConfig?.displayType === DisplayType.Number &&
isBuilderChartConfig(queriedConfig) && (
<DBNumberChart
key={`${keyPrefix}-${chart.id}`}
enabled
sourceId={chart.config.source}
config={{
...queriedConfig,
orderBy: [
{
ordering: 'DESC',
valueExpression: getFirstTimestampValueExpression(
queriedConfig.timestampValueExpression,
),
},
],
dateRange,
select:
queriedConfig.select ||
source?.defaultTableSelectExpression ||
'',
groupBy: undefined,
granularity: undefined,
}}
isLive={false}
queryKeyPrefix={'search'}
variant="muted"
title={title}
toolbarPrefix={toolbar}
config={queriedConfig}
/>
</ChartContainer>
)}
)}
{queriedConfig?.displayType === DisplayType.Pie &&
isBuilderChartConfig(queriedConfig) && (
<DBPieChart
key={`${keyPrefix}-${chart.id}`}
title={title}
toolbarPrefix={toolbar}
config={queriedConfig}
/>
)}
{effectiveMarkdownConfig?.displayType === DisplayType.Markdown &&
'markdown' in effectiveMarkdownConfig && (
<HDXMarkdownChart
key={`${keyPrefix}-${chart.id}`}
title={title}
toolbarItems={toolbar}
config={effectiveMarkdownConfig}
/>
)}
{queriedConfig?.displayType === DisplayType.Search &&
isBuilderChartConfig(queriedConfig) &&
isBuilderSavedChartConfig(chart.config) && (
<ChartContainer
title={title}
toolbarItems={toolbar}
disableReactiveContainer
>
<DBSqlRowTableWithSideBar
key={`${keyPrefix}-${chart.id}`}
enabled
sourceId={chart.config.source}
config={{
...queriedConfig,
orderBy: [
{
ordering: 'DESC',
valueExpression: getFirstTimestampValueExpression(
queriedConfig.timestampValueExpression,
),
},
],
dateRange,
select:
queriedConfig.select ||
source?.defaultTableSelectExpression ||
'',
groupBy: undefined,
granularity: undefined,
}}
isLive={false}
queryKeyPrefix={'search'}
variant="muted"
/>
</ChartContainer>
)}
</ErrorBoundary>
);
},
@ -717,6 +740,7 @@ function DBDashboardPage({ presetConfig }: { presetConfig?: Dashboard }) {
const tc: TableConnection[] = [];
for (const { config } of dashboard.tiles) {
if (!isBuilderSavedChartConfig(config)) continue;
const source = sources?.find(v => v.id === config.source);
if (!source) continue;
// TODO: will need to update this when we allow for multiple metrics per chart

3
packages/app/src/DBSearchPage.tsx
View file

@ -32,6 +32,7 @@ import {
splitAndTrimWithBracket,
} from '@hyperdx/common-utils/dist/core/utils';
import {
BuilderChartConfigWithDateRange,
ChartConfigWithDateRange,
DisplayType,
Filter,
@ -1465,7 +1466,7 @@ function DBSearchPage() {
[onTimeRangeSelect, setIsLive],
);
const filtersChartConfig = useMemo<ChartConfigWithDateRange>(() => {
const filtersChartConfig = useMemo<BuilderChartConfigWithDateRange>(() => {
const overrides = {
orderBy: undefined,
dateRange: searchedTimeRange,

19
packages/app/src/ServicesDashboardPage.tsx
View file

@ -11,8 +11,7 @@ import { UseControllerProps, useForm, useWatch } from 'react-hook-form';
import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata';
import { convertDateRangeToGranularityString } from '@hyperdx/common-utils/dist/core/utils';
import {
ChartConfigWithDateRange,
ChartConfigWithOptDateRange,
BuilderChartConfigWithDateRange,
CteChartConfig,
DisplayType,
Filter,
@ -370,7 +369,7 @@ function HttpTab({
}, []);
const requestErrorRateConfig =
useMemo<ChartConfigWithDateRange | null>(() => {
useMemo<BuilderChartConfigWithDateRange | null>(() => {
if (!source || !expressions) return null;
if (reqChartType === 'overall') {
return {
@ -407,7 +406,7 @@ function HttpTab({
numberFormat: ERROR_RATE_PERCENTAGE_NUMBER_FORMAT,
filters: getScopedFilters({ appliedConfig, expressions }),
dateRange: searchedTimeRange,
} satisfies ChartConfigWithDateRange;
} satisfies BuilderChartConfigWithDateRange;
}
return {
timestampValueExpression: 'series_time_bucket',
@ -465,7 +464,7 @@ function HttpTab({
dateRange: searchedTimeRange,
granularity:
convertDateRangeToGranularityString(searchedTimeRange),
} as ChartConfigWithOptDateRange,
} as CteChartConfig,
isSubquery: true,
},
// Select the top N series from the search as we don't want to crash the browser.
@ -539,7 +538,7 @@ function HttpTab({
displayType: DisplayType.Line,
numberFormat: ERROR_RATE_PERCENTAGE_NUMBER_FORMAT,
groupBy: 'zipped, endpoint',
} satisfies ChartConfigWithDateRange;
} satisfies BuilderChartConfigWithDateRange;
}, [source, searchedTimeRange, appliedConfig, expressions, reqChartType]);
return (
@ -872,7 +871,7 @@ function DatabaseTab({
}, []);
const totalTimePerQueryConfig =
useMemo<ChartConfigWithDateRange | null>(() => {
useMemo<BuilderChartConfigWithDateRange | null>(() => {
if (!source || !expressions) return null;
return {
@ -991,11 +990,11 @@ function DatabaseTab({
timestampValueExpression: 'series_time_bucket',
connection: source.connection,
source: source.id,
} satisfies ChartConfigWithDateRange;
} satisfies BuilderChartConfigWithDateRange;
}, [appliedConfig, expressions, searchedTimeRange, source]);
const totalThroughputPerQueryConfig =
useMemo<ChartConfigWithDateRange | null>(() => {
useMemo<BuilderChartConfigWithDateRange | null>(() => {
if (!source || !expressions) return null;
return {
@ -1112,7 +1111,7 @@ function DatabaseTab({
timestampValueExpression: 'series_time_bucket',
connection: source.connection,
source: source.id,
} satisfies ChartConfigWithDateRange;
} satisfies BuilderChartConfigWithDateRange;
}, [appliedConfig, expressions, searchedTimeRange, source]);
const displaySwitcher = (

16
packages/app/src/__tests__/ChartUtils.test.ts
View file

@ -1,5 +1,5 @@
import {
ChartConfigWithDateRange,
BuilderChartConfigWithDateRange,
SourceKind,
TSource,
} from '@hyperdx/common-utils/dist/types';
@ -640,7 +640,7 @@ describe('ChartUtils', () => {
new Date('2025-11-26T00:00:00Z'),
new Date('2025-11-27T00:00:00Z'),
],
} as ChartConfigWithDateRange;
} as BuilderChartConfigWithDateRange;
const granularityFromFunction =
convertToTimeChartConfig(config).granularity;
@ -654,7 +654,7 @@ describe('ChartUtils', () => {
new Date('2025-11-26T00:00:00Z'),
new Date('2025-11-27T00:00:00Z'),
],
} as ChartConfigWithDateRange;
} as BuilderChartConfigWithDateRange;
const granularityFromFunction =
convertToTimeChartConfig(config).granularity;
@ -669,7 +669,7 @@ describe('ChartUtils', () => {
new Date('2025-11-26T00:00:00Z'),
new Date('2025-11-27T00:00:00Z'),
],
} as ChartConfigWithDateRange;
} as BuilderChartConfigWithDateRange;
const granularityFromFunction =
convertToTimeChartConfig(config).granularity;
@ -687,7 +687,7 @@ describe('ChartUtils', () => {
new Date('2025-11-26T00:00:00Z'),
new Date('2025-11-27T00:00:00Z'),
],
} as ChartConfigWithDateRange;
} as BuilderChartConfigWithDateRange;
const convertedConfig = convertToNumberChartConfig(config);
@ -704,7 +704,7 @@ describe('ChartUtils', () => {
new Date('2025-11-26T00:00:00Z'),
new Date('2025-11-27T00:00:00Z'),
],
} as ChartConfigWithDateRange;
} as BuilderChartConfigWithDateRange;
const convertedConfig = convertToTableChartConfig(config);
@ -718,7 +718,7 @@ describe('ChartUtils', () => {
new Date('2025-11-26T00:00:00Z'),
new Date('2025-11-27T00:00:00Z'),
],
} as ChartConfigWithDateRange;
} as BuilderChartConfigWithDateRange;
const convertedConfig = convertToTableChartConfig(config);
@ -732,7 +732,7 @@ describe('ChartUtils', () => {
new Date('2025-11-26T00:00:00Z'),
new Date('2025-11-27T00:00:00Z'),
],
} as ChartConfigWithDateRange;
} as BuilderChartConfigWithDateRange;
const convertedConfig = convertToTableChartConfig(config);

54
packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx Normal file
View file

@ -0,0 +1,54 @@
import { Control } from 'react-hook-form';
import { Box, Button, Group, Stack, Text } from '@mantine/core';
import useResizable from '@/hooks/useResizable';
import { ConnectionSelectControlled } from '../ConnectionSelect';
import { SQLEditorControlled } from '../SQLEditor';
import { ChartEditorFormState } from './types';
import resizeStyles from '@/../styles/ResizablePanel.module.scss';
export default function RawSqlChartEditor({
control,
onOpenDisplaySettings,
}: {
control: Control<ChartEditorFormState>;
onOpenDisplaySettings: () => void;
}) {
const { size, startResize } = useResizable(20, 'bottom');
return (
<Stack>
<Group mb="md" align="center">
<Text pe="md" size="sm">
Connection
</Text>
<ConnectionSelectControlled
control={control}
name="connection"
size="xs"
/>
</Group>
<Box style={{ position: 'relative' }}>
<SQLEditorControlled
control={control}
name="sqlTemplate"
height={`${size}vh`}
enableLineWrapping
/>
<div className={resizeStyles.resizeYHandle} onMouseDown={startResize} />
</Box>
<Group justify="flex-end">
<Button
onClick={onOpenDisplaySettings}
size="compact-sm"
variant="secondary"
>
Display Settings
</Button>
</Group>
</Stack>
);
}

405
packages/app/src/components/ChartEditor/__tests__/utils.test.ts Normal file
View file

@ -0,0 +1,405 @@
import type {
BuilderChartConfig,
BuilderSavedChartConfig,
RawSqlSavedChartConfig,
TSource,
} from '@hyperdx/common-utils/dist/types';
import { DisplayType, SourceKind } from '@hyperdx/common-utils/dist/types';
import { DEFAULT_CHART_CONFIG } from '@/ChartUtils';
import type { ChartEditorFormState } from '../types';
import {
convertFormStateToChartConfig,
convertFormStateToSavedChartConfig,
convertSavedChartConfigToFormState,
} from '../utils';
jest.mock('../../SearchInput', () => ({
getStoredLanguage: jest.fn().mockReturnValue('lucene'),
}));
const dateRange: [Date, Date] = [
new Date('2024-01-01'),
new Date('2024-01-02'),
];
const logSource: TSource = {
id: 'source-log',
name: 'Log Source',
kind: SourceKind.Log,
connection: 'conn-1',
from: { databaseName: 'db', tableName: 'logs' },
timestampValueExpression: 'Timestamp',
defaultTableSelectExpression: 'Body, SeverityText',
implicitColumnExpression: 'Body',
};
const metricSource: TSource = {
id: 'source-metric',
name: 'Metric Source',
kind: SourceKind.Metric,
connection: 'conn-1',
from: { databaseName: 'db', tableName: '' },
timestampValueExpression: 'TimeUnix',
metricTables: { gauge: 'gauge_table' } as TSource['metricTables'],
resourceAttributesExpression: 'ResourceAttributes',
};
const seriesItem = {
aggFn: 'count' as const,
valueExpression: '*',
aggCondition: '',
aggConditionLanguage: 'lucene' as const,
};
describe('convertFormStateToSavedChartConfig', () => {
it('returns undefined when no source and configType is not sql', () => {
const form: ChartEditorFormState = {
displayType: DisplayType.Line,
series: [seriesItem],
};
expect(convertFormStateToSavedChartConfig(form, undefined)).toBeUndefined();
});
it('returns RawSqlSavedChartConfig for sql+table config', () => {
const form: ChartEditorFormState = {
configType: 'sql',
displayType: DisplayType.Table,
sqlTemplate: 'SELECT 1',
connection: 'conn-1',
name: 'My Chart',
series: [],
};
const result = convertFormStateToSavedChartConfig(form, undefined);
expect(result).toEqual({
configType: 'sql',
displayType: DisplayType.Table,
sqlTemplate: 'SELECT 1',
connection: 'conn-1',
name: 'My Chart',
});
});
it('returns undefined for sql config without Table displayType', () => {
const form: ChartEditorFormState = {
configType: 'sql',
displayType: DisplayType.Line,
sqlTemplate: 'SELECT 1',
connection: 'conn-1',
series: [],
};
expect(convertFormStateToSavedChartConfig(form, undefined)).toBeUndefined();
});
it('uses sqlTemplate empty string as default when undefined', () => {
const form: ChartEditorFormState = {
configType: 'sql',
displayType: DisplayType.Table,
series: [],
};
const result = convertFormStateToSavedChartConfig(
form,
undefined,
) as RawSqlSavedChartConfig;
expect(result.sqlTemplate).toBe('');
expect(result.connection).toBe('');
});
it('maps series to select for builder config', () => {
const form: ChartEditorFormState = {
displayType: DisplayType.Line,
where: 'status = 200',
series: [seriesItem],
};
const result = convertFormStateToSavedChartConfig(
form,
logSource,
) as BuilderSavedChartConfig;
expect(result.select).toEqual([seriesItem]);
expect('series' in result).toBe(false);
expect(result.source).toBe('source-log');
});
it('uses form.select string for Search displayType', () => {
const form: ChartEditorFormState = {
displayType: DisplayType.Search,
select: 'Body, SeverityText',
series: [seriesItem],
};
const result = convertFormStateToSavedChartConfig(
form,
logSource,
) as BuilderSavedChartConfig;
expect(result.select).toBe('Body, SeverityText');
expect('series' in result).toBe(false);
});
it('uses empty string for Search displayType when select is not a string', () => {
const form: ChartEditorFormState = {
displayType: DisplayType.Search,
select: [seriesItem],
series: [],
};
const result = convertFormStateToSavedChartConfig(
form,
logSource,
) as BuilderSavedChartConfig;
expect(result.select).toBe('');
expect('series' in result).toBe(false);
});
it('strips metricName and metricType from select for non-metric source', () => {
const form: ChartEditorFormState = {
displayType: DisplayType.Line,
series: [
{
...seriesItem,
metricName: 'cpu.usage',
metricType: 'gauge' as any,
},
],
};
const result = convertFormStateToSavedChartConfig(
form,
logSource,
) as BuilderSavedChartConfig;
const select = result.select as (typeof seriesItem)[];
expect(select[0]).not.toHaveProperty('metricName');
expect(select[0]).not.toHaveProperty('metricType');
});
it('preserves form metricTables for metric source', () => {
const formMetricTables = {
gauge: 'gauge_table',
} as BuilderSavedChartConfig['metricTables'];
const form: ChartEditorFormState = {
displayType: DisplayType.Line,
series: [seriesItem],
metricTables: formMetricTables,
};
const result = convertFormStateToSavedChartConfig(
form,
metricSource,
) as BuilderSavedChartConfig;
expect(result.metricTables).toEqual(formMetricTables);
});
it('strips metricTables for non-metric source', () => {
const form: ChartEditorFormState = {
displayType: DisplayType.Line,
series: [seriesItem],
metricTables: { gauge: 'gauge_table' } as any,
};
const result = convertFormStateToSavedChartConfig(
form,
logSource,
) as BuilderSavedChartConfig;
expect(result.metricTables).toBeUndefined();
});
it('preserves having and orderBy only for Table displayType', () => {
const having = 'count > 5';
const orderBy = [
{
aggFn: 'count',
valueExpression: '*',
aggCondition: '',
ordering: 'DESC' as const,
},
];
const form: ChartEditorFormState = {
displayType: DisplayType.Table,
series: [seriesItem],
having,
orderBy,
};
const tableResult = convertFormStateToSavedChartConfig(
form,
logSource,
) as BuilderSavedChartConfig;
expect(tableResult.having).toBe(having);
expect(tableResult.orderBy).toEqual(orderBy);
const lineResult = convertFormStateToSavedChartConfig(
{ ...form, displayType: DisplayType.Line },
logSource,
) as BuilderSavedChartConfig;
expect(lineResult.having).toBeUndefined();
expect(lineResult.orderBy).toBeUndefined();
});
it('defaults where to empty string when undefined', () => {
const form: ChartEditorFormState = {
displayType: DisplayType.Line,
series: [seriesItem],
};
const result = convertFormStateToSavedChartConfig(
form,
logSource,
) as BuilderSavedChartConfig;
expect(result.where).toBe('');
});
});
describe('convertFormStateToChartConfig', () => {
it('returns undefined when no source and configType is not sql', () => {
const form: ChartEditorFormState = {
displayType: DisplayType.Line,
series: [seriesItem],
};
expect(
convertFormStateToChartConfig(form, dateRange, undefined),
).toBeUndefined();
});
it('returns RawSqlChartConfig with dateRange for sql config', () => {
const form: ChartEditorFormState = {
configType: 'sql',
displayType: DisplayType.Table,
sqlTemplate: 'SELECT now()',
connection: 'conn-1',
series: [],
};
const result = convertFormStateToChartConfig(form, dateRange, undefined);
expect(result).toMatchObject({
configType: 'sql',
sqlTemplate: 'SELECT now()',
connection: 'conn-1',
displayType: DisplayType.Table,
dateRange,
});
});
it('returns builder config with source fields merged', () => {
const form: ChartEditorFormState = {
displayType: DisplayType.Line,
where: 'status = 200',
series: [seriesItem],
};
const result = convertFormStateToChartConfig(form, dateRange, logSource);
expect(result).toMatchObject({
from: logSource.from,
timestampValueExpression: logSource.timestampValueExpression,
connection: logSource.connection,
implicitColumnExpression: logSource.implicitColumnExpression,
dateRange,
where: 'status = 200',
});
});
it('falls back to defaultTableSelectExpression when series is empty', () => {
const form: ChartEditorFormState = {
displayType: DisplayType.Line,
series: [],
};
const result = convertFormStateToChartConfig(
form,
dateRange,
logSource,
) as BuilderChartConfig;
expect(result?.select).toBe(logSource.defaultTableSelectExpression);
});
it('uses series as select for non-Search displayType', () => {
const form: ChartEditorFormState = {
displayType: DisplayType.Line,
series: [seriesItem],
};
const result = convertFormStateToChartConfig(
form,
dateRange,
logSource,
) as BuilderChartConfig;
expect(result?.select).toEqual([seriesItem]);
});
it('uses form.select for Search displayType', () => {
const form: ChartEditorFormState = {
displayType: DisplayType.Search,
select: 'Body',
series: [],
};
const result = convertFormStateToChartConfig(
form,
dateRange,
logSource,
) as BuilderChartConfig;
expect(result?.select).toBe('Body');
});
});
describe('convertSavedChartConfigToFormState', () => {
it('sets configType to sql for RawSqlSavedChartConfig', () => {
const config: RawSqlSavedChartConfig = {
configType: 'sql',
displayType: DisplayType.Table,
sqlTemplate: 'SELECT 1',
connection: 'conn-1',
};
const result = convertSavedChartConfigToFormState(config);
expect(result.configType).toBe('sql');
expect(result.series).toEqual([]);
});
it('sets configType to builder for BuilderSavedChartConfig', () => {
const config: BuilderSavedChartConfig = {
source: 'source-1',
displayType: DisplayType.Line,
select: [seriesItem],
where: '',
};
const result = convertSavedChartConfigToFormState(config);
expect(result.configType).toBe('builder');
});
it('maps array select to series with aggConditionLanguage defaulted', () => {
const selectItem = {
aggFn: 'count' as const,
valueExpression: '*',
aggCondition: '',
};
const config: BuilderSavedChartConfig = {
source: 'source-1',
select: [selectItem],
where: '',
};
const result = convertSavedChartConfigToFormState(config);
expect(result.series).toHaveLength(1);
expect(result.series[0].aggConditionLanguage).toBe('lucene');
});
it('preserves existing aggConditionLanguage when already set', () => {
const config: BuilderSavedChartConfig = {
source: 'source-1',
select: [{ ...seriesItem, aggConditionLanguage: 'sql' as const }],
where: '',
};
const result = convertSavedChartConfigToFormState(config);
expect(result.series[0].aggConditionLanguage).toBe('sql');
});
it('sets series to empty array when select is a string', () => {
const config: BuilderSavedChartConfig = {
source: 'source-1',
select: 'Body, SeverityText',
where: '',
};
const result = convertSavedChartConfigToFormState(config);
expect(result.series).toEqual([]);
});
it('preserves other config fields in the form state', () => {
const config: BuilderSavedChartConfig = {
source: 'source-1',
name: 'My Chart',
displayType: DisplayType.Table,
select: [seriesItem],
where: 'status = 200',
};
const result = convertSavedChartConfigToFormState(config);
expect(result.name).toBe('My Chart');
expect(result.displayType).toBe(DisplayType.Table);
expect(result.where).toBe('status = 200');
});
});

29
packages/app/src/components/ChartEditor/types.ts Normal file
View file

@ -0,0 +1,29 @@
import {
BuilderSavedChartConfig,
RawSqlSavedChartConfig,
} from '@hyperdx/common-utils/dist/types';
export type SavedChartConfigWithSelectArray = Omit<
BuilderSavedChartConfig,
'select'
> & {
select: NonNullable<Exclude<BuilderSavedChartConfig['select'], string>>;
};
/**
* A type that flattens the SavedChartConfig union so that the form can include
* properties from both BuilderChartConfig and RawSqlSavedChartConfig without
* type errors.
*
* All fields are optional since the form may be in either builder or raw SQL
* mode at any given time. `configType?: 'sql'` is the discriminator.
*
* Additionally, 'series' is added as a separate field that is always an array,
* to work around the fact that useFieldArray only works with fields which are *always*
* arrays. `series` stores the array `select` data for the form.
**/
export type ChartEditorFormState = Partial<BuilderSavedChartConfig> &
Partial<Omit<RawSqlSavedChartConfig, 'configType'>> & {
series: SavedChartConfigWithSelectArray['select'];
configType?: 'sql' | 'builder';
};

148
packages/app/src/components/ChartEditor/utils.ts Normal file
View file

@ -0,0 +1,148 @@
import { omit, pick } from 'lodash';
import {
isBuilderSavedChartConfig,
isRawSqlSavedChartConfig,
} from '@hyperdx/common-utils/dist/guards';
import {
BuilderSavedChartConfig,
ChartConfigWithDateRange,
DisplayType,
RawSqlChartConfig,
RawSqlSavedChartConfig,
SavedChartConfig,
SourceKind,
TSource,
} from '@hyperdx/common-utils/dist/types';
import { getStoredLanguage } from '../SearchInput';
import { ChartEditorFormState } from './types';
function normalizeChartConfig<
C extends Pick<
BuilderSavedChartConfig,
'select' | 'having' | 'orderBy' | 'displayType' | 'metricTables'
>,
>(config: C, source: TSource): C {
const isMetricSource = source.kind === SourceKind.Metric;
return {
...config,
// Strip out metric-specific fields for non-metric sources
select:
!isMetricSource && Array.isArray(config.select)
? config.select.map(s => omit(s, ['metricName', 'metricType']))
: config.select,
metricTables: isMetricSource ? config.metricTables : undefined,
// Order By and Having can only be set by the user for table charts
having:
config.displayType === DisplayType.Table ? config.having : undefined,
orderBy:
config.displayType === DisplayType.Table ? config.orderBy : undefined,
};
}
export function convertFormStateToSavedChartConfig(
form: ChartEditorFormState,
source: TSource | undefined,
): SavedChartConfig | undefined {
if (form.configType === 'sql' && form.displayType === DisplayType.Table) {
const rawSqlConfig: RawSqlSavedChartConfig = {
configType: 'sql',
...pick(form, [
'name',
'displayType',
'numberFormat',
'granularity',
'compareToPreviousPeriod',
'fillNulls',
'alignDateRangeToGranularity',
]),
sqlTemplate: form.sqlTemplate ?? '',
connection: form.connection ?? '',
};
return rawSqlConfig;
}
if (source) {
// Merge the series and select fields back together, and prevent the series field from being submitted
const config: BuilderSavedChartConfig = {
...omit(form, ['series', 'configType', 'sqlTemplate']),
// If the chart type is search, we need to ensure the select is a string
select:
form.displayType === DisplayType.Search
? typeof form.select === 'string'
? form.select
: ''
: form.series,
where: form.where ?? '',
source: source.id,
};
return normalizeChartConfig(config, source);
}
}
export function convertFormStateToChartConfig(
form: ChartEditorFormState,
dateRange: ChartConfigWithDateRange['dateRange'],
source: TSource | undefined,
): ChartConfigWithDateRange | undefined {
if (form.configType === 'sql' && form.displayType === DisplayType.Table) {
const rawSqlConfig: RawSqlChartConfig = {
configType: 'sql',
...pick(form, [
'name',
'displayType',
'numberFormat',
'granularity',
'compareToPreviousPeriod',
'fillNulls',
'alignDateRangeToGranularity',
]),
sqlTemplate: form.sqlTemplate ?? '',
connection: form.connection ?? '',
};
return { ...rawSqlConfig, dateRange };
}
if (source) {
// Merge the series and select fields back together, and prevent the series field from being submitted
const mergedSelect =
form.displayType === DisplayType.Search ? form.select : form.series;
const isSelectEmpty = !mergedSelect || mergedSelect.length === 0;
const newConfig: ChartConfigWithDateRange = {
...omit(form, ['series', 'configType', 'sqlTemplate']),
from: source.from,
timestampValueExpression: source.timestampValueExpression,
dateRange,
connection: source.connection,
implicitColumnExpression: source.implicitColumnExpression,
metricTables: source.metricTables,
where: form.where ?? '',
select: isSelectEmpty
? source.defaultTableSelectExpression || ''
: mergedSelect,
};
return structuredClone(normalizeChartConfig(newConfig, source));
}
}
export function convertSavedChartConfigToFormState(
config: SavedChartConfig,
): ChartEditorFormState {
return {
...config,
configType: isRawSqlSavedChartConfig(config) ? 'sql' : 'builder',
series:
isBuilderSavedChartConfig(config) && Array.isArray(config.select)
? config.select.map(s => ({
...s,
aggConditionLanguage:
s.aggConditionLanguage ?? getStoredLanguage() ?? 'lucene',
}))
: [],
};
}

4
packages/app/src/components/ContextSidePanel.tsx
View file

@ -5,7 +5,7 @@ import { useQueryState } from 'nuqs';
import { useForm, useWatch } from 'react-hook-form';
import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata';
import {
ChartConfigWithDateRange,
BuilderChartConfigWithDateRange,
TSource,
} from '@hyperdx/common-utils/dist/types';
import { Badge, Flex, Group, SegmentedControl } from '@mantine/core';
@ -38,7 +38,7 @@ enum ContextBy {
interface ContextSubpanelProps {
source: TSource;
dbSqlRowTableConfig: ChartConfigWithDateRange | undefined;
dbSqlRowTableConfig: BuilderChartConfigWithDateRange | undefined;
rowData: Record<string, any>;
rowId: string | undefined;
breadcrumbPath?: BreadcrumbPath;

7
packages/app/src/components/DBDeltaChart.tsx
View file

@ -1,8 +1,7 @@
import { useEffect, useMemo, useState } from 'react';
import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse';
import {
ChartConfigWithDateRange,
ChartConfigWithOptDateRange,
BuilderChartConfigWithDateRange,
Filter,
} from '@hyperdx/common-utils/dist/types';
import {
@ -46,7 +45,7 @@ export default function DBDeltaChart({
yMax,
spanIdExpression,
}: {
config: ChartConfigWithDateRange;
config: BuilderChartConfigWithDateRange;
valueExpr: string;
xMin: number;
xMax: number;
@ -102,7 +101,7 @@ export default function DBDeltaChart({
// Helper to build WITH clauses for a query (outlier or inlier)
const buildWithClauses = (
isOutlier: boolean,
): NonNullable<ChartConfigWithOptDateRange['with']> => {
): NonNullable<BuilderChartConfigWithDateRange['with']> => {
const aggregatedTimestampsCTE = buildAggregatedTimestampsCTE();
// Build the SQL condition for filtering

331
packages/app/src/components/DBEditTimeChartForm.tsx
View file

@ -1,5 +1,4 @@
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { omit } from 'lodash';
import {
Control,
Controller,
@ -15,6 +14,11 @@ import { NativeSelect, NumberInput } from 'react-hook-form-mantine';
import z from 'zod';
import { zodResolver } from '@hookform/resolvers/zod';
import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata';
import {
isBuilderChartConfig,
isRawSqlChartConfig,
isRawSqlSavedChartConfig,
} from '@hyperdx/common-utils/dist/guards';
import {
ChartAlertBaseSchema,
ChartConfigWithDateRange,
@ -39,6 +43,7 @@ import {
Group,
Menu,
Paper,
SegmentedControl,
Stack,
Switch,
Tabs,
@ -83,7 +88,7 @@ import SearchWhereInput, {
} from '@/components/SearchInput/SearchWhereInput';
import { SQLInlineEditorControlled } from '@/components/SearchInput/SQLInlineEditor';
import { TimePicker } from '@/components/TimePicker';
import { IS_LOCAL_MODE } from '@/config';
import { IS_LOCAL_MODE, IS_SQL_CHARTS_ENABLED } from '@/config';
import { GranularityPickerControlled } from '@/GranularityPicker';
import { useFetchMetricMetadata } from '@/hooks/useFetchMetricMetadata';
import {
@ -108,6 +113,16 @@ import {
import HDXMarkdownChart from '../HDXMarkdownChart';
import RawSqlChartEditor from './ChartEditor/RawSqlChartEditor';
import {
ChartEditorFormState,
SavedChartConfigWithSelectArray,
} from './ChartEditor/types';
import {
convertFormStateToChartConfig,
convertFormStateToSavedChartConfig,
convertSavedChartConfigToFormState,
} from './ChartEditor/utils';
import { ErrorBoundary } from './Error/ErrorBoundary';
import MVOptimizationIndicator from './MaterializedViews/MVOptimizationIndicator';
import { AggFnSelectControlled } from './AggFnSelect';
@ -128,13 +143,20 @@ import SaveToDashboardModal from './SaveToDashboardModal';
import SourceSchemaPreview from './SourceSchemaPreview';
import { SourceSelectControlled } from './SourceSelect';
const isQueryReady = (queriedConfig: ChartConfigWithDateRange | undefined) =>
((queriedConfig?.select?.length ?? 0) > 0 ||
typeof queriedConfig?.select === 'string') &&
queriedConfig?.from?.databaseName &&
// tableName is empty for metric sources
(queriedConfig?.from?.tableName || queriedConfig?.metricTables) &&
queriedConfig?.timestampValueExpression;
const isQueryReady = (queriedConfig: ChartConfigWithDateRange | undefined) => {
if (!queriedConfig) return false;
if (isRawSqlChartConfig(queriedConfig)) {
return !!(queriedConfig.sqlTemplate && queriedConfig.connection);
}
return (
((queriedConfig.select?.length ?? 0) > 0 ||
typeof queriedConfig.select === 'string') &&
queriedConfig.from?.databaseName &&
// tableName is empty for metric sources
(queriedConfig.from?.tableName || queriedConfig.metricTables) &&
queriedConfig.timestampValueExpression
);
};
const MINIMUM_THRESHOLD_VALUE = 0.0000000001; // to make alert input > 0
@ -142,39 +164,17 @@ const MINIMUM_THRESHOLD_VALUE = 0.0000000001; // to make alert input > 0
const getSeriesFieldPath = (
namePrefix: string,
fieldName: string,
): FieldPath<SavedChartConfigWithSeries> => {
return `${namePrefix}${fieldName}` as FieldPath<SavedChartConfigWithSeries>;
): FieldPath<ChartEditorFormState> => {
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
return `${namePrefix}${fieldName}` as FieldPath<ChartEditorFormState>;
};
export function normalizeChartConfig<
C extends Pick<
SavedChartConfig,
'select' | 'having' | 'orderBy' | 'displayType' | 'metricTables'
>,
>(config: C, source: TSource): C {
const isMetricSource = source.kind === SourceKind.Metric;
return {
...config,
// Strip out metric-specific fields for non-metric sources
select:
!isMetricSource && Array.isArray(config.select)
? config.select.map(s => omit(s, ['metricName', 'metricType']))
: config.select,
metricTables: isMetricSource ? config.metricTables : undefined,
// Order By and Having can only be set by the user for table charts
having:
config.displayType === DisplayType.Table ? config.having : undefined,
orderBy:
config.displayType === DisplayType.Table ? config.orderBy : undefined,
};
}
// Helper function to validate metric names for metric sources
const validateMetricNames = (
tableSource: TSource | undefined,
series: SavedChartConfigWithSelectArray['select'] | undefined,
setError: (
name: FieldPath<SavedChartConfigWithSeries>,
name: FieldPath<ChartEditorFormState>,
error: { type: string; message: string },
) => void,
): boolean => {
@ -235,7 +235,7 @@ function ChartSeriesEditorComponent({
length: number;
tableSource?: TSource;
errors?: FieldErrors<SeriesItem>;
clearErrors: UseFormClearErrors<SavedChartConfigWithSeries>;
clearErrors: UseFormClearErrors<ChartEditorFormState>;
}) {
const aggFn = useWatch({ control, name: `${namePrefix}aggFn` });
const aggConditionLanguage = useWatch({
@ -537,17 +537,6 @@ const zSavedChartConfig = z
})
.passthrough();
export type SavedChartConfigWithSelectArray = Omit<
SavedChartConfig,
'select'
> & {
select: Exclude<SavedChartConfig['select'], string>;
};
type SavedChartConfigWithSeries = SavedChartConfig & {
series: SavedChartConfigWithSelectArray['select'];
};
export default function EditTimeChartForm({
dashboardId,
chartConfig,
@ -579,19 +568,8 @@ export default function EditTimeChartForm({
'data-testid'?: string;
submitRef?: React.MutableRefObject<(() => void) | undefined>;
}) {
// useFieldArray only supports array type fields, and select can be either a string or array.
// To solve for this, we maintain an extra form field called 'series' which is always an array.
const configWithSeries: SavedChartConfigWithSeries = useMemo(
() => ({
...chartConfig,
series: Array.isArray(chartConfig.select)
? chartConfig.select.map(s => ({
...s,
aggConditionLanguage:
s.aggConditionLanguage ?? getStoredLanguage() ?? 'lucene',
}))
: [],
}),
const formValue: ChartEditorFormState = useMemo(
() => convertSavedChartConfigToFormState(chartConfig),
[chartConfig],
);
@ -603,9 +581,9 @@ export default function EditTimeChartForm({
setError,
clearErrors,
formState: { errors, isDirty },
} = useForm<SavedChartConfigWithSeries>({
defaultValues: configWithSeries,
values: configWithSeries,
} = useForm<ChartEditorFormState>({
defaultValues: formValue,
values: formValue,
resolver: zodResolver(zSavedChartConfig),
});
@ -615,7 +593,7 @@ export default function EditTimeChartForm({
remove: removeSeries,
swap: swapSeries,
} = useFieldArray({
control: control as Control<SavedChartConfigWithSeries>,
control,
name: 'series',
});
@ -635,6 +613,10 @@ export default function EditTimeChartForm({
const markdown = useWatch({ control, name: 'markdown' });
const alertChannelType = useWatch({ control, name: 'alert.channel.type' });
const granularity = useWatch({ control, name: 'granularity' });
const configType = useWatch({ control, name: 'configType' });
const isRawSqlInput =
configType === 'sql' && displayType === DisplayType.Table;
const { data: tableSource } = useSource({ id: sourceId });
const databaseName = tableSource?.from.databaseName;
@ -668,8 +650,10 @@ export default function EditTimeChartForm({
const showGeneratedSql = ['table', 'time', 'number', 'pie'].includes(
activeTab,
); // Whether to show the generated SQL preview
const showSampleEvents = tableSource?.kind !== SourceKind.Metric;
);
const showSampleEvents =
tableSource?.kind !== SourceKind.Metric && !isRawSqlInput;
const [
alignDateRangeToGranularity,
@ -717,7 +701,7 @@ export default function EditTimeChartForm({
);
const setQueriedConfigAndSource = useCallback(
(config: ChartConfigWithDateRange, source: TSource) => {
(config: ChartConfigWithDateRange, source: TSource | undefined) => {
setQueriedConfig(config);
setQueriedSource(source);
},
@ -725,7 +709,7 @@ export default function EditTimeChartForm({
);
const dbTimeChartConfig = useMemo(() => {
if (!queriedConfig) {
if (!queriedConfig || !isBuilderChartConfig(queriedConfig)) {
return undefined;
}
@ -745,40 +729,28 @@ export default function EditTimeChartForm({
const onSubmit = useCallback(() => {
handleSubmit(form => {
// Validate metric sources have metric names selected
if (validateMetricNames(tableSource, form.series, setError)) {
const isRawSqlChart =
form.configType === 'sql' && form.displayType === DisplayType.Table;
if (
!isRawSqlChart &&
validateMetricNames(tableSource, form.series, setError)
) {
return;
}
// Merge the series and select fields back together, and prevent the series field from being submitted
const config = {
...omit(form, ['series']),
select:
form.displayType === DisplayType.Search ? form.select : form.series,
};
const savedConfig = convertFormStateToSavedChartConfig(form, tableSource);
const queriedConfig = convertFormStateToChartConfig(
form,
dateRange,
tableSource,
);
setChartConfig?.(config);
if (tableSource != null) {
const isSelectEmpty = !config.select || config.select.length === 0; // select is string or array
const newConfig = {
...config,
from: tableSource.from,
timestampValueExpression: tableSource.timestampValueExpression,
dateRange,
connection: tableSource.connection,
implicitColumnExpression: tableSource.implicitColumnExpression,
metricTables: tableSource.metricTables,
select: isSelectEmpty
? tableSource.defaultTableSelectExpression || ''
: config.select,
};
if (savedConfig && queriedConfig) {
setChartConfig?.(savedConfig);
setQueriedConfigAndSource(
// WARNING: DON'T JUST ASSIGN OBJECTS OR DO SPREAD OPERATOR STUFF WHEN
// YOUR STATE IS AN OBJECT. YOU'RE COPYING BY REFERENCE WHICH MIGHT
// ACCIDENTALLY CAUSE A useQuery SOMEWHERE TO FIRE A REQUEST EVERY TIME
// AN INPUT CHANGES. USE structuredClone TO PERFORM A DEEP COPY INSTEAD
structuredClone(normalizeChartConfig(newConfig, tableSource)),
tableSource,
queriedConfig,
isRawSqlChart ? undefined : tableSource,
);
}
})();
@ -801,7 +773,10 @@ export default function EditTimeChartForm({
const tableSortState = useMemo(
() =>
queriedConfig?.orderBy && typeof queriedConfig.orderBy === 'string'
queriedConfig != null &&
isBuilderChartConfig(queriedConfig) &&
queriedConfig.orderBy &&
typeof queriedConfig.orderBy === 'string'
? orderByStringToSortingState(queriedConfig.orderBy)
: undefined,
[queriedConfig],
@ -814,38 +789,32 @@ export default function EditTimeChartForm({
}, [onSubmit, submitRef]);
const handleSave = useCallback(
(v: SavedChartConfigWithSeries) => {
if (tableSource != null) {
// Validate metric sources have metric names selected
if (validateMetricNames(tableSource, v.series, setError)) {
return;
}
(form: ChartEditorFormState) => {
const isRawSqlChart =
form.configType === 'sql' && form.displayType === DisplayType.Table;
// If the chart type is search, we need to ensure the select is a string
if (
displayType === DisplayType.Search &&
typeof v.select !== 'string'
) {
v.select = '';
} else if (displayType !== DisplayType.Search) {
v.select = v.series;
}
const normalizedChartConfig = normalizeChartConfig(
// Avoid saving the series field. Series should be persisted in the select field.
omit(v, ['series']),
tableSource,
);
onSave?.(normalizedChartConfig);
// Validate metric sources have metric names selected
if (
!isRawSqlChart &&
validateMetricNames(tableSource, form.series, setError)
) {
return;
}
const savedChartConfig = convertFormStateToSavedChartConfig(
form,
tableSource,
);
if (savedChartConfig) onSave?.(savedChartConfig);
},
[onSave, displayType, tableSource, setError],
[onSave, tableSource, setError],
);
// Track previous values for detecting changes
const prevGranularityRef = useRef(granularity);
const prevDisplayTypeRef = useRef(displayType);
const prevConfigTypeRef = useRef(configType);
useEffect(() => {
// Emulate the granularity picker auto-searching similar to dashboards
@ -856,14 +825,18 @@ export default function EditTimeChartForm({
}, [granularity, onSubmit]);
useEffect(() => {
if (displayType !== prevDisplayTypeRef.current) {
if (
displayType !== prevDisplayTypeRef.current ||
configType !== prevConfigTypeRef.current
) {
prevDisplayTypeRef.current = displayType;
prevConfigTypeRef.current = configType;
if (displayType === DisplayType.Search && typeof select !== 'string') {
setValue('select', '');
setValue('series', []);
}
if (displayType !== DisplayType.Search && typeof select === 'string') {
if (displayType !== DisplayType.Search && !Array.isArray(select)) {
const defaultSeries: SavedChartConfigWithSelectArray['select'] = [
{
aggFn: 'count',
@ -878,7 +851,7 @@ export default function EditTimeChartForm({
}
onSubmit();
}
}, [displayType, select, setValue, onSubmit]);
}, [displayType, select, setValue, onSubmit, configType]);
// Emulate the date range picker auto-searching similar to dashboards
useEffect(() => {
@ -900,11 +873,17 @@ export default function EditTimeChartForm({
// and explaining whether a MV can be used.
const chartConfigForExplanations: ChartConfigWithOptTimestamp | undefined =
useMemo(() => {
if (queriedConfig && isRawSqlChartConfig(queriedConfig))
return { ...queriedConfig, dateRange };
if (chartConfig && isRawSqlSavedChartConfig(chartConfig))
return { ...chartConfig, dateRange };
const userHasSubmittedQuery = !!queriedConfig;
const queriedSourceMatchesSelectedSource =
queriedSource?.id === tableSource?.id;
const urlParamsSourceMatchesSelectedSource =
chartConfig?.source === tableSource?.id;
chartConfig.source === tableSource?.id;
const effectiveQueriedConfig =
activeTab === 'time' ? dbTimeChartConfig : queriedConfig;
@ -912,7 +891,7 @@ export default function EditTimeChartForm({
const config =
userHasSubmittedQuery && queriedSourceMatchesSelectedSource
? effectiveQueriedConfig
: chartConfig && urlParamsSourceMatchesSelectedSource
: chartConfig && urlParamsSourceMatchesSelectedSource && tableSource
? {
...chartConfig,
dateRange,
@ -922,7 +901,7 @@ export default function EditTimeChartForm({
}
: undefined;
if (!config) {
if (!config || isRawSqlChartConfig(config)) {
return undefined;
}
@ -954,7 +933,10 @@ export default function EditTimeChartForm({
const sampleEventsConfig = useMemo(
() =>
tableSource != null && queriedConfig != null && queryReady
tableSource != null &&
queriedConfig != null &&
isBuilderChartConfig(queriedConfig) &&
queryReady
? {
...queriedConfig,
orderBy: [
@ -1067,11 +1049,27 @@ export default function EditTimeChartForm({
<InputControlled
name="name"
control={control}
w="100%"
flex={1}
type="text"
placeholder="My Chart Name"
data-testid="chart-name-input"
/>
{IS_SQL_CHARTS_ENABLED && displayType === DisplayType.Table && (
<Controller
control={control}
name="configType"
render={({ field: { onChange, value } }) => (
<SegmentedControl
value={value ?? 'builder'}
onChange={onChange}
data={[
{ label: 'Builder', value: 'builder' },
{ label: 'SQL', value: 'sql' },
]}
/>
)}
/>
)}
</Flex>
<Divider my="md" />
{activeTab === 'markdown' ? (
@ -1095,9 +1093,14 @@ export default function EditTimeChartForm({
/>
</Box>
</div>
) : isRawSqlInput ? (
<RawSqlChartEditor
control={control}
onOpenDisplaySettings={openDisplaySettings}
/>
) : (
<>
<Flex mb="md" align="center" gap="sm" justify="space-between">
<Flex mb="md" align="center" justify="space-between">
<Group>
<Text pe="md" size="sm">
Data Source
@ -1112,14 +1115,18 @@ export default function EditTimeChartForm({
}
/>
</Group>
{tableSource && activeTab !== 'search' && (
<MVOptimizationIndicator
source={tableSource}
config={chartConfigForExplanations}
/>
)}
<Group>
{tableSource &&
activeTab !== 'search' &&
chartConfigForExplanations &&
isBuilderChartConfig(chartConfigForExplanations) && (
<MVOptimizationIndicator
source={tableSource}
config={chartConfigForExplanations}
/>
)}
</Group>
</Flex>
{displayType !== DisplayType.Search && Array.isArray(select) ? (
<>
{fields.map((field, index) => (
@ -1393,7 +1400,7 @@ export default function EditTimeChartForm({
)}
</Flex>
<Flex gap="sm" mb="sm" align="center" justify="end">
{activeTab === 'table' && (
{activeTab === 'table' && !isRawSqlInput && (
<div style={{ width: 400 }}>
<SQLInlineEditorControlled
parentRef={parentRef}
@ -1475,13 +1482,16 @@ export default function EditTimeChartForm({
<div className="flex-grow-1 d-flex flex-column" style={{ height: 400 }}>
<DBTableChart
config={queriedConfig}
getRowSearchLink={row =>
buildTableRowSearchUrl({
row,
source: tableSource,
config: queriedConfig,
dateRange: queriedConfig.dateRange,
})
getRowSearchLink={
isBuilderChartConfig(queriedConfig)
? row =>
buildTableRowSearchUrl({
row,
source: tableSource,
config: queriedConfig,
dateRange: queriedConfig.dateRange,
})
: undefined
}
onSortingChange={onTableSortingChange}
sort={tableSortState}
@ -1514,24 +1524,31 @@ export default function EditTimeChartForm({
/>
</div>
)}
{queryReady && queriedConfig != null && activeTab === 'number' && (
<div className="flex-grow-1 d-flex flex-column" style={{ height: 400 }}>
<DBNumberChart
config={queriedConfig}
showMVOptimizationIndicator={false}
/>
</div>
)}
{queryReady &&
queriedConfig != null &&
isBuilderChartConfig(queriedConfig) &&
activeTab === 'number' && (
<div
className="flex-grow-1 d-flex flex-column"
style={{ height: 400 }}
>
<DBNumberChart
config={queriedConfig}
showMVOptimizationIndicator={false}
/>
</div>
)}
{queryReady &&
tableSource &&
queriedConfig != null &&
isBuilderChartConfig(queriedConfig) &&
activeTab === 'search' && (
<div
className="flex-grow-1 d-flex flex-column"
style={{ height: 400 }}
>
<DBSqlRowTableWithSideBar
sourceId={sourceId}
sourceId={tableSource.id}
config={{
...queriedConfig,
orderBy: [
@ -1578,13 +1595,13 @@ export default function EditTimeChartForm({
</Text>
</Accordion.Control>
<Accordion.Panel>
{sampleEventsConfig != null && (
{sampleEventsConfig != null && tableSource && (
<div
className="flex-grow-1 d-flex flex-column"
style={{ height: 400 }}
>
<DBSqlRowTableWithSideBar
sourceId={sourceId}
sourceId={tableSource.id}
config={sampleEventsConfig}
enabled={isSampleEventsOpen}
isLive={false}

22
packages/app/src/components/DBHeatmapChart.tsx
View file

@ -8,7 +8,7 @@ import {
inferTimestampColumn,
} from '@hyperdx/common-utils/dist/clickhouse';
import { convertDateRangeToGranularityString } from '@hyperdx/common-utils/dist/core/utils';
import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
import { DisplayType } from '@hyperdx/common-utils/dist/types';
import { Box, Button, Code, Divider, Group, Modal, Text } from '@mantine/core';
import { useDisclosure, useElementSize } from '@mantine/hooks';
@ -269,15 +269,15 @@ type HeatmapChartConfig = {
countExpression?: string;
},
];
from: ChartConfigWithDateRange['from'];
where: ChartConfigWithDateRange['where'];
dateRange: ChartConfigWithDateRange['dateRange'];
granularity: ChartConfigWithDateRange['granularity'];
timestampValueExpression: ChartConfigWithDateRange['timestampValueExpression'];
numberFormat?: ChartConfigWithDateRange['numberFormat'];
filters?: ChartConfigWithDateRange['filters'];
from: BuilderChartConfigWithDateRange['from'];
where: BuilderChartConfigWithDateRange['where'];
dateRange: BuilderChartConfigWithDateRange['dateRange'];
granularity: BuilderChartConfigWithDateRange['granularity'];
timestampValueExpression: BuilderChartConfigWithDateRange['timestampValueExpression'];
numberFormat?: BuilderChartConfigWithDateRange['numberFormat'];
filters?: BuilderChartConfigWithDateRange['filters'];
connection: string;
with?: ChartConfigWithDateRange['with'];
with?: BuilderChartConfigWithDateRange['with'];
};
function HeatmapContainer({
@ -306,7 +306,7 @@ function HeatmapContainer({
// When valueExpression is an aggregate like count(), we need to use a CTE to calculate the heatmap
const isAggregateExpression = isAggregateFunction(valueExpression);
const minMaxConfig: ChartConfigWithDateRange = isAggregateExpression
const minMaxConfig: BuilderChartConfigWithDateRange = isAggregateExpression
? {
...config,
where: '',
@ -372,7 +372,7 @@ function HeatmapContainer({
const min = Number.parseInt(minMaxData?.data?.[0]?.['min'] ?? '0', 10);
const max = Number.parseInt(minMaxData?.data?.[0]?.['max'] ?? '0', 10);
const bucketConfig: ChartConfigWithDateRange = isAggregateExpression
const bucketConfig: BuilderChartConfigWithDateRange = isAggregateExpression
? {
...config,
where: '',

4
packages/app/src/components/DBHistogramChart.tsx
View file

@ -11,7 +11,7 @@ import {
} from 'recharts';
import { CategoricalChartState } from 'recharts/types/chart/types';
import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse';
import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
import { Box, Code, Text } from '@mantine/core';
import { buildMVDateRangeIndicator } from '@/ChartUtils';
@ -199,7 +199,7 @@ export default function DBHistogramChart({
toolbarSuffix,
showMVOptimizationIndicator = true,
}: {
config: ChartConfigWithDateRange;
config: BuilderChartConfigWithDateRange;
onSettled?: () => void;
queryKeyPrefix?: string;
enabled?: boolean;

4
packages/app/src/components/DBListBarChart.tsx
View file

@ -1,7 +1,7 @@
import { useMemo } from 'react';
import Link from 'next/link';
import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse';
import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
import type { FloatingPosition } from '@mantine/core';
import { Box, Code, Flex, HoverCard, Text } from '@mantine/core';
@ -185,7 +185,7 @@ export default function DBListBarChart({
toolbarItems,
showMVOptimizationIndicator = true,
}: {
config: ChartConfigWithDateRange;
config: BuilderChartConfigWithDateRange;
onSettled?: () => void;
getRowSearchLink?: (row: any) => string;
hoverCardPosition?: FloatingPosition;

4
packages/app/src/components/DBNumberChart.tsx
View file

@ -1,6 +1,6 @@
import { useMemo } from 'react';
import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse';
import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
import { Box, Code, Flex, Text } from '@mantine/core';
import {
@ -25,7 +25,7 @@ export default function DBNumberChart({
toolbarSuffix,
showMVOptimizationIndicator = true,
}: {
config: ChartConfigWithDateRange;
config: BuilderChartConfigWithDateRange;
queryKeyPrefix?: string;
enabled?: boolean;
title?: React.ReactNode;

4
packages/app/src/components/DBPieChart.tsx
View file

@ -1,7 +1,7 @@
import { memo, useMemo } from 'react';
import { Cell, Pie, PieChart, ResponsiveContainer, Tooltip } from 'recharts';
import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse';
import { ChartConfigWithOptTimestamp } from '@hyperdx/common-utils/dist/types';
import { BuilderChartConfigWithOptTimestamp } from '@hyperdx/common-utils/dist/types';
import { Box, Code, Flex, Text } from '@mantine/core';
import {
@ -55,7 +55,7 @@ export const DBPieChart = ({
toolbarPrefix,
toolbarSuffix,
}: {
config: ChartConfigWithOptTimestamp;
config: BuilderChartConfigWithOptTimestamp;
title?: React.ReactNode;
enabled?: boolean;
queryKeyPrefix?: string;

4
packages/app/src/components/DBRowSidePanel.tsx
View file

@ -13,7 +13,7 @@ import { parseAsStringEnum, useQueryState } from 'nuqs';
import { ErrorBoundary } from 'react-error-boundary';
import { useHotkeys } from 'react-hotkeys-hook';
import { SourceKind, TSource } from '@hyperdx/common-utils/dist/types';
import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
import { Box, Drawer, Flex, Stack } from '@mantine/core';
import DBRowSidePanelHeader, {
@ -63,7 +63,7 @@ export type RowSidePanelContextProps = {
displayedColumns?: string[];
toggleColumn?: (column: string) => void;
shareUrl?: string;
dbSqlRowTableConfig?: ChartConfigWithDateRange;
dbSqlRowTableConfig?: BuilderChartConfigWithDateRange;
isChildModalOpen?: boolean;
setChildModalOpen?: (open: boolean) => void;
source?: TSource;

10
packages/app/src/components/DBRowTable.tsx
View file

@ -32,7 +32,7 @@ import {
} from '@hyperdx/common-utils/dist/clickhouse';
import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils';
import {
ChartConfigWithDateRange,
BuilderChartConfigWithDateRange,
SelectList,
TSource,
} from '@hyperdx/common-utils/dist/types';
@ -271,7 +271,7 @@ const SqlModal = ({
}: {
opened: boolean;
onClose: () => void;
config: ChartConfigWithDateRange;
config: BuilderChartConfigWithDateRange;
}) => {
const { data: sql, isLoading: isLoadingSql } = useRenderedSqlChartConfig(
config,
@ -368,7 +368,7 @@ export const RawLogTable = memo(
error?: ClickHouseQueryError | Error;
dateRange?: [Date, Date];
loadingDate?: Date;
config?: ChartConfigWithDateRange;
config?: BuilderChartConfigWithDateRange;
onChildModalOpen?: (open: boolean) => void;
source?: TSource;
onExpandedRowsChange?: (hasExpandedRows: boolean) => void;
@ -1290,7 +1290,7 @@ function getSelectLength(select: SelectList): number {
}
export function useConfigWithPrimaryAndPartitionKey(
config: ChartConfigWithDateRange,
config: BuilderChartConfigWithDateRange,
) {
const { data: tableMetadata } = useTableMetadata({
databaseName: config.from.databaseName,
@ -1365,7 +1365,7 @@ function DBSqlRowTableComponent({
initialSortBy,
variant = 'default',
}: {
config: ChartConfigWithDateRange;
config: BuilderChartConfigWithDateRange;
sourceId?: string;
onRowDetailsClick?: (rowWhere: RowWhereResult) => void;
highlightedLineId?: string;

6
packages/app/src/components/DBSearchPageFilters.tsx
View file

@ -5,7 +5,7 @@ import {
tcFromSource,
} from '@hyperdx/common-utils/dist/core/metadata';
import {
ChartConfigWithDateRange,
BuilderChartConfigWithDateRange,
SourceKind,
} from '@hyperdx/common-utils/dist/types';
import {
@ -346,7 +346,7 @@ export type FilterGroupProps = {
hasLoadedMore: boolean;
isDefaultExpanded?: boolean;
'data-testid'?: string;
chartConfig: ChartConfigWithDateRange;
chartConfig: BuilderChartConfigWithDateRange;
isLive?: boolean;
onRangeChange?: (range: { min: number; max: number }) => void;
distributionKey?: string; // Optional key to use for distribution queries, defaults to name
@ -848,7 +848,7 @@ const DBSearchPageFiltersComponent = ({
analysisMode: 'results' | 'delta' | 'pattern';
setAnalysisMode: (mode: 'results' | 'delta' | 'pattern') => void;
isLive: boolean;
chartConfig: ChartConfigWithDateRange;
chartConfig: BuilderChartConfigWithDateRange;
sourceId?: string;
showDelta: boolean;
denoiseResults: boolean;

4
packages/app/src/components/DBSqlRowTableWithSidebar.tsx
View file

@ -2,7 +2,7 @@ import { useCallback, useState } from 'react';
import { useQueryState } from 'nuqs';
import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse';
import {
ChartConfigWithDateRange,
BuilderChartConfigWithDateRange,
TSource,
} from '@hyperdx/common-utils/dist/types';
import { SortingState } from '@tanstack/react-table';
@ -25,7 +25,7 @@ import { DBRowTableVariant, DBSqlRowTable } from './DBRowTable';
interface Props {
sourceId: string;
config: ChartConfigWithDateRange;
config: BuilderChartConfigWithDateRange;
onError?: (error: Error | ClickHouseQueryError) => void;
onScroll?: (scrollTop: number) => void;
onSidebarOpen?: (rowId: string) => void;

42
packages/app/src/components/DBTableChart.tsx
View file

@ -1,5 +1,9 @@
import { useCallback, useMemo, useState } from 'react';
import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse';
import {
isBuilderChartConfig,
isRawSqlChartConfig,
} from '@hyperdx/common-utils/dist/guards';
import { ChartConfigWithOptTimestamp } from '@hyperdx/common-utils/dist/types';
import { Box, Code, Text } from '@mantine/core';
import { SortingState } from '@tanstack/react-table';
@ -48,7 +52,9 @@ export default function DBTableChart({
}) {
const [sort, setSort] = useState<SortingState>([]);
const { data: source } = useSource({ id: config.source });
const { data: source } = useSource({
id: isBuilderChartConfig(config) ? config.source : undefined,
});
const effectiveSort = useMemo(
() => controlledSort || sort,
@ -66,6 +72,8 @@ export default function DBTableChart({
);
const queriedConfig = useMemo(() => {
if (isRawSqlChartConfig(config)) return config;
const _config = convertToTableChartConfig(config);
if (effectiveSort.length) {
@ -79,8 +87,9 @@ export default function DBTableChart({
return _config;
}, [config, effectiveSort]);
const { data: mvOptimizationData } =
useMVOptimizationExplanation(queriedConfig);
const { data: mvOptimizationData } = useMVOptimizationExplanation(
isBuilderChartConfig(queriedConfig) ? queriedConfig : undefined,
);
const { data, fetchNextPage, hasNextPage, isLoading, isError, error } =
useOffsetPaginatedQuery(queriedConfig, {
@ -91,6 +100,10 @@ export default function DBTableChart({
// Returns an array of aliases, so we can check if something is using an alias
const aliasMap = useMemo(() => {
if (isRawSqlChartConfig(config)) {
return [];
}
// If the config.select is a string, we can't infer this.
// One day, we could potentially run this through chSqlToAliasMap but AST parsing
// doesn't work for most DBTableChart queries.
@ -103,7 +116,8 @@ export default function DBTableChart({
}
return acc;
}, [] as string[]);
}, [config.select]);
}, [config]);
const columns = useMemo(() => {
const rows = data?.data ?? [];
if (rows.length === 0) {
@ -111,7 +125,11 @@ export default function DBTableChart({
}
let groupByKeys: string[] = [];
if (queriedConfig.groupBy && typeof queriedConfig.groupBy === 'string') {
if (
isBuilderChartConfig(queriedConfig) &&
queriedConfig.groupBy &&
typeof queriedConfig.groupBy === 'string'
) {
groupByKeys = queriedConfig.groupBy.split(',').map(v => v.trim());
}
@ -126,13 +144,7 @@ export default function DBTableChart({
? undefined
: config.numberFormat,
}));
}, [
config.numberFormat,
aliasMap,
queriedConfig.groupBy,
data,
hiddenColumns,
]);
}, [config.numberFormat, aliasMap, queriedConfig, data, hiddenColumns]);
const toolbarItemsMemo = useMemo(() => {
const allToolbarItems = [];
@ -141,7 +153,11 @@ export default function DBTableChart({
allToolbarItems.push(...toolbarPrefix);
}
if (source && showMVOptimizationIndicator) {
if (
source &&
showMVOptimizationIndicator &&
isBuilderChartConfig(queriedConfig)
) {
allToolbarItems.push(
<MVOptimizationIndicator
key="db-table-chart-mv-indicator"

3
packages/app/src/components/DBTimeChart.tsx
View file

@ -4,6 +4,7 @@ import { add, differenceInSeconds } from 'date-fns';
import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse';
import { getAlignedDateRange } from '@hyperdx/common-utils/dist/core/utils';
import {
BuilderChartConfigWithDateRange,
ChartConfigWithDateRange,
DisplayType,
} from '@hyperdx/common-utils/dist/types';
@ -199,7 +200,7 @@ function ActiveTimeTooltip({
}
type DBTimeChartComponentProps = {
config: ChartConfigWithDateRange;
config: BuilderChartConfigWithDateRange;
disableQueryChunking?: boolean;
disableDrillDown?: boolean;
enableParallelQueries?: boolean;

4
packages/app/src/components/KubeComponents.tsx
View file

@ -4,7 +4,7 @@ import { sub } from 'date-fns';
import type { ResponseJSON } from '@hyperdx/common-utils/dist/clickhouse';
import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig';
import {
ChartConfigWithDateRange,
BuilderChartConfigWithDateRange,
DateRange,
SearchCondition,
SearchConditionLanguage,
@ -47,7 +47,7 @@ export const useV2LogBatch = <T = any,>(
whereLanguage,
}: {
dateRange: DateRange['dateRange'];
extraSelects?: ChartConfigWithDateRange['select'];
extraSelects?: BuilderChartConfigWithDateRange['select'];
limit?: number;
logSource: TSource;
order: 'asc' | 'desc';

6
packages/app/src/components/KubernetesFilters.tsx
View file

@ -2,7 +2,7 @@ import React, { useEffect, useMemo, useRef, useState } from 'react';
import { useForm, useWatch } from 'react-hook-form';
import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata';
import {
ChartConfigWithDateRange,
BuilderChartConfigWithDateRange,
TSource,
} from '@hyperdx/common-utils/dist/types';
import { Box, Group, Select } from '@mantine/core';
@ -23,7 +23,7 @@ type FilterSelectProps = {
fieldName: string;
value: string | null;
onChange: (value: string | null) => void;
chartConfig: ChartConfigWithDateRange;
chartConfig: BuilderChartConfigWithDateRange;
dataTestId?: string;
};
@ -157,7 +157,7 @@ export const KubernetesFilters: React.FC<KubernetesFiltersProps> = ({
}, [searchQuery, metricSource.resourceAttributesExpression]);
// Create chart config for fetching key values
const chartConfig: ChartConfigWithDateRange = {
const chartConfig: BuilderChartConfigWithDateRange = {
from: {
databaseName: metricSource.from.databaseName,
tableName: metricSource.metricTables?.gauge || '',

4
packages/app/src/components/MaterializedViews/MVOptimizationIndicator.tsx
View file

@ -1,6 +1,6 @@
import { useState } from 'react';
import {
ChartConfigWithOptDateRange,
BuilderChartConfigWithOptDateRange,
TSource,
} from '@hyperdx/common-utils/dist/types';
import { ActionIcon, Badge, Tooltip } from '@mantine/core';
@ -59,7 +59,7 @@ export default function MVOptimizationIndicator({
variant = 'badge',
}: {
source: TSource;
config: ChartConfigWithOptDateRange | undefined;
config: BuilderChartConfigWithOptDateRange | undefined;
variant?: 'badge' | 'icon';
}) {
const [modalOpen, setModalOpen] = useState(false);

6
packages/app/src/components/PatternTable.tsx
View file

@ -1,6 +1,6 @@
import { useMemo, useState } from 'react';
import {
ChartConfigWithDateRange,
BuilderChartConfigWithDateRange,
TSource,
} from '@hyperdx/common-utils/dist/types';
@ -19,8 +19,8 @@ export default function PatternTable({
bodyValueExpression,
source,
}: {
config: ChartConfigWithDateRange;
totalCountConfig: ChartConfigWithDateRange;
config: BuilderChartConfigWithDateRange;
totalCountConfig: BuilderChartConfigWithDateRange;
bodyValueExpression: string;
totalCountQueryKeyPrefix: string;
source?: TSource;

18
packages/app/src/components/SQLEditor.tsx
View file

@ -1,13 +1,11 @@
import { useCallback, useEffect, useMemo, useRef } from 'react';
import { useRef } from 'react';
import { useController, UseControllerProps } from 'react-hook-form';
import { acceptCompletion, startCompletion } from '@codemirror/autocomplete';
import { sql, SQLDialect } from '@codemirror/lang-sql';
import { Flex, Group, Paper, Text, useMantineColorScheme } from '@mantine/core';
import { startCompletion } from '@codemirror/autocomplete';
import { sql } from '@codemirror/lang-sql';
import { Paper, useMantineColorScheme } from '@mantine/core';
import CodeMirror, {
Compartment,
EditorView,
keymap,
Prec,
ReactCodeMirrorRef,
} from '@uiw/react-codemirror';
@ -15,6 +13,8 @@ type SQLInlineEditorProps = {
value: string;
onChange: (value: string) => void;
placeholder?: string;
height?: string;
enableLineWrapping?: boolean;
};
const styleTheme = EditorView.baseTheme({
@ -33,6 +33,8 @@ export default function SQLEditor({
onChange,
placeholder,
value,
height,
enableLineWrapping = false,
}: SQLInlineEditorProps) {
const { colorScheme } = useMantineColorScheme();
const ref = useRef<ReactCodeMirrorRef>(null);
@ -57,6 +59,7 @@ export default function SQLEditor({
value={value}
onChange={onChange}
theme={colorScheme === 'dark' ? 'dark' : 'light'}
height={height}
minHeight={'100px'}
extensions={[
styleTheme,
@ -66,6 +69,7 @@ export default function SQLEditor({
upperCaseKeywords: true,
}),
),
...(enableLineWrapping ? [EditorView.lineWrapping] : []),
]}
onUpdate={update => {
// Always open completion window as much as possible
@ -92,6 +96,7 @@ export default function SQLEditor({
export function SQLEditorControlled({
placeholder,
height,
...props
}: Omit<SQLInlineEditorProps, 'value' | 'onChange'> & UseControllerProps<any>) {
const { field } = useController(props);
@ -101,6 +106,7 @@ export function SQLEditorControlled({
onChange={field.onChange}
placeholder={placeholder}
value={field.value}
height={height}
{...props}
/>
);

4
packages/app/src/components/Search/DBSearchHeatmapChart.tsx
View file

@ -8,7 +8,7 @@ import {
tcFromSource,
} from '@hyperdx/common-utils/dist/core/metadata';
import {
ChartConfigWithDateRange,
BuilderChartConfigWithDateRange,
DisplayType,
TSource,
} from '@hyperdx/common-utils/dist/types';
@ -34,7 +34,7 @@ export function DBSearchHeatmapChart({
source,
isReady,
}: {
chartConfig: ChartConfigWithDateRange;
chartConfig: BuilderChartConfigWithDateRange;
source: TSource;
isReady: boolean;
}) {

6
packages/app/src/components/SearchTotalCountChart.tsx
View file

@ -4,7 +4,7 @@ import {
JSDataType,
ResponseJSON,
} from '@hyperdx/common-utils/dist/clickhouse';
import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
import { Text } from '@mantine/core';
import { keepPreviousData } from '@tanstack/react-query';
@ -25,7 +25,7 @@ function inferCountColumn(meta: ResponseJSON['meta'] | undefined): string {
}
export function useSearchTotalCount(
config: ChartConfigWithDateRange,
config: BuilderChartConfigWithDateRange,
queryKeyPrefix: string,
{
disableQueryChunking,
@ -88,7 +88,7 @@ export default function SearchTotalCountChart({
disableQueryChunking,
enableParallelQueries,
}: {
config: ChartConfigWithDateRange;
config: BuilderChartConfigWithDateRange;
queryKeyPrefix: string;
disableQueryChunking?: boolean;
enableParallelQueries?: boolean;

2
packages/app/src/config.ts
View file

@ -37,3 +37,5 @@ export const IS_K8S_DASHBOARD_ENABLED = true;
export const IS_METRICS_ENABLED = true;
export const IS_MTVIEWS_ENABLED = false;
export const IS_SESSIONS_ENABLED = true;
export const IS_SQL_CHARTS_ENABLED =
process.env.NEXT_PUBLIC_IS_SQL_CHARTS_ENABLED === 'true';

4
packages/app/src/defaults.ts
View file

@ -1,4 +1,4 @@
import type { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
import type { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
// Limit defaults
export const DEFAULT_SEARCH_ROW_LIMIT = 200;
@ -6,7 +6,7 @@ export const DEFAULT_QUERY_TIMEOUT = 60; // max_execution_time, seconds
export function searchChartConfigDefaults(
team: any | undefined | null,
): Partial<ChartConfigWithDateRange> {
): Partial<BuilderChartConfigWithDateRange> {
return {
limit: {
limit: team?.searchRowLimit ?? DEFAULT_SEARCH_ROW_LIMIT,

6
packages/app/src/hdxMTViews.ts
View file

@ -13,7 +13,7 @@ import {
} from '@hyperdx/common-utils/dist/core/renderChartConfig';
import {
AggregateFunction,
ChartConfigWithOptDateRange,
BuilderChartConfigWithOptDateRange,
DerivedColumn,
QuerySettings,
SQLInterval,
@ -58,7 +58,7 @@ const getAggFn = (
const buildMTViewDataTableDDL = (
table: string,
chartConfig: ChartConfigWithOptDateRange,
chartConfig: BuilderChartConfigWithOptDateRange,
) => {
if (!Array.isArray(chartConfig.select)) {
throw new Error('Only array select is supported');
@ -96,7 +96,7 @@ const buildMTViewDDL = (name: string, table: string, query: ChSql) => {
};
export const buildMTViewSelectQuery = async (
chartConfig: ChartConfigWithOptDateRange,
chartConfig: BuilderChartConfigWithOptDateRange,
metadata: Metadata,
querySettings: QuerySettings | undefined,
customGranularity?: SQLInterval,

4
packages/app/src/hooks/__tests__/useChartConfig.test.tsx
View file

@ -1,6 +1,7 @@
import React from 'react';
import { ResponseJSON } from '@hyperdx/common-utils/dist/clickhouse';
import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/browser';
import { isBuilderChartConfig } from '@hyperdx/common-utils/dist/guards';
import {
ChartConfigWithDateRange,
ChartConfigWithOptDateRange,
@ -1398,6 +1399,9 @@ describe('useChartConfig', () => {
// Verify the query used the optimized config (materialized view)
const queryCall = mockClickhouseClient.queryChartConfig.mock.calls[0][0];
if (!isBuilderChartConfig(queryCall.config)) {
throw new Error('Expected a BuilderChartConfig');
}
expect(queryCall.config.from.tableName).toBe('metrics_rollup_1h');
expect(result2.current.data?.data).toBeDefined();

8
packages/app/src/hooks/__tests__/useMetadata.test.tsx
View file

@ -5,7 +5,7 @@ import {
Metadata,
MetadataCache,
} from '@hyperdx/common-utils/dist/core/metadata';
import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
import { renderHook, waitFor } from '@testing-library/react';
@ -19,8 +19,8 @@ import {
// Create a mock ChartConfig based on the Zod schema
const createMockChartConfig = (
overrides: Partial<ChartConfigWithDateRange> = {},
): ChartConfigWithDateRange =>
overrides: Partial<BuilderChartConfigWithDateRange> = {},
): BuilderChartConfigWithDateRange =>
({
timestampValueExpression: '',
connection: 'foo',
@ -29,7 +29,7 @@ const createMockChartConfig = (
tableName: 'traces',
},
...overrides,
}) as ChartConfigWithDateRange;
}) as BuilderChartConfigWithDateRange;
jest.mock('@/source', () => ({
useSources: jest.fn().mockReturnValue({

60
packages/app/src/hooks/__tests__/useOffsetPaginatedQuery.test.tsx
View file

@ -49,6 +49,7 @@ jest.mock('@hyperdx/common-utils/dist/core/renderChartConfig', () => ({
import { getClickhouseClient } from '@hyperdx/app/src/clickhouse';
import { MVOptimizationExplanation } from '@hyperdx/common-utils/dist/core/materializedViews';
import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig';
import { isBuilderChartConfig } from '@hyperdx/common-utils/dist/guards';
import {
MVOptimizationExplanationResult,
@ -763,6 +764,57 @@ describe('useOffsetPaginatedQuery', () => {
});
});
describe('RawSqlChartConfig', () => {
it('should execute raw SQL query without time windowing and not paginate', async () => {
const rawSqlConfig = {
configType: 'sql' as const,
sqlTemplate: 'SELECT status, count() FROM logs GROUP BY status',
connection: 'conn-1',
displayType: undefined,
dateRange: [
new Date('2024-01-01T00:00:00Z'),
new Date('2024-01-02T00:00:00Z'),
] as [Date, Date],
};
mockReader.read
.mockResolvedValueOnce({
done: false,
value: [
{ json: () => ['status', 'count()'] },
{ json: () => ['String', 'UInt64'] },
{ json: () => ['error', 42] },
{ json: () => ['info', 100] },
],
})
.mockResolvedValueOnce({ done: true });
const { result } = renderHook(
() => useOffsetPaginatedQuery(rawSqlConfig),
{ wrapper },
);
await waitFor(() => expect(result.current.isLoading).toBe(false));
// Raw SQL config should be passed through to renderChartConfig unchanged
expect(renderChartConfig).toHaveBeenCalledTimes(1);
expect(jest.mocked(renderChartConfig).mock.calls[0][0]).toMatchObject({
configType: 'sql',
sqlTemplate: 'SELECT status, count() FROM logs GROUP BY status',
});
// Should have data
expect(result.current.data?.data).toHaveLength(2);
expect(result.current.data?.data[0]).toEqual({
status: 'error',
'count()': 42,
});
// Pagination is disabled for raw SQL
expect(result.current.hasNextPage).toBe(false);
});
});
describe('MV Optimization Integration', () => {
it('should optimize queries using MVs when possible', async () => {
const config = createMockChartConfig({
@ -826,7 +878,9 @@ describe('useOffsetPaginatedQuery', () => {
jest
.mocked(renderChartConfig)
.mock.calls.every(
call => call[0].from.tableName === 'metrics_rollup_1m',
call =>
isBuilderChartConfig(call[0]) &&
call[0].from.tableName === 'metrics_rollup_1m',
),
).toBeTruthy();
@ -919,7 +973,9 @@ describe('useOffsetPaginatedQuery', () => {
jest
.mocked(renderChartConfig)
.mock.calls.every(
call => call[0].from.tableName === 'metrics_rollup_1m',
call =>
isBuilderChartConfig(call[0]) &&
call[0].from.tableName === 'metrics_rollup_1m',
),
).toBeTruthy();

32
packages/app/src/hooks/useAutoCompleteOptions.tsx
View file

@ -3,7 +3,7 @@ import {
Field,
TableConnection,
} from '@hyperdx/common-utils/dist/core/metadata';
import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
import {
deduplicate2dArray,
@ -103,21 +103,21 @@ export function useAutoCompleteOptions(
);
// hooks to get key values
const chartConfigs: ChartConfigWithDateRange[] = toArray(tableConnection).map(
({ databaseName, tableName, connectionId }) => ({
connection: connectionId,
from: {
databaseName,
tableName,
},
timestampValueExpression: '',
select: '',
where: '',
// TODO: Pull in date for query as arg
// just assuming 1/2 day is okay to query over right now
dateRange: [new Date(NOW - (86400 * 1000) / 2), new Date(NOW)],
}),
);
const chartConfigs: BuilderChartConfigWithDateRange[] = toArray(
tableConnection,
).map(({ databaseName, tableName, connectionId }) => ({
connection: connectionId,
from: {
databaseName,
tableName,
},
timestampValueExpression: '',
select: '',
where: '',
// TODO: Pull in date for query as arg
// just assuming 1/2 day is okay to query over right now
dateRange: [new Date(NOW - (86400 * 1000) / 2), new Date(NOW)],
}));
const { data: keyVals } = useMultipleGetKeyValues({
chartConfigs,
keys: searchKeys,

32
packages/app/src/hooks/useChartConfig.tsx
View file

@ -12,8 +12,13 @@ import {
renderChartConfig,
} from '@hyperdx/common-utils/dist/core/renderChartConfig';
import { convertDateRangeToGranularityString } from '@hyperdx/common-utils/dist/core/utils';
import {
isBuilderChartConfig,
isRawSqlChartConfig,
} from '@hyperdx/common-utils/dist/guards';
import { format } from '@hyperdx/common-utils/dist/sqlFormatter';
import {
BuilderChartConfigWithOptDateRange,
ChartConfigWithDateRange,
ChartConfigWithOptDateRange,
QuerySettings,
@ -65,6 +70,9 @@ const shouldUseChunking = (
): config is ChartConfigWithDateRange & {
granularity: string;
} => {
// Avoid chunking for raw SQL charts since they can include arbitrary window functions, etc.
if (isRawSqlChartConfig(config)) return false;
// Granularity is required for chunking, otherwise we could break other group-bys.
if (!isUsingGranularity(config)) return false;
@ -143,7 +151,7 @@ async function* fetchDataInChunks({
? getGranularityAlignedTimeWindows(config)
: [undefined];
if (IS_MTVIEWS_ENABLED) {
if (IS_MTVIEWS_ENABLED && isBuilderChartConfig(config)) {
const { dataTableDDL, mtViewDDL, renderMTViewConfig } =
await buildMTViewSelectQuery(config, metadata, querySettings);
// TODO: show the DDLs in the UI so users can run commands manually
@ -154,6 +162,11 @@ async function* fetchDataInChunks({
await renderMTViewConfig();
}
// Readonly = 2 means the query is readonly but can still specify query settings.
const clickHouseSettings = isRawSqlChartConfig(config)
? { readonly: '2' }
: {};
if (enableParallelQueries) {
// fetch in parallel
const promises = windows.map(async (w, index) => {
@ -168,6 +181,7 @@ async function* fetchDataInChunks({
metadata,
opts: {
abort_signal: signal,
clickhouse_settings: clickHouseSettings,
},
querySettings,
}),
@ -258,14 +272,15 @@ export function useQueriedChartConfig(
const queryClient = useQueryClient();
const metadata = useMetadataWithSettings();
const builderConfig = isBuilderChartConfig(config) ? config : undefined;
const { data: mvOptimizationData, isLoading: isLoadingMVOptimization } =
useMVOptimizationExplanation(config, {
enabled: !!enabled,
useMVOptimizationExplanation(builderConfig, {
enabled: !!enabled && !!builderConfig,
placeholderData: undefined,
});
const { data: source, isLoading: isSourceLoading } = useSource({
id: config.source,
id: builderConfig?.source,
});
const query = useQuery<TQueryFnData, ClickHouseQueryError | Error>({
@ -351,14 +366,15 @@ export function useRenderedSqlChartConfig(
const metadata = useMetadataWithSettings();
const builderConfig = isBuilderChartConfig(config) ? config : undefined;
const { data: mvOptimizationData, isLoading: isLoadingMVOptimization } =
useMVOptimizationExplanation(config, {
enabled: !!enabled,
useMVOptimizationExplanation(builderConfig, {
enabled: !!enabled && !!builderConfig,
placeholderData: undefined,
});
const { data: source, isLoading: isSourceLoading } = useSource({
id: config.source,
id: builderConfig?.source,
});
const query = useQuery({
@ -383,7 +399,7 @@ export function useRenderedSqlChartConfig(
}
export function useAliasMapFromChartConfig(
config: ChartConfigWithOptDateRange | undefined,
config: BuilderChartConfigWithOptDateRange | undefined,
options?: UseQueryOptions<Record<string, string>>,
) {
// For granularity: 'auto', the bucket size depends on dateRange duration (not absolute times).

91
packages/app/src/hooks/useDashboardFilterValues.tsx
View file

@ -5,7 +5,7 @@ import {
optimizeGetKeyValuesCalls,
} from '@hyperdx/common-utils/dist/core/materializedViews';
import {
ChartConfigWithDateRange,
BuilderChartConfigWithDateRange,
DashboardFilter,
} from '@hyperdx/common-utils/dist/types';
import {
@ -57,53 +57,54 @@ function useOptimizedKeyValuesCalls({
return filtersBySourceIdAndMetric;
}, [filters]);
const results: UseQueryResult<GetKeyValueCall<ChartConfigWithDateRange>[]>[] =
useQueries({
queries: Array.from(filtersBySourceIdAndMetric.entries())
.filter(([key]) =>
sources?.some(s => s.id === filterFromKey(key).sourceId),
)
.map(([key, filters]) => {
const { sourceId, metricType } = filterFromKey(key);
const source = sources!.find(s => s.id === sourceId)!;
const keys = filters.map(f => f.expression);
const tableName = getMetricTableName(source, metricType) ?? '';
const results: UseQueryResult<
GetKeyValueCall<BuilderChartConfigWithDateRange>[]
>[] = useQueries({
queries: Array.from(filtersBySourceIdAndMetric.entries())
.filter(([key]) =>
sources?.some(s => s.id === filterFromKey(key).sourceId),
)
.map(([key, filters]) => {
const { sourceId, metricType } = filterFromKey(key);
const source = sources!.find(s => s.id === sourceId)!;
const keys = filters.map(f => f.expression);
const tableName = getMetricTableName(source, metricType) ?? '';
const chartConfig: ChartConfigWithDateRange = {
...pick(source, ['timestampValueExpression', 'connection']),
from: {
databaseName: source.from.databaseName,
tableName,
},
const chartConfig: BuilderChartConfigWithDateRange = {
...pick(source, ['timestampValueExpression', 'connection']),
from: {
databaseName: source.from.databaseName,
tableName,
},
dateRange,
source: source.id,
where: '',
whereLanguage: 'sql',
select: '',
};
return {
queryKey: [
'dashboard-filters-key-value-calls',
sourceId,
metricType,
dateRange,
source: source.id,
where: '',
whereLanguage: 'sql',
select: '',
};
return {
queryKey: [
'dashboard-filters-key-value-calls',
sourceId,
metricType,
dateRange,
keys,
],
enabled: !isLoadingSources,
staleTime: 1000 * 60 * 5, // Cache every 5 min
queryFn: async ({ signal }) =>
await optimizeGetKeyValuesCalls({
chartConfig,
source,
clickhouseClient,
metadata,
keys,
],
enabled: !isLoadingSources,
staleTime: 1000 * 60 * 5, // Cache every 5 min
queryFn: async ({ signal }) =>
await optimizeGetKeyValuesCalls({
chartConfig,
source,
clickhouseClient,
metadata,
keys,
signal,
}),
};
}),
});
signal,
}),
};
}),
});
return {
data: results.map(r => r.data ?? []).flat(),

3
packages/app/src/hooks/useExplainQuery.tsx
View file

@ -1,4 +1,5 @@
import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig';
import { isBuilderChartConfig } from '@hyperdx/common-utils/dist/guards';
import { ChartConfigWithOptDateRange } from '@hyperdx/common-utils/dist/types';
import { useQuery, UseQueryOptions } from '@tanstack/react-query';
@ -20,7 +21,7 @@ export function useExplainQuery(
const metadata = useMetadataWithSettings();
const { data: source, isLoading: isSourceLoading } = useSource({
id: config?.source,
id: isBuilderChartConfig(config) ? config.source : undefined,
});
return useQuery({

7
packages/app/src/hooks/useMVOptimizationExplanation.tsx
View file

@ -2,7 +2,7 @@ import {
MVOptimizationExplanation,
tryOptimizeConfigWithMaterializedViewWithExplanations,
} from '@hyperdx/common-utils/dist/core/materializedViews';
import { ChartConfigWithOptDateRange } from '@hyperdx/common-utils/dist/types';
import { BuilderChartConfigWithOptDateRange } from '@hyperdx/common-utils/dist/types';
import {
keepPreviousData,
useQuery,
@ -15,14 +15,15 @@ import { useSource } from '@/source';
import { useMetadataWithSettings } from './useMetadata';
export interface MVOptimizationExplanationResult<
C extends ChartConfigWithOptDateRange = ChartConfigWithOptDateRange,
C extends
BuilderChartConfigWithOptDateRange = BuilderChartConfigWithOptDateRange,
> {
optimizedConfig?: C;
explanations: MVOptimizationExplanation[];
}
export function useMVOptimizationExplanation<
C extends ChartConfigWithOptDateRange,
C extends BuilderChartConfigWithOptDateRange,
>(
config: C | undefined,
options?: Partial<UseQueryOptions<MVOptimizationExplanationResult<C>>>,

13
packages/app/src/hooks/useMetadata.tsx
View file

@ -10,10 +10,7 @@ import {
TableConnection,
TableMetadata,
} from '@hyperdx/common-utils/dist/core/metadata';
import {
ChartConfigWithDateRange,
TSource,
} from '@hyperdx/common-utils/dist/types';
import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
import {
keepPreviousData,
useQuery,
@ -204,7 +201,9 @@ export function useMultipleGetKeyValues(
limit,
disableRowLimit,
}: {
chartConfigs: ChartConfigWithDateRange | ChartConfigWithDateRange[];
chartConfigs:
| BuilderChartConfigWithDateRange
| BuilderChartConfigWithDateRange[];
keys: string[];
limit?: number;
disableRowLimit?: boolean;
@ -261,7 +260,7 @@ export function useGetValuesDistribution(
key,
limit,
}: {
chartConfig: ChartConfigWithDateRange;
chartConfig: BuilderChartConfigWithDateRange;
key: string;
limit: number;
},
@ -297,7 +296,7 @@ export function useGetKeyValues(
limit,
disableRowLimit,
}: {
chartConfig?: ChartConfigWithDateRange;
chartConfig?: BuilderChartConfigWithDateRange;
keys: string[];
limit?: number;
disableRowLimit?: boolean;

61
packages/app/src/hooks/useOffsetPaginatedQuery.tsx
View file

@ -12,6 +12,10 @@ import {
isFirstOrderByAscending,
isTimestampExpressionInFirstOrderBy,
} from '@hyperdx/common-utils/dist/core/utils';
import {
isBuilderChartConfig,
isRawSqlChartConfig,
} from '@hyperdx/common-utils/dist/guards';
import {
ChartConfigWithOptTimestamp,
TSource,
@ -72,6 +76,7 @@ type QueryMeta = {
metadata: Metadata;
optimizedConfig?: ChartConfigWithOptTimestamp;
source: TSource | undefined;
readonly: boolean;
};
// Get time window from page param
@ -80,9 +85,10 @@ function getTimeWindowFromPageParam(
pageParam: TPageParam,
): TimeWindow {
const [startDate, endDate] = config.dateRange;
const windows = isFirstOrderByAscending(config.orderBy)
? generateTimeWindowsAscending(startDate, endDate)
: generateTimeWindowsDescending(startDate, endDate);
const windows =
isBuilderChartConfig(config) && isFirstOrderByAscending(config.orderBy)
? generateTimeWindowsAscending(startDate, endDate)
: generateTimeWindowsDescending(startDate, endDate);
const window = windows[pageParam.windowIndex];
if (window == null) {
throw new Error('Invalid time window for page param');
@ -96,7 +102,8 @@ function getNextPageParam(
allPages: TQueryFnData[],
config: ChartConfigWithOptTimestamp,
): TPageParam | undefined {
if (lastPage == null) {
// Pagination is not supported for raw SQL tables since they may not be ordered at all.
if (lastPage == null || isRawSqlChartConfig(config)) {
return undefined;
}
@ -124,7 +131,8 @@ function getNextPageParam(
}
// If no more results in current window, move to next window (if windowing is being used)
const shouldUseWindowing = isTimestampExpressionInFirstOrderBy(config);
const shouldUseWindowing =
isBuilderChartConfig(config) && isTimestampExpressionInFirstOrderBy(config);
const nextWindowIndex = currentWindow.windowIndex + 1;
if (shouldUseWindowing && nextWindowIndex < windows.length) {
return {
@ -146,8 +154,14 @@ const queryFn: QueryFunction<TQueryFnData, TQueryKey, TPageParam> = async ({
throw new Error('Query missing client meta');
}
const { queryClient, metadata, hasPreviousQueries, optimizedConfig, source } =
meta as QueryMeta;
const {
queryClient,
metadata,
hasPreviousQueries,
optimizedConfig,
source,
readonly,
} = meta as QueryMeta;
// Only stream incrementally if this is a fresh query with no previous
// response or if it's a paginated query
@ -162,7 +176,8 @@ const queryFn: QueryFunction<TQueryFnData, TQueryKey, TPageParam> = async ({
const config = optimizedConfig ?? rawConfig;
// Get the time window for this page
const shouldUseWindowing = isTimestampExpressionInFirstOrderBy(config);
const shouldUseWindowing =
isBuilderChartConfig(config) && isTimestampExpressionInFirstOrderBy(config);
const timeWindow = shouldUseWindowing
? getTimeWindowFromPageParam(config, pageParam)
: {
@ -173,14 +188,16 @@ const queryFn: QueryFunction<TQueryFnData, TQueryKey, TPageParam> = async ({
};
// Create config with windowed date range
const windowedConfig = {
...config,
dateRange: [timeWindow.startTime, timeWindow.endTime] as [Date, Date],
limit: {
limit: config.limit?.limit,
offset: pageParam.offset,
},
};
const windowedConfig = isBuilderChartConfig(config)
? {
...config,
dateRange: [timeWindow.startTime, timeWindow.endTime] as [Date, Date],
limit: {
limit: config.limit?.limit,
offset: pageParam.offset,
},
}
: config;
const query = await renderChartConfig(
windowedConfig,
@ -194,6 +211,8 @@ const queryFn: QueryFunction<TQueryFnData, TQueryKey, TPageParam> = async ({
setTimeout(() => abortController.abort(), queryTimeout * 1000);
}
// Readonly = 2 means the query is readonly but can still specify query settings.
const clickHouseSettings = readonly ? { readonly: '2' } : {};
const resultSet =
await clickhouseClient.query<'JSONCompactEachRowWithNamesAndTypes'>({
query: query.sql,
@ -201,6 +220,7 @@ const queryFn: QueryFunction<TQueryFnData, TQueryKey, TPageParam> = async ({
format: 'JSONCompactEachRowWithNamesAndTypes',
abort_signal: abortController?.signal || signal,
connectionId: config.connection,
clickhouse_settings: clickHouseSettings,
});
const stream = resultSet.stream();
@ -402,14 +422,15 @@ export default function useOffsetPaginatedQuery(
const hasPreviousQueries =
matchedQueries.filter(([_, data]) => data != null).length > 0;
const builderConfig = isBuilderChartConfig(config) ? config : undefined;
const { data: mvOptimizationData, isLoading: isLoadingMVOptimization } =
useMVOptimizationExplanation(config, {
enabled: !!enabled,
useMVOptimizationExplanation(builderConfig, {
enabled: !!enabled && !!builderConfig,
placeholderData: undefined,
});
const { data: source, isLoading: isSourceLoading } = useSource({
id: config?.source,
id: builderConfig?.source,
});
const {
@ -445,6 +466,8 @@ export default function useOffsetPaginatedQuery(
metadata,
optimizedConfig: mvOptimizationData?.optimizedConfig,
source,
// Additional readonly protection when the user is running a raw SQL query
readonly: isRawSqlChartConfig(config),
} satisfies QueryMeta,
queryFn,
gcTime: isLive ? ms('30s') : ms('5m'), // more aggressive gc for live data, since it can end up holding lots of data

11
packages/app/src/hooks/usePatterns.tsx
View file

@ -1,14 +1,11 @@
import { useMemo } from 'react';
import stripAnsi from 'strip-ansi';
import { convertDateRangeToGranularityString } from '@hyperdx/common-utils/dist/core/utils';
import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
import { useQuery } from '@tanstack/react-query';
import { timeBucketByGranularity, toStartOfInterval } from '@/ChartUtils';
import {
selectColumnMapWithoutAdditionalKeys,
useConfigWithPrimaryAndPartitionKey,
} from '@/components/DBRowTable';
import { useConfigWithPrimaryAndPartitionKey } from '@/components/DBRowTable';
import { useQueriedChartConfig } from '@/hooks/useChartConfig';
import { getFirstTimestampValueExpression } from '@/source';
@ -130,7 +127,7 @@ function usePatterns({
statusCodeExpression,
enabled = true,
}: {
config: ChartConfigWithDateRange;
config: BuilderChartConfigWithDateRange;
samples: number;
bodyValueExpression: string;
severityTextExpression?: string;
@ -220,7 +217,7 @@ export function useGroupedPatterns({
totalCount,
enabled = true,
}: {
config: ChartConfigWithDateRange;
config: BuilderChartConfigWithDateRange;
samples: number;
bodyValueExpression: string;
severityTextExpression?: string;

4
packages/app/src/hooks/useRowWhere.tsx
View file

@ -7,12 +7,12 @@ import {
JSDataType,
} from '@hyperdx/common-utils/dist/clickhouse';
import { aliasMapToWithClauses } from '@hyperdx/common-utils/dist/core/utils';
import { ChartConfig } from '@hyperdx/common-utils/dist/types';
import { BuilderChartConfig } from '@hyperdx/common-utils/dist/types';
const MAX_STRING_LENGTH = 512;
// Type for WITH clause entries, derived from ChartConfig's with property
export type WithClause = NonNullable<ChartConfig['with']>[number];
export type WithClause = NonNullable<BuilderChartConfig['with']>[number];
// Internal row field names used by the table component for row tracking
export const INTERNAL_ROW_FIELDS = {

6
packages/app/src/types.ts
View file

@ -2,7 +2,7 @@ import { z } from 'zod';
import {
Alert,
AlertHistory,
ChartConfig,
BuilderChartConfig,
DashboardSchema,
Filter,
NumberFormat as _NumberFormat,
@ -63,8 +63,8 @@ export type SavedSearchWithEnhancedAlerts = Omit<SavedSearch, 'alerts'> & {
export type SearchConfig = {
select?: string | null;
source?: string | null;
where?: ChartConfig['where'] | null;
whereLanguage?: ChartConfig['whereLanguage'] | null;
where?: BuilderChartConfig['where'] | null;
whereLanguage?: BuilderChartConfig['whereLanguage'] | null;
filters?: Filter[] | null;
orderBy?: string | null;
};

13
packages/common-utils/src/__tests__/metadata.test.ts
View file

@ -1,7 +1,8 @@
import { ClickhouseClient } from '../clickhouse/node';
import { Metadata, MetadataCache } from '../core/metadata';
import * as renderChartConfigModule from '../core/renderChartConfig';
import { ChartConfigWithDateRange, TSource } from '../types';
import { isBuilderChartConfig } from '../guards';
import { BuilderChartConfigWithDateRange, TSource } from '../types';
// Mock ClickhouseClient
const mockClickhouseClient = {
@ -232,7 +233,7 @@ describe('Metadata', () => {
});
describe('getKeyValues', () => {
const mockChartConfig: ChartConfigWithDateRange = {
const mockChartConfig: BuilderChartConfigWithDateRange = {
from: {
databaseName: 'test_db',
tableName: 'test_table',
@ -372,7 +373,7 @@ describe('Metadata', () => {
});
describe('getValuesDistribution', () => {
const mockChartConfig: ChartConfigWithDateRange = {
const mockChartConfig: BuilderChartConfigWithDateRange = {
from: {
databaseName: 'test_db',
tableName: 'test_table',
@ -462,6 +463,8 @@ describe('Metadata', () => {
});
const actualConfig = renderChartConfigSpy.mock.calls[0][0];
if (!isBuilderChartConfig(actualConfig))
throw new Error('Expected builder config');
expect(actualConfig.with).toContainEqual({
name: 'service',
sql: {
@ -480,7 +483,7 @@ describe('Metadata', () => {
});
it('should include filters from the config in the query', async () => {
const configWithFilters: ChartConfigWithDateRange = {
const configWithFilters: BuilderChartConfigWithDateRange = {
...mockChartConfig,
filters: [
{
@ -502,6 +505,8 @@ describe('Metadata', () => {
});
const actualConfig = renderChartConfigSpy.mock.calls[0][0];
if (!isBuilderChartConfig(actualConfig))
throw new Error('Expected builder config');
expect(actualConfig.filters).toContainEqual({
type: 'sql',
condition: "ServiceName IN ('clickhouse')",

17
packages/common-utils/src/__tests__/renderChartConfig.test.ts
View file

@ -1427,4 +1427,21 @@ describe('renderChartConfig', () => {
expect(actual).toMatchSnapshot();
});
});
it('returns sqlTemplate verbatim for raw sql config', async () => {
const rawSqlConfig: ChartConfigWithOptDateRangeEx = {
configType: 'sql',
sqlTemplate: 'SELECT count() FROM logs WHERE level = {level:String}',
connection: 'conn-1',
};
const result = await renderChartConfig(
rawSqlConfig,
mockMetadata,
undefined,
);
expect(result.sql).toBe(
'SELECT count() FROM logs WHERE level = {level:String}',
);
expect(result.params).toEqual({});
});
});

45
packages/common-utils/src/__tests__/utils.test.ts
View file

@ -1,7 +1,8 @@
import { z } from 'zod';
import { isBuilderSavedChartConfig } from '@/guards';
import {
ChartConfigWithDateRange,
BuilderChartConfigWithDateRange,
DashboardSchema,
MetricsDataType,
SourceKind,
@ -272,7 +273,10 @@ describe('utils', () => {
});
it('should return the first column name for an array of objects input', () => {
const orderBy: Exclude<ChartConfigWithDateRange['orderBy'], string> = [
const orderBy: Exclude<
BuilderChartConfigWithDateRange['orderBy'],
string
> = [
{ valueExpression: 'column1', ordering: 'ASC' },
{ valueExpression: 'column2', ordering: 'ASC' },
];
@ -288,7 +292,7 @@ describe('utils', () => {
const config = {
timestampValueExpression: 'Timestamp',
orderBy: undefined,
} as ChartConfigWithDateRange;
} as BuilderChartConfigWithDateRange;
expect(isTimestampExpressionInFirstOrderBy(config)).toBe(false);
});
@ -297,7 +301,7 @@ describe('utils', () => {
const config = {
timestampValueExpression: 'Timestamp',
orderBy: '',
} as ChartConfigWithDateRange;
} as BuilderChartConfigWithDateRange;
expect(isTimestampExpressionInFirstOrderBy(config)).toBe(false);
});
@ -306,7 +310,7 @@ describe('utils', () => {
const config = {
timestampValueExpression: 'Timestamp',
orderBy: 'ServiceName',
} as ChartConfigWithDateRange;
} as BuilderChartConfigWithDateRange;
expect(isTimestampExpressionInFirstOrderBy(config)).toBe(false);
});
@ -315,7 +319,7 @@ describe('utils', () => {
const config = {
timestampValueExpression: 'Timestamp',
orderBy: 'ServiceName ASC, Timestamp',
} as ChartConfigWithDateRange;
} as BuilderChartConfigWithDateRange;
expect(isTimestampExpressionInFirstOrderBy(config)).toBe(false);
});
@ -324,7 +328,7 @@ describe('utils', () => {
const config = {
timestampValueExpression: 'Timestamp',
orderBy: 'Timestamp',
} as ChartConfigWithDateRange;
} as BuilderChartConfigWithDateRange;
expect(isTimestampExpressionInFirstOrderBy(config)).toBe(true);
});
@ -333,7 +337,7 @@ describe('utils', () => {
const config = {
timestampValueExpression: 'Timestamp',
orderBy: 'Timestamp DESC, ServiceName',
} as ChartConfigWithDateRange;
} as BuilderChartConfigWithDateRange;
expect(isTimestampExpressionInFirstOrderBy(config)).toBe(true);
});
@ -342,7 +346,7 @@ describe('utils', () => {
const config = {
timestampValueExpression: 'Timestamp',
orderBy: 'Timestamp desc, ServiceName',
} as ChartConfigWithDateRange;
} as BuilderChartConfigWithDateRange;
expect(isTimestampExpressionInFirstOrderBy(config)).toBe(true);
});
@ -354,7 +358,7 @@ describe('utils', () => {
{ valueExpression: 'Timestamp', ordering: 'ASC' },
{ valueExpression: 'ServiceName', ordering: 'ASC' },
],
} as ChartConfigWithDateRange;
} as BuilderChartConfigWithDateRange;
expect(isTimestampExpressionInFirstOrderBy(config)).toBe(true);
});
@ -363,7 +367,7 @@ describe('utils', () => {
const config = {
timestampValueExpression: 'toStartOfDay(Timestamp), Timestamp',
orderBy: '(toStartOfDay(Timestamp)) DESC, Timestamp',
} as ChartConfigWithDateRange;
} as BuilderChartConfigWithDateRange;
expect(isTimestampExpressionInFirstOrderBy(config)).toBe(true);
});
@ -372,7 +376,7 @@ describe('utils', () => {
const config = {
timestampValueExpression: 'toStartOfDay(Timestamp), Timestamp',
orderBy: '(toStartOfHour(TimestampTime), TimestampTime) DESC',
} as ChartConfigWithDateRange;
} as BuilderChartConfigWithDateRange;
expect(isTimestampExpressionInFirstOrderBy(config)).toBe(true);
});
@ -382,7 +386,7 @@ describe('utils', () => {
timestampValueExpression:
'toStartOfInterval(TimestampTime, INTERVAL 1 DAY)',
orderBy: 'toStartOfInterval(TimestampTime, INTERVAL 1 DAY) DESC',
} as ChartConfigWithDateRange;
} as BuilderChartConfigWithDateRange;
expect(isTimestampExpressionInFirstOrderBy(config)).toBe(true);
});
@ -412,7 +416,10 @@ describe('utils', () => {
});
it('should return true for ascending order in object input', () => {
const orderBy: Exclude<ChartConfigWithDateRange['orderBy'], string> = [
const orderBy: Exclude<
BuilderChartConfigWithDateRange['orderBy'],
string
> = [
{ valueExpression: 'column1', ordering: 'ASC' },
{ valueExpression: 'column2', ordering: 'DESC' },
];
@ -420,7 +427,10 @@ describe('utils', () => {
});
it('should return false for descending order in object input', () => {
const orderBy: Exclude<ChartConfigWithDateRange['orderBy'], string> = [
const orderBy: Exclude<
BuilderChartConfigWithDateRange['orderBy'],
string
> = [
{ valueExpression: 'column1', ordering: 'DESC' },
{ valueExpression: 'column2', ordering: 'ASC' },
];
@ -731,7 +741,10 @@ describe('utils', () => {
];
const template = convertToDashboardTemplate(dashboard, sources);
const selectList = template.tiles[0].config.select;
const tileConfig = template.tiles[0].config;
if (!isBuilderSavedChartConfig(tileConfig))
throw new Error('Expected builder config');
const selectList = tileConfig.select;
expect(Array.isArray(selectList)).toBe(true);
expect((selectList as any[])[0]).toMatchObject({
aggFn: 'quantile',

17
packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts
View file

@ -6,10 +6,10 @@ import {
tryOptimizeConfigWithMaterializedViewWithExplanations,
} from '@/core/materializedViews';
import { Metadata } from '@/core/metadata';
import { isBuilderChartConfig } from '@/guards';
import {
ChartConfigWithOptDateRange,
MaterializedViewConfiguration,
QuerySettings,
TSource,
} from '@/types';
@ -1713,7 +1713,10 @@ describe('materializedViews', () => {
it('should optimize a config with the MV that will scan the fewest rows, if multiple MVs could be used', async () => {
mockClickHouseClient.testChartConfigValidity.mockImplementation(
({ config }) => {
if (config.from.tableName === 'db_statement_rollup_1s') {
if (
isBuilderChartConfig(config) &&
config.from.tableName === 'db_statement_rollup_1s'
) {
return Promise.resolve({
isValid: true,
rowEstimate: 1000,
@ -1964,7 +1967,10 @@ describe('materializedViews', () => {
Promise.resolve({
isValid: true,
rowEstimate:
config.from.tableName === 'logs_rollup_1h' ? 500 : 1000,
isBuilderChartConfig(config) &&
config.from.tableName === 'logs_rollup_1h'
? 500
: 1000,
}),
);
@ -2124,7 +2130,10 @@ describe('materializedViews', () => {
Promise.resolve({
isValid: true,
rowEstimate:
config.from.tableName === 'logs_rollup_1h' ? 500 : 1000,
isBuilderChartConfig(config) &&
config.from.tableName === 'logs_rollup_1h'
? 500
: 1000,
}),
);

7
packages/common-utils/src/clickhouse/index.ts
View file

@ -23,6 +23,7 @@ import {
replaceJsonExpressions,
splitAndTrimWithBracket,
} from '@/core/utils';
import { isBuilderChartConfig } from '@/guards';
import { ChartConfigWithOptDateRange, QuerySettings } from '@/types';
// export @clickhouse/client-common types
@ -625,7 +626,9 @@ export abstract class BaseClickhouseClient {
};
querySettings: QuerySettings | undefined;
}): Promise<ResponseJSON<Record<string, string | number>>> {
config = setChartSelectsAlias(config);
config = isBuilderChartConfig(config)
? setChartSelectsAlias(config)
: config;
const queries: ChSql[] = await Promise.all(
splitChartConfigs(config).map(c =>
renderChartConfig(c, metadata, querySettings),
@ -652,7 +655,7 @@ export abstract class BaseClickhouseClient {
return resultSets[0];
}
// metrics -> join resultSets
else if (resultSets.length > 1) {
else if (isBuilderChartConfig(config) && resultSets.length > 1) {
const metaSet = new Map<string, { name: string; type: string }>();
const tsBucketMap = new Map<string, Record<string, string | number>>();
for (const resultSet of resultSets) {

6
packages/common-utils/src/core/histogram.ts
View file

@ -1,14 +1,14 @@
import { ChSql, chSql } from '@/clickhouse';
import { ChartConfig } from '@/types';
import { BuilderChartConfig } from '@/types';
type WithClauses = ChartConfig['with'];
type WithClauses = BuilderChartConfig['with'];
type TemplatedInput = ChSql | string;
export const translateHistogram = ({
select,
...rest
}: {
select: Exclude<ChartConfig['select'], string>[number];
select: Exclude<BuilderChartConfig['select'], string>[number];
timeBucketSelect: TemplatedInput;
groupBy?: TemplatedInput;
from: TemplatedInput;

20
packages/common-utils/src/core/materializedViews.ts
View file

@ -2,7 +2,7 @@ import { differenceInSeconds } from 'date-fns';
import { BaseClickhouseClient } from '@/clickhouse';
import {
ChartConfigWithOptDateRange,
BuilderChartConfigWithOptDateRange,
CteChartConfig,
InternalAggregateFunction,
InternalAggregateFunctionSchema,
@ -19,7 +19,7 @@ import {
} from './utils';
type SelectItem = Exclude<
ChartConfigWithOptDateRange['select'],
BuilderChartConfigWithOptDateRange['select'],
string
>[number];
@ -123,7 +123,7 @@ function getAggregatedColumnConfig(
**/
function mvConfigSupportsGranularity(
mvConfig: MaterializedViewConfiguration,
chartConfig: ChartConfigWithOptDateRange,
chartConfig: BuilderChartConfigWithOptDateRange,
): boolean {
if (!chartConfig.granularity && !chartConfig.dateRange) {
return true;
@ -171,7 +171,7 @@ function countIntervalsInDateRange(
function mvConfigSupportsDateRange(
mvConfig: MaterializedViewConfiguration,
chartConfig: ChartConfigWithOptDateRange,
chartConfig: BuilderChartConfigWithOptDateRange,
) {
if (mvConfig.minDate && !chartConfig.dateRange) {
return false;
@ -287,7 +287,7 @@ export type MVOptimizationExplanation = {
};
export async function tryConvertConfigToMaterializedViewSelect<
C extends ChartConfigWithOptDateRange | CteChartConfig,
C extends BuilderChartConfigWithOptDateRange | CteChartConfig,
>(
chartConfig: C,
mvConfig: MaterializedViewConfiguration,
@ -377,7 +377,7 @@ export async function tryConvertConfigToMaterializedViewSelect<
}
/** Attempts to optimize a config with a single MV Config */
async function tryOptimizeConfig<C extends ChartConfigWithOptDateRange>(
async function tryOptimizeConfig<C extends BuilderChartConfigWithOptDateRange>(
config: C,
metadata: Metadata,
clickhouseClient: BaseClickhouseClient,
@ -481,7 +481,7 @@ async function tryOptimizeConfig<C extends ChartConfigWithOptDateRange>(
/** Attempts to optimize a config with each of the provided MV Configs */
export async function tryOptimizeConfigWithMaterializedViewWithExplanations<
C extends ChartConfigWithOptDateRange,
C extends BuilderChartConfigWithOptDateRange,
>(
config: C,
metadata: Metadata,
@ -535,7 +535,7 @@ export async function tryOptimizeConfigWithMaterializedViewWithExplanations<
}
export async function tryOptimizeConfigWithMaterializedView<
C extends ChartConfigWithOptDateRange,
C extends BuilderChartConfigWithOptDateRange,
>(
config: C,
metadata: Metadata,
@ -580,13 +580,13 @@ function toMvId(
return `${mv.databaseName}.${mv.tableName}`;
}
export interface GetKeyValueCall<C extends ChartConfigWithOptDateRange> {
export interface GetKeyValueCall<C extends BuilderChartConfigWithOptDateRange> {
chartConfig: C;
keys: string[];
}
export async function optimizeGetKeyValuesCalls<
C extends ChartConfigWithOptDateRange,
C extends BuilderChartConfigWithOptDateRange,
>({
chartConfig,
keys,

18
packages/common-utils/src/core/metadata.ts
View file

@ -13,7 +13,11 @@ import {
tableExpr,
} from '@/clickhouse';
import { renderChartConfig } from '@/core/renderChartConfig';
import type { ChartConfig, ChartConfigWithDateRange, TSource } from '@/types';
import type {
BuilderChartConfig,
BuilderChartConfigWithDateRange,
TSource,
} from '@/types';
import { optimizeGetKeyValuesCalls } from './materializedViews';
import { objectHash } from './utils';
@ -923,7 +927,7 @@ export class Metadata {
limit = 100,
source,
}: {
chartConfig: ChartConfigWithDateRange;
chartConfig: BuilderChartConfigWithDateRange;
key: string;
samples?: number;
limit?: number;
@ -940,7 +944,7 @@ export class Metadata {
return this.cache.getOrFetch(
`${objectHash(cacheKeyConfig)}.${key}.valuesDistribution`,
async () => {
const config: ChartConfigWithDateRange = {
const config: BuilderChartConfigWithDateRange = {
...chartConfig,
with: [
...(chartConfig.with || []),
@ -1013,7 +1017,7 @@ export class Metadata {
signal,
source,
}: {
chartConfig: ChartConfigWithDateRange;
chartConfig: BuilderChartConfigWithDateRange;
keys: string[];
limit?: number;
disableRowLimit?: boolean;
@ -1132,7 +1136,7 @@ export class Metadata {
disableRowLimit,
signal,
}: {
chartConfig: ChartConfigWithDateRange;
chartConfig: BuilderChartConfigWithDateRange;
keys: string[];
source: TSource | undefined;
limit?: number;
@ -1210,7 +1214,9 @@ export type TableConnectionChoice =
tableConnections?: never;
};
export function tcFromChartConfig(config?: ChartConfig): TableConnection {
export function tcFromChartConfig(
config?: BuilderChartConfig,
): TableConnection {
return {
databaseName: config?.from?.databaseName ?? '',
tableName: config?.from?.tableName ?? '',

104
packages/common-utils/src/core/renderChartConfig.ts
View file

@ -16,18 +16,22 @@ import {
parseToStartOfFunction,
splitAndTrimWithBracket,
} from '@/core/utils';
import { isBuilderChartConfig, isRawSqlChartConfig } from '@/guards';
import { CustomSchemaSQLSerializerV2, SearchQueryBuilder } from '@/queryParser';
import {
AggregateFunction,
AggregateFunctionWithCombinators,
BuilderChartConfigWithDateRange,
BuilderChartConfigWithOptDateRange,
ChartConfig,
ChartConfigSchema,
ChartConfigWithDateRange,
ChartConfigWithOptDateRange,
ChSqlSchema,
CteChartConfig,
DateRange,
MetricsDataType,
QuerySettings,
RawSqlChartConfig,
SearchCondition,
SearchConditionLanguage,
SelectList,
@ -71,23 +75,23 @@ const DEFAULT_METRIC_TABLE_TIME_COLUMN = 'TimeUnix';
export const FIXED_TIME_BUCKET_EXPR_ALIAS = '__hdx_time_bucket';
export function isUsingGroupBy(
chartConfig: ChartConfigWithOptDateRange,
): chartConfig is Omit<ChartConfigWithDateRange, 'groupBy'> & {
groupBy: NonNullable<ChartConfigWithDateRange['groupBy']>;
chartConfig: BuilderChartConfigWithOptDateRange,
): chartConfig is Omit<BuilderChartConfigWithDateRange, 'groupBy'> & {
groupBy: NonNullable<BuilderChartConfigWithDateRange['groupBy']>;
} {
return chartConfig.groupBy != null && chartConfig.groupBy.length > 0;
}
export function isUsingGranularity(
chartConfig: ChartConfigWithOptDateRange,
chartConfig: BuilderChartConfigWithOptDateRange,
): chartConfig is Omit<
Omit<Omit<ChartConfigWithDateRange, 'granularity'>, 'dateRange'>,
Omit<Omit<BuilderChartConfigWithDateRange, 'granularity'>, 'dateRange'>,
'timestampValueExpression'
> & {
granularity: NonNullable<ChartConfigWithDateRange['granularity']>;
dateRange: NonNullable<ChartConfigWithDateRange['dateRange']>;
granularity: NonNullable<BuilderChartConfigWithDateRange['granularity']>;
dateRange: NonNullable<BuilderChartConfigWithDateRange['dateRange']>;
timestampValueExpression: NonNullable<
ChartConfigWithDateRange['timestampValueExpression']
BuilderChartConfigWithDateRange['timestampValueExpression']
>;
} {
return (
@ -97,13 +101,17 @@ export function isUsingGranularity(
}
export const isMetricChartConfig = (
chartConfig: ChartConfigWithOptDateRange,
) => {
chartConfig: BuilderChartConfigWithOptDateRange,
): chartConfig is BuilderChartConfigWithOptDateRange & {
metricTables: NonNullable<BuilderChartConfigWithOptDateRange['metricTables']>;
} => {
return chartConfig.metricTables != null;
};
// TODO: apply this to all chart configs
export const setChartSelectsAlias = (config: ChartConfigWithOptDateRange) => {
export const setChartSelectsAlias = (
config: BuilderChartConfigWithOptDateRange,
) => {
if (Array.isArray(config.select) && isMetricChartConfig(config)) {
return {
...config,
@ -120,10 +128,16 @@ export const setChartSelectsAlias = (config: ChartConfigWithOptDateRange) => {
return config;
};
export const splitChartConfigs = (config: ChartConfigWithOptDateRange) => {
export const splitChartConfigs = (
config: ChartConfigWithOptDateRange,
): ChartConfigWithOptDateRangeEx[] => {
// only split metric queries for now
if (isMetricChartConfig(config) && Array.isArray(config.select)) {
const _configs: ChartConfigWithOptDateRange[] = [];
if (
isBuilderChartConfig(config) &&
isMetricChartConfig(config) &&
Array.isArray(config.select)
) {
const _configs: BuilderChartConfigWithOptDateRange[] = [];
// split the query into multiple queries
for (const select of config.select) {
_configs.push({
@ -133,7 +147,12 @@ export const splitChartConfigs = (config: ChartConfigWithOptDateRange) => {
}
return _configs;
}
return [config];
if (isRawSqlChartConfig(config) || isBuilderChartConfig(config)) {
return [config]; // narrowed to BuilderChartConfig or RawSqlChartConfig, assignable to RawSqlChartConfigEx
}
throw new Error(`Unexpected chart config type: ${JSON.stringify(config)}`);
};
const INVERSE_OPERATOR_MAP = {
@ -391,7 +410,7 @@ const aggFnExpr = ({
async function renderSelectList(
selectList: SelectList,
chartConfig: ChartConfigWithOptDateRangeEx,
chartConfig: BuilderChartConfigWithOptDateRangeEx,
metadata: Metadata,
) {
if (typeof selectList === 'string') {
@ -546,7 +565,7 @@ export async function timeFilterExpr({
metadata: Metadata;
tableName: string;
timestampValueExpression: string;
with?: ChartConfigWithDateRange['with'];
with?: BuilderChartConfigWithDateRange['with'];
}) {
const startTime = dateRange[0].getTime();
const endTime = dateRange[1].getTime();
@ -634,7 +653,7 @@ export async function timeFilterExpr({
}
async function renderSelect(
chartConfig: ChartConfigWithOptDateRangeEx,
chartConfig: BuilderChartConfigWithOptDateRangeEx,
metadata: Metadata,
): Promise<ChSql> {
/**
@ -666,7 +685,7 @@ async function renderSelect(
function renderFrom({
from,
}: {
from: ChartConfigWithDateRange['from'];
from: BuilderChartConfigWithDateRange['from'];
}): ChSql {
return concatChSql(
'.',
@ -689,10 +708,10 @@ async function renderWhereExpression({
condition: SearchCondition;
language: SearchConditionLanguage;
metadata: Metadata;
from: ChartConfigWithDateRange['from'];
from: BuilderChartConfigWithDateRange['from'];
implicitColumnExpression?: string;
connectionId: string;
with?: ChartConfigWithDateRange['with'];
with?: BuilderChartConfigWithDateRange['with'];
}): Promise<ChSql> {
let _condition = condition;
if (language === 'lucene') {
@ -740,7 +759,7 @@ async function renderWhereExpression({
}
async function renderWhere(
chartConfig: ChartConfigWithOptDateRangeEx,
chartConfig: BuilderChartConfigWithOptDateRangeEx,
metadata: Metadata,
): Promise<ChSql> {
let whereSearchCondition: ChSql | [] = [];
@ -847,7 +866,7 @@ async function renderWhere(
}
async function renderGroupBy(
chartConfig: ChartConfigWithOptDateRange,
chartConfig: BuilderChartConfigWithOptDateRange,
metadata: Metadata,
): Promise<ChSql | undefined> {
return concatChSql(
@ -866,7 +885,7 @@ async function renderGroupBy(
}
async function renderHaving(
chartConfig: ChartConfigWithOptDateRangeEx,
chartConfig: BuilderChartConfigWithOptDateRangeEx,
metadata: Metadata,
): Promise<ChSql | undefined> {
if (!isNonEmptyWhereExpr(chartConfig.having)) {
@ -885,7 +904,7 @@ async function renderHaving(
}
function renderOrderBy(
chartConfig: ChartConfigWithOptDateRange,
chartConfig: BuilderChartConfigWithOptDateRange,
): ChSql | undefined {
const isIncludingTimeBucket = isUsingGranularity(chartConfig);
@ -909,7 +928,7 @@ function renderOrderBy(
}
function renderLimit(
chartConfig: ChartConfigWithOptDateRange,
chartConfig: BuilderChartConfigWithOptDateRange,
): ChSql | undefined {
if (chartConfig.limit == null || chartConfig.limit.limit == null) {
return undefined;
@ -924,7 +943,7 @@ function renderLimit(
}
function renderSettings(
chartConfig: ChartConfigWithOptDateRangeEx,
chartConfig: BuilderChartConfigWithOptDateRangeEx,
querySettings: QuerySettings | undefined,
) {
const querySettingsJoined = joinQuerySettings(querySettings);
@ -937,13 +956,24 @@ function renderSettings(
// includedDataInterval isn't exported at this time. It's only used internally
// for metric SQL generation.
export type ChartConfigWithOptDateRangeEx = ChartConfigWithOptDateRange & {
type InternalChartFields = {
includedDataInterval?: string;
settings?: ChSql;
};
type BuilderChartConfigWithOptDateRangeEx = BuilderChartConfigWithOptDateRange &
InternalChartFields;
type RawSqlChartConfigEx = RawSqlChartConfig &
Partial<DateRange> &
InternalChartFields;
export type ChartConfigWithOptDateRangeEx =
| BuilderChartConfigWithOptDateRangeEx
| RawSqlChartConfigEx;
async function renderWith(
chartConfig: ChartConfigWithOptDateRangeEx,
chartConfig: BuilderChartConfigWithOptDateRangeEx,
metadata: Metadata,
querySettings: QuerySettings | undefined,
): Promise<ChSql | undefined> {
@ -1031,7 +1061,7 @@ function intervalToSeconds(interval: SQLInterval): number {
}
function renderFill(
chartConfig: ChartConfigWithOptDateRangeEx,
chartConfig: BuilderChartConfigWithOptDateRangeEx,
): ChSql | undefined {
const { granularity, dateRange } = chartConfig;
if (dateRange && granularity && granularity !== 'auto') {
@ -1049,7 +1079,7 @@ function renderFill(
}
function renderDeltaExpression(
chartConfig: ChartConfigWithOptDateRange,
chartConfig: BuilderChartConfigWithOptDateRange,
valueExpression: string,
) {
const interval =
@ -1067,9 +1097,9 @@ function renderDeltaExpression(
}
async function translateMetricChartConfig(
chartConfig: ChartConfigWithOptDateRange,
chartConfig: BuilderChartConfigWithOptDateRangeEx,
metadata: Metadata,
): Promise<ChartConfigWithOptDateRangeEx> {
): Promise<BuilderChartConfigWithOptDateRangeEx> {
const metricTables = chartConfig.metricTables;
if (!metricTables) {
return chartConfig;
@ -1318,7 +1348,7 @@ async function translateMetricChartConfig(
Array.isArray(chartConfig.dateRange)
? convertDateRangeToGranularityString(chartConfig.dateRange)
: chartConfig.granularity,
} as ChartConfigWithOptDateRangeEx;
} as BuilderChartConfigWithOptDateRangeEx;
const timeBucketSelect = isUsingGranularity(cteChartConfig)
? timeBucketExpr({
@ -1377,6 +1407,10 @@ export async function renderChartConfig(
metadata: Metadata,
querySettings: QuerySettings | undefined,
): Promise<ChSql> {
if (isRawSqlChartConfig(rawChartConfig)) {
return chSql`${{ UNSAFE_RAW_SQL: rawChartConfig.sqlTemplate ?? '' }}`;
}
// metric types require more rewriting since we know more about the schema
// but goes through the same generation process
const chartConfig = isMetricChartConfig(rawChartConfig)

25
packages/common-utils/src/core/utils.ts
View file

@ -5,10 +5,11 @@ import { z } from 'zod';
export { default as objectHash } from 'object-hash';
import { isBuilderSavedChartConfig } from '@/guards';
import {
ChartConfig,
ChartConfigWithDateRange,
ChartConfigWithOptTimestamp,
BuilderChartConfig,
BuilderChartConfigWithDateRange,
BuilderChartConfigWithOptTimestamp,
DashboardFilter,
DashboardFilterSchema,
DashboardSchema,
@ -472,9 +473,13 @@ export function convertToDashboardTemplate(
): TileTemplate => {
const tile = TileTemplateSchema.strip().parse(structuredClone(input));
// Extract name from source or default to '' if not found
tile.config.source = (
sources.find(source => source.id === tile.config.source) ?? { name: '' }
).name;
// Raw SQL configs don't have a source field, so only update builder configs
const tileConfig = tile.config;
if (isBuilderSavedChartConfig(tileConfig)) {
tileConfig.source = (
sources.find(source => source.id === tileConfig.source) ?? { name: '' }
).name;
}
return tile;
};
@ -539,7 +544,7 @@ export function convertToDashboardDocument(
}
export const getFirstOrderingItem = (
orderBy: ChartConfigWithDateRange['orderBy'],
orderBy: BuilderChartConfigWithDateRange['orderBy'],
) => {
if (!orderBy || orderBy.length === 0) return undefined;
@ -560,7 +565,7 @@ export const removeTrailingDirection = (s: string) => {
};
export const isTimestampExpressionInFirstOrderBy = (
config: ChartConfigWithOptTimestamp,
config: BuilderChartConfigWithOptTimestamp,
) => {
const firstOrderingItem = getFirstOrderingItem(config.orderBy);
if (!firstOrderingItem || config.timestampValueExpression == null)
@ -581,7 +586,7 @@ export const isTimestampExpressionInFirstOrderBy = (
};
export const isFirstOrderByAscending = (
orderBy: ChartConfigWithDateRange['orderBy'],
orderBy: BuilderChartConfigWithDateRange['orderBy'],
): boolean => {
const primaryOrderingItem = getFirstOrderingItem(orderBy);
@ -936,7 +941,7 @@ export function parseTokenizerFromTextIndex({
*/
export function aliasMapToWithClauses(
aliasMap: Record<string, string | undefined> | undefined,
): ChartConfig['with'] {
): BuilderChartConfig['with'] {
if (!aliasMap) {
return undefined;
}

33
packages/common-utils/src/guards.ts Normal file
View file

@ -0,0 +1,33 @@
import {
BuilderChartConfig,
BuilderSavedChartConfig,
ChartConfig,
ChartConfigWithOptDateRange,
RawSqlChartConfig,
RawSqlSavedChartConfig,
SavedChartConfig,
} from './types';
export function isRawSqlChartConfig(
chartConfig: ChartConfig | ChartConfigWithOptDateRange,
): chartConfig is RawSqlChartConfig {
return 'configType' in chartConfig && chartConfig.configType === 'sql';
}
export function isBuilderChartConfig(
chartConfig: ChartConfig | ChartConfigWithOptDateRange,
): chartConfig is BuilderChartConfig {
return !isRawSqlChartConfig(chartConfig);
}
export function isRawSqlSavedChartConfig(
chartConfig: SavedChartConfig,
): chartConfig is RawSqlSavedChartConfig {
return 'configType' in chartConfig && chartConfig.configType === 'sql';
}
export function isBuilderSavedChartConfig(
chartConfig: SavedChartConfig,
): chartConfig is BuilderSavedChartConfig {
return !isRawSqlSavedChartConfig(chartConfig);
}

94
packages/common-utils/src/types.ts
View file

@ -416,29 +416,37 @@ export type NumberFormat = z.infer<typeof NumberFormatSchema>;
// When making changes here, consider if they need to be made to the external API
// schema as well (packages/api/src/utils/zod.ts).
export const _ChartConfigSchema = z.object({
/**
* Schema describing display settings which are shared between Raw SQL
* chart configs and Structured ChartBuilder chart configs
**/
const SharedChartDisplaySettingsSchema = z.object({
displayType: z.nativeEnum(DisplayType).optional(),
numberFormat: NumberFormatSchema.optional(),
granularity: z.union([SQLIntervalSchema, z.literal('auto')]).optional(),
compareToPreviousPeriod: z.boolean().optional(),
fillNulls: z.union([z.number(), z.literal(false)]).optional(),
alignDateRangeToGranularity: z.boolean().optional(),
});
export const _ChartConfigSchema = SharedChartDisplaySettingsSchema.extend({
timestampValueExpression: z.string(),
implicitColumnExpression: z.string().optional(),
granularity: z.union([SQLIntervalSchema, z.literal('auto')]).optional(),
markdown: z.string().optional(),
filtersLogicalOperator: z.enum(['AND', 'OR']).optional(),
filters: z.array(FilterSchema).optional(),
connection: z.string(),
fillNulls: z.union([z.number(), z.literal(false)]).optional(),
selectGroupBy: z.boolean().optional(),
metricTables: MetricTableSchema.optional(),
seriesReturnType: z.enum(['ratio', 'column']).optional(),
// Used to preserve original table select string when chart overrides it (e.g., histograms)
eventTableSelect: z.string().optional(),
compareToPreviousPeriod: z.boolean().optional(),
source: z.string().optional(),
alignDateRangeToGranularity: z.boolean().optional(),
});
// This is a ChartConfig type without the `with` CTE clause included.
// It needs to be a separate, named schema to avoid use ot z.lazy(...),
// It needs to be a separate, named schema to avoid use of z.lazy(...),
// use of which allows for type mistakes to make it past linting.
export const CteChartConfigSchema = z.intersection(
_ChartConfigSchema.partial({ timestampValueExpression: true }),
@ -451,7 +459,7 @@ export type CteChartConfig = z.infer<typeof CteChartConfigSchema>;
// non-recursive chart config so that it can reference a complete chart config
// schema. This structure does mean that we cannot nest `with` clauses but does
// ensure the type system can catch more issues in the build pipeline.
export const ChartConfigSchema = z.intersection(
const BuilderChartConfigSchema = z.intersection(
z.intersection(_ChartConfigSchema, SelectSQLStatementSchema),
z
.object({
@ -477,6 +485,22 @@ export const ChartConfigSchema = z.intersection(
.partial(),
);
export type BuilderChartConfig = z.infer<typeof BuilderChartConfigSchema>;
/** Schema describing Raw SQL chart configs */
const RawSqlChartConfigSchema = SharedChartDisplaySettingsSchema.extend({
configType: z.literal('sql'),
sqlTemplate: z.string(),
connection: z.string(),
});
export type RawSqlChartConfig = z.infer<typeof RawSqlChartConfigSchema>;
export const ChartConfigSchema = z.union([
BuilderChartConfigSchema,
RawSqlChartConfigSchema,
]);
export type ChartConfig = z.infer<typeof ChartConfigSchema>;
export type DateRange = {
@ -486,31 +510,38 @@ export type DateRange = {
};
export type ChartConfigWithDateRange = ChartConfig & DateRange;
export type BuilderChartConfigWithDateRange = BuilderChartConfig & DateRange;
export type RawSqlConfigWithDateRange = RawSqlChartConfig & DateRange;
export type ChartConfigWithOptTimestamp = Omit<
ChartConfigWithDateRange,
export type BuilderChartConfigWithOptTimestamp = Omit<
BuilderChartConfigWithDateRange,
'timestampValueExpression'
> & {
timestampValueExpression?: string;
};
export type ChartConfigWithOptTimestamp =
| BuilderChartConfigWithOptTimestamp
| RawSqlConfigWithDateRange;
// For non-time-based searches (ex. grab 1 row)
export type ChartConfigWithOptDateRange = Omit<
ChartConfig,
export type BuilderChartConfigWithOptDateRange = Omit<
BuilderChartConfig,
'timestampValueExpression'
> & {
timestampValueExpression?: string;
} & Partial<DateRange>;
export type ChartConfigWithOptDateRange =
| BuilderChartConfigWithOptDateRange
| (RawSqlChartConfig & Partial<DateRange>);
// When making changes here, consider if they need to be made to the external API
// schema as well (packages/api/src/utils/zod.ts).
export const SavedChartConfigSchema = z
const BuilderSavedChartConfigWithoutAlertSchema = z
.object({
name: z.string().optional(),
source: z.string(),
alert: z.union([
AlertBaseSchema.optional(),
ChartAlertBaseSchema.optional(),
]),
})
.extend(
_ChartConfigSchema.omit({
@ -525,6 +556,31 @@ export const SavedChartConfigSchema = z
}).shape,
);
const BuilderSavedChartConfigSchema =
BuilderSavedChartConfigWithoutAlertSchema.extend({
alert: z.union([
AlertBaseSchema.optional(),
ChartAlertBaseSchema.optional(),
]),
});
export type BuilderSavedChartConfig = z.infer<
typeof BuilderSavedChartConfigSchema
>;
const RawSqlSavedChartConfigSchema = RawSqlChartConfigSchema.extend({
name: z.string().optional(),
});
export const SavedChartConfigSchema = z.union([
BuilderSavedChartConfigSchema,
RawSqlSavedChartConfigSchema,
]);
export type RawSqlSavedChartConfig = z.infer<
typeof RawSqlSavedChartConfigSchema
>;
export type SavedChartConfig = z.infer<typeof SavedChartConfigSchema>;
export const TileSchema = z.object({
@ -535,8 +591,12 @@ export const TileSchema = z.object({
h: z.number(),
config: SavedChartConfigSchema,
});
export const TileTemplateSchema = TileSchema.extend({
config: TileSchema.shape.config.omit({ alert: true }),
config: z.union([
BuilderSavedChartConfigWithoutAlertSchema,
RawSqlSavedChartConfigSchema,
]),
});
export type Tile = z.infer<typeof TileSchema>;