diff --git a/.changeset/fast-swans-brake.md b/.changeset/fast-swans-brake.md new file mode 100644 index 00000000..1bab238d --- /dev/null +++ b/.changeset/fast-swans-brake.md @@ -0,0 +1,7 @@ +--- +"@hyperdx/common-utils": patch +"@hyperdx/api": patch +"@hyperdx/app": patch +--- + +feat: Add RawSqlChartConfig types for SQL-based Table diff --git a/packages/api/src/controllers/dashboard.ts b/packages/api/src/controllers/dashboard.ts index ded883fa..d56163f2 100644 --- a/packages/api/src/controllers/dashboard.ts +++ b/packages/api/src/controllers/dashboard.ts @@ -1,4 +1,6 @@ +import { isBuilderSavedChartConfig } from '@hyperdx/common-utils/dist/guards'; import { + BuilderSavedChartConfig, DashboardWithoutIdSchema, Tile, } from '@hyperdx/common-utils/dist/types'; @@ -17,7 +19,7 @@ import Dashboard from '@/models/dashboard'; function pickAlertsByTile(tiles: Tile[]) { return tiles.reduce((acc, tile) => { - if (tile.config.alert) { + if (isBuilderSavedChartConfig(tile.config) && tile.config.alert) { acc[tile.id] = tile.config.alert; } return acc; @@ -25,7 +27,9 @@ function pickAlertsByTile(tiles: Tile[]) { } type TileForAlertSync = Pick & { - config?: Pick | { alert?: IAlert | AlertDocument }; + config?: + | Pick + | { alert?: IAlert | AlertDocument }; }; function extractTileAlertData(tiles: TileForAlertSync[]): { @@ -48,8 +52,15 @@ async function syncDashboardAlerts( ): Promise { const { tileIds: oldTileIds, tileIdsWithAlerts: oldTileIdsWithAlerts } = extractTileAlertData(oldTiles); + + const newTilesForAlertSync: TileForAlertSync[] = newTiles.map(t => ({ + id: t.id, + config: isBuilderSavedChartConfig(t.config) + ? { alert: t.config.alert } + : {}, + })); const { tileIds: newTileIds, tileIdsWithAlerts: newTileIdsWithAlerts } = - extractTileAlertData(newTiles); + extractTileAlertData(newTilesForAlertSync); // 1. Create/update alerts for tiles that have alerts const alertsByTile = pickAlertsByTile(newTiles); diff --git a/packages/api/src/fixtures.ts b/packages/api/src/fixtures.ts index a3e5742a..70f15df0 100644 --- a/packages/api/src/fixtures.ts +++ b/packages/api/src/fixtures.ts @@ -1,5 +1,6 @@ import { createNativeClient } from '@hyperdx/common-utils/dist/clickhouse/node'; import { + BuilderSavedChartConfig, DisplayType, SavedChartConfig, Tile, @@ -421,7 +422,8 @@ export const randomMongoId = () => export const makeTile = (opts?: { id?: string; - alert?: SavedChartConfig['alert']; + alert?: BuilderSavedChartConfig['alert']; + sourceId?: string; }): Tile => ({ id: opts?.id ?? randomMongoId(), x: 1, @@ -433,10 +435,11 @@ export const makeTile = (opts?: { export const makeChartConfig = (opts?: { id?: string; - alert?: SavedChartConfig['alert']; + alert?: BuilderSavedChartConfig['alert']; + sourceId?: string; }): SavedChartConfig => ({ name: 'Test Chart', - source: 'test-source', + source: opts?.sourceId ?? 'test-source', displayType: DisplayType.Line, select: [ { diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index 25b2ceba..a7ac5d10 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -1,5 +1,7 @@ +import { isRawSqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards'; import { AggregateFunctionSchema, + BuilderSavedChartConfig, DisplayType, SavedChartConfig, } from '@hyperdx/common-utils/dist/types'; @@ -62,7 +64,7 @@ const DEFAULT_SELECT_ITEM: ExternalDashboardSelectItem = { }; const convertToExternalSelectItem = ( - item: Exclude, + item: Exclude, ): ExternalDashboardSelectItem => { const parsedAggFn = AggregateFunctionSchema.safeParse(item.aggFn); const aggFn = parsedAggFn.success ? parsedAggFn.data : 'none'; @@ -84,6 +86,9 @@ const convertToExternalSelectItem = ( const convertToExternalTileChartConfig = ( config: SavedChartConfig, ): ExternalDashboardTileConfig | undefined => { + // HDX-3582: Implement this for Raw SQL charts + if (isRawSqlSavedChartConfig(config)) return undefined; + const sourceId = config.source?.toString() ?? ''; const stringValueOrDefault = ( @@ -175,7 +180,10 @@ const convertToExternalTileChartConfig = ( function convertTileToExternalChart( tile: DashboardDocument['tiles'][number], -): ExternalDashboardTileWithId { +): ExternalDashboardTileWithId | undefined { + // HDX-3582: Implement this for Raw SQL charts + if (isRawSqlSavedChartConfig(tile.config)) return undefined; + // Returned in case of a failure converting the saved chart config const defaultTileConfig: ExternalDashboardTileConfig = { displayType: 'line', @@ -196,7 +204,9 @@ export function convertToExternalDashboard( return { id: dashboard._id.toString(), name: dashboard.name, - tiles: dashboard.tiles.map(convertTileToExternalChart), + tiles: dashboard.tiles + .map(convertTileToExternalChart) + .filter(t => t !== undefined), tags: dashboard.tags || [], filters: dashboard.filters?.map(translateFilterToExternalFilter) || [], savedQuery: dashboard.savedQuery ?? null, @@ -211,7 +221,7 @@ export function convertToExternalDashboard( const convertToInternalSelectItem = ( item: ExternalDashboardSelectItem, -): Exclude => { +): Exclude => { return { ...pick(item, ['alias', 'metricType', 'metricName', 'aggFn', 'level']), aggCondition: item.where, diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 86f639c0..cad316a3 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -17,7 +17,11 @@ import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartCo import { aliasMapToWithClauses } from '@hyperdx/common-utils/dist/core/utils'; import { timeBucketByGranularity } from '@hyperdx/common-utils/dist/core/utils'; import { - ChartConfigWithOptDateRange, + isBuilderSavedChartConfig, + isRawSqlSavedChartConfig, +} from '@hyperdx/common-utils/dist/guards'; +import { + BuilderChartConfigWithOptDateRange, DisplayType, } from '@hyperdx/common-utils/dist/types'; import * as fns from 'date-fns'; @@ -67,6 +71,7 @@ export const alertHasGroupBy = (details: AlertDetails): boolean => { } if ( details.taskType === AlertTaskType.TILE && + isBuilderSavedChartConfig(details.tile.config) && details.tile.config.groupBy && details.tile.config.groupBy.length > 0 ) { @@ -84,10 +89,10 @@ export async function computeAliasWithClauses( savedSearch: Pick, source: ISource, metadata: Metadata, -): Promise { +): Promise { const resolvedSelect = savedSearch.select || source.defaultTableSelectExpression || ''; - const config: ChartConfigWithOptDateRange = { + const config: BuilderChartConfigWithOptDateRange = { connection: '', displayType: DisplayType.Search, from: source.from, @@ -317,7 +322,7 @@ const getChartConfigFromAlert = ( connection: string, dateRange: [Date, Date], windowSizeInMins: number, -): ChartConfigWithOptDateRange | undefined => { +): BuilderChartConfigWithOptDateRange | undefined => { const { alert, source } = details; if (details.taskType === AlertTaskType.SAVED_SEARCH) { const savedSearch = details.savedSearch; @@ -344,6 +349,10 @@ const getChartConfigFromAlert = ( }; } else if (details.taskType === AlertTaskType.TILE) { const tile = details.tile; + + // Alerts are not supported for raw sql based charts + if (isRawSqlSavedChartConfig(tile.config)) return undefined; + // Doesn't work for metric alerts yet if ( tile.config.displayType === DisplayType.Line || diff --git a/packages/api/src/tasks/checkAlerts/providers/__tests__/default.test.ts b/packages/api/src/tasks/checkAlerts/providers/__tests__/default.test.ts index 945589e7..d2d270b2 100644 --- a/packages/api/src/tasks/checkAlerts/providers/__tests__/default.test.ts +++ b/packages/api/src/tasks/checkAlerts/providers/__tests__/default.test.ts @@ -148,8 +148,10 @@ describe('DefaultAlertProvider', () => { }); // Create tile with source - const tile = makeTile({ id: 'test-tile-123' }); - tile.config.source = source._id.toString(); + const tile = makeTile({ + id: 'test-tile-123', + sourceId: source._id.toString(), + }); // Create dashboard const dashboard = await Dashboard.create({ @@ -301,8 +303,10 @@ describe('DefaultAlertProvider', () => { ); // Create tile and alert - const tile = makeTile({ id: 'test-tile-123' }); - tile.config.source = source._id.toString(); + const tile = makeTile({ + id: 'test-tile-123', + sourceId: source._id.toString(), + }); const dashboard = await Dashboard.create({ team: team._id, @@ -503,8 +507,10 @@ describe('DefaultAlertProvider', () => { it('should skip alerts with missing source', async () => { const team = await createTeam({ name: 'Test Team' }); - const tile = makeTile({ id: 'test-tile' }); - tile.config.source = new mongoose.Types.ObjectId().toString(); // Non-existent source + const tile = makeTile({ + id: 'test-tile', + sourceId: new mongoose.Types.ObjectId().toString(), // Non-existent source + }); const dashboard = await Dashboard.create({ team: team._id, @@ -684,8 +690,10 @@ describe('DefaultAlertProvider', () => { }); // Create tile with source - const tile = makeTile({ id: 'test-tile-123' }); - tile.config.source = source._id.toString(); + const tile = makeTile({ + id: 'test-tile-123', + sourceId: source._id.toString(), + }); // Create dashboard const dashboard = await Dashboard.create({ diff --git a/packages/api/src/tasks/checkAlerts/providers/default.ts b/packages/api/src/tasks/checkAlerts/providers/default.ts index 5eb5d354..ace87f21 100644 --- a/packages/api/src/tasks/checkAlerts/providers/default.ts +++ b/packages/api/src/tasks/checkAlerts/providers/default.ts @@ -1,4 +1,5 @@ import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; +import { isRawSqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards'; import { Tile } from '@hyperdx/common-utils/dist/types'; import mongoose from 'mongoose'; import ms from 'ms'; @@ -106,6 +107,16 @@ async function getTileDetails( return []; } + if (isRawSqlSavedChartConfig(tile.config)) { + logger.warn({ + tileId, + dashboardId: dashboard._id, + alertId: alert.id, + message: 'skipping alert with raw sql chart config, not supported', + }); + return []; + } + const source = await Source.findOne({ _id: tile.config.source, team: alert.team, diff --git a/packages/api/src/utils/externalApi.ts b/packages/api/src/utils/externalApi.ts index f19ba049..38f93598 100644 --- a/packages/api/src/utils/externalApi.ts +++ b/packages/api/src/utils/externalApi.ts @@ -1,10 +1,8 @@ -import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils'; import { - AggregateFunctionSchema, + BuilderSavedChartConfig, DashboardFilter, DisplayType, SavedChartConfig, - SelectList, } from '@hyperdx/common-utils/dist/types'; import { omit } from 'lodash'; import { FlattenMaps, LeanDocument } from 'mongoose'; @@ -18,42 +16,7 @@ import { } from '@/models/alert'; import type { DashboardDocument } from '@/models/dashboard'; import { SeriesTile } from '@/routers/external-api/v2/utils/dashboards'; -import { - ChartSeries, - ExternalDashboardFilterWithId, - MarkdownChartSeries, - NumberChartSeries, - SearchChartSeries, - TableChartSeries, - TimeChartSeries, -} from '@/utils/zod'; - -import logger from './logger'; - -type NonStringSelectItem = Exclude; -type NonStringSelectWithLevel = NonStringSelectItem & { level: number }; - -function hasLevel( - series: NonStringSelectItem, -): series is NonStringSelectWithLevel { - return 'level' in series && typeof series.level === 'number'; -} - -function isSortOrderDesc(config: SavedChartConfig): boolean { - if (!config.orderBy) { - return false; - } - - if (typeof config.orderBy === 'string') { - return config.orderBy.toLowerCase().endsWith(' desc'); - } - - if (Array.isArray(config.orderBy) && config.orderBy.length === 0) { - return false; - } - - return Array.isArray(config.orderBy) && config.orderBy[0].ordering === 'DESC'; -} +import { ExternalDashboardFilterWithId } from '@/utils/zod'; /** Returns a new object containing only the truthy, requested keys from the original object */ const pickIfTruthy = (obj: T, keys: K[]): Partial => { @@ -66,143 +29,6 @@ const pickIfTruthy = (obj: T, keys: K[]): Partial => { return result; }; -const convertChartConfigToExternalChartSeries = ( - config: SavedChartConfig, -): ChartSeries[] => { - const { - displayType, - source: sourceId, - select, - groupBy, - numberFormat, - } = config; - const isSelectArray = Array.isArray(select); - const convertedGroupBy = Array.isArray(groupBy) - ? groupBy.map(g => g.valueExpression) - : splitAndTrimWithBracket(groupBy ?? ''); - - switch (displayType) { - case 'line': - case 'stacked_bar': - if (!isSelectArray) { - logger.error(`Expected array select for displayType ${displayType}`); - return []; - } - - return select.map(s => { - const aggFnSanitized = AggregateFunctionSchema.safeParse( - s.aggFn ?? 'none', - ); - return { - aggFn: aggFnSanitized.success ? aggFnSanitized.data : 'none', - alias: s.alias ?? undefined, - type: 'time', - sourceId, - displayType, - level: hasLevel(s) ? s.level : undefined, - field: s.valueExpression, - where: s.aggCondition ?? '', - whereLanguage: s.aggConditionLanguage ?? 'lucene', - groupBy: convertedGroupBy, - metricName: s.metricName ?? undefined, - metricDataType: s.metricType ?? undefined, - numberFormat: numberFormat ?? undefined, - } satisfies TimeChartSeries; - }); - - case 'table': - if (!isSelectArray) { - logger.error(`Expected array select for displayType ${displayType}`); - return []; - } - - return select.map(s => { - const aggFnSanitized = AggregateFunctionSchema.safeParse( - s.aggFn ?? 'none', - ); - return { - aggFn: aggFnSanitized.success ? aggFnSanitized.data : 'none', - alias: s.alias ?? undefined, - type: 'table', - sourceId, - level: hasLevel(s) ? s.level : undefined, - field: s.valueExpression, - where: s.aggCondition ?? '', - whereLanguage: s.aggConditionLanguage ?? 'lucene', - groupBy: convertedGroupBy, - metricName: s.metricName ?? undefined, - metricDataType: s.metricType ?? undefined, - sortOrder: isSortOrderDesc(config) ? 'desc' : 'asc', - numberFormat: numberFormat ?? undefined, - } satisfies TableChartSeries; - }); - - case 'number': { - if (!isSelectArray || select.length === 0) { - logger.error( - `Expected non-empty array select for displayType ${displayType}`, - ); - return []; - } - - const firstSelect = select[0]; - const aggFnSanitized = AggregateFunctionSchema.safeParse( - firstSelect.aggFn ?? 'none', - ); - - return [ - { - alias: firstSelect.alias ?? undefined, - aggFn: aggFnSanitized.success ? aggFnSanitized.data : 'none', - type: 'number', - sourceId, - level: hasLevel(firstSelect) ? firstSelect.level : undefined, - field: firstSelect.valueExpression, - where: firstSelect.aggCondition ?? '', - whereLanguage: firstSelect.aggConditionLanguage ?? 'lucene', - metricName: firstSelect.metricName ?? undefined, - metricDataType: firstSelect.metricType ?? undefined, - numberFormat: numberFormat ?? undefined, - }, - ] satisfies [NumberChartSeries]; - } - - case 'search': { - if (isSelectArray) { - logger.error( - `Expected non-array select for displayType ${displayType}`, - ); - return []; - } - - return [ - { - type: 'search', - sourceId, - fields: splitAndTrimWithBracket(select ?? ''), - where: config.where ?? '', - whereLanguage: config.whereLanguage ?? 'lucene', - }, - ] satisfies [SearchChartSeries]; - } - - case 'markdown': - return [ - { - type: 'markdown', - content: config.markdown || '', - }, - ] satisfies [MarkdownChartSeries]; - - case 'heatmap': // Heatmap is not supported in external API, and should not be present in dashboards - default: - logger.error( - `DisplayType ${displayType} is not supported in external API`, - ); - return []; - } -}; - export function translateExternalChartToTileConfig( chart: SeriesTile, ): DashboardDocument['tiles'][number] { @@ -218,14 +44,14 @@ export function translateExternalChartToTileConfig( // Determine the sourceId and displayType based on series type let sourceId: string = firstSeries.type === 'markdown' ? '' : firstSeries.sourceId; - let select: SavedChartConfig['select'] = ''; - let displayType: SavedChartConfig['displayType']; - let groupBy: SavedChartConfig['groupBy'] = ''; - let where: SavedChartConfig['where'] = ''; - let whereLanguage: SavedChartConfig['whereLanguage'] = 'lucene'; - let orderBy: SavedChartConfig['orderBy'] = ''; - let markdown: SavedChartConfig['markdown'] = ''; - let numberFormat: SavedChartConfig['numberFormat'] = undefined; + let select: BuilderSavedChartConfig['select'] = ''; + let displayType: BuilderSavedChartConfig['displayType']; + let groupBy: BuilderSavedChartConfig['groupBy'] = ''; + let where: BuilderSavedChartConfig['where'] = ''; + let whereLanguage: BuilderSavedChartConfig['whereLanguage'] = 'lucene'; + let orderBy: BuilderSavedChartConfig['orderBy'] = ''; + let markdown: BuilderSavedChartConfig['markdown'] = ''; + let numberFormat: BuilderSavedChartConfig['numberFormat'] = undefined; switch (firstSeries.type) { case 'time': { diff --git a/packages/app/.env.development b/packages/app/.env.development index 0c63da39..f48409e7 100644 --- a/packages/app/.env.development +++ b/packages/app/.env.development @@ -7,4 +7,5 @@ OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4318" OTEL_SERVICE_NAME="hdx-oss-dev-app" PORT=${HYPERDX_APP_PORT} NODE_OPTIONS="--max-http-header-size=131072" -NEXT_PUBLIC_HYPERDX_BASE_PATH= \ No newline at end of file +NEXT_PUBLIC_HYPERDX_BASE_PATH= +NEXT_PUBLIC_IS_SQL_CHARTS_ENABLED=true \ No newline at end of file diff --git a/packages/app/src/ChartUtils.tsx b/packages/app/src/ChartUtils.tsx index c76510cc..cfb4fd78 100644 --- a/packages/app/src/ChartUtils.tsx +++ b/packages/app/src/ChartUtils.tsx @@ -18,13 +18,13 @@ import { } from '@hyperdx/common-utils/dist/core/utils'; import { AggregateFunction as AggFnV2, - ChartConfigWithDateRange, + BuilderChartConfigWithDateRange, + BuilderChartConfigWithOptTimestamp, + BuilderSavedChartConfig, ChartConfigWithOptDateRange, - ChartConfigWithOptTimestamp, DisplayType, Filter, MetricsDataType as MetricsDataTypeV2, - SavedChartConfig, SourceKind, SQLInterval, TSource, @@ -109,7 +109,7 @@ export const getMetricAggFns = ( }; export const DEFAULT_CHART_CONFIG: Omit< - SavedChartConfig, + BuilderSavedChartConfig, 'source' | 'connection' > = { name: '', @@ -132,7 +132,9 @@ export const isGranularity = (value: string): value is Granularity => { return Object.values(Granularity).includes(value as Granularity); }; -export function convertToTimeChartConfig(config: ChartConfigWithDateRange) { +export function convertToTimeChartConfig( + config: BuilderChartConfigWithDateRange, +) { const granularity = config.granularity === 'auto' || config.granularity == null ? convertDateRangeToGranularityString(config.dateRange, 80) @@ -152,7 +154,9 @@ export function convertToTimeChartConfig(config: ChartConfigWithDateRange) { }; } -export function useTimeChartSettings(chartConfig: ChartConfigWithDateRange) { +export function useTimeChartSettings( + chartConfig: BuilderChartConfigWithDateRange, +) { return useMemo(() => { const convertedConfig = convertToTimeChartConfig(chartConfig); @@ -884,7 +888,7 @@ export const convertV1ChartConfigToV2 = ( metric?: TSource; trace?: TSource; }, -): ChartConfigWithDateRange => { +): BuilderChartConfigWithDateRange => { const { series, granularity, @@ -959,7 +963,7 @@ export function buildEventsSearchUrl({ valueRangeFilter, }: { source: TSource; - config: ChartConfigWithDateRange; + config: BuilderChartConfigWithDateRange; dateRange: [Date, Date]; groupFilters?: Array<{ column: string; value: any }>; valueRangeFilter?: { expression: string; value: number; threshold?: number }; @@ -1056,7 +1060,7 @@ export function buildEventsSearchUrl({ * Handles both string format ("col1, col2") and array format ([{ valueExpression: "col1" }, ...]) */ function extractGroupColumns( - groupBy: ChartConfigWithDateRange['groupBy'], + groupBy: BuilderChartConfigWithDateRange['groupBy'], ): string[] { if (!groupBy) return []; @@ -1081,7 +1085,7 @@ export function buildTableRowSearchUrl({ }: { row: Record; source: TSource | undefined; - config: ChartConfigWithDateRange; + config: BuilderChartConfigWithDateRange; dateRange: [Date, Date]; }): string | null { if (!source?.id) { @@ -1143,20 +1147,20 @@ export function buildTableRowSearchUrl({ } export function convertToNumberChartConfig( - config: ChartConfigWithDateRange, -): ChartConfigWithOptTimestamp { + config: BuilderChartConfigWithDateRange, +): BuilderChartConfigWithOptTimestamp { return omit(config, ['granularity', 'groupBy']); } export function convertToPieChartConfig( - config: ChartConfigWithOptTimestamp, -): ChartConfigWithOptTimestamp { + config: BuilderChartConfigWithOptTimestamp, +): BuilderChartConfigWithOptTimestamp { return omit(config, ['granularity']); } export function convertToTableChartConfig( - config: ChartConfigWithOptTimestamp, -): ChartConfigWithOptTimestamp { + config: BuilderChartConfigWithOptTimestamp, +): BuilderChartConfigWithOptTimestamp { const convertedConfig = structuredClone(omit(config, ['granularity'])); // Set a default limit if not already set diff --git a/packages/app/src/DBChartPage.tsx b/packages/app/src/DBChartPage.tsx index bf918f37..045ed411 100644 --- a/packages/app/src/DBChartPage.tsx +++ b/packages/app/src/DBChartPage.tsx @@ -226,6 +226,7 @@ function DBChartExplorerPage() { parseAsJson().withDefault({ ...DEFAULT_CHART_CONFIG, source: sources?.[0]?.id ?? '', + connection: sources?.[0]?.connection, }), ); diff --git a/packages/app/src/DBDashboardImportPage.tsx b/packages/app/src/DBDashboardImportPage.tsx index 2e46a6b7..a6512a9c 100644 --- a/packages/app/src/DBDashboardImportPage.tsx +++ b/packages/app/src/DBDashboardImportPage.tsx @@ -8,6 +8,7 @@ import { StringParam, useQueryParam } from 'use-query-params'; import { z } from 'zod'; import { zodResolver } from '@hookform/resolvers/zod'; import { convertToDashboardDocument } from '@hyperdx/common-utils/dist/core/utils'; +import { isRawSqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards'; import { DashboardTemplateSchema } from '@hyperdx/common-utils/dist/types'; import { Button, @@ -207,9 +208,13 @@ function Mapping({ input }: { input: Input }) { const sourceMappings = input.tiles.map(tile => { // find matching source name + const configSource = !isRawSqlSavedChartConfig(tile.config) + ? tile.config.source + : undefined; const match = sources.find( source => - source.name.toLowerCase() === tile.config.source.toLowerCase(), + configSource && + source.name.toLowerCase() === configSource.toLowerCase(), ); return match?.id || ''; }); @@ -230,6 +235,7 @@ function Mapping({ input }: { input: Input }) { const sourceMappings = useWatch({ control, name: 'sourceMappings' }); const prevSourceMappingsRef = useRef(sourceMappings); + // HDX-3583: Extend this to support connection matching for Raw SQL-based charts. useEffect(() => { if (isUpdatingRef.current) return; if (!sourceMappings || !input.tiles) return; @@ -245,15 +251,22 @@ function Mapping({ input }: { input: Input }) { const inputTile = input.tiles[changedIdx]; if (!inputTile) return; const sourceId = sourceMappings[changedIdx] ?? ''; + const inputTileSource = !isRawSqlSavedChartConfig(inputTile.config) + ? inputTile.config.source + : undefined; const keysForTilesWithMatchingSource = input.tiles .map((tile, index) => ({ ...tile, index })) - .filter(tile => tile.config.source === inputTile.config.source) + .filter( + tile => + !isRawSqlSavedChartConfig(tile.config) && + tile.config.source === inputTileSource, + ) .map(({ index }) => `sourceMappings.${index}` as const); const keysForFiltersWithMatchingSource = input.filters ?.map((filter, index) => ({ ...filter, index })) - .filter(f => f.source === inputTile.config.source) + .filter(f => f.source === inputTileSource) .map(({ index }) => `filterSourceMappings.${index}` as const) ?? []; isUpdatingRef.current = true; @@ -362,7 +375,11 @@ function Mapping({ input }: { input: Input }) { {input.tiles.map((tile, i) => ( {tile.config.name} - {tile.config.source} + + {!isRawSqlSavedChartConfig(tile.config) + ? tile.config.source + : ''} + (undefined); const { data: source } = useSource({ - id: chart.config.source, + id: isBuilderSavedChartConfig(chart.config) + ? chart.config.source + : undefined, }); useEffect(() => { - if (source != null) { + if (isRawSqlSavedChartConfig(chart.config)) { + setQueriedConfig({ ...chart.config, dateRange, granularity }); + return; + } + + if (source != null && isBuilderSavedChartConfig(chart.config)) { const isMetricSource = source.kind === SourceKind.Metric; // TODO: will need to update this when we allow for multiple metrics per chart @@ -223,7 +234,9 @@ const Tile = forwardRef( const [hovered, setHovered] = useState(false); - const alert = chart.config.alert; + const alert = isBuilderSavedChartConfig(chart.config) + ? chart.config.alert + : undefined; const alertIndicatorColor = useMemo(() => { if (!alert) { return 'transparent'; @@ -349,6 +362,9 @@ const Tile = forwardRef( const toolbar = hideToolbar ? [] : [hoverToolbar]; const keyPrefix = isFullscreenView ? 'fullscreen' : 'tile'; + // Markdown charts may not have queriedConfig, if config.source is not set + const effectiveMarkdownConfig = queriedConfig ?? chart.config; + return ( {(queriedConfig?.displayType === DisplayType.Line || - queriedConfig?.displayType === DisplayType.StackedBar) && ( - { - onUpdateChart?.({ - ...chart, - config: { - ...chart.config, - displayType, - }, - }); - }} - /> - )} + queriedConfig?.displayType === DisplayType.StackedBar) && + isBuilderChartConfig(queriedConfig) && + isBuilderSavedChartConfig(chart.config) && ( + { + onUpdateChart?.({ + ...chart, + config: { + ...chart.config, + displayType, + }, + }); + }} + /> + )} {queriedConfig?.displayType === DisplayType.Table && ( - buildTableRowSearchUrl({ - row, - source, - config: queriedConfig, - dateRange: dateRange, - }) + getRowSearchLink={ + isBuilderChartConfig(queriedConfig) + ? row => + buildTableRowSearchUrl({ + row, + source, + config: queriedConfig, + dateRange: dateRange, + }) + : undefined } /> )} - {queriedConfig?.displayType === DisplayType.Number && ( - - )} - {queriedConfig?.displayType === DisplayType.Pie && ( - - )} - {/* Markdown charts may not have queriedConfig, if source is not set */} - {(queriedConfig?.displayType === DisplayType.Markdown || - (!queriedConfig && - chart.config.displayType === DisplayType.Markdown)) && ( - - )} - {queriedConfig?.displayType === DisplayType.Search && ( - - - - )} + )} + {queriedConfig?.displayType === DisplayType.Pie && + isBuilderChartConfig(queriedConfig) && ( + + )} + {effectiveMarkdownConfig?.displayType === DisplayType.Markdown && + 'markdown' in effectiveMarkdownConfig && ( + + )} + {queriedConfig?.displayType === DisplayType.Search && + isBuilderChartConfig(queriedConfig) && + isBuilderSavedChartConfig(chart.config) && ( + + + + )} ); }, @@ -717,6 +740,7 @@ function DBDashboardPage({ presetConfig }: { presetConfig?: Dashboard }) { const tc: TableConnection[] = []; for (const { config } of dashboard.tiles) { + if (!isBuilderSavedChartConfig(config)) continue; const source = sources?.find(v => v.id === config.source); if (!source) continue; // TODO: will need to update this when we allow for multiple metrics per chart diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index 9bf77774..1c443f43 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -32,6 +32,7 @@ import { splitAndTrimWithBracket, } from '@hyperdx/common-utils/dist/core/utils'; import { + BuilderChartConfigWithDateRange, ChartConfigWithDateRange, DisplayType, Filter, @@ -1465,7 +1466,7 @@ function DBSearchPage() { [onTimeRangeSelect, setIsLive], ); - const filtersChartConfig = useMemo(() => { + const filtersChartConfig = useMemo(() => { const overrides = { orderBy: undefined, dateRange: searchedTimeRange, diff --git a/packages/app/src/ServicesDashboardPage.tsx b/packages/app/src/ServicesDashboardPage.tsx index b1b6d480..a971ee06 100644 --- a/packages/app/src/ServicesDashboardPage.tsx +++ b/packages/app/src/ServicesDashboardPage.tsx @@ -11,8 +11,7 @@ import { UseControllerProps, useForm, useWatch } from 'react-hook-form'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; import { convertDateRangeToGranularityString } from '@hyperdx/common-utils/dist/core/utils'; import { - ChartConfigWithDateRange, - ChartConfigWithOptDateRange, + BuilderChartConfigWithDateRange, CteChartConfig, DisplayType, Filter, @@ -370,7 +369,7 @@ function HttpTab({ }, []); const requestErrorRateConfig = - useMemo(() => { + useMemo(() => { if (!source || !expressions) return null; if (reqChartType === 'overall') { return { @@ -407,7 +406,7 @@ function HttpTab({ numberFormat: ERROR_RATE_PERCENTAGE_NUMBER_FORMAT, filters: getScopedFilters({ appliedConfig, expressions }), dateRange: searchedTimeRange, - } satisfies ChartConfigWithDateRange; + } satisfies BuilderChartConfigWithDateRange; } return { timestampValueExpression: 'series_time_bucket', @@ -465,7 +464,7 @@ function HttpTab({ dateRange: searchedTimeRange, granularity: convertDateRangeToGranularityString(searchedTimeRange), - } as ChartConfigWithOptDateRange, + } as CteChartConfig, isSubquery: true, }, // Select the top N series from the search as we don't want to crash the browser. @@ -539,7 +538,7 @@ function HttpTab({ displayType: DisplayType.Line, numberFormat: ERROR_RATE_PERCENTAGE_NUMBER_FORMAT, groupBy: 'zipped, endpoint', - } satisfies ChartConfigWithDateRange; + } satisfies BuilderChartConfigWithDateRange; }, [source, searchedTimeRange, appliedConfig, expressions, reqChartType]); return ( @@ -872,7 +871,7 @@ function DatabaseTab({ }, []); const totalTimePerQueryConfig = - useMemo(() => { + useMemo(() => { if (!source || !expressions) return null; return { @@ -991,11 +990,11 @@ function DatabaseTab({ timestampValueExpression: 'series_time_bucket', connection: source.connection, source: source.id, - } satisfies ChartConfigWithDateRange; + } satisfies BuilderChartConfigWithDateRange; }, [appliedConfig, expressions, searchedTimeRange, source]); const totalThroughputPerQueryConfig = - useMemo(() => { + useMemo(() => { if (!source || !expressions) return null; return { @@ -1112,7 +1111,7 @@ function DatabaseTab({ timestampValueExpression: 'series_time_bucket', connection: source.connection, source: source.id, - } satisfies ChartConfigWithDateRange; + } satisfies BuilderChartConfigWithDateRange; }, [appliedConfig, expressions, searchedTimeRange, source]); const displaySwitcher = ( diff --git a/packages/app/src/__tests__/ChartUtils.test.ts b/packages/app/src/__tests__/ChartUtils.test.ts index 4180c0cf..cce61988 100644 --- a/packages/app/src/__tests__/ChartUtils.test.ts +++ b/packages/app/src/__tests__/ChartUtils.test.ts @@ -1,5 +1,5 @@ import { - ChartConfigWithDateRange, + BuilderChartConfigWithDateRange, SourceKind, TSource, } from '@hyperdx/common-utils/dist/types'; @@ -640,7 +640,7 @@ describe('ChartUtils', () => { new Date('2025-11-26T00:00:00Z'), new Date('2025-11-27T00:00:00Z'), ], - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; const granularityFromFunction = convertToTimeChartConfig(config).granularity; @@ -654,7 +654,7 @@ describe('ChartUtils', () => { new Date('2025-11-26T00:00:00Z'), new Date('2025-11-27T00:00:00Z'), ], - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; const granularityFromFunction = convertToTimeChartConfig(config).granularity; @@ -669,7 +669,7 @@ describe('ChartUtils', () => { new Date('2025-11-26T00:00:00Z'), new Date('2025-11-27T00:00:00Z'), ], - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; const granularityFromFunction = convertToTimeChartConfig(config).granularity; @@ -687,7 +687,7 @@ describe('ChartUtils', () => { new Date('2025-11-26T00:00:00Z'), new Date('2025-11-27T00:00:00Z'), ], - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; const convertedConfig = convertToNumberChartConfig(config); @@ -704,7 +704,7 @@ describe('ChartUtils', () => { new Date('2025-11-26T00:00:00Z'), new Date('2025-11-27T00:00:00Z'), ], - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; const convertedConfig = convertToTableChartConfig(config); @@ -718,7 +718,7 @@ describe('ChartUtils', () => { new Date('2025-11-26T00:00:00Z'), new Date('2025-11-27T00:00:00Z'), ], - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; const convertedConfig = convertToTableChartConfig(config); @@ -732,7 +732,7 @@ describe('ChartUtils', () => { new Date('2025-11-26T00:00:00Z'), new Date('2025-11-27T00:00:00Z'), ], - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; const convertedConfig = convertToTableChartConfig(config); diff --git a/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx b/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx new file mode 100644 index 00000000..57121dd6 --- /dev/null +++ b/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx @@ -0,0 +1,54 @@ +import { Control } from 'react-hook-form'; +import { Box, Button, Group, Stack, Text } from '@mantine/core'; + +import useResizable from '@/hooks/useResizable'; + +import { ConnectionSelectControlled } from '../ConnectionSelect'; +import { SQLEditorControlled } from '../SQLEditor'; + +import { ChartEditorFormState } from './types'; + +import resizeStyles from '@/../styles/ResizablePanel.module.scss'; + +export default function RawSqlChartEditor({ + control, + onOpenDisplaySettings, +}: { + control: Control; + onOpenDisplaySettings: () => void; +}) { + const { size, startResize } = useResizable(20, 'bottom'); + + return ( + + + + Connection + + + + + +
+ + + + + + ); +} diff --git a/packages/app/src/components/ChartEditor/__tests__/utils.test.ts b/packages/app/src/components/ChartEditor/__tests__/utils.test.ts new file mode 100644 index 00000000..75ee5b9b --- /dev/null +++ b/packages/app/src/components/ChartEditor/__tests__/utils.test.ts @@ -0,0 +1,405 @@ +import type { + BuilderChartConfig, + BuilderSavedChartConfig, + RawSqlSavedChartConfig, + TSource, +} from '@hyperdx/common-utils/dist/types'; +import { DisplayType, SourceKind } from '@hyperdx/common-utils/dist/types'; + +import { DEFAULT_CHART_CONFIG } from '@/ChartUtils'; + +import type { ChartEditorFormState } from '../types'; +import { + convertFormStateToChartConfig, + convertFormStateToSavedChartConfig, + convertSavedChartConfigToFormState, +} from '../utils'; + +jest.mock('../../SearchInput', () => ({ + getStoredLanguage: jest.fn().mockReturnValue('lucene'), +})); + +const dateRange: [Date, Date] = [ + new Date('2024-01-01'), + new Date('2024-01-02'), +]; + +const logSource: TSource = { + id: 'source-log', + name: 'Log Source', + kind: SourceKind.Log, + connection: 'conn-1', + from: { databaseName: 'db', tableName: 'logs' }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: 'Body, SeverityText', + implicitColumnExpression: 'Body', +}; + +const metricSource: TSource = { + id: 'source-metric', + name: 'Metric Source', + kind: SourceKind.Metric, + connection: 'conn-1', + from: { databaseName: 'db', tableName: '' }, + timestampValueExpression: 'TimeUnix', + metricTables: { gauge: 'gauge_table' } as TSource['metricTables'], + resourceAttributesExpression: 'ResourceAttributes', +}; + +const seriesItem = { + aggFn: 'count' as const, + valueExpression: '*', + aggCondition: '', + aggConditionLanguage: 'lucene' as const, +}; + +describe('convertFormStateToSavedChartConfig', () => { + it('returns undefined when no source and configType is not sql', () => { + const form: ChartEditorFormState = { + displayType: DisplayType.Line, + series: [seriesItem], + }; + expect(convertFormStateToSavedChartConfig(form, undefined)).toBeUndefined(); + }); + + it('returns RawSqlSavedChartConfig for sql+table config', () => { + const form: ChartEditorFormState = { + configType: 'sql', + displayType: DisplayType.Table, + sqlTemplate: 'SELECT 1', + connection: 'conn-1', + name: 'My Chart', + series: [], + }; + const result = convertFormStateToSavedChartConfig(form, undefined); + expect(result).toEqual({ + configType: 'sql', + displayType: DisplayType.Table, + sqlTemplate: 'SELECT 1', + connection: 'conn-1', + name: 'My Chart', + }); + }); + + it('returns undefined for sql config without Table displayType', () => { + const form: ChartEditorFormState = { + configType: 'sql', + displayType: DisplayType.Line, + sqlTemplate: 'SELECT 1', + connection: 'conn-1', + series: [], + }; + expect(convertFormStateToSavedChartConfig(form, undefined)).toBeUndefined(); + }); + + it('uses sqlTemplate empty string as default when undefined', () => { + const form: ChartEditorFormState = { + configType: 'sql', + displayType: DisplayType.Table, + series: [], + }; + const result = convertFormStateToSavedChartConfig( + form, + undefined, + ) as RawSqlSavedChartConfig; + expect(result.sqlTemplate).toBe(''); + expect(result.connection).toBe(''); + }); + + it('maps series to select for builder config', () => { + const form: ChartEditorFormState = { + displayType: DisplayType.Line, + where: 'status = 200', + series: [seriesItem], + }; + const result = convertFormStateToSavedChartConfig( + form, + logSource, + ) as BuilderSavedChartConfig; + expect(result.select).toEqual([seriesItem]); + expect('series' in result).toBe(false); + expect(result.source).toBe('source-log'); + }); + + it('uses form.select string for Search displayType', () => { + const form: ChartEditorFormState = { + displayType: DisplayType.Search, + select: 'Body, SeverityText', + series: [seriesItem], + }; + const result = convertFormStateToSavedChartConfig( + form, + logSource, + ) as BuilderSavedChartConfig; + expect(result.select).toBe('Body, SeverityText'); + expect('series' in result).toBe(false); + }); + + it('uses empty string for Search displayType when select is not a string', () => { + const form: ChartEditorFormState = { + displayType: DisplayType.Search, + select: [seriesItem], + series: [], + }; + const result = convertFormStateToSavedChartConfig( + form, + logSource, + ) as BuilderSavedChartConfig; + expect(result.select).toBe(''); + expect('series' in result).toBe(false); + }); + + it('strips metricName and metricType from select for non-metric source', () => { + const form: ChartEditorFormState = { + displayType: DisplayType.Line, + series: [ + { + ...seriesItem, + metricName: 'cpu.usage', + metricType: 'gauge' as any, + }, + ], + }; + const result = convertFormStateToSavedChartConfig( + form, + logSource, + ) as BuilderSavedChartConfig; + const select = result.select as (typeof seriesItem)[]; + expect(select[0]).not.toHaveProperty('metricName'); + expect(select[0]).not.toHaveProperty('metricType'); + }); + + it('preserves form metricTables for metric source', () => { + const formMetricTables = { + gauge: 'gauge_table', + } as BuilderSavedChartConfig['metricTables']; + const form: ChartEditorFormState = { + displayType: DisplayType.Line, + series: [seriesItem], + metricTables: formMetricTables, + }; + const result = convertFormStateToSavedChartConfig( + form, + metricSource, + ) as BuilderSavedChartConfig; + expect(result.metricTables).toEqual(formMetricTables); + }); + + it('strips metricTables for non-metric source', () => { + const form: ChartEditorFormState = { + displayType: DisplayType.Line, + series: [seriesItem], + metricTables: { gauge: 'gauge_table' } as any, + }; + const result = convertFormStateToSavedChartConfig( + form, + logSource, + ) as BuilderSavedChartConfig; + expect(result.metricTables).toBeUndefined(); + }); + + it('preserves having and orderBy only for Table displayType', () => { + const having = 'count > 5'; + const orderBy = [ + { + aggFn: 'count', + valueExpression: '*', + aggCondition: '', + ordering: 'DESC' as const, + }, + ]; + const form: ChartEditorFormState = { + displayType: DisplayType.Table, + series: [seriesItem], + having, + orderBy, + }; + const tableResult = convertFormStateToSavedChartConfig( + form, + logSource, + ) as BuilderSavedChartConfig; + expect(tableResult.having).toBe(having); + expect(tableResult.orderBy).toEqual(orderBy); + + const lineResult = convertFormStateToSavedChartConfig( + { ...form, displayType: DisplayType.Line }, + logSource, + ) as BuilderSavedChartConfig; + expect(lineResult.having).toBeUndefined(); + expect(lineResult.orderBy).toBeUndefined(); + }); + + it('defaults where to empty string when undefined', () => { + const form: ChartEditorFormState = { + displayType: DisplayType.Line, + series: [seriesItem], + }; + const result = convertFormStateToSavedChartConfig( + form, + logSource, + ) as BuilderSavedChartConfig; + expect(result.where).toBe(''); + }); +}); + +describe('convertFormStateToChartConfig', () => { + it('returns undefined when no source and configType is not sql', () => { + const form: ChartEditorFormState = { + displayType: DisplayType.Line, + series: [seriesItem], + }; + expect( + convertFormStateToChartConfig(form, dateRange, undefined), + ).toBeUndefined(); + }); + + it('returns RawSqlChartConfig with dateRange for sql config', () => { + const form: ChartEditorFormState = { + configType: 'sql', + displayType: DisplayType.Table, + sqlTemplate: 'SELECT now()', + connection: 'conn-1', + series: [], + }; + const result = convertFormStateToChartConfig(form, dateRange, undefined); + expect(result).toMatchObject({ + configType: 'sql', + sqlTemplate: 'SELECT now()', + connection: 'conn-1', + displayType: DisplayType.Table, + dateRange, + }); + }); + + it('returns builder config with source fields merged', () => { + const form: ChartEditorFormState = { + displayType: DisplayType.Line, + where: 'status = 200', + series: [seriesItem], + }; + const result = convertFormStateToChartConfig(form, dateRange, logSource); + expect(result).toMatchObject({ + from: logSource.from, + timestampValueExpression: logSource.timestampValueExpression, + connection: logSource.connection, + implicitColumnExpression: logSource.implicitColumnExpression, + dateRange, + where: 'status = 200', + }); + }); + + it('falls back to defaultTableSelectExpression when series is empty', () => { + const form: ChartEditorFormState = { + displayType: DisplayType.Line, + series: [], + }; + const result = convertFormStateToChartConfig( + form, + dateRange, + logSource, + ) as BuilderChartConfig; + expect(result?.select).toBe(logSource.defaultTableSelectExpression); + }); + + it('uses series as select for non-Search displayType', () => { + const form: ChartEditorFormState = { + displayType: DisplayType.Line, + series: [seriesItem], + }; + const result = convertFormStateToChartConfig( + form, + dateRange, + logSource, + ) as BuilderChartConfig; + expect(result?.select).toEqual([seriesItem]); + }); + + it('uses form.select for Search displayType', () => { + const form: ChartEditorFormState = { + displayType: DisplayType.Search, + select: 'Body', + series: [], + }; + const result = convertFormStateToChartConfig( + form, + dateRange, + logSource, + ) as BuilderChartConfig; + expect(result?.select).toBe('Body'); + }); +}); + +describe('convertSavedChartConfigToFormState', () => { + it('sets configType to sql for RawSqlSavedChartConfig', () => { + const config: RawSqlSavedChartConfig = { + configType: 'sql', + displayType: DisplayType.Table, + sqlTemplate: 'SELECT 1', + connection: 'conn-1', + }; + const result = convertSavedChartConfigToFormState(config); + expect(result.configType).toBe('sql'); + expect(result.series).toEqual([]); + }); + + it('sets configType to builder for BuilderSavedChartConfig', () => { + const config: BuilderSavedChartConfig = { + source: 'source-1', + displayType: DisplayType.Line, + select: [seriesItem], + where: '', + }; + const result = convertSavedChartConfigToFormState(config); + expect(result.configType).toBe('builder'); + }); + + it('maps array select to series with aggConditionLanguage defaulted', () => { + const selectItem = { + aggFn: 'count' as const, + valueExpression: '*', + aggCondition: '', + }; + const config: BuilderSavedChartConfig = { + source: 'source-1', + select: [selectItem], + where: '', + }; + const result = convertSavedChartConfigToFormState(config); + expect(result.series).toHaveLength(1); + expect(result.series[0].aggConditionLanguage).toBe('lucene'); + }); + + it('preserves existing aggConditionLanguage when already set', () => { + const config: BuilderSavedChartConfig = { + source: 'source-1', + select: [{ ...seriesItem, aggConditionLanguage: 'sql' as const }], + where: '', + }; + const result = convertSavedChartConfigToFormState(config); + expect(result.series[0].aggConditionLanguage).toBe('sql'); + }); + + it('sets series to empty array when select is a string', () => { + const config: BuilderSavedChartConfig = { + source: 'source-1', + select: 'Body, SeverityText', + where: '', + }; + const result = convertSavedChartConfigToFormState(config); + expect(result.series).toEqual([]); + }); + + it('preserves other config fields in the form state', () => { + const config: BuilderSavedChartConfig = { + source: 'source-1', + name: 'My Chart', + displayType: DisplayType.Table, + select: [seriesItem], + where: 'status = 200', + }; + const result = convertSavedChartConfigToFormState(config); + expect(result.name).toBe('My Chart'); + expect(result.displayType).toBe(DisplayType.Table); + expect(result.where).toBe('status = 200'); + }); +}); diff --git a/packages/app/src/components/ChartEditor/types.ts b/packages/app/src/components/ChartEditor/types.ts new file mode 100644 index 00000000..80312856 --- /dev/null +++ b/packages/app/src/components/ChartEditor/types.ts @@ -0,0 +1,29 @@ +import { + BuilderSavedChartConfig, + RawSqlSavedChartConfig, +} from '@hyperdx/common-utils/dist/types'; + +export type SavedChartConfigWithSelectArray = Omit< + BuilderSavedChartConfig, + 'select' +> & { + select: NonNullable>; +}; + +/** + * A type that flattens the SavedChartConfig union so that the form can include + * properties from both BuilderChartConfig and RawSqlSavedChartConfig without + * type errors. + * + * All fields are optional since the form may be in either builder or raw SQL + * mode at any given time. `configType?: 'sql'` is the discriminator. + * + * Additionally, 'series' is added as a separate field that is always an array, + * to work around the fact that useFieldArray only works with fields which are *always* + * arrays. `series` stores the array `select` data for the form. + **/ +export type ChartEditorFormState = Partial & + Partial> & { + series: SavedChartConfigWithSelectArray['select']; + configType?: 'sql' | 'builder'; + }; diff --git a/packages/app/src/components/ChartEditor/utils.ts b/packages/app/src/components/ChartEditor/utils.ts new file mode 100644 index 00000000..817c5e3f --- /dev/null +++ b/packages/app/src/components/ChartEditor/utils.ts @@ -0,0 +1,148 @@ +import { omit, pick } from 'lodash'; +import { + isBuilderSavedChartConfig, + isRawSqlSavedChartConfig, +} from '@hyperdx/common-utils/dist/guards'; +import { + BuilderSavedChartConfig, + ChartConfigWithDateRange, + DisplayType, + RawSqlChartConfig, + RawSqlSavedChartConfig, + SavedChartConfig, + SourceKind, + TSource, +} from '@hyperdx/common-utils/dist/types'; + +import { getStoredLanguage } from '../SearchInput'; + +import { ChartEditorFormState } from './types'; + +function normalizeChartConfig< + C extends Pick< + BuilderSavedChartConfig, + 'select' | 'having' | 'orderBy' | 'displayType' | 'metricTables' + >, +>(config: C, source: TSource): C { + const isMetricSource = source.kind === SourceKind.Metric; + return { + ...config, + // Strip out metric-specific fields for non-metric sources + select: + !isMetricSource && Array.isArray(config.select) + ? config.select.map(s => omit(s, ['metricName', 'metricType'])) + : config.select, + metricTables: isMetricSource ? config.metricTables : undefined, + // Order By and Having can only be set by the user for table charts + having: + config.displayType === DisplayType.Table ? config.having : undefined, + orderBy: + config.displayType === DisplayType.Table ? config.orderBy : undefined, + }; +} + +export function convertFormStateToSavedChartConfig( + form: ChartEditorFormState, + source: TSource | undefined, +): SavedChartConfig | undefined { + if (form.configType === 'sql' && form.displayType === DisplayType.Table) { + const rawSqlConfig: RawSqlSavedChartConfig = { + configType: 'sql', + ...pick(form, [ + 'name', + 'displayType', + 'numberFormat', + 'granularity', + 'compareToPreviousPeriod', + 'fillNulls', + 'alignDateRangeToGranularity', + ]), + sqlTemplate: form.sqlTemplate ?? '', + connection: form.connection ?? '', + }; + return rawSqlConfig; + } + + if (source) { + // Merge the series and select fields back together, and prevent the series field from being submitted + const config: BuilderSavedChartConfig = { + ...omit(form, ['series', 'configType', 'sqlTemplate']), + // If the chart type is search, we need to ensure the select is a string + select: + form.displayType === DisplayType.Search + ? typeof form.select === 'string' + ? form.select + : '' + : form.series, + where: form.where ?? '', + source: source.id, + }; + + return normalizeChartConfig(config, source); + } +} + +export function convertFormStateToChartConfig( + form: ChartEditorFormState, + dateRange: ChartConfigWithDateRange['dateRange'], + source: TSource | undefined, +): ChartConfigWithDateRange | undefined { + if (form.configType === 'sql' && form.displayType === DisplayType.Table) { + const rawSqlConfig: RawSqlChartConfig = { + configType: 'sql', + ...pick(form, [ + 'name', + 'displayType', + 'numberFormat', + 'granularity', + 'compareToPreviousPeriod', + 'fillNulls', + 'alignDateRangeToGranularity', + ]), + sqlTemplate: form.sqlTemplate ?? '', + connection: form.connection ?? '', + }; + + return { ...rawSqlConfig, dateRange }; + } + + if (source) { + // Merge the series and select fields back together, and prevent the series field from being submitted + const mergedSelect = + form.displayType === DisplayType.Search ? form.select : form.series; + const isSelectEmpty = !mergedSelect || mergedSelect.length === 0; + + const newConfig: ChartConfigWithDateRange = { + ...omit(form, ['series', 'configType', 'sqlTemplate']), + from: source.from, + timestampValueExpression: source.timestampValueExpression, + dateRange, + connection: source.connection, + implicitColumnExpression: source.implicitColumnExpression, + metricTables: source.metricTables, + where: form.where ?? '', + select: isSelectEmpty + ? source.defaultTableSelectExpression || '' + : mergedSelect, + }; + + return structuredClone(normalizeChartConfig(newConfig, source)); + } +} + +export function convertSavedChartConfigToFormState( + config: SavedChartConfig, +): ChartEditorFormState { + return { + ...config, + configType: isRawSqlSavedChartConfig(config) ? 'sql' : 'builder', + series: + isBuilderSavedChartConfig(config) && Array.isArray(config.select) + ? config.select.map(s => ({ + ...s, + aggConditionLanguage: + s.aggConditionLanguage ?? getStoredLanguage() ?? 'lucene', + })) + : [], + }; +} diff --git a/packages/app/src/components/ContextSidePanel.tsx b/packages/app/src/components/ContextSidePanel.tsx index efc59957..375b2043 100644 --- a/packages/app/src/components/ContextSidePanel.tsx +++ b/packages/app/src/components/ContextSidePanel.tsx @@ -5,7 +5,7 @@ import { useQueryState } from 'nuqs'; import { useForm, useWatch } from 'react-hook-form'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; import { - ChartConfigWithDateRange, + BuilderChartConfigWithDateRange, TSource, } from '@hyperdx/common-utils/dist/types'; import { Badge, Flex, Group, SegmentedControl } from '@mantine/core'; @@ -38,7 +38,7 @@ enum ContextBy { interface ContextSubpanelProps { source: TSource; - dbSqlRowTableConfig: ChartConfigWithDateRange | undefined; + dbSqlRowTableConfig: BuilderChartConfigWithDateRange | undefined; rowData: Record; rowId: string | undefined; breadcrumbPath?: BreadcrumbPath; diff --git a/packages/app/src/components/DBDeltaChart.tsx b/packages/app/src/components/DBDeltaChart.tsx index d38d44ff..9ff7f9f2 100644 --- a/packages/app/src/components/DBDeltaChart.tsx +++ b/packages/app/src/components/DBDeltaChart.tsx @@ -1,8 +1,7 @@ import { useEffect, useMemo, useState } from 'react'; import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse'; import { - ChartConfigWithDateRange, - ChartConfigWithOptDateRange, + BuilderChartConfigWithDateRange, Filter, } from '@hyperdx/common-utils/dist/types'; import { @@ -46,7 +45,7 @@ export default function DBDeltaChart({ yMax, spanIdExpression, }: { - config: ChartConfigWithDateRange; + config: BuilderChartConfigWithDateRange; valueExpr: string; xMin: number; xMax: number; @@ -102,7 +101,7 @@ export default function DBDeltaChart({ // Helper to build WITH clauses for a query (outlier or inlier) const buildWithClauses = ( isOutlier: boolean, - ): NonNullable => { + ): NonNullable => { const aggregatedTimestampsCTE = buildAggregatedTimestampsCTE(); // Build the SQL condition for filtering diff --git a/packages/app/src/components/DBEditTimeChartForm.tsx b/packages/app/src/components/DBEditTimeChartForm.tsx index ba5d9d7c..a5955a1b 100644 --- a/packages/app/src/components/DBEditTimeChartForm.tsx +++ b/packages/app/src/components/DBEditTimeChartForm.tsx @@ -1,5 +1,4 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; -import { omit } from 'lodash'; import { Control, Controller, @@ -15,6 +14,11 @@ import { NativeSelect, NumberInput } from 'react-hook-form-mantine'; import z from 'zod'; import { zodResolver } from '@hookform/resolvers/zod'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; +import { + isBuilderChartConfig, + isRawSqlChartConfig, + isRawSqlSavedChartConfig, +} from '@hyperdx/common-utils/dist/guards'; import { ChartAlertBaseSchema, ChartConfigWithDateRange, @@ -39,6 +43,7 @@ import { Group, Menu, Paper, + SegmentedControl, Stack, Switch, Tabs, @@ -83,7 +88,7 @@ import SearchWhereInput, { } from '@/components/SearchInput/SearchWhereInput'; import { SQLInlineEditorControlled } from '@/components/SearchInput/SQLInlineEditor'; import { TimePicker } from '@/components/TimePicker'; -import { IS_LOCAL_MODE } from '@/config'; +import { IS_LOCAL_MODE, IS_SQL_CHARTS_ENABLED } from '@/config'; import { GranularityPickerControlled } from '@/GranularityPicker'; import { useFetchMetricMetadata } from '@/hooks/useFetchMetricMetadata'; import { @@ -108,6 +113,16 @@ import { import HDXMarkdownChart from '../HDXMarkdownChart'; +import RawSqlChartEditor from './ChartEditor/RawSqlChartEditor'; +import { + ChartEditorFormState, + SavedChartConfigWithSelectArray, +} from './ChartEditor/types'; +import { + convertFormStateToChartConfig, + convertFormStateToSavedChartConfig, + convertSavedChartConfigToFormState, +} from './ChartEditor/utils'; import { ErrorBoundary } from './Error/ErrorBoundary'; import MVOptimizationIndicator from './MaterializedViews/MVOptimizationIndicator'; import { AggFnSelectControlled } from './AggFnSelect'; @@ -128,13 +143,20 @@ import SaveToDashboardModal from './SaveToDashboardModal'; import SourceSchemaPreview from './SourceSchemaPreview'; import { SourceSelectControlled } from './SourceSelect'; -const isQueryReady = (queriedConfig: ChartConfigWithDateRange | undefined) => - ((queriedConfig?.select?.length ?? 0) > 0 || - typeof queriedConfig?.select === 'string') && - queriedConfig?.from?.databaseName && - // tableName is empty for metric sources - (queriedConfig?.from?.tableName || queriedConfig?.metricTables) && - queriedConfig?.timestampValueExpression; +const isQueryReady = (queriedConfig: ChartConfigWithDateRange | undefined) => { + if (!queriedConfig) return false; + if (isRawSqlChartConfig(queriedConfig)) { + return !!(queriedConfig.sqlTemplate && queriedConfig.connection); + } + return ( + ((queriedConfig.select?.length ?? 0) > 0 || + typeof queriedConfig.select === 'string') && + queriedConfig.from?.databaseName && + // tableName is empty for metric sources + (queriedConfig.from?.tableName || queriedConfig.metricTables) && + queriedConfig.timestampValueExpression + ); +}; const MINIMUM_THRESHOLD_VALUE = 0.0000000001; // to make alert input > 0 @@ -142,39 +164,17 @@ const MINIMUM_THRESHOLD_VALUE = 0.0000000001; // to make alert input > 0 const getSeriesFieldPath = ( namePrefix: string, fieldName: string, -): FieldPath => { - return `${namePrefix}${fieldName}` as FieldPath; +): FieldPath => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + return `${namePrefix}${fieldName}` as FieldPath; }; -export function normalizeChartConfig< - C extends Pick< - SavedChartConfig, - 'select' | 'having' | 'orderBy' | 'displayType' | 'metricTables' - >, ->(config: C, source: TSource): C { - const isMetricSource = source.kind === SourceKind.Metric; - return { - ...config, - // Strip out metric-specific fields for non-metric sources - select: - !isMetricSource && Array.isArray(config.select) - ? config.select.map(s => omit(s, ['metricName', 'metricType'])) - : config.select, - metricTables: isMetricSource ? config.metricTables : undefined, - // Order By and Having can only be set by the user for table charts - having: - config.displayType === DisplayType.Table ? config.having : undefined, - orderBy: - config.displayType === DisplayType.Table ? config.orderBy : undefined, - }; -} - // Helper function to validate metric names for metric sources const validateMetricNames = ( tableSource: TSource | undefined, series: SavedChartConfigWithSelectArray['select'] | undefined, setError: ( - name: FieldPath, + name: FieldPath, error: { type: string; message: string }, ) => void, ): boolean => { @@ -235,7 +235,7 @@ function ChartSeriesEditorComponent({ length: number; tableSource?: TSource; errors?: FieldErrors; - clearErrors: UseFormClearErrors; + clearErrors: UseFormClearErrors; }) { const aggFn = useWatch({ control, name: `${namePrefix}aggFn` }); const aggConditionLanguage = useWatch({ @@ -537,17 +537,6 @@ const zSavedChartConfig = z }) .passthrough(); -export type SavedChartConfigWithSelectArray = Omit< - SavedChartConfig, - 'select' -> & { - select: Exclude; -}; - -type SavedChartConfigWithSeries = SavedChartConfig & { - series: SavedChartConfigWithSelectArray['select']; -}; - export default function EditTimeChartForm({ dashboardId, chartConfig, @@ -579,19 +568,8 @@ export default function EditTimeChartForm({ 'data-testid'?: string; submitRef?: React.MutableRefObject<(() => void) | undefined>; }) { - // useFieldArray only supports array type fields, and select can be either a string or array. - // To solve for this, we maintain an extra form field called 'series' which is always an array. - const configWithSeries: SavedChartConfigWithSeries = useMemo( - () => ({ - ...chartConfig, - series: Array.isArray(chartConfig.select) - ? chartConfig.select.map(s => ({ - ...s, - aggConditionLanguage: - s.aggConditionLanguage ?? getStoredLanguage() ?? 'lucene', - })) - : [], - }), + const formValue: ChartEditorFormState = useMemo( + () => convertSavedChartConfigToFormState(chartConfig), [chartConfig], ); @@ -603,9 +581,9 @@ export default function EditTimeChartForm({ setError, clearErrors, formState: { errors, isDirty }, - } = useForm({ - defaultValues: configWithSeries, - values: configWithSeries, + } = useForm({ + defaultValues: formValue, + values: formValue, resolver: zodResolver(zSavedChartConfig), }); @@ -615,7 +593,7 @@ export default function EditTimeChartForm({ remove: removeSeries, swap: swapSeries, } = useFieldArray({ - control: control as Control, + control, name: 'series', }); @@ -635,6 +613,10 @@ export default function EditTimeChartForm({ const markdown = useWatch({ control, name: 'markdown' }); const alertChannelType = useWatch({ control, name: 'alert.channel.type' }); const granularity = useWatch({ control, name: 'granularity' }); + const configType = useWatch({ control, name: 'configType' }); + + const isRawSqlInput = + configType === 'sql' && displayType === DisplayType.Table; const { data: tableSource } = useSource({ id: sourceId }); const databaseName = tableSource?.from.databaseName; @@ -668,8 +650,10 @@ export default function EditTimeChartForm({ const showGeneratedSql = ['table', 'time', 'number', 'pie'].includes( activeTab, - ); // Whether to show the generated SQL preview - const showSampleEvents = tableSource?.kind !== SourceKind.Metric; + ); + + const showSampleEvents = + tableSource?.kind !== SourceKind.Metric && !isRawSqlInput; const [ alignDateRangeToGranularity, @@ -717,7 +701,7 @@ export default function EditTimeChartForm({ ); const setQueriedConfigAndSource = useCallback( - (config: ChartConfigWithDateRange, source: TSource) => { + (config: ChartConfigWithDateRange, source: TSource | undefined) => { setQueriedConfig(config); setQueriedSource(source); }, @@ -725,7 +709,7 @@ export default function EditTimeChartForm({ ); const dbTimeChartConfig = useMemo(() => { - if (!queriedConfig) { + if (!queriedConfig || !isBuilderChartConfig(queriedConfig)) { return undefined; } @@ -745,40 +729,28 @@ export default function EditTimeChartForm({ const onSubmit = useCallback(() => { handleSubmit(form => { - // Validate metric sources have metric names selected - if (validateMetricNames(tableSource, form.series, setError)) { + const isRawSqlChart = + form.configType === 'sql' && form.displayType === DisplayType.Table; + + if ( + !isRawSqlChart && + validateMetricNames(tableSource, form.series, setError) + ) { return; } - // Merge the series and select fields back together, and prevent the series field from being submitted - const config = { - ...omit(form, ['series']), - select: - form.displayType === DisplayType.Search ? form.select : form.series, - }; + const savedConfig = convertFormStateToSavedChartConfig(form, tableSource); + const queriedConfig = convertFormStateToChartConfig( + form, + dateRange, + tableSource, + ); - setChartConfig?.(config); - if (tableSource != null) { - const isSelectEmpty = !config.select || config.select.length === 0; // select is string or array - const newConfig = { - ...config, - from: tableSource.from, - timestampValueExpression: tableSource.timestampValueExpression, - dateRange, - connection: tableSource.connection, - implicitColumnExpression: tableSource.implicitColumnExpression, - metricTables: tableSource.metricTables, - select: isSelectEmpty - ? tableSource.defaultTableSelectExpression || '' - : config.select, - }; + if (savedConfig && queriedConfig) { + setChartConfig?.(savedConfig); setQueriedConfigAndSource( - // WARNING: DON'T JUST ASSIGN OBJECTS OR DO SPREAD OPERATOR STUFF WHEN - // YOUR STATE IS AN OBJECT. YOU'RE COPYING BY REFERENCE WHICH MIGHT - // ACCIDENTALLY CAUSE A useQuery SOMEWHERE TO FIRE A REQUEST EVERY TIME - // AN INPUT CHANGES. USE structuredClone TO PERFORM A DEEP COPY INSTEAD - structuredClone(normalizeChartConfig(newConfig, tableSource)), - tableSource, + queriedConfig, + isRawSqlChart ? undefined : tableSource, ); } })(); @@ -801,7 +773,10 @@ export default function EditTimeChartForm({ const tableSortState = useMemo( () => - queriedConfig?.orderBy && typeof queriedConfig.orderBy === 'string' + queriedConfig != null && + isBuilderChartConfig(queriedConfig) && + queriedConfig.orderBy && + typeof queriedConfig.orderBy === 'string' ? orderByStringToSortingState(queriedConfig.orderBy) : undefined, [queriedConfig], @@ -814,38 +789,32 @@ export default function EditTimeChartForm({ }, [onSubmit, submitRef]); const handleSave = useCallback( - (v: SavedChartConfigWithSeries) => { - if (tableSource != null) { - // Validate metric sources have metric names selected - if (validateMetricNames(tableSource, v.series, setError)) { - return; - } + (form: ChartEditorFormState) => { + const isRawSqlChart = + form.configType === 'sql' && form.displayType === DisplayType.Table; - // If the chart type is search, we need to ensure the select is a string - if ( - displayType === DisplayType.Search && - typeof v.select !== 'string' - ) { - v.select = ''; - } else if (displayType !== DisplayType.Search) { - v.select = v.series; - } - - const normalizedChartConfig = normalizeChartConfig( - // Avoid saving the series field. Series should be persisted in the select field. - omit(v, ['series']), - tableSource, - ); - - onSave?.(normalizedChartConfig); + // Validate metric sources have metric names selected + if ( + !isRawSqlChart && + validateMetricNames(tableSource, form.series, setError) + ) { + return; } + + const savedChartConfig = convertFormStateToSavedChartConfig( + form, + tableSource, + ); + + if (savedChartConfig) onSave?.(savedChartConfig); }, - [onSave, displayType, tableSource, setError], + [onSave, tableSource, setError], ); // Track previous values for detecting changes const prevGranularityRef = useRef(granularity); const prevDisplayTypeRef = useRef(displayType); + const prevConfigTypeRef = useRef(configType); useEffect(() => { // Emulate the granularity picker auto-searching similar to dashboards @@ -856,14 +825,18 @@ export default function EditTimeChartForm({ }, [granularity, onSubmit]); useEffect(() => { - if (displayType !== prevDisplayTypeRef.current) { + if ( + displayType !== prevDisplayTypeRef.current || + configType !== prevConfigTypeRef.current + ) { prevDisplayTypeRef.current = displayType; + prevConfigTypeRef.current = configType; if (displayType === DisplayType.Search && typeof select !== 'string') { setValue('select', ''); setValue('series', []); } - if (displayType !== DisplayType.Search && typeof select === 'string') { + if (displayType !== DisplayType.Search && !Array.isArray(select)) { const defaultSeries: SavedChartConfigWithSelectArray['select'] = [ { aggFn: 'count', @@ -878,7 +851,7 @@ export default function EditTimeChartForm({ } onSubmit(); } - }, [displayType, select, setValue, onSubmit]); + }, [displayType, select, setValue, onSubmit, configType]); // Emulate the date range picker auto-searching similar to dashboards useEffect(() => { @@ -900,11 +873,17 @@ export default function EditTimeChartForm({ // and explaining whether a MV can be used. const chartConfigForExplanations: ChartConfigWithOptTimestamp | undefined = useMemo(() => { + if (queriedConfig && isRawSqlChartConfig(queriedConfig)) + return { ...queriedConfig, dateRange }; + + if (chartConfig && isRawSqlSavedChartConfig(chartConfig)) + return { ...chartConfig, dateRange }; + const userHasSubmittedQuery = !!queriedConfig; const queriedSourceMatchesSelectedSource = queriedSource?.id === tableSource?.id; const urlParamsSourceMatchesSelectedSource = - chartConfig?.source === tableSource?.id; + chartConfig.source === tableSource?.id; const effectiveQueriedConfig = activeTab === 'time' ? dbTimeChartConfig : queriedConfig; @@ -912,7 +891,7 @@ export default function EditTimeChartForm({ const config = userHasSubmittedQuery && queriedSourceMatchesSelectedSource ? effectiveQueriedConfig - : chartConfig && urlParamsSourceMatchesSelectedSource + : chartConfig && urlParamsSourceMatchesSelectedSource && tableSource ? { ...chartConfig, dateRange, @@ -922,7 +901,7 @@ export default function EditTimeChartForm({ } : undefined; - if (!config) { + if (!config || isRawSqlChartConfig(config)) { return undefined; } @@ -954,7 +933,10 @@ export default function EditTimeChartForm({ const sampleEventsConfig = useMemo( () => - tableSource != null && queriedConfig != null && queryReady + tableSource != null && + queriedConfig != null && + isBuilderChartConfig(queriedConfig) && + queryReady ? { ...queriedConfig, orderBy: [ @@ -1067,11 +1049,27 @@ export default function EditTimeChartForm({ + {IS_SQL_CHARTS_ENABLED && displayType === DisplayType.Table && ( + ( + + )} + /> + )} {activeTab === 'markdown' ? ( @@ -1095,9 +1093,14 @@ export default function EditTimeChartForm({ />
+ ) : isRawSqlInput ? ( + ) : ( <> - + Data Source @@ -1112,14 +1115,18 @@ export default function EditTimeChartForm({ } /> - {tableSource && activeTab !== 'search' && ( - - )} + + {tableSource && + activeTab !== 'search' && + chartConfigForExplanations && + isBuilderChartConfig(chartConfigForExplanations) && ( + + )} + - {displayType !== DisplayType.Search && Array.isArray(select) ? ( <> {fields.map((field, index) => ( @@ -1393,7 +1400,7 @@ export default function EditTimeChartForm({ )} - {activeTab === 'table' && ( + {activeTab === 'table' && !isRawSqlInput && (
- buildTableRowSearchUrl({ - row, - source: tableSource, - config: queriedConfig, - dateRange: queriedConfig.dateRange, - }) + getRowSearchLink={ + isBuilderChartConfig(queriedConfig) + ? row => + buildTableRowSearchUrl({ + row, + source: tableSource, + config: queriedConfig, + dateRange: queriedConfig.dateRange, + }) + : undefined } onSortingChange={onTableSortingChange} sort={tableSortState} @@ -1514,24 +1524,31 @@ export default function EditTimeChartForm({ />
)} - {queryReady && queriedConfig != null && activeTab === 'number' && ( -
- -
- )} + {queryReady && + queriedConfig != null && + isBuilderChartConfig(queriedConfig) && + activeTab === 'number' && ( +
+ +
+ )} {queryReady && tableSource && queriedConfig != null && + isBuilderChartConfig(queriedConfig) && activeTab === 'search' && (
- {sampleEventsConfig != null && ( + {sampleEventsConfig != null && tableSource && (
void; queryKeyPrefix?: string; enabled?: boolean; diff --git a/packages/app/src/components/DBListBarChart.tsx b/packages/app/src/components/DBListBarChart.tsx index 25d322f4..b33caae1 100644 --- a/packages/app/src/components/DBListBarChart.tsx +++ b/packages/app/src/components/DBListBarChart.tsx @@ -1,7 +1,7 @@ import { useMemo } from 'react'; import Link from 'next/link'; import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse'; -import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import type { FloatingPosition } from '@mantine/core'; import { Box, Code, Flex, HoverCard, Text } from '@mantine/core'; @@ -185,7 +185,7 @@ export default function DBListBarChart({ toolbarItems, showMVOptimizationIndicator = true, }: { - config: ChartConfigWithDateRange; + config: BuilderChartConfigWithDateRange; onSettled?: () => void; getRowSearchLink?: (row: any) => string; hoverCardPosition?: FloatingPosition; diff --git a/packages/app/src/components/DBNumberChart.tsx b/packages/app/src/components/DBNumberChart.tsx index 9c752b1a..b179deb7 100644 --- a/packages/app/src/components/DBNumberChart.tsx +++ b/packages/app/src/components/DBNumberChart.tsx @@ -1,6 +1,6 @@ import { useMemo } from 'react'; import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse'; -import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import { Box, Code, Flex, Text } from '@mantine/core'; import { @@ -25,7 +25,7 @@ export default function DBNumberChart({ toolbarSuffix, showMVOptimizationIndicator = true, }: { - config: ChartConfigWithDateRange; + config: BuilderChartConfigWithDateRange; queryKeyPrefix?: string; enabled?: boolean; title?: React.ReactNode; diff --git a/packages/app/src/components/DBPieChart.tsx b/packages/app/src/components/DBPieChart.tsx index ac7869a1..22832020 100644 --- a/packages/app/src/components/DBPieChart.tsx +++ b/packages/app/src/components/DBPieChart.tsx @@ -1,7 +1,7 @@ import { memo, useMemo } from 'react'; import { Cell, Pie, PieChart, ResponsiveContainer, Tooltip } from 'recharts'; import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse'; -import { ChartConfigWithOptTimestamp } from '@hyperdx/common-utils/dist/types'; +import { BuilderChartConfigWithOptTimestamp } from '@hyperdx/common-utils/dist/types'; import { Box, Code, Flex, Text } from '@mantine/core'; import { @@ -55,7 +55,7 @@ export const DBPieChart = ({ toolbarPrefix, toolbarSuffix, }: { - config: ChartConfigWithOptTimestamp; + config: BuilderChartConfigWithOptTimestamp; title?: React.ReactNode; enabled?: boolean; queryKeyPrefix?: string; diff --git a/packages/app/src/components/DBRowSidePanel.tsx b/packages/app/src/components/DBRowSidePanel.tsx index c0a31401..8de925a0 100644 --- a/packages/app/src/components/DBRowSidePanel.tsx +++ b/packages/app/src/components/DBRowSidePanel.tsx @@ -13,7 +13,7 @@ import { parseAsStringEnum, useQueryState } from 'nuqs'; import { ErrorBoundary } from 'react-error-boundary'; import { useHotkeys } from 'react-hotkeys-hook'; import { SourceKind, TSource } from '@hyperdx/common-utils/dist/types'; -import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import { Box, Drawer, Flex, Stack } from '@mantine/core'; import DBRowSidePanelHeader, { @@ -63,7 +63,7 @@ export type RowSidePanelContextProps = { displayedColumns?: string[]; toggleColumn?: (column: string) => void; shareUrl?: string; - dbSqlRowTableConfig?: ChartConfigWithDateRange; + dbSqlRowTableConfig?: BuilderChartConfigWithDateRange; isChildModalOpen?: boolean; setChildModalOpen?: (open: boolean) => void; source?: TSource; diff --git a/packages/app/src/components/DBRowTable.tsx b/packages/app/src/components/DBRowTable.tsx index 612e93e6..02d7f7b8 100644 --- a/packages/app/src/components/DBRowTable.tsx +++ b/packages/app/src/components/DBRowTable.tsx @@ -32,7 +32,7 @@ import { } from '@hyperdx/common-utils/dist/clickhouse'; import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils'; import { - ChartConfigWithDateRange, + BuilderChartConfigWithDateRange, SelectList, TSource, } from '@hyperdx/common-utils/dist/types'; @@ -271,7 +271,7 @@ const SqlModal = ({ }: { opened: boolean; onClose: () => void; - config: ChartConfigWithDateRange; + config: BuilderChartConfigWithDateRange; }) => { const { data: sql, isLoading: isLoadingSql } = useRenderedSqlChartConfig( config, @@ -368,7 +368,7 @@ export const RawLogTable = memo( error?: ClickHouseQueryError | Error; dateRange?: [Date, Date]; loadingDate?: Date; - config?: ChartConfigWithDateRange; + config?: BuilderChartConfigWithDateRange; onChildModalOpen?: (open: boolean) => void; source?: TSource; onExpandedRowsChange?: (hasExpandedRows: boolean) => void; @@ -1290,7 +1290,7 @@ function getSelectLength(select: SelectList): number { } export function useConfigWithPrimaryAndPartitionKey( - config: ChartConfigWithDateRange, + config: BuilderChartConfigWithDateRange, ) { const { data: tableMetadata } = useTableMetadata({ databaseName: config.from.databaseName, @@ -1365,7 +1365,7 @@ function DBSqlRowTableComponent({ initialSortBy, variant = 'default', }: { - config: ChartConfigWithDateRange; + config: BuilderChartConfigWithDateRange; sourceId?: string; onRowDetailsClick?: (rowWhere: RowWhereResult) => void; highlightedLineId?: string; diff --git a/packages/app/src/components/DBSearchPageFilters.tsx b/packages/app/src/components/DBSearchPageFilters.tsx index 890d3e56..dc64d103 100644 --- a/packages/app/src/components/DBSearchPageFilters.tsx +++ b/packages/app/src/components/DBSearchPageFilters.tsx @@ -5,7 +5,7 @@ import { tcFromSource, } from '@hyperdx/common-utils/dist/core/metadata'; import { - ChartConfigWithDateRange, + BuilderChartConfigWithDateRange, SourceKind, } from '@hyperdx/common-utils/dist/types'; import { @@ -346,7 +346,7 @@ export type FilterGroupProps = { hasLoadedMore: boolean; isDefaultExpanded?: boolean; 'data-testid'?: string; - chartConfig: ChartConfigWithDateRange; + chartConfig: BuilderChartConfigWithDateRange; isLive?: boolean; onRangeChange?: (range: { min: number; max: number }) => void; distributionKey?: string; // Optional key to use for distribution queries, defaults to name @@ -848,7 +848,7 @@ const DBSearchPageFiltersComponent = ({ analysisMode: 'results' | 'delta' | 'pattern'; setAnalysisMode: (mode: 'results' | 'delta' | 'pattern') => void; isLive: boolean; - chartConfig: ChartConfigWithDateRange; + chartConfig: BuilderChartConfigWithDateRange; sourceId?: string; showDelta: boolean; denoiseResults: boolean; diff --git a/packages/app/src/components/DBSqlRowTableWithSidebar.tsx b/packages/app/src/components/DBSqlRowTableWithSidebar.tsx index bd24a643..89744266 100644 --- a/packages/app/src/components/DBSqlRowTableWithSidebar.tsx +++ b/packages/app/src/components/DBSqlRowTableWithSidebar.tsx @@ -2,7 +2,7 @@ import { useCallback, useState } from 'react'; import { useQueryState } from 'nuqs'; import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse'; import { - ChartConfigWithDateRange, + BuilderChartConfigWithDateRange, TSource, } from '@hyperdx/common-utils/dist/types'; import { SortingState } from '@tanstack/react-table'; @@ -25,7 +25,7 @@ import { DBRowTableVariant, DBSqlRowTable } from './DBRowTable'; interface Props { sourceId: string; - config: ChartConfigWithDateRange; + config: BuilderChartConfigWithDateRange; onError?: (error: Error | ClickHouseQueryError) => void; onScroll?: (scrollTop: number) => void; onSidebarOpen?: (rowId: string) => void; diff --git a/packages/app/src/components/DBTableChart.tsx b/packages/app/src/components/DBTableChart.tsx index d56f540e..53d6723a 100644 --- a/packages/app/src/components/DBTableChart.tsx +++ b/packages/app/src/components/DBTableChart.tsx @@ -1,5 +1,9 @@ import { useCallback, useMemo, useState } from 'react'; import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse'; +import { + isBuilderChartConfig, + isRawSqlChartConfig, +} from '@hyperdx/common-utils/dist/guards'; import { ChartConfigWithOptTimestamp } from '@hyperdx/common-utils/dist/types'; import { Box, Code, Text } from '@mantine/core'; import { SortingState } from '@tanstack/react-table'; @@ -48,7 +52,9 @@ export default function DBTableChart({ }) { const [sort, setSort] = useState([]); - const { data: source } = useSource({ id: config.source }); + const { data: source } = useSource({ + id: isBuilderChartConfig(config) ? config.source : undefined, + }); const effectiveSort = useMemo( () => controlledSort || sort, @@ -66,6 +72,8 @@ export default function DBTableChart({ ); const queriedConfig = useMemo(() => { + if (isRawSqlChartConfig(config)) return config; + const _config = convertToTableChartConfig(config); if (effectiveSort.length) { @@ -79,8 +87,9 @@ export default function DBTableChart({ return _config; }, [config, effectiveSort]); - const { data: mvOptimizationData } = - useMVOptimizationExplanation(queriedConfig); + const { data: mvOptimizationData } = useMVOptimizationExplanation( + isBuilderChartConfig(queriedConfig) ? queriedConfig : undefined, + ); const { data, fetchNextPage, hasNextPage, isLoading, isError, error } = useOffsetPaginatedQuery(queriedConfig, { @@ -91,6 +100,10 @@ export default function DBTableChart({ // Returns an array of aliases, so we can check if something is using an alias const aliasMap = useMemo(() => { + if (isRawSqlChartConfig(config)) { + return []; + } + // If the config.select is a string, we can't infer this. // One day, we could potentially run this through chSqlToAliasMap but AST parsing // doesn't work for most DBTableChart queries. @@ -103,7 +116,8 @@ export default function DBTableChart({ } return acc; }, [] as string[]); - }, [config.select]); + }, [config]); + const columns = useMemo(() => { const rows = data?.data ?? []; if (rows.length === 0) { @@ -111,7 +125,11 @@ export default function DBTableChart({ } let groupByKeys: string[] = []; - if (queriedConfig.groupBy && typeof queriedConfig.groupBy === 'string') { + if ( + isBuilderChartConfig(queriedConfig) && + queriedConfig.groupBy && + typeof queriedConfig.groupBy === 'string' + ) { groupByKeys = queriedConfig.groupBy.split(',').map(v => v.trim()); } @@ -126,13 +144,7 @@ export default function DBTableChart({ ? undefined : config.numberFormat, })); - }, [ - config.numberFormat, - aliasMap, - queriedConfig.groupBy, - data, - hiddenColumns, - ]); + }, [config.numberFormat, aliasMap, queriedConfig, data, hiddenColumns]); const toolbarItemsMemo = useMemo(() => { const allToolbarItems = []; @@ -141,7 +153,11 @@ export default function DBTableChart({ allToolbarItems.push(...toolbarPrefix); } - if (source && showMVOptimizationIndicator) { + if ( + source && + showMVOptimizationIndicator && + isBuilderChartConfig(queriedConfig) + ) { allToolbarItems.push( ( whereLanguage, }: { dateRange: DateRange['dateRange']; - extraSelects?: ChartConfigWithDateRange['select']; + extraSelects?: BuilderChartConfigWithDateRange['select']; limit?: number; logSource: TSource; order: 'asc' | 'desc'; diff --git a/packages/app/src/components/KubernetesFilters.tsx b/packages/app/src/components/KubernetesFilters.tsx index 521d0cc3..34900108 100644 --- a/packages/app/src/components/KubernetesFilters.tsx +++ b/packages/app/src/components/KubernetesFilters.tsx @@ -2,7 +2,7 @@ import React, { useEffect, useMemo, useRef, useState } from 'react'; import { useForm, useWatch } from 'react-hook-form'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; import { - ChartConfigWithDateRange, + BuilderChartConfigWithDateRange, TSource, } from '@hyperdx/common-utils/dist/types'; import { Box, Group, Select } from '@mantine/core'; @@ -23,7 +23,7 @@ type FilterSelectProps = { fieldName: string; value: string | null; onChange: (value: string | null) => void; - chartConfig: ChartConfigWithDateRange; + chartConfig: BuilderChartConfigWithDateRange; dataTestId?: string; }; @@ -157,7 +157,7 @@ export const KubernetesFilters: React.FC = ({ }, [searchQuery, metricSource.resourceAttributesExpression]); // Create chart config for fetching key values - const chartConfig: ChartConfigWithDateRange = { + const chartConfig: BuilderChartConfigWithDateRange = { from: { databaseName: metricSource.from.databaseName, tableName: metricSource.metricTables?.gauge || '', diff --git a/packages/app/src/components/MaterializedViews/MVOptimizationIndicator.tsx b/packages/app/src/components/MaterializedViews/MVOptimizationIndicator.tsx index 022d333b..dfb3beaa 100644 --- a/packages/app/src/components/MaterializedViews/MVOptimizationIndicator.tsx +++ b/packages/app/src/components/MaterializedViews/MVOptimizationIndicator.tsx @@ -1,6 +1,6 @@ import { useState } from 'react'; import { - ChartConfigWithOptDateRange, + BuilderChartConfigWithOptDateRange, TSource, } from '@hyperdx/common-utils/dist/types'; import { ActionIcon, Badge, Tooltip } from '@mantine/core'; @@ -59,7 +59,7 @@ export default function MVOptimizationIndicator({ variant = 'badge', }: { source: TSource; - config: ChartConfigWithOptDateRange | undefined; + config: BuilderChartConfigWithOptDateRange | undefined; variant?: 'badge' | 'icon'; }) { const [modalOpen, setModalOpen] = useState(false); diff --git a/packages/app/src/components/PatternTable.tsx b/packages/app/src/components/PatternTable.tsx index 1feadf63..7f9931d6 100644 --- a/packages/app/src/components/PatternTable.tsx +++ b/packages/app/src/components/PatternTable.tsx @@ -1,6 +1,6 @@ import { useMemo, useState } from 'react'; import { - ChartConfigWithDateRange, + BuilderChartConfigWithDateRange, TSource, } from '@hyperdx/common-utils/dist/types'; @@ -19,8 +19,8 @@ export default function PatternTable({ bodyValueExpression, source, }: { - config: ChartConfigWithDateRange; - totalCountConfig: ChartConfigWithDateRange; + config: BuilderChartConfigWithDateRange; + totalCountConfig: BuilderChartConfigWithDateRange; bodyValueExpression: string; totalCountQueryKeyPrefix: string; source?: TSource; diff --git a/packages/app/src/components/SQLEditor.tsx b/packages/app/src/components/SQLEditor.tsx index 54e9a616..2c06b133 100644 --- a/packages/app/src/components/SQLEditor.tsx +++ b/packages/app/src/components/SQLEditor.tsx @@ -1,13 +1,11 @@ -import { useCallback, useEffect, useMemo, useRef } from 'react'; +import { useRef } from 'react'; import { useController, UseControllerProps } from 'react-hook-form'; -import { acceptCompletion, startCompletion } from '@codemirror/autocomplete'; -import { sql, SQLDialect } from '@codemirror/lang-sql'; -import { Flex, Group, Paper, Text, useMantineColorScheme } from '@mantine/core'; +import { startCompletion } from '@codemirror/autocomplete'; +import { sql } from '@codemirror/lang-sql'; +import { Paper, useMantineColorScheme } from '@mantine/core'; import CodeMirror, { Compartment, EditorView, - keymap, - Prec, ReactCodeMirrorRef, } from '@uiw/react-codemirror'; @@ -15,6 +13,8 @@ type SQLInlineEditorProps = { value: string; onChange: (value: string) => void; placeholder?: string; + height?: string; + enableLineWrapping?: boolean; }; const styleTheme = EditorView.baseTheme({ @@ -33,6 +33,8 @@ export default function SQLEditor({ onChange, placeholder, value, + height, + enableLineWrapping = false, }: SQLInlineEditorProps) { const { colorScheme } = useMantineColorScheme(); const ref = useRef(null); @@ -57,6 +59,7 @@ export default function SQLEditor({ value={value} onChange={onChange} theme={colorScheme === 'dark' ? 'dark' : 'light'} + height={height} minHeight={'100px'} extensions={[ styleTheme, @@ -66,6 +69,7 @@ export default function SQLEditor({ upperCaseKeywords: true, }), ), + ...(enableLineWrapping ? [EditorView.lineWrapping] : []), ]} onUpdate={update => { // Always open completion window as much as possible @@ -92,6 +96,7 @@ export default function SQLEditor({ export function SQLEditorControlled({ placeholder, + height, ...props }: Omit & UseControllerProps) { const { field } = useController(props); @@ -101,6 +106,7 @@ export function SQLEditorControlled({ onChange={field.onChange} placeholder={placeholder} value={field.value} + height={height} {...props} /> ); diff --git a/packages/app/src/components/Search/DBSearchHeatmapChart.tsx b/packages/app/src/components/Search/DBSearchHeatmapChart.tsx index fea4425c..e027199d 100644 --- a/packages/app/src/components/Search/DBSearchHeatmapChart.tsx +++ b/packages/app/src/components/Search/DBSearchHeatmapChart.tsx @@ -8,7 +8,7 @@ import { tcFromSource, } from '@hyperdx/common-utils/dist/core/metadata'; import { - ChartConfigWithDateRange, + BuilderChartConfigWithDateRange, DisplayType, TSource, } from '@hyperdx/common-utils/dist/types'; @@ -34,7 +34,7 @@ export function DBSearchHeatmapChart({ source, isReady, }: { - chartConfig: ChartConfigWithDateRange; + chartConfig: BuilderChartConfigWithDateRange; source: TSource; isReady: boolean; }) { diff --git a/packages/app/src/components/SearchTotalCountChart.tsx b/packages/app/src/components/SearchTotalCountChart.tsx index f7604b7a..ce2fde71 100644 --- a/packages/app/src/components/SearchTotalCountChart.tsx +++ b/packages/app/src/components/SearchTotalCountChart.tsx @@ -4,7 +4,7 @@ import { JSDataType, ResponseJSON, } from '@hyperdx/common-utils/dist/clickhouse'; -import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import { Text } from '@mantine/core'; import { keepPreviousData } from '@tanstack/react-query'; @@ -25,7 +25,7 @@ function inferCountColumn(meta: ResponseJSON['meta'] | undefined): string { } export function useSearchTotalCount( - config: ChartConfigWithDateRange, + config: BuilderChartConfigWithDateRange, queryKeyPrefix: string, { disableQueryChunking, @@ -88,7 +88,7 @@ export default function SearchTotalCountChart({ disableQueryChunking, enableParallelQueries, }: { - config: ChartConfigWithDateRange; + config: BuilderChartConfigWithDateRange; queryKeyPrefix: string; disableQueryChunking?: boolean; enableParallelQueries?: boolean; diff --git a/packages/app/src/config.ts b/packages/app/src/config.ts index 055897d7..191d7117 100644 --- a/packages/app/src/config.ts +++ b/packages/app/src/config.ts @@ -37,3 +37,5 @@ export const IS_K8S_DASHBOARD_ENABLED = true; export const IS_METRICS_ENABLED = true; export const IS_MTVIEWS_ENABLED = false; export const IS_SESSIONS_ENABLED = true; +export const IS_SQL_CHARTS_ENABLED = + process.env.NEXT_PUBLIC_IS_SQL_CHARTS_ENABLED === 'true'; diff --git a/packages/app/src/defaults.ts b/packages/app/src/defaults.ts index 7b202101..0b37eac3 100644 --- a/packages/app/src/defaults.ts +++ b/packages/app/src/defaults.ts @@ -1,4 +1,4 @@ -import type { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import type { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; // Limit defaults export const DEFAULT_SEARCH_ROW_LIMIT = 200; @@ -6,7 +6,7 @@ export const DEFAULT_QUERY_TIMEOUT = 60; // max_execution_time, seconds export function searchChartConfigDefaults( team: any | undefined | null, -): Partial { +): Partial { return { limit: { limit: team?.searchRowLimit ?? DEFAULT_SEARCH_ROW_LIMIT, diff --git a/packages/app/src/hdxMTViews.ts b/packages/app/src/hdxMTViews.ts index 5b3d00f4..220770a6 100644 --- a/packages/app/src/hdxMTViews.ts +++ b/packages/app/src/hdxMTViews.ts @@ -13,7 +13,7 @@ import { } from '@hyperdx/common-utils/dist/core/renderChartConfig'; import { AggregateFunction, - ChartConfigWithOptDateRange, + BuilderChartConfigWithOptDateRange, DerivedColumn, QuerySettings, SQLInterval, @@ -58,7 +58,7 @@ const getAggFn = ( const buildMTViewDataTableDDL = ( table: string, - chartConfig: ChartConfigWithOptDateRange, + chartConfig: BuilderChartConfigWithOptDateRange, ) => { if (!Array.isArray(chartConfig.select)) { throw new Error('Only array select is supported'); @@ -96,7 +96,7 @@ const buildMTViewDDL = (name: string, table: string, query: ChSql) => { }; export const buildMTViewSelectQuery = async ( - chartConfig: ChartConfigWithOptDateRange, + chartConfig: BuilderChartConfigWithOptDateRange, metadata: Metadata, querySettings: QuerySettings | undefined, customGranularity?: SQLInterval, diff --git a/packages/app/src/hooks/__tests__/useChartConfig.test.tsx b/packages/app/src/hooks/__tests__/useChartConfig.test.tsx index a7bd7732..11b07c0a 100644 --- a/packages/app/src/hooks/__tests__/useChartConfig.test.tsx +++ b/packages/app/src/hooks/__tests__/useChartConfig.test.tsx @@ -1,6 +1,7 @@ import React from 'react'; import { ResponseJSON } from '@hyperdx/common-utils/dist/clickhouse'; import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/browser'; +import { isBuilderChartConfig } from '@hyperdx/common-utils/dist/guards'; import { ChartConfigWithDateRange, ChartConfigWithOptDateRange, @@ -1398,6 +1399,9 @@ describe('useChartConfig', () => { // Verify the query used the optimized config (materialized view) const queryCall = mockClickhouseClient.queryChartConfig.mock.calls[0][0]; + if (!isBuilderChartConfig(queryCall.config)) { + throw new Error('Expected a BuilderChartConfig'); + } expect(queryCall.config.from.tableName).toBe('metrics_rollup_1h'); expect(result2.current.data?.data).toBeDefined(); diff --git a/packages/app/src/hooks/__tests__/useMetadata.test.tsx b/packages/app/src/hooks/__tests__/useMetadata.test.tsx index 5ad7e902..6a55f3cd 100644 --- a/packages/app/src/hooks/__tests__/useMetadata.test.tsx +++ b/packages/app/src/hooks/__tests__/useMetadata.test.tsx @@ -5,7 +5,7 @@ import { Metadata, MetadataCache, } from '@hyperdx/common-utils/dist/core/metadata'; -import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import { renderHook, waitFor } from '@testing-library/react'; @@ -19,8 +19,8 @@ import { // Create a mock ChartConfig based on the Zod schema const createMockChartConfig = ( - overrides: Partial = {}, -): ChartConfigWithDateRange => + overrides: Partial = {}, +): BuilderChartConfigWithDateRange => ({ timestampValueExpression: '', connection: 'foo', @@ -29,7 +29,7 @@ const createMockChartConfig = ( tableName: 'traces', }, ...overrides, - }) as ChartConfigWithDateRange; + }) as BuilderChartConfigWithDateRange; jest.mock('@/source', () => ({ useSources: jest.fn().mockReturnValue({ diff --git a/packages/app/src/hooks/__tests__/useOffsetPaginatedQuery.test.tsx b/packages/app/src/hooks/__tests__/useOffsetPaginatedQuery.test.tsx index 157b2c37..947ef8e5 100644 --- a/packages/app/src/hooks/__tests__/useOffsetPaginatedQuery.test.tsx +++ b/packages/app/src/hooks/__tests__/useOffsetPaginatedQuery.test.tsx @@ -49,6 +49,7 @@ jest.mock('@hyperdx/common-utils/dist/core/renderChartConfig', () => ({ import { getClickhouseClient } from '@hyperdx/app/src/clickhouse'; import { MVOptimizationExplanation } from '@hyperdx/common-utils/dist/core/materializedViews'; import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig'; +import { isBuilderChartConfig } from '@hyperdx/common-utils/dist/guards'; import { MVOptimizationExplanationResult, @@ -763,6 +764,57 @@ describe('useOffsetPaginatedQuery', () => { }); }); + describe('RawSqlChartConfig', () => { + it('should execute raw SQL query without time windowing and not paginate', async () => { + const rawSqlConfig = { + configType: 'sql' as const, + sqlTemplate: 'SELECT status, count() FROM logs GROUP BY status', + connection: 'conn-1', + displayType: undefined, + dateRange: [ + new Date('2024-01-01T00:00:00Z'), + new Date('2024-01-02T00:00:00Z'), + ] as [Date, Date], + }; + + mockReader.read + .mockResolvedValueOnce({ + done: false, + value: [ + { json: () => ['status', 'count()'] }, + { json: () => ['String', 'UInt64'] }, + { json: () => ['error', 42] }, + { json: () => ['info', 100] }, + ], + }) + .mockResolvedValueOnce({ done: true }); + + const { result } = renderHook( + () => useOffsetPaginatedQuery(rawSqlConfig), + { wrapper }, + ); + + await waitFor(() => expect(result.current.isLoading).toBe(false)); + + // Raw SQL config should be passed through to renderChartConfig unchanged + expect(renderChartConfig).toHaveBeenCalledTimes(1); + expect(jest.mocked(renderChartConfig).mock.calls[0][0]).toMatchObject({ + configType: 'sql', + sqlTemplate: 'SELECT status, count() FROM logs GROUP BY status', + }); + + // Should have data + expect(result.current.data?.data).toHaveLength(2); + expect(result.current.data?.data[0]).toEqual({ + status: 'error', + 'count()': 42, + }); + + // Pagination is disabled for raw SQL + expect(result.current.hasNextPage).toBe(false); + }); + }); + describe('MV Optimization Integration', () => { it('should optimize queries using MVs when possible', async () => { const config = createMockChartConfig({ @@ -826,7 +878,9 @@ describe('useOffsetPaginatedQuery', () => { jest .mocked(renderChartConfig) .mock.calls.every( - call => call[0].from.tableName === 'metrics_rollup_1m', + call => + isBuilderChartConfig(call[0]) && + call[0].from.tableName === 'metrics_rollup_1m', ), ).toBeTruthy(); @@ -919,7 +973,9 @@ describe('useOffsetPaginatedQuery', () => { jest .mocked(renderChartConfig) .mock.calls.every( - call => call[0].from.tableName === 'metrics_rollup_1m', + call => + isBuilderChartConfig(call[0]) && + call[0].from.tableName === 'metrics_rollup_1m', ), ).toBeTruthy(); diff --git a/packages/app/src/hooks/useAutoCompleteOptions.tsx b/packages/app/src/hooks/useAutoCompleteOptions.tsx index ed4f6688..2973fa9e 100644 --- a/packages/app/src/hooks/useAutoCompleteOptions.tsx +++ b/packages/app/src/hooks/useAutoCompleteOptions.tsx @@ -3,7 +3,7 @@ import { Field, TableConnection, } from '@hyperdx/common-utils/dist/core/metadata'; -import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import { deduplicate2dArray, @@ -103,21 +103,21 @@ export function useAutoCompleteOptions( ); // hooks to get key values - const chartConfigs: ChartConfigWithDateRange[] = toArray(tableConnection).map( - ({ databaseName, tableName, connectionId }) => ({ - connection: connectionId, - from: { - databaseName, - tableName, - }, - timestampValueExpression: '', - select: '', - where: '', - // TODO: Pull in date for query as arg - // just assuming 1/2 day is okay to query over right now - dateRange: [new Date(NOW - (86400 * 1000) / 2), new Date(NOW)], - }), - ); + const chartConfigs: BuilderChartConfigWithDateRange[] = toArray( + tableConnection, + ).map(({ databaseName, tableName, connectionId }) => ({ + connection: connectionId, + from: { + databaseName, + tableName, + }, + timestampValueExpression: '', + select: '', + where: '', + // TODO: Pull in date for query as arg + // just assuming 1/2 day is okay to query over right now + dateRange: [new Date(NOW - (86400 * 1000) / 2), new Date(NOW)], + })); const { data: keyVals } = useMultipleGetKeyValues({ chartConfigs, keys: searchKeys, diff --git a/packages/app/src/hooks/useChartConfig.tsx b/packages/app/src/hooks/useChartConfig.tsx index a0d9838b..5e8e9aa8 100644 --- a/packages/app/src/hooks/useChartConfig.tsx +++ b/packages/app/src/hooks/useChartConfig.tsx @@ -12,8 +12,13 @@ import { renderChartConfig, } from '@hyperdx/common-utils/dist/core/renderChartConfig'; import { convertDateRangeToGranularityString } from '@hyperdx/common-utils/dist/core/utils'; +import { + isBuilderChartConfig, + isRawSqlChartConfig, +} from '@hyperdx/common-utils/dist/guards'; import { format } from '@hyperdx/common-utils/dist/sqlFormatter'; import { + BuilderChartConfigWithOptDateRange, ChartConfigWithDateRange, ChartConfigWithOptDateRange, QuerySettings, @@ -65,6 +70,9 @@ const shouldUseChunking = ( ): config is ChartConfigWithDateRange & { granularity: string; } => { + // Avoid chunking for raw SQL charts since they can include arbitrary window functions, etc. + if (isRawSqlChartConfig(config)) return false; + // Granularity is required for chunking, otherwise we could break other group-bys. if (!isUsingGranularity(config)) return false; @@ -143,7 +151,7 @@ async function* fetchDataInChunks({ ? getGranularityAlignedTimeWindows(config) : [undefined]; - if (IS_MTVIEWS_ENABLED) { + if (IS_MTVIEWS_ENABLED && isBuilderChartConfig(config)) { const { dataTableDDL, mtViewDDL, renderMTViewConfig } = await buildMTViewSelectQuery(config, metadata, querySettings); // TODO: show the DDLs in the UI so users can run commands manually @@ -154,6 +162,11 @@ async function* fetchDataInChunks({ await renderMTViewConfig(); } + // Readonly = 2 means the query is readonly but can still specify query settings. + const clickHouseSettings = isRawSqlChartConfig(config) + ? { readonly: '2' } + : {}; + if (enableParallelQueries) { // fetch in parallel const promises = windows.map(async (w, index) => { @@ -168,6 +181,7 @@ async function* fetchDataInChunks({ metadata, opts: { abort_signal: signal, + clickhouse_settings: clickHouseSettings, }, querySettings, }), @@ -258,14 +272,15 @@ export function useQueriedChartConfig( const queryClient = useQueryClient(); const metadata = useMetadataWithSettings(); + const builderConfig = isBuilderChartConfig(config) ? config : undefined; const { data: mvOptimizationData, isLoading: isLoadingMVOptimization } = - useMVOptimizationExplanation(config, { - enabled: !!enabled, + useMVOptimizationExplanation(builderConfig, { + enabled: !!enabled && !!builderConfig, placeholderData: undefined, }); const { data: source, isLoading: isSourceLoading } = useSource({ - id: config.source, + id: builderConfig?.source, }); const query = useQuery({ @@ -351,14 +366,15 @@ export function useRenderedSqlChartConfig( const metadata = useMetadataWithSettings(); + const builderConfig = isBuilderChartConfig(config) ? config : undefined; const { data: mvOptimizationData, isLoading: isLoadingMVOptimization } = - useMVOptimizationExplanation(config, { - enabled: !!enabled, + useMVOptimizationExplanation(builderConfig, { + enabled: !!enabled && !!builderConfig, placeholderData: undefined, }); const { data: source, isLoading: isSourceLoading } = useSource({ - id: config.source, + id: builderConfig?.source, }); const query = useQuery({ @@ -383,7 +399,7 @@ export function useRenderedSqlChartConfig( } export function useAliasMapFromChartConfig( - config: ChartConfigWithOptDateRange | undefined, + config: BuilderChartConfigWithOptDateRange | undefined, options?: UseQueryOptions>, ) { // For granularity: 'auto', the bucket size depends on dateRange duration (not absolute times). diff --git a/packages/app/src/hooks/useDashboardFilterValues.tsx b/packages/app/src/hooks/useDashboardFilterValues.tsx index 3464f542..ddddcb66 100644 --- a/packages/app/src/hooks/useDashboardFilterValues.tsx +++ b/packages/app/src/hooks/useDashboardFilterValues.tsx @@ -5,7 +5,7 @@ import { optimizeGetKeyValuesCalls, } from '@hyperdx/common-utils/dist/core/materializedViews'; import { - ChartConfigWithDateRange, + BuilderChartConfigWithDateRange, DashboardFilter, } from '@hyperdx/common-utils/dist/types'; import { @@ -57,53 +57,54 @@ function useOptimizedKeyValuesCalls({ return filtersBySourceIdAndMetric; }, [filters]); - const results: UseQueryResult[]>[] = - useQueries({ - queries: Array.from(filtersBySourceIdAndMetric.entries()) - .filter(([key]) => - sources?.some(s => s.id === filterFromKey(key).sourceId), - ) - .map(([key, filters]) => { - const { sourceId, metricType } = filterFromKey(key); - const source = sources!.find(s => s.id === sourceId)!; - const keys = filters.map(f => f.expression); - const tableName = getMetricTableName(source, metricType) ?? ''; + const results: UseQueryResult< + GetKeyValueCall[] + >[] = useQueries({ + queries: Array.from(filtersBySourceIdAndMetric.entries()) + .filter(([key]) => + sources?.some(s => s.id === filterFromKey(key).sourceId), + ) + .map(([key, filters]) => { + const { sourceId, metricType } = filterFromKey(key); + const source = sources!.find(s => s.id === sourceId)!; + const keys = filters.map(f => f.expression); + const tableName = getMetricTableName(source, metricType) ?? ''; - const chartConfig: ChartConfigWithDateRange = { - ...pick(source, ['timestampValueExpression', 'connection']), - from: { - databaseName: source.from.databaseName, - tableName, - }, + const chartConfig: BuilderChartConfigWithDateRange = { + ...pick(source, ['timestampValueExpression', 'connection']), + from: { + databaseName: source.from.databaseName, + tableName, + }, + dateRange, + source: source.id, + where: '', + whereLanguage: 'sql', + select: '', + }; + + return { + queryKey: [ + 'dashboard-filters-key-value-calls', + sourceId, + metricType, dateRange, - source: source.id, - where: '', - whereLanguage: 'sql', - select: '', - }; - - return { - queryKey: [ - 'dashboard-filters-key-value-calls', - sourceId, - metricType, - dateRange, + keys, + ], + enabled: !isLoadingSources, + staleTime: 1000 * 60 * 5, // Cache every 5 min + queryFn: async ({ signal }) => + await optimizeGetKeyValuesCalls({ + chartConfig, + source, + clickhouseClient, + metadata, keys, - ], - enabled: !isLoadingSources, - staleTime: 1000 * 60 * 5, // Cache every 5 min - queryFn: async ({ signal }) => - await optimizeGetKeyValuesCalls({ - chartConfig, - source, - clickhouseClient, - metadata, - keys, - signal, - }), - }; - }), - }); + signal, + }), + }; + }), + }); return { data: results.map(r => r.data ?? []).flat(), diff --git a/packages/app/src/hooks/useExplainQuery.tsx b/packages/app/src/hooks/useExplainQuery.tsx index 81e0c181..1dfe8e3a 100644 --- a/packages/app/src/hooks/useExplainQuery.tsx +++ b/packages/app/src/hooks/useExplainQuery.tsx @@ -1,4 +1,5 @@ import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig'; +import { isBuilderChartConfig } from '@hyperdx/common-utils/dist/guards'; import { ChartConfigWithOptDateRange } from '@hyperdx/common-utils/dist/types'; import { useQuery, UseQueryOptions } from '@tanstack/react-query'; @@ -20,7 +21,7 @@ export function useExplainQuery( const metadata = useMetadataWithSettings(); const { data: source, isLoading: isSourceLoading } = useSource({ - id: config?.source, + id: isBuilderChartConfig(config) ? config.source : undefined, }); return useQuery({ diff --git a/packages/app/src/hooks/useMVOptimizationExplanation.tsx b/packages/app/src/hooks/useMVOptimizationExplanation.tsx index 311de77f..a5c2455e 100644 --- a/packages/app/src/hooks/useMVOptimizationExplanation.tsx +++ b/packages/app/src/hooks/useMVOptimizationExplanation.tsx @@ -2,7 +2,7 @@ import { MVOptimizationExplanation, tryOptimizeConfigWithMaterializedViewWithExplanations, } from '@hyperdx/common-utils/dist/core/materializedViews'; -import { ChartConfigWithOptDateRange } from '@hyperdx/common-utils/dist/types'; +import { BuilderChartConfigWithOptDateRange } from '@hyperdx/common-utils/dist/types'; import { keepPreviousData, useQuery, @@ -15,14 +15,15 @@ import { useSource } from '@/source'; import { useMetadataWithSettings } from './useMetadata'; export interface MVOptimizationExplanationResult< - C extends ChartConfigWithOptDateRange = ChartConfigWithOptDateRange, + C extends + BuilderChartConfigWithOptDateRange = BuilderChartConfigWithOptDateRange, > { optimizedConfig?: C; explanations: MVOptimizationExplanation[]; } export function useMVOptimizationExplanation< - C extends ChartConfigWithOptDateRange, + C extends BuilderChartConfigWithOptDateRange, >( config: C | undefined, options?: Partial>>, diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx index 05e253a9..11e004b6 100644 --- a/packages/app/src/hooks/useMetadata.tsx +++ b/packages/app/src/hooks/useMetadata.tsx @@ -10,10 +10,7 @@ import { TableConnection, TableMetadata, } from '@hyperdx/common-utils/dist/core/metadata'; -import { - ChartConfigWithDateRange, - TSource, -} from '@hyperdx/common-utils/dist/types'; +import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import { keepPreviousData, useQuery, @@ -204,7 +201,9 @@ export function useMultipleGetKeyValues( limit, disableRowLimit, }: { - chartConfigs: ChartConfigWithDateRange | ChartConfigWithDateRange[]; + chartConfigs: + | BuilderChartConfigWithDateRange + | BuilderChartConfigWithDateRange[]; keys: string[]; limit?: number; disableRowLimit?: boolean; @@ -261,7 +260,7 @@ export function useGetValuesDistribution( key, limit, }: { - chartConfig: ChartConfigWithDateRange; + chartConfig: BuilderChartConfigWithDateRange; key: string; limit: number; }, @@ -297,7 +296,7 @@ export function useGetKeyValues( limit, disableRowLimit, }: { - chartConfig?: ChartConfigWithDateRange; + chartConfig?: BuilderChartConfigWithDateRange; keys: string[]; limit?: number; disableRowLimit?: boolean; diff --git a/packages/app/src/hooks/useOffsetPaginatedQuery.tsx b/packages/app/src/hooks/useOffsetPaginatedQuery.tsx index 217ad509..ea042862 100644 --- a/packages/app/src/hooks/useOffsetPaginatedQuery.tsx +++ b/packages/app/src/hooks/useOffsetPaginatedQuery.tsx @@ -12,6 +12,10 @@ import { isFirstOrderByAscending, isTimestampExpressionInFirstOrderBy, } from '@hyperdx/common-utils/dist/core/utils'; +import { + isBuilderChartConfig, + isRawSqlChartConfig, +} from '@hyperdx/common-utils/dist/guards'; import { ChartConfigWithOptTimestamp, TSource, @@ -72,6 +76,7 @@ type QueryMeta = { metadata: Metadata; optimizedConfig?: ChartConfigWithOptTimestamp; source: TSource | undefined; + readonly: boolean; }; // Get time window from page param @@ -80,9 +85,10 @@ function getTimeWindowFromPageParam( pageParam: TPageParam, ): TimeWindow { const [startDate, endDate] = config.dateRange; - const windows = isFirstOrderByAscending(config.orderBy) - ? generateTimeWindowsAscending(startDate, endDate) - : generateTimeWindowsDescending(startDate, endDate); + const windows = + isBuilderChartConfig(config) && isFirstOrderByAscending(config.orderBy) + ? generateTimeWindowsAscending(startDate, endDate) + : generateTimeWindowsDescending(startDate, endDate); const window = windows[pageParam.windowIndex]; if (window == null) { throw new Error('Invalid time window for page param'); @@ -96,7 +102,8 @@ function getNextPageParam( allPages: TQueryFnData[], config: ChartConfigWithOptTimestamp, ): TPageParam | undefined { - if (lastPage == null) { + // Pagination is not supported for raw SQL tables since they may not be ordered at all. + if (lastPage == null || isRawSqlChartConfig(config)) { return undefined; } @@ -124,7 +131,8 @@ function getNextPageParam( } // If no more results in current window, move to next window (if windowing is being used) - const shouldUseWindowing = isTimestampExpressionInFirstOrderBy(config); + const shouldUseWindowing = + isBuilderChartConfig(config) && isTimestampExpressionInFirstOrderBy(config); const nextWindowIndex = currentWindow.windowIndex + 1; if (shouldUseWindowing && nextWindowIndex < windows.length) { return { @@ -146,8 +154,14 @@ const queryFn: QueryFunction = async ({ throw new Error('Query missing client meta'); } - const { queryClient, metadata, hasPreviousQueries, optimizedConfig, source } = - meta as QueryMeta; + const { + queryClient, + metadata, + hasPreviousQueries, + optimizedConfig, + source, + readonly, + } = meta as QueryMeta; // Only stream incrementally if this is a fresh query with no previous // response or if it's a paginated query @@ -162,7 +176,8 @@ const queryFn: QueryFunction = async ({ const config = optimizedConfig ?? rawConfig; // Get the time window for this page - const shouldUseWindowing = isTimestampExpressionInFirstOrderBy(config); + const shouldUseWindowing = + isBuilderChartConfig(config) && isTimestampExpressionInFirstOrderBy(config); const timeWindow = shouldUseWindowing ? getTimeWindowFromPageParam(config, pageParam) : { @@ -173,14 +188,16 @@ const queryFn: QueryFunction = async ({ }; // Create config with windowed date range - const windowedConfig = { - ...config, - dateRange: [timeWindow.startTime, timeWindow.endTime] as [Date, Date], - limit: { - limit: config.limit?.limit, - offset: pageParam.offset, - }, - }; + const windowedConfig = isBuilderChartConfig(config) + ? { + ...config, + dateRange: [timeWindow.startTime, timeWindow.endTime] as [Date, Date], + limit: { + limit: config.limit?.limit, + offset: pageParam.offset, + }, + } + : config; const query = await renderChartConfig( windowedConfig, @@ -194,6 +211,8 @@ const queryFn: QueryFunction = async ({ setTimeout(() => abortController.abort(), queryTimeout * 1000); } + // Readonly = 2 means the query is readonly but can still specify query settings. + const clickHouseSettings = readonly ? { readonly: '2' } : {}; const resultSet = await clickhouseClient.query<'JSONCompactEachRowWithNamesAndTypes'>({ query: query.sql, @@ -201,6 +220,7 @@ const queryFn: QueryFunction = async ({ format: 'JSONCompactEachRowWithNamesAndTypes', abort_signal: abortController?.signal || signal, connectionId: config.connection, + clickhouse_settings: clickHouseSettings, }); const stream = resultSet.stream(); @@ -402,14 +422,15 @@ export default function useOffsetPaginatedQuery( const hasPreviousQueries = matchedQueries.filter(([_, data]) => data != null).length > 0; + const builderConfig = isBuilderChartConfig(config) ? config : undefined; const { data: mvOptimizationData, isLoading: isLoadingMVOptimization } = - useMVOptimizationExplanation(config, { - enabled: !!enabled, + useMVOptimizationExplanation(builderConfig, { + enabled: !!enabled && !!builderConfig, placeholderData: undefined, }); const { data: source, isLoading: isSourceLoading } = useSource({ - id: config?.source, + id: builderConfig?.source, }); const { @@ -445,6 +466,8 @@ export default function useOffsetPaginatedQuery( metadata, optimizedConfig: mvOptimizationData?.optimizedConfig, source, + // Additional readonly protection when the user is running a raw SQL query + readonly: isRawSqlChartConfig(config), } satisfies QueryMeta, queryFn, gcTime: isLive ? ms('30s') : ms('5m'), // more aggressive gc for live data, since it can end up holding lots of data diff --git a/packages/app/src/hooks/usePatterns.tsx b/packages/app/src/hooks/usePatterns.tsx index 10647074..03c53d0d 100644 --- a/packages/app/src/hooks/usePatterns.tsx +++ b/packages/app/src/hooks/usePatterns.tsx @@ -1,14 +1,11 @@ import { useMemo } from 'react'; import stripAnsi from 'strip-ansi'; import { convertDateRangeToGranularityString } from '@hyperdx/common-utils/dist/core/utils'; -import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import { useQuery } from '@tanstack/react-query'; import { timeBucketByGranularity, toStartOfInterval } from '@/ChartUtils'; -import { - selectColumnMapWithoutAdditionalKeys, - useConfigWithPrimaryAndPartitionKey, -} from '@/components/DBRowTable'; +import { useConfigWithPrimaryAndPartitionKey } from '@/components/DBRowTable'; import { useQueriedChartConfig } from '@/hooks/useChartConfig'; import { getFirstTimestampValueExpression } from '@/source'; @@ -130,7 +127,7 @@ function usePatterns({ statusCodeExpression, enabled = true, }: { - config: ChartConfigWithDateRange; + config: BuilderChartConfigWithDateRange; samples: number; bodyValueExpression: string; severityTextExpression?: string; @@ -220,7 +217,7 @@ export function useGroupedPatterns({ totalCount, enabled = true, }: { - config: ChartConfigWithDateRange; + config: BuilderChartConfigWithDateRange; samples: number; bodyValueExpression: string; severityTextExpression?: string; diff --git a/packages/app/src/hooks/useRowWhere.tsx b/packages/app/src/hooks/useRowWhere.tsx index a2bf2aba..cc2bb810 100644 --- a/packages/app/src/hooks/useRowWhere.tsx +++ b/packages/app/src/hooks/useRowWhere.tsx @@ -7,12 +7,12 @@ import { JSDataType, } from '@hyperdx/common-utils/dist/clickhouse'; import { aliasMapToWithClauses } from '@hyperdx/common-utils/dist/core/utils'; -import { ChartConfig } from '@hyperdx/common-utils/dist/types'; +import { BuilderChartConfig } from '@hyperdx/common-utils/dist/types'; const MAX_STRING_LENGTH = 512; // Type for WITH clause entries, derived from ChartConfig's with property -export type WithClause = NonNullable[number]; +export type WithClause = NonNullable[number]; // Internal row field names used by the table component for row tracking export const INTERNAL_ROW_FIELDS = { diff --git a/packages/app/src/types.ts b/packages/app/src/types.ts index 77deb603..16194ac9 100644 --- a/packages/app/src/types.ts +++ b/packages/app/src/types.ts @@ -2,7 +2,7 @@ import { z } from 'zod'; import { Alert, AlertHistory, - ChartConfig, + BuilderChartConfig, DashboardSchema, Filter, NumberFormat as _NumberFormat, @@ -63,8 +63,8 @@ export type SavedSearchWithEnhancedAlerts = Omit & { export type SearchConfig = { select?: string | null; source?: string | null; - where?: ChartConfig['where'] | null; - whereLanguage?: ChartConfig['whereLanguage'] | null; + where?: BuilderChartConfig['where'] | null; + whereLanguage?: BuilderChartConfig['whereLanguage'] | null; filters?: Filter[] | null; orderBy?: string | null; }; diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index a5f2f9fd..19d8eea0 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1,7 +1,8 @@ import { ClickhouseClient } from '../clickhouse/node'; import { Metadata, MetadataCache } from '../core/metadata'; import * as renderChartConfigModule from '../core/renderChartConfig'; -import { ChartConfigWithDateRange, TSource } from '../types'; +import { isBuilderChartConfig } from '../guards'; +import { BuilderChartConfigWithDateRange, TSource } from '../types'; // Mock ClickhouseClient const mockClickhouseClient = { @@ -232,7 +233,7 @@ describe('Metadata', () => { }); describe('getKeyValues', () => { - const mockChartConfig: ChartConfigWithDateRange = { + const mockChartConfig: BuilderChartConfigWithDateRange = { from: { databaseName: 'test_db', tableName: 'test_table', @@ -372,7 +373,7 @@ describe('Metadata', () => { }); describe('getValuesDistribution', () => { - const mockChartConfig: ChartConfigWithDateRange = { + const mockChartConfig: BuilderChartConfigWithDateRange = { from: { databaseName: 'test_db', tableName: 'test_table', @@ -462,6 +463,8 @@ describe('Metadata', () => { }); const actualConfig = renderChartConfigSpy.mock.calls[0][0]; + if (!isBuilderChartConfig(actualConfig)) + throw new Error('Expected builder config'); expect(actualConfig.with).toContainEqual({ name: 'service', sql: { @@ -480,7 +483,7 @@ describe('Metadata', () => { }); it('should include filters from the config in the query', async () => { - const configWithFilters: ChartConfigWithDateRange = { + const configWithFilters: BuilderChartConfigWithDateRange = { ...mockChartConfig, filters: [ { @@ -502,6 +505,8 @@ describe('Metadata', () => { }); const actualConfig = renderChartConfigSpy.mock.calls[0][0]; + if (!isBuilderChartConfig(actualConfig)) + throw new Error('Expected builder config'); expect(actualConfig.filters).toContainEqual({ type: 'sql', condition: "ServiceName IN ('clickhouse')", diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index fe6bf728..06af9605 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -1427,4 +1427,21 @@ describe('renderChartConfig', () => { expect(actual).toMatchSnapshot(); }); }); + + it('returns sqlTemplate verbatim for raw sql config', async () => { + const rawSqlConfig: ChartConfigWithOptDateRangeEx = { + configType: 'sql', + sqlTemplate: 'SELECT count() FROM logs WHERE level = {level:String}', + connection: 'conn-1', + }; + const result = await renderChartConfig( + rawSqlConfig, + mockMetadata, + undefined, + ); + expect(result.sql).toBe( + 'SELECT count() FROM logs WHERE level = {level:String}', + ); + expect(result.params).toEqual({}); + }); }); diff --git a/packages/common-utils/src/__tests__/utils.test.ts b/packages/common-utils/src/__tests__/utils.test.ts index 16925394..07eef790 100644 --- a/packages/common-utils/src/__tests__/utils.test.ts +++ b/packages/common-utils/src/__tests__/utils.test.ts @@ -1,7 +1,8 @@ import { z } from 'zod'; +import { isBuilderSavedChartConfig } from '@/guards'; import { - ChartConfigWithDateRange, + BuilderChartConfigWithDateRange, DashboardSchema, MetricsDataType, SourceKind, @@ -272,7 +273,10 @@ describe('utils', () => { }); it('should return the first column name for an array of objects input', () => { - const orderBy: Exclude = [ + const orderBy: Exclude< + BuilderChartConfigWithDateRange['orderBy'], + string + > = [ { valueExpression: 'column1', ordering: 'ASC' }, { valueExpression: 'column2', ordering: 'ASC' }, ]; @@ -288,7 +292,7 @@ describe('utils', () => { const config = { timestampValueExpression: 'Timestamp', orderBy: undefined, - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; expect(isTimestampExpressionInFirstOrderBy(config)).toBe(false); }); @@ -297,7 +301,7 @@ describe('utils', () => { const config = { timestampValueExpression: 'Timestamp', orderBy: '', - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; expect(isTimestampExpressionInFirstOrderBy(config)).toBe(false); }); @@ -306,7 +310,7 @@ describe('utils', () => { const config = { timestampValueExpression: 'Timestamp', orderBy: 'ServiceName', - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; expect(isTimestampExpressionInFirstOrderBy(config)).toBe(false); }); @@ -315,7 +319,7 @@ describe('utils', () => { const config = { timestampValueExpression: 'Timestamp', orderBy: 'ServiceName ASC, Timestamp', - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; expect(isTimestampExpressionInFirstOrderBy(config)).toBe(false); }); @@ -324,7 +328,7 @@ describe('utils', () => { const config = { timestampValueExpression: 'Timestamp', orderBy: 'Timestamp', - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; expect(isTimestampExpressionInFirstOrderBy(config)).toBe(true); }); @@ -333,7 +337,7 @@ describe('utils', () => { const config = { timestampValueExpression: 'Timestamp', orderBy: 'Timestamp DESC, ServiceName', - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; expect(isTimestampExpressionInFirstOrderBy(config)).toBe(true); }); @@ -342,7 +346,7 @@ describe('utils', () => { const config = { timestampValueExpression: 'Timestamp', orderBy: 'Timestamp desc, ServiceName', - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; expect(isTimestampExpressionInFirstOrderBy(config)).toBe(true); }); @@ -354,7 +358,7 @@ describe('utils', () => { { valueExpression: 'Timestamp', ordering: 'ASC' }, { valueExpression: 'ServiceName', ordering: 'ASC' }, ], - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; expect(isTimestampExpressionInFirstOrderBy(config)).toBe(true); }); @@ -363,7 +367,7 @@ describe('utils', () => { const config = { timestampValueExpression: 'toStartOfDay(Timestamp), Timestamp', orderBy: '(toStartOfDay(Timestamp)) DESC, Timestamp', - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; expect(isTimestampExpressionInFirstOrderBy(config)).toBe(true); }); @@ -372,7 +376,7 @@ describe('utils', () => { const config = { timestampValueExpression: 'toStartOfDay(Timestamp), Timestamp', orderBy: '(toStartOfHour(TimestampTime), TimestampTime) DESC', - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; expect(isTimestampExpressionInFirstOrderBy(config)).toBe(true); }); @@ -382,7 +386,7 @@ describe('utils', () => { timestampValueExpression: 'toStartOfInterval(TimestampTime, INTERVAL 1 DAY)', orderBy: 'toStartOfInterval(TimestampTime, INTERVAL 1 DAY) DESC', - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; expect(isTimestampExpressionInFirstOrderBy(config)).toBe(true); }); @@ -412,7 +416,10 @@ describe('utils', () => { }); it('should return true for ascending order in object input', () => { - const orderBy: Exclude = [ + const orderBy: Exclude< + BuilderChartConfigWithDateRange['orderBy'], + string + > = [ { valueExpression: 'column1', ordering: 'ASC' }, { valueExpression: 'column2', ordering: 'DESC' }, ]; @@ -420,7 +427,10 @@ describe('utils', () => { }); it('should return false for descending order in object input', () => { - const orderBy: Exclude = [ + const orderBy: Exclude< + BuilderChartConfigWithDateRange['orderBy'], + string + > = [ { valueExpression: 'column1', ordering: 'DESC' }, { valueExpression: 'column2', ordering: 'ASC' }, ]; @@ -731,7 +741,10 @@ describe('utils', () => { ]; const template = convertToDashboardTemplate(dashboard, sources); - const selectList = template.tiles[0].config.select; + const tileConfig = template.tiles[0].config; + if (!isBuilderSavedChartConfig(tileConfig)) + throw new Error('Expected builder config'); + const selectList = tileConfig.select; expect(Array.isArray(selectList)).toBe(true); expect((selectList as any[])[0]).toMatchObject({ aggFn: 'quantile', diff --git a/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts b/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts index 574b020e..696b494a 100644 --- a/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts +++ b/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts @@ -6,10 +6,10 @@ import { tryOptimizeConfigWithMaterializedViewWithExplanations, } from '@/core/materializedViews'; import { Metadata } from '@/core/metadata'; +import { isBuilderChartConfig } from '@/guards'; import { ChartConfigWithOptDateRange, MaterializedViewConfiguration, - QuerySettings, TSource, } from '@/types'; @@ -1713,7 +1713,10 @@ describe('materializedViews', () => { it('should optimize a config with the MV that will scan the fewest rows, if multiple MVs could be used', async () => { mockClickHouseClient.testChartConfigValidity.mockImplementation( ({ config }) => { - if (config.from.tableName === 'db_statement_rollup_1s') { + if ( + isBuilderChartConfig(config) && + config.from.tableName === 'db_statement_rollup_1s' + ) { return Promise.resolve({ isValid: true, rowEstimate: 1000, @@ -1964,7 +1967,10 @@ describe('materializedViews', () => { Promise.resolve({ isValid: true, rowEstimate: - config.from.tableName === 'logs_rollup_1h' ? 500 : 1000, + isBuilderChartConfig(config) && + config.from.tableName === 'logs_rollup_1h' + ? 500 + : 1000, }), ); @@ -2124,7 +2130,10 @@ describe('materializedViews', () => { Promise.resolve({ isValid: true, rowEstimate: - config.from.tableName === 'logs_rollup_1h' ? 500 : 1000, + isBuilderChartConfig(config) && + config.from.tableName === 'logs_rollup_1h' + ? 500 + : 1000, }), ); diff --git a/packages/common-utils/src/clickhouse/index.ts b/packages/common-utils/src/clickhouse/index.ts index cc3fa209..9e30f6f2 100644 --- a/packages/common-utils/src/clickhouse/index.ts +++ b/packages/common-utils/src/clickhouse/index.ts @@ -23,6 +23,7 @@ import { replaceJsonExpressions, splitAndTrimWithBracket, } from '@/core/utils'; +import { isBuilderChartConfig } from '@/guards'; import { ChartConfigWithOptDateRange, QuerySettings } from '@/types'; // export @clickhouse/client-common types @@ -625,7 +626,9 @@ export abstract class BaseClickhouseClient { }; querySettings: QuerySettings | undefined; }): Promise>> { - config = setChartSelectsAlias(config); + config = isBuilderChartConfig(config) + ? setChartSelectsAlias(config) + : config; const queries: ChSql[] = await Promise.all( splitChartConfigs(config).map(c => renderChartConfig(c, metadata, querySettings), @@ -652,7 +655,7 @@ export abstract class BaseClickhouseClient { return resultSets[0]; } // metrics -> join resultSets - else if (resultSets.length > 1) { + else if (isBuilderChartConfig(config) && resultSets.length > 1) { const metaSet = new Map(); const tsBucketMap = new Map>(); for (const resultSet of resultSets) { diff --git a/packages/common-utils/src/core/histogram.ts b/packages/common-utils/src/core/histogram.ts index b64c2577..d84453e8 100644 --- a/packages/common-utils/src/core/histogram.ts +++ b/packages/common-utils/src/core/histogram.ts @@ -1,14 +1,14 @@ import { ChSql, chSql } from '@/clickhouse'; -import { ChartConfig } from '@/types'; +import { BuilderChartConfig } from '@/types'; -type WithClauses = ChartConfig['with']; +type WithClauses = BuilderChartConfig['with']; type TemplatedInput = ChSql | string; export const translateHistogram = ({ select, ...rest }: { - select: Exclude[number]; + select: Exclude[number]; timeBucketSelect: TemplatedInput; groupBy?: TemplatedInput; from: TemplatedInput; diff --git a/packages/common-utils/src/core/materializedViews.ts b/packages/common-utils/src/core/materializedViews.ts index 6f16fb40..38ad542f 100644 --- a/packages/common-utils/src/core/materializedViews.ts +++ b/packages/common-utils/src/core/materializedViews.ts @@ -2,7 +2,7 @@ import { differenceInSeconds } from 'date-fns'; import { BaseClickhouseClient } from '@/clickhouse'; import { - ChartConfigWithOptDateRange, + BuilderChartConfigWithOptDateRange, CteChartConfig, InternalAggregateFunction, InternalAggregateFunctionSchema, @@ -19,7 +19,7 @@ import { } from './utils'; type SelectItem = Exclude< - ChartConfigWithOptDateRange['select'], + BuilderChartConfigWithOptDateRange['select'], string >[number]; @@ -123,7 +123,7 @@ function getAggregatedColumnConfig( **/ function mvConfigSupportsGranularity( mvConfig: MaterializedViewConfiguration, - chartConfig: ChartConfigWithOptDateRange, + chartConfig: BuilderChartConfigWithOptDateRange, ): boolean { if (!chartConfig.granularity && !chartConfig.dateRange) { return true; @@ -171,7 +171,7 @@ function countIntervalsInDateRange( function mvConfigSupportsDateRange( mvConfig: MaterializedViewConfiguration, - chartConfig: ChartConfigWithOptDateRange, + chartConfig: BuilderChartConfigWithOptDateRange, ) { if (mvConfig.minDate && !chartConfig.dateRange) { return false; @@ -287,7 +287,7 @@ export type MVOptimizationExplanation = { }; export async function tryConvertConfigToMaterializedViewSelect< - C extends ChartConfigWithOptDateRange | CteChartConfig, + C extends BuilderChartConfigWithOptDateRange | CteChartConfig, >( chartConfig: C, mvConfig: MaterializedViewConfiguration, @@ -377,7 +377,7 @@ export async function tryConvertConfigToMaterializedViewSelect< } /** Attempts to optimize a config with a single MV Config */ -async function tryOptimizeConfig( +async function tryOptimizeConfig( config: C, metadata: Metadata, clickhouseClient: BaseClickhouseClient, @@ -481,7 +481,7 @@ async function tryOptimizeConfig( /** Attempts to optimize a config with each of the provided MV Configs */ export async function tryOptimizeConfigWithMaterializedViewWithExplanations< - C extends ChartConfigWithOptDateRange, + C extends BuilderChartConfigWithOptDateRange, >( config: C, metadata: Metadata, @@ -535,7 +535,7 @@ export async function tryOptimizeConfigWithMaterializedViewWithExplanations< } export async function tryOptimizeConfigWithMaterializedView< - C extends ChartConfigWithOptDateRange, + C extends BuilderChartConfigWithOptDateRange, >( config: C, metadata: Metadata, @@ -580,13 +580,13 @@ function toMvId( return `${mv.databaseName}.${mv.tableName}`; } -export interface GetKeyValueCall { +export interface GetKeyValueCall { chartConfig: C; keys: string[]; } export async function optimizeGetKeyValuesCalls< - C extends ChartConfigWithOptDateRange, + C extends BuilderChartConfigWithOptDateRange, >({ chartConfig, keys, diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 6a239361..64db7aec 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -13,7 +13,11 @@ import { tableExpr, } from '@/clickhouse'; import { renderChartConfig } from '@/core/renderChartConfig'; -import type { ChartConfig, ChartConfigWithDateRange, TSource } from '@/types'; +import type { + BuilderChartConfig, + BuilderChartConfigWithDateRange, + TSource, +} from '@/types'; import { optimizeGetKeyValuesCalls } from './materializedViews'; import { objectHash } from './utils'; @@ -923,7 +927,7 @@ export class Metadata { limit = 100, source, }: { - chartConfig: ChartConfigWithDateRange; + chartConfig: BuilderChartConfigWithDateRange; key: string; samples?: number; limit?: number; @@ -940,7 +944,7 @@ export class Metadata { return this.cache.getOrFetch( `${objectHash(cacheKeyConfig)}.${key}.valuesDistribution`, async () => { - const config: ChartConfigWithDateRange = { + const config: BuilderChartConfigWithDateRange = { ...chartConfig, with: [ ...(chartConfig.with || []), @@ -1013,7 +1017,7 @@ export class Metadata { signal, source, }: { - chartConfig: ChartConfigWithDateRange; + chartConfig: BuilderChartConfigWithDateRange; keys: string[]; limit?: number; disableRowLimit?: boolean; @@ -1132,7 +1136,7 @@ export class Metadata { disableRowLimit, signal, }: { - chartConfig: ChartConfigWithDateRange; + chartConfig: BuilderChartConfigWithDateRange; keys: string[]; source: TSource | undefined; limit?: number; @@ -1210,7 +1214,9 @@ export type TableConnectionChoice = tableConnections?: never; }; -export function tcFromChartConfig(config?: ChartConfig): TableConnection { +export function tcFromChartConfig( + config?: BuilderChartConfig, +): TableConnection { return { databaseName: config?.from?.databaseName ?? '', tableName: config?.from?.tableName ?? '', diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts index 2cdd6fff..ca1ed94c 100644 --- a/packages/common-utils/src/core/renderChartConfig.ts +++ b/packages/common-utils/src/core/renderChartConfig.ts @@ -16,18 +16,22 @@ import { parseToStartOfFunction, splitAndTrimWithBracket, } from '@/core/utils'; +import { isBuilderChartConfig, isRawSqlChartConfig } from '@/guards'; import { CustomSchemaSQLSerializerV2, SearchQueryBuilder } from '@/queryParser'; import { AggregateFunction, AggregateFunctionWithCombinators, + BuilderChartConfigWithDateRange, + BuilderChartConfigWithOptDateRange, ChartConfig, ChartConfigSchema, - ChartConfigWithDateRange, ChartConfigWithOptDateRange, ChSqlSchema, CteChartConfig, + DateRange, MetricsDataType, QuerySettings, + RawSqlChartConfig, SearchCondition, SearchConditionLanguage, SelectList, @@ -71,23 +75,23 @@ const DEFAULT_METRIC_TABLE_TIME_COLUMN = 'TimeUnix'; export const FIXED_TIME_BUCKET_EXPR_ALIAS = '__hdx_time_bucket'; export function isUsingGroupBy( - chartConfig: ChartConfigWithOptDateRange, -): chartConfig is Omit & { - groupBy: NonNullable; + chartConfig: BuilderChartConfigWithOptDateRange, +): chartConfig is Omit & { + groupBy: NonNullable; } { return chartConfig.groupBy != null && chartConfig.groupBy.length > 0; } export function isUsingGranularity( - chartConfig: ChartConfigWithOptDateRange, + chartConfig: BuilderChartConfigWithOptDateRange, ): chartConfig is Omit< - Omit, 'dateRange'>, + Omit, 'dateRange'>, 'timestampValueExpression' > & { - granularity: NonNullable; - dateRange: NonNullable; + granularity: NonNullable; + dateRange: NonNullable; timestampValueExpression: NonNullable< - ChartConfigWithDateRange['timestampValueExpression'] + BuilderChartConfigWithDateRange['timestampValueExpression'] >; } { return ( @@ -97,13 +101,17 @@ export function isUsingGranularity( } export const isMetricChartConfig = ( - chartConfig: ChartConfigWithOptDateRange, -) => { + chartConfig: BuilderChartConfigWithOptDateRange, +): chartConfig is BuilderChartConfigWithOptDateRange & { + metricTables: NonNullable; +} => { return chartConfig.metricTables != null; }; // TODO: apply this to all chart configs -export const setChartSelectsAlias = (config: ChartConfigWithOptDateRange) => { +export const setChartSelectsAlias = ( + config: BuilderChartConfigWithOptDateRange, +) => { if (Array.isArray(config.select) && isMetricChartConfig(config)) { return { ...config, @@ -120,10 +128,16 @@ export const setChartSelectsAlias = (config: ChartConfigWithOptDateRange) => { return config; }; -export const splitChartConfigs = (config: ChartConfigWithOptDateRange) => { +export const splitChartConfigs = ( + config: ChartConfigWithOptDateRange, +): ChartConfigWithOptDateRangeEx[] => { // only split metric queries for now - if (isMetricChartConfig(config) && Array.isArray(config.select)) { - const _configs: ChartConfigWithOptDateRange[] = []; + if ( + isBuilderChartConfig(config) && + isMetricChartConfig(config) && + Array.isArray(config.select) + ) { + const _configs: BuilderChartConfigWithOptDateRange[] = []; // split the query into multiple queries for (const select of config.select) { _configs.push({ @@ -133,7 +147,12 @@ export const splitChartConfigs = (config: ChartConfigWithOptDateRange) => { } return _configs; } - return [config]; + + if (isRawSqlChartConfig(config) || isBuilderChartConfig(config)) { + return [config]; // narrowed to BuilderChartConfig or RawSqlChartConfig, assignable to RawSqlChartConfigEx + } + + throw new Error(`Unexpected chart config type: ${JSON.stringify(config)}`); }; const INVERSE_OPERATOR_MAP = { @@ -391,7 +410,7 @@ const aggFnExpr = ({ async function renderSelectList( selectList: SelectList, - chartConfig: ChartConfigWithOptDateRangeEx, + chartConfig: BuilderChartConfigWithOptDateRangeEx, metadata: Metadata, ) { if (typeof selectList === 'string') { @@ -546,7 +565,7 @@ export async function timeFilterExpr({ metadata: Metadata; tableName: string; timestampValueExpression: string; - with?: ChartConfigWithDateRange['with']; + with?: BuilderChartConfigWithDateRange['with']; }) { const startTime = dateRange[0].getTime(); const endTime = dateRange[1].getTime(); @@ -634,7 +653,7 @@ export async function timeFilterExpr({ } async function renderSelect( - chartConfig: ChartConfigWithOptDateRangeEx, + chartConfig: BuilderChartConfigWithOptDateRangeEx, metadata: Metadata, ): Promise { /** @@ -666,7 +685,7 @@ async function renderSelect( function renderFrom({ from, }: { - from: ChartConfigWithDateRange['from']; + from: BuilderChartConfigWithDateRange['from']; }): ChSql { return concatChSql( '.', @@ -689,10 +708,10 @@ async function renderWhereExpression({ condition: SearchCondition; language: SearchConditionLanguage; metadata: Metadata; - from: ChartConfigWithDateRange['from']; + from: BuilderChartConfigWithDateRange['from']; implicitColumnExpression?: string; connectionId: string; - with?: ChartConfigWithDateRange['with']; + with?: BuilderChartConfigWithDateRange['with']; }): Promise { let _condition = condition; if (language === 'lucene') { @@ -740,7 +759,7 @@ async function renderWhereExpression({ } async function renderWhere( - chartConfig: ChartConfigWithOptDateRangeEx, + chartConfig: BuilderChartConfigWithOptDateRangeEx, metadata: Metadata, ): Promise { let whereSearchCondition: ChSql | [] = []; @@ -847,7 +866,7 @@ async function renderWhere( } async function renderGroupBy( - chartConfig: ChartConfigWithOptDateRange, + chartConfig: BuilderChartConfigWithOptDateRange, metadata: Metadata, ): Promise { return concatChSql( @@ -866,7 +885,7 @@ async function renderGroupBy( } async function renderHaving( - chartConfig: ChartConfigWithOptDateRangeEx, + chartConfig: BuilderChartConfigWithOptDateRangeEx, metadata: Metadata, ): Promise { if (!isNonEmptyWhereExpr(chartConfig.having)) { @@ -885,7 +904,7 @@ async function renderHaving( } function renderOrderBy( - chartConfig: ChartConfigWithOptDateRange, + chartConfig: BuilderChartConfigWithOptDateRange, ): ChSql | undefined { const isIncludingTimeBucket = isUsingGranularity(chartConfig); @@ -909,7 +928,7 @@ function renderOrderBy( } function renderLimit( - chartConfig: ChartConfigWithOptDateRange, + chartConfig: BuilderChartConfigWithOptDateRange, ): ChSql | undefined { if (chartConfig.limit == null || chartConfig.limit.limit == null) { return undefined; @@ -924,7 +943,7 @@ function renderLimit( } function renderSettings( - chartConfig: ChartConfigWithOptDateRangeEx, + chartConfig: BuilderChartConfigWithOptDateRangeEx, querySettings: QuerySettings | undefined, ) { const querySettingsJoined = joinQuerySettings(querySettings); @@ -937,13 +956,24 @@ function renderSettings( // includedDataInterval isn't exported at this time. It's only used internally // for metric SQL generation. -export type ChartConfigWithOptDateRangeEx = ChartConfigWithOptDateRange & { +type InternalChartFields = { includedDataInterval?: string; settings?: ChSql; }; +type BuilderChartConfigWithOptDateRangeEx = BuilderChartConfigWithOptDateRange & + InternalChartFields; + +type RawSqlChartConfigEx = RawSqlChartConfig & + Partial & + InternalChartFields; + +export type ChartConfigWithOptDateRangeEx = + | BuilderChartConfigWithOptDateRangeEx + | RawSqlChartConfigEx; + async function renderWith( - chartConfig: ChartConfigWithOptDateRangeEx, + chartConfig: BuilderChartConfigWithOptDateRangeEx, metadata: Metadata, querySettings: QuerySettings | undefined, ): Promise { @@ -1031,7 +1061,7 @@ function intervalToSeconds(interval: SQLInterval): number { } function renderFill( - chartConfig: ChartConfigWithOptDateRangeEx, + chartConfig: BuilderChartConfigWithOptDateRangeEx, ): ChSql | undefined { const { granularity, dateRange } = chartConfig; if (dateRange && granularity && granularity !== 'auto') { @@ -1049,7 +1079,7 @@ function renderFill( } function renderDeltaExpression( - chartConfig: ChartConfigWithOptDateRange, + chartConfig: BuilderChartConfigWithOptDateRange, valueExpression: string, ) { const interval = @@ -1067,9 +1097,9 @@ function renderDeltaExpression( } async function translateMetricChartConfig( - chartConfig: ChartConfigWithOptDateRange, + chartConfig: BuilderChartConfigWithOptDateRangeEx, metadata: Metadata, -): Promise { +): Promise { const metricTables = chartConfig.metricTables; if (!metricTables) { return chartConfig; @@ -1318,7 +1348,7 @@ async function translateMetricChartConfig( Array.isArray(chartConfig.dateRange) ? convertDateRangeToGranularityString(chartConfig.dateRange) : chartConfig.granularity, - } as ChartConfigWithOptDateRangeEx; + } as BuilderChartConfigWithOptDateRangeEx; const timeBucketSelect = isUsingGranularity(cteChartConfig) ? timeBucketExpr({ @@ -1377,6 +1407,10 @@ export async function renderChartConfig( metadata: Metadata, querySettings: QuerySettings | undefined, ): Promise { + if (isRawSqlChartConfig(rawChartConfig)) { + return chSql`${{ UNSAFE_RAW_SQL: rawChartConfig.sqlTemplate ?? '' }}`; + } + // metric types require more rewriting since we know more about the schema // but goes through the same generation process const chartConfig = isMetricChartConfig(rawChartConfig) diff --git a/packages/common-utils/src/core/utils.ts b/packages/common-utils/src/core/utils.ts index 906bdc69..b8c3f3a6 100644 --- a/packages/common-utils/src/core/utils.ts +++ b/packages/common-utils/src/core/utils.ts @@ -5,10 +5,11 @@ import { z } from 'zod'; export { default as objectHash } from 'object-hash'; +import { isBuilderSavedChartConfig } from '@/guards'; import { - ChartConfig, - ChartConfigWithDateRange, - ChartConfigWithOptTimestamp, + BuilderChartConfig, + BuilderChartConfigWithDateRange, + BuilderChartConfigWithOptTimestamp, DashboardFilter, DashboardFilterSchema, DashboardSchema, @@ -472,9 +473,13 @@ export function convertToDashboardTemplate( ): TileTemplate => { const tile = TileTemplateSchema.strip().parse(structuredClone(input)); // Extract name from source or default to '' if not found - tile.config.source = ( - sources.find(source => source.id === tile.config.source) ?? { name: '' } - ).name; + // Raw SQL configs don't have a source field, so only update builder configs + const tileConfig = tile.config; + if (isBuilderSavedChartConfig(tileConfig)) { + tileConfig.source = ( + sources.find(source => source.id === tileConfig.source) ?? { name: '' } + ).name; + } return tile; }; @@ -539,7 +544,7 @@ export function convertToDashboardDocument( } export const getFirstOrderingItem = ( - orderBy: ChartConfigWithDateRange['orderBy'], + orderBy: BuilderChartConfigWithDateRange['orderBy'], ) => { if (!orderBy || orderBy.length === 0) return undefined; @@ -560,7 +565,7 @@ export const removeTrailingDirection = (s: string) => { }; export const isTimestampExpressionInFirstOrderBy = ( - config: ChartConfigWithOptTimestamp, + config: BuilderChartConfigWithOptTimestamp, ) => { const firstOrderingItem = getFirstOrderingItem(config.orderBy); if (!firstOrderingItem || config.timestampValueExpression == null) @@ -581,7 +586,7 @@ export const isTimestampExpressionInFirstOrderBy = ( }; export const isFirstOrderByAscending = ( - orderBy: ChartConfigWithDateRange['orderBy'], + orderBy: BuilderChartConfigWithDateRange['orderBy'], ): boolean => { const primaryOrderingItem = getFirstOrderingItem(orderBy); @@ -936,7 +941,7 @@ export function parseTokenizerFromTextIndex({ */ export function aliasMapToWithClauses( aliasMap: Record | undefined, -): ChartConfig['with'] { +): BuilderChartConfig['with'] { if (!aliasMap) { return undefined; } diff --git a/packages/common-utils/src/guards.ts b/packages/common-utils/src/guards.ts new file mode 100644 index 00000000..62621db5 --- /dev/null +++ b/packages/common-utils/src/guards.ts @@ -0,0 +1,33 @@ +import { + BuilderChartConfig, + BuilderSavedChartConfig, + ChartConfig, + ChartConfigWithOptDateRange, + RawSqlChartConfig, + RawSqlSavedChartConfig, + SavedChartConfig, +} from './types'; + +export function isRawSqlChartConfig( + chartConfig: ChartConfig | ChartConfigWithOptDateRange, +): chartConfig is RawSqlChartConfig { + return 'configType' in chartConfig && chartConfig.configType === 'sql'; +} + +export function isBuilderChartConfig( + chartConfig: ChartConfig | ChartConfigWithOptDateRange, +): chartConfig is BuilderChartConfig { + return !isRawSqlChartConfig(chartConfig); +} + +export function isRawSqlSavedChartConfig( + chartConfig: SavedChartConfig, +): chartConfig is RawSqlSavedChartConfig { + return 'configType' in chartConfig && chartConfig.configType === 'sql'; +} + +export function isBuilderSavedChartConfig( + chartConfig: SavedChartConfig, +): chartConfig is BuilderSavedChartConfig { + return !isRawSqlSavedChartConfig(chartConfig); +} diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index d829eabb..662f4650 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -416,29 +416,37 @@ export type NumberFormat = z.infer; // When making changes here, consider if they need to be made to the external API // schema as well (packages/api/src/utils/zod.ts). -export const _ChartConfigSchema = z.object({ + +/** + * Schema describing display settings which are shared between Raw SQL + * chart configs and Structured ChartBuilder chart configs + **/ +const SharedChartDisplaySettingsSchema = z.object({ displayType: z.nativeEnum(DisplayType).optional(), numberFormat: NumberFormatSchema.optional(), + granularity: z.union([SQLIntervalSchema, z.literal('auto')]).optional(), + compareToPreviousPeriod: z.boolean().optional(), + fillNulls: z.union([z.number(), z.literal(false)]).optional(), + alignDateRangeToGranularity: z.boolean().optional(), +}); + +export const _ChartConfigSchema = SharedChartDisplaySettingsSchema.extend({ timestampValueExpression: z.string(), implicitColumnExpression: z.string().optional(), - granularity: z.union([SQLIntervalSchema, z.literal('auto')]).optional(), markdown: z.string().optional(), filtersLogicalOperator: z.enum(['AND', 'OR']).optional(), filters: z.array(FilterSchema).optional(), connection: z.string(), - fillNulls: z.union([z.number(), z.literal(false)]).optional(), selectGroupBy: z.boolean().optional(), metricTables: MetricTableSchema.optional(), seriesReturnType: z.enum(['ratio', 'column']).optional(), // Used to preserve original table select string when chart overrides it (e.g., histograms) eventTableSelect: z.string().optional(), - compareToPreviousPeriod: z.boolean().optional(), source: z.string().optional(), - alignDateRangeToGranularity: z.boolean().optional(), }); // This is a ChartConfig type without the `with` CTE clause included. -// It needs to be a separate, named schema to avoid use ot z.lazy(...), +// It needs to be a separate, named schema to avoid use of z.lazy(...), // use of which allows for type mistakes to make it past linting. export const CteChartConfigSchema = z.intersection( _ChartConfigSchema.partial({ timestampValueExpression: true }), @@ -451,7 +459,7 @@ export type CteChartConfig = z.infer; // non-recursive chart config so that it can reference a complete chart config // schema. This structure does mean that we cannot nest `with` clauses but does // ensure the type system can catch more issues in the build pipeline. -export const ChartConfigSchema = z.intersection( +const BuilderChartConfigSchema = z.intersection( z.intersection(_ChartConfigSchema, SelectSQLStatementSchema), z .object({ @@ -477,6 +485,22 @@ export const ChartConfigSchema = z.intersection( .partial(), ); +export type BuilderChartConfig = z.infer; + +/** Schema describing Raw SQL chart configs */ +const RawSqlChartConfigSchema = SharedChartDisplaySettingsSchema.extend({ + configType: z.literal('sql'), + sqlTemplate: z.string(), + connection: z.string(), +}); + +export type RawSqlChartConfig = z.infer; + +export const ChartConfigSchema = z.union([ + BuilderChartConfigSchema, + RawSqlChartConfigSchema, +]); + export type ChartConfig = z.infer; export type DateRange = { @@ -486,31 +510,38 @@ export type DateRange = { }; export type ChartConfigWithDateRange = ChartConfig & DateRange; +export type BuilderChartConfigWithDateRange = BuilderChartConfig & DateRange; +export type RawSqlConfigWithDateRange = RawSqlChartConfig & DateRange; -export type ChartConfigWithOptTimestamp = Omit< - ChartConfigWithDateRange, +export type BuilderChartConfigWithOptTimestamp = Omit< + BuilderChartConfigWithDateRange, 'timestampValueExpression' > & { timestampValueExpression?: string; }; + +export type ChartConfigWithOptTimestamp = + | BuilderChartConfigWithOptTimestamp + | RawSqlConfigWithDateRange; + // For non-time-based searches (ex. grab 1 row) -export type ChartConfigWithOptDateRange = Omit< - ChartConfig, +export type BuilderChartConfigWithOptDateRange = Omit< + BuilderChartConfig, 'timestampValueExpression' > & { timestampValueExpression?: string; } & Partial; +export type ChartConfigWithOptDateRange = + | BuilderChartConfigWithOptDateRange + | (RawSqlChartConfig & Partial); + // When making changes here, consider if they need to be made to the external API // schema as well (packages/api/src/utils/zod.ts). -export const SavedChartConfigSchema = z +const BuilderSavedChartConfigWithoutAlertSchema = z .object({ name: z.string().optional(), source: z.string(), - alert: z.union([ - AlertBaseSchema.optional(), - ChartAlertBaseSchema.optional(), - ]), }) .extend( _ChartConfigSchema.omit({ @@ -525,6 +556,31 @@ export const SavedChartConfigSchema = z }).shape, ); +const BuilderSavedChartConfigSchema = + BuilderSavedChartConfigWithoutAlertSchema.extend({ + alert: z.union([ + AlertBaseSchema.optional(), + ChartAlertBaseSchema.optional(), + ]), + }); + +export type BuilderSavedChartConfig = z.infer< + typeof BuilderSavedChartConfigSchema +>; + +const RawSqlSavedChartConfigSchema = RawSqlChartConfigSchema.extend({ + name: z.string().optional(), +}); + +export const SavedChartConfigSchema = z.union([ + BuilderSavedChartConfigSchema, + RawSqlSavedChartConfigSchema, +]); + +export type RawSqlSavedChartConfig = z.infer< + typeof RawSqlSavedChartConfigSchema +>; + export type SavedChartConfig = z.infer; export const TileSchema = z.object({ @@ -535,8 +591,12 @@ export const TileSchema = z.object({ h: z.number(), config: SavedChartConfigSchema, }); + export const TileTemplateSchema = TileSchema.extend({ - config: TileSchema.shape.config.omit({ alert: true }), + config: z.union([ + BuilderSavedChartConfigWithoutAlertSchema, + RawSqlSavedChartConfigSchema, + ]), }); export type Tile = z.infer;