From 32f1189a7d64e3fa01fd9df6833f8e5b6896c048 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Thu, 5 Mar 2026 15:30:58 -0500 Subject: [PATCH] feat: Add RawSqlChartConfig types for SQL-based Table (#1846) ## Summary This PR is the first step towards raw SQL-driven charts. - It introduces updated ChartConfig types, which are now unions of `BuilderChartConfig` (which is unchanged from the current `ChartConfig` types` and `RawSqlChartConfig` types which represent sql-driven charts. - It adds _very basic_ support for SQL-driven tables in the Chart Explorer and Dashboard pages. This is currently behind a feature toggle and enabled only in preview environments and for local development. The changes in most of the files in this PR are either type updates or the addition of type guards to handle the new ChartConfig union type. The DBEditTimeChartForm has been updated significantly to (a) add the Raw SQL option to the table chart editor and (b) handle conversion from internal form state (which can now include properties from either branch of the ChartConfig union) to valid SavedChartConfigs (which may only include properties from one branch). Significant changes are in: - packages/app/src/components/ChartEditor/types.ts - packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx - packages/app/src/components/ChartEditor/utils.ts - packages/app/src/components/DBEditTimeChartForm.tsx - packages/app/src/components/DBTableChart.tsx - packages/app/src/components/SQLEditor.tsx - packages/app/src/hooks/useOffsetPaginatedQuery.tsx Future PRs will add templating to the Raw SQL driven charts for date range and granularity injection; support for other chart types driven by SQL; improved placeholder, validation, and error states; and improved support in the external API and import/export. ### Screenshots or video https://github.com/user-attachments/assets/008579cc-ef3c-496e-9899-88bbb21eaa5e ### How to test locally or on Vercel The SQL-driven table can be tested in the preview environment or locally. ### References - Linear Issue: HDX-3580 - Related PRs: --- .changeset/fast-swans-brake.md | 7 + packages/api/src/controllers/dashboard.ts | 17 +- packages/api/src/fixtures.ts | 9 +- .../external-api/v2/utils/dashboards.ts | 18 +- packages/api/src/tasks/checkAlerts/index.ts | 17 +- .../providers/__tests__/default.test.ts | 24 +- .../tasks/checkAlerts/providers/default.ts | 11 + packages/api/src/utils/externalApi.ts | 194 +-------- packages/app/.env.development | 3 +- packages/app/src/ChartUtils.tsx | 36 +- packages/app/src/DBChartPage.tsx | 1 + packages/app/src/DBDashboardImportPage.tsx | 25 +- packages/app/src/DBDashboardPage.tsx | 212 +++++---- packages/app/src/DBSearchPage.tsx | 3 +- packages/app/src/ServicesDashboardPage.tsx | 19 +- packages/app/src/__tests__/ChartUtils.test.ts | 16 +- .../ChartEditor/RawSqlChartEditor.tsx | 54 +++ .../ChartEditor/__tests__/utils.test.ts | 405 ++++++++++++++++++ .../app/src/components/ChartEditor/types.ts | 29 ++ .../app/src/components/ChartEditor/utils.ts | 148 +++++++ .../app/src/components/ContextSidePanel.tsx | 4 +- packages/app/src/components/DBDeltaChart.tsx | 7 +- .../src/components/DBEditTimeChartForm.tsx | 331 +++++++------- .../app/src/components/DBHeatmapChart.tsx | 22 +- .../app/src/components/DBHistogramChart.tsx | 4 +- .../app/src/components/DBListBarChart.tsx | 4 +- packages/app/src/components/DBNumberChart.tsx | 4 +- packages/app/src/components/DBPieChart.tsx | 4 +- .../app/src/components/DBRowSidePanel.tsx | 4 +- packages/app/src/components/DBRowTable.tsx | 10 +- .../src/components/DBSearchPageFilters.tsx | 6 +- .../components/DBSqlRowTableWithSidebar.tsx | 4 +- packages/app/src/components/DBTableChart.tsx | 42 +- packages/app/src/components/DBTimeChart.tsx | 3 +- .../app/src/components/KubeComponents.tsx | 4 +- .../app/src/components/KubernetesFilters.tsx | 6 +- .../MVOptimizationIndicator.tsx | 4 +- packages/app/src/components/PatternTable.tsx | 6 +- packages/app/src/components/SQLEditor.tsx | 18 +- .../Search/DBSearchHeatmapChart.tsx | 4 +- .../src/components/SearchTotalCountChart.tsx | 6 +- packages/app/src/config.ts | 2 + packages/app/src/defaults.ts | 4 +- packages/app/src/hdxMTViews.ts | 6 +- .../hooks/__tests__/useChartConfig.test.tsx | 4 + .../src/hooks/__tests__/useMetadata.test.tsx | 8 +- .../useOffsetPaginatedQuery.test.tsx | 60 ++- .../app/src/hooks/useAutoCompleteOptions.tsx | 32 +- packages/app/src/hooks/useChartConfig.tsx | 32 +- .../src/hooks/useDashboardFilterValues.tsx | 91 ++-- packages/app/src/hooks/useExplainQuery.tsx | 3 +- .../hooks/useMVOptimizationExplanation.tsx | 7 +- packages/app/src/hooks/useMetadata.tsx | 13 +- .../app/src/hooks/useOffsetPaginatedQuery.tsx | 61 ++- packages/app/src/hooks/usePatterns.tsx | 11 +- packages/app/src/hooks/useRowWhere.tsx | 4 +- packages/app/src/types.ts | 6 +- .../src/__tests__/metadata.test.ts | 13 +- .../src/__tests__/renderChartConfig.test.ts | 17 + .../common-utils/src/__tests__/utils.test.ts | 45 +- .../__tests__/materializedViews.test.ts | 17 +- packages/common-utils/src/clickhouse/index.ts | 7 +- packages/common-utils/src/core/histogram.ts | 6 +- .../src/core/materializedViews.ts | 20 +- packages/common-utils/src/core/metadata.ts | 18 +- .../src/core/renderChartConfig.ts | 104 +++-- packages/common-utils/src/core/utils.ts | 25 +- packages/common-utils/src/guards.ts | 33 ++ packages/common-utils/src/types.ts | 94 +++- 69 files changed, 1690 insertions(+), 798 deletions(-) create mode 100644 .changeset/fast-swans-brake.md create mode 100644 packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx create mode 100644 packages/app/src/components/ChartEditor/__tests__/utils.test.ts create mode 100644 packages/app/src/components/ChartEditor/types.ts create mode 100644 packages/app/src/components/ChartEditor/utils.ts create mode 100644 packages/common-utils/src/guards.ts diff --git a/.changeset/fast-swans-brake.md b/.changeset/fast-swans-brake.md new file mode 100644 index 00000000..1bab238d --- /dev/null +++ b/.changeset/fast-swans-brake.md @@ -0,0 +1,7 @@ +--- +"@hyperdx/common-utils": patch +"@hyperdx/api": patch +"@hyperdx/app": patch +--- + +feat: Add RawSqlChartConfig types for SQL-based Table diff --git a/packages/api/src/controllers/dashboard.ts b/packages/api/src/controllers/dashboard.ts index ded883fa..d56163f2 100644 --- a/packages/api/src/controllers/dashboard.ts +++ b/packages/api/src/controllers/dashboard.ts @@ -1,4 +1,6 @@ +import { isBuilderSavedChartConfig } from '@hyperdx/common-utils/dist/guards'; import { + BuilderSavedChartConfig, DashboardWithoutIdSchema, Tile, } from '@hyperdx/common-utils/dist/types'; @@ -17,7 +19,7 @@ import Dashboard from '@/models/dashboard'; function pickAlertsByTile(tiles: Tile[]) { return tiles.reduce((acc, tile) => { - if (tile.config.alert) { + if (isBuilderSavedChartConfig(tile.config) && tile.config.alert) { acc[tile.id] = tile.config.alert; } return acc; @@ -25,7 +27,9 @@ function pickAlertsByTile(tiles: Tile[]) { } type TileForAlertSync = Pick & { - config?: Pick | { alert?: IAlert | AlertDocument }; + config?: + | Pick + | { alert?: IAlert | AlertDocument }; }; function extractTileAlertData(tiles: TileForAlertSync[]): { @@ -48,8 +52,15 @@ async function syncDashboardAlerts( ): Promise { const { tileIds: oldTileIds, tileIdsWithAlerts: oldTileIdsWithAlerts } = extractTileAlertData(oldTiles); + + const newTilesForAlertSync: TileForAlertSync[] = newTiles.map(t => ({ + id: t.id, + config: isBuilderSavedChartConfig(t.config) + ? { alert: t.config.alert } + : {}, + })); const { tileIds: newTileIds, tileIdsWithAlerts: newTileIdsWithAlerts } = - extractTileAlertData(newTiles); + extractTileAlertData(newTilesForAlertSync); // 1. Create/update alerts for tiles that have alerts const alertsByTile = pickAlertsByTile(newTiles); diff --git a/packages/api/src/fixtures.ts b/packages/api/src/fixtures.ts index a3e5742a..70f15df0 100644 --- a/packages/api/src/fixtures.ts +++ b/packages/api/src/fixtures.ts @@ -1,5 +1,6 @@ import { createNativeClient } from '@hyperdx/common-utils/dist/clickhouse/node'; import { + BuilderSavedChartConfig, DisplayType, SavedChartConfig, Tile, @@ -421,7 +422,8 @@ export const randomMongoId = () => export const makeTile = (opts?: { id?: string; - alert?: SavedChartConfig['alert']; + alert?: BuilderSavedChartConfig['alert']; + sourceId?: string; }): Tile => ({ id: opts?.id ?? randomMongoId(), x: 1, @@ -433,10 +435,11 @@ export const makeTile = (opts?: { export const makeChartConfig = (opts?: { id?: string; - alert?: SavedChartConfig['alert']; + alert?: BuilderSavedChartConfig['alert']; + sourceId?: string; }): SavedChartConfig => ({ name: 'Test Chart', - source: 'test-source', + source: opts?.sourceId ?? 'test-source', displayType: DisplayType.Line, select: [ { diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index 25b2ceba..a7ac5d10 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -1,5 +1,7 @@ +import { isRawSqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards'; import { AggregateFunctionSchema, + BuilderSavedChartConfig, DisplayType, SavedChartConfig, } from '@hyperdx/common-utils/dist/types'; @@ -62,7 +64,7 @@ const DEFAULT_SELECT_ITEM: ExternalDashboardSelectItem = { }; const convertToExternalSelectItem = ( - item: Exclude, + item: Exclude, ): ExternalDashboardSelectItem => { const parsedAggFn = AggregateFunctionSchema.safeParse(item.aggFn); const aggFn = parsedAggFn.success ? parsedAggFn.data : 'none'; @@ -84,6 +86,9 @@ const convertToExternalSelectItem = ( const convertToExternalTileChartConfig = ( config: SavedChartConfig, ): ExternalDashboardTileConfig | undefined => { + // HDX-3582: Implement this for Raw SQL charts + if (isRawSqlSavedChartConfig(config)) return undefined; + const sourceId = config.source?.toString() ?? ''; const stringValueOrDefault = ( @@ -175,7 +180,10 @@ const convertToExternalTileChartConfig = ( function convertTileToExternalChart( tile: DashboardDocument['tiles'][number], -): ExternalDashboardTileWithId { +): ExternalDashboardTileWithId | undefined { + // HDX-3582: Implement this for Raw SQL charts + if (isRawSqlSavedChartConfig(tile.config)) return undefined; + // Returned in case of a failure converting the saved chart config const defaultTileConfig: ExternalDashboardTileConfig = { displayType: 'line', @@ -196,7 +204,9 @@ export function convertToExternalDashboard( return { id: dashboard._id.toString(), name: dashboard.name, - tiles: dashboard.tiles.map(convertTileToExternalChart), + tiles: dashboard.tiles + .map(convertTileToExternalChart) + .filter(t => t !== undefined), tags: dashboard.tags || [], filters: dashboard.filters?.map(translateFilterToExternalFilter) || [], savedQuery: dashboard.savedQuery ?? null, @@ -211,7 +221,7 @@ export function convertToExternalDashboard( const convertToInternalSelectItem = ( item: ExternalDashboardSelectItem, -): Exclude => { +): Exclude => { return { ...pick(item, ['alias', 'metricType', 'metricName', 'aggFn', 'level']), aggCondition: item.where, diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 86f639c0..cad316a3 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -17,7 +17,11 @@ import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartCo import { aliasMapToWithClauses } from '@hyperdx/common-utils/dist/core/utils'; import { timeBucketByGranularity } from '@hyperdx/common-utils/dist/core/utils'; import { - ChartConfigWithOptDateRange, + isBuilderSavedChartConfig, + isRawSqlSavedChartConfig, +} from '@hyperdx/common-utils/dist/guards'; +import { + BuilderChartConfigWithOptDateRange, DisplayType, } from '@hyperdx/common-utils/dist/types'; import * as fns from 'date-fns'; @@ -67,6 +71,7 @@ export const alertHasGroupBy = (details: AlertDetails): boolean => { } if ( details.taskType === AlertTaskType.TILE && + isBuilderSavedChartConfig(details.tile.config) && details.tile.config.groupBy && details.tile.config.groupBy.length > 0 ) { @@ -84,10 +89,10 @@ export async function computeAliasWithClauses( savedSearch: Pick, source: ISource, metadata: Metadata, -): Promise { +): Promise { const resolvedSelect = savedSearch.select || source.defaultTableSelectExpression || ''; - const config: ChartConfigWithOptDateRange = { + const config: BuilderChartConfigWithOptDateRange = { connection: '', displayType: DisplayType.Search, from: source.from, @@ -317,7 +322,7 @@ const getChartConfigFromAlert = ( connection: string, dateRange: [Date, Date], windowSizeInMins: number, -): ChartConfigWithOptDateRange | undefined => { +): BuilderChartConfigWithOptDateRange | undefined => { const { alert, source } = details; if (details.taskType === AlertTaskType.SAVED_SEARCH) { const savedSearch = details.savedSearch; @@ -344,6 +349,10 @@ const getChartConfigFromAlert = ( }; } else if (details.taskType === AlertTaskType.TILE) { const tile = details.tile; + + // Alerts are not supported for raw sql based charts + if (isRawSqlSavedChartConfig(tile.config)) return undefined; + // Doesn't work for metric alerts yet if ( tile.config.displayType === DisplayType.Line || diff --git a/packages/api/src/tasks/checkAlerts/providers/__tests__/default.test.ts b/packages/api/src/tasks/checkAlerts/providers/__tests__/default.test.ts index 945589e7..d2d270b2 100644 --- a/packages/api/src/tasks/checkAlerts/providers/__tests__/default.test.ts +++ b/packages/api/src/tasks/checkAlerts/providers/__tests__/default.test.ts @@ -148,8 +148,10 @@ describe('DefaultAlertProvider', () => { }); // Create tile with source - const tile = makeTile({ id: 'test-tile-123' }); - tile.config.source = source._id.toString(); + const tile = makeTile({ + id: 'test-tile-123', + sourceId: source._id.toString(), + }); // Create dashboard const dashboard = await Dashboard.create({ @@ -301,8 +303,10 @@ describe('DefaultAlertProvider', () => { ); // Create tile and alert - const tile = makeTile({ id: 'test-tile-123' }); - tile.config.source = source._id.toString(); + const tile = makeTile({ + id: 'test-tile-123', + sourceId: source._id.toString(), + }); const dashboard = await Dashboard.create({ team: team._id, @@ -503,8 +507,10 @@ describe('DefaultAlertProvider', () => { it('should skip alerts with missing source', async () => { const team = await createTeam({ name: 'Test Team' }); - const tile = makeTile({ id: 'test-tile' }); - tile.config.source = new mongoose.Types.ObjectId().toString(); // Non-existent source + const tile = makeTile({ + id: 'test-tile', + sourceId: new mongoose.Types.ObjectId().toString(), // Non-existent source + }); const dashboard = await Dashboard.create({ team: team._id, @@ -684,8 +690,10 @@ describe('DefaultAlertProvider', () => { }); // Create tile with source - const tile = makeTile({ id: 'test-tile-123' }); - tile.config.source = source._id.toString(); + const tile = makeTile({ + id: 'test-tile-123', + sourceId: source._id.toString(), + }); // Create dashboard const dashboard = await Dashboard.create({ diff --git a/packages/api/src/tasks/checkAlerts/providers/default.ts b/packages/api/src/tasks/checkAlerts/providers/default.ts index 5eb5d354..ace87f21 100644 --- a/packages/api/src/tasks/checkAlerts/providers/default.ts +++ b/packages/api/src/tasks/checkAlerts/providers/default.ts @@ -1,4 +1,5 @@ import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; +import { isRawSqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards'; import { Tile } from '@hyperdx/common-utils/dist/types'; import mongoose from 'mongoose'; import ms from 'ms'; @@ -106,6 +107,16 @@ async function getTileDetails( return []; } + if (isRawSqlSavedChartConfig(tile.config)) { + logger.warn({ + tileId, + dashboardId: dashboard._id, + alertId: alert.id, + message: 'skipping alert with raw sql chart config, not supported', + }); + return []; + } + const source = await Source.findOne({ _id: tile.config.source, team: alert.team, diff --git a/packages/api/src/utils/externalApi.ts b/packages/api/src/utils/externalApi.ts index f19ba049..38f93598 100644 --- a/packages/api/src/utils/externalApi.ts +++ b/packages/api/src/utils/externalApi.ts @@ -1,10 +1,8 @@ -import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils'; import { - AggregateFunctionSchema, + BuilderSavedChartConfig, DashboardFilter, DisplayType, SavedChartConfig, - SelectList, } from '@hyperdx/common-utils/dist/types'; import { omit } from 'lodash'; import { FlattenMaps, LeanDocument } from 'mongoose'; @@ -18,42 +16,7 @@ import { } from '@/models/alert'; import type { DashboardDocument } from '@/models/dashboard'; import { SeriesTile } from '@/routers/external-api/v2/utils/dashboards'; -import { - ChartSeries, - ExternalDashboardFilterWithId, - MarkdownChartSeries, - NumberChartSeries, - SearchChartSeries, - TableChartSeries, - TimeChartSeries, -} from '@/utils/zod'; - -import logger from './logger'; - -type NonStringSelectItem = Exclude; -type NonStringSelectWithLevel = NonStringSelectItem & { level: number }; - -function hasLevel( - series: NonStringSelectItem, -): series is NonStringSelectWithLevel { - return 'level' in series && typeof series.level === 'number'; -} - -function isSortOrderDesc(config: SavedChartConfig): boolean { - if (!config.orderBy) { - return false; - } - - if (typeof config.orderBy === 'string') { - return config.orderBy.toLowerCase().endsWith(' desc'); - } - - if (Array.isArray(config.orderBy) && config.orderBy.length === 0) { - return false; - } - - return Array.isArray(config.orderBy) && config.orderBy[0].ordering === 'DESC'; -} +import { ExternalDashboardFilterWithId } from '@/utils/zod'; /** Returns a new object containing only the truthy, requested keys from the original object */ const pickIfTruthy = (obj: T, keys: K[]): Partial => { @@ -66,143 +29,6 @@ const pickIfTruthy = (obj: T, keys: K[]): Partial => { return result; }; -const convertChartConfigToExternalChartSeries = ( - config: SavedChartConfig, -): ChartSeries[] => { - const { - displayType, - source: sourceId, - select, - groupBy, - numberFormat, - } = config; - const isSelectArray = Array.isArray(select); - const convertedGroupBy = Array.isArray(groupBy) - ? groupBy.map(g => g.valueExpression) - : splitAndTrimWithBracket(groupBy ?? ''); - - switch (displayType) { - case 'line': - case 'stacked_bar': - if (!isSelectArray) { - logger.error(`Expected array select for displayType ${displayType}`); - return []; - } - - return select.map(s => { - const aggFnSanitized = AggregateFunctionSchema.safeParse( - s.aggFn ?? 'none', - ); - return { - aggFn: aggFnSanitized.success ? aggFnSanitized.data : 'none', - alias: s.alias ?? undefined, - type: 'time', - sourceId, - displayType, - level: hasLevel(s) ? s.level : undefined, - field: s.valueExpression, - where: s.aggCondition ?? '', - whereLanguage: s.aggConditionLanguage ?? 'lucene', - groupBy: convertedGroupBy, - metricName: s.metricName ?? undefined, - metricDataType: s.metricType ?? undefined, - numberFormat: numberFormat ?? undefined, - } satisfies TimeChartSeries; - }); - - case 'table': - if (!isSelectArray) { - logger.error(`Expected array select for displayType ${displayType}`); - return []; - } - - return select.map(s => { - const aggFnSanitized = AggregateFunctionSchema.safeParse( - s.aggFn ?? 'none', - ); - return { - aggFn: aggFnSanitized.success ? aggFnSanitized.data : 'none', - alias: s.alias ?? undefined, - type: 'table', - sourceId, - level: hasLevel(s) ? s.level : undefined, - field: s.valueExpression, - where: s.aggCondition ?? '', - whereLanguage: s.aggConditionLanguage ?? 'lucene', - groupBy: convertedGroupBy, - metricName: s.metricName ?? undefined, - metricDataType: s.metricType ?? undefined, - sortOrder: isSortOrderDesc(config) ? 'desc' : 'asc', - numberFormat: numberFormat ?? undefined, - } satisfies TableChartSeries; - }); - - case 'number': { - if (!isSelectArray || select.length === 0) { - logger.error( - `Expected non-empty array select for displayType ${displayType}`, - ); - return []; - } - - const firstSelect = select[0]; - const aggFnSanitized = AggregateFunctionSchema.safeParse( - firstSelect.aggFn ?? 'none', - ); - - return [ - { - alias: firstSelect.alias ?? undefined, - aggFn: aggFnSanitized.success ? aggFnSanitized.data : 'none', - type: 'number', - sourceId, - level: hasLevel(firstSelect) ? firstSelect.level : undefined, - field: firstSelect.valueExpression, - where: firstSelect.aggCondition ?? '', - whereLanguage: firstSelect.aggConditionLanguage ?? 'lucene', - metricName: firstSelect.metricName ?? undefined, - metricDataType: firstSelect.metricType ?? undefined, - numberFormat: numberFormat ?? undefined, - }, - ] satisfies [NumberChartSeries]; - } - - case 'search': { - if (isSelectArray) { - logger.error( - `Expected non-array select for displayType ${displayType}`, - ); - return []; - } - - return [ - { - type: 'search', - sourceId, - fields: splitAndTrimWithBracket(select ?? ''), - where: config.where ?? '', - whereLanguage: config.whereLanguage ?? 'lucene', - }, - ] satisfies [SearchChartSeries]; - } - - case 'markdown': - return [ - { - type: 'markdown', - content: config.markdown || '', - }, - ] satisfies [MarkdownChartSeries]; - - case 'heatmap': // Heatmap is not supported in external API, and should not be present in dashboards - default: - logger.error( - `DisplayType ${displayType} is not supported in external API`, - ); - return []; - } -}; - export function translateExternalChartToTileConfig( chart: SeriesTile, ): DashboardDocument['tiles'][number] { @@ -218,14 +44,14 @@ export function translateExternalChartToTileConfig( // Determine the sourceId and displayType based on series type let sourceId: string = firstSeries.type === 'markdown' ? '' : firstSeries.sourceId; - let select: SavedChartConfig['select'] = ''; - let displayType: SavedChartConfig['displayType']; - let groupBy: SavedChartConfig['groupBy'] = ''; - let where: SavedChartConfig['where'] = ''; - let whereLanguage: SavedChartConfig['whereLanguage'] = 'lucene'; - let orderBy: SavedChartConfig['orderBy'] = ''; - let markdown: SavedChartConfig['markdown'] = ''; - let numberFormat: SavedChartConfig['numberFormat'] = undefined; + let select: BuilderSavedChartConfig['select'] = ''; + let displayType: BuilderSavedChartConfig['displayType']; + let groupBy: BuilderSavedChartConfig['groupBy'] = ''; + let where: BuilderSavedChartConfig['where'] = ''; + let whereLanguage: BuilderSavedChartConfig['whereLanguage'] = 'lucene'; + let orderBy: BuilderSavedChartConfig['orderBy'] = ''; + let markdown: BuilderSavedChartConfig['markdown'] = ''; + let numberFormat: BuilderSavedChartConfig['numberFormat'] = undefined; switch (firstSeries.type) { case 'time': { diff --git a/packages/app/.env.development b/packages/app/.env.development index 0c63da39..f48409e7 100644 --- a/packages/app/.env.development +++ b/packages/app/.env.development @@ -7,4 +7,5 @@ OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4318" OTEL_SERVICE_NAME="hdx-oss-dev-app" PORT=${HYPERDX_APP_PORT} NODE_OPTIONS="--max-http-header-size=131072" -NEXT_PUBLIC_HYPERDX_BASE_PATH= \ No newline at end of file +NEXT_PUBLIC_HYPERDX_BASE_PATH= +NEXT_PUBLIC_IS_SQL_CHARTS_ENABLED=true \ No newline at end of file diff --git a/packages/app/src/ChartUtils.tsx b/packages/app/src/ChartUtils.tsx index c76510cc..cfb4fd78 100644 --- a/packages/app/src/ChartUtils.tsx +++ b/packages/app/src/ChartUtils.tsx @@ -18,13 +18,13 @@ import { } from '@hyperdx/common-utils/dist/core/utils'; import { AggregateFunction as AggFnV2, - ChartConfigWithDateRange, + BuilderChartConfigWithDateRange, + BuilderChartConfigWithOptTimestamp, + BuilderSavedChartConfig, ChartConfigWithOptDateRange, - ChartConfigWithOptTimestamp, DisplayType, Filter, MetricsDataType as MetricsDataTypeV2, - SavedChartConfig, SourceKind, SQLInterval, TSource, @@ -109,7 +109,7 @@ export const getMetricAggFns = ( }; export const DEFAULT_CHART_CONFIG: Omit< - SavedChartConfig, + BuilderSavedChartConfig, 'source' | 'connection' > = { name: '', @@ -132,7 +132,9 @@ export const isGranularity = (value: string): value is Granularity => { return Object.values(Granularity).includes(value as Granularity); }; -export function convertToTimeChartConfig(config: ChartConfigWithDateRange) { +export function convertToTimeChartConfig( + config: BuilderChartConfigWithDateRange, +) { const granularity = config.granularity === 'auto' || config.granularity == null ? convertDateRangeToGranularityString(config.dateRange, 80) @@ -152,7 +154,9 @@ export function convertToTimeChartConfig(config: ChartConfigWithDateRange) { }; } -export function useTimeChartSettings(chartConfig: ChartConfigWithDateRange) { +export function useTimeChartSettings( + chartConfig: BuilderChartConfigWithDateRange, +) { return useMemo(() => { const convertedConfig = convertToTimeChartConfig(chartConfig); @@ -884,7 +888,7 @@ export const convertV1ChartConfigToV2 = ( metric?: TSource; trace?: TSource; }, -): ChartConfigWithDateRange => { +): BuilderChartConfigWithDateRange => { const { series, granularity, @@ -959,7 +963,7 @@ export function buildEventsSearchUrl({ valueRangeFilter, }: { source: TSource; - config: ChartConfigWithDateRange; + config: BuilderChartConfigWithDateRange; dateRange: [Date, Date]; groupFilters?: Array<{ column: string; value: any }>; valueRangeFilter?: { expression: string; value: number; threshold?: number }; @@ -1056,7 +1060,7 @@ export function buildEventsSearchUrl({ * Handles both string format ("col1, col2") and array format ([{ valueExpression: "col1" }, ...]) */ function extractGroupColumns( - groupBy: ChartConfigWithDateRange['groupBy'], + groupBy: BuilderChartConfigWithDateRange['groupBy'], ): string[] { if (!groupBy) return []; @@ -1081,7 +1085,7 @@ export function buildTableRowSearchUrl({ }: { row: Record; source: TSource | undefined; - config: ChartConfigWithDateRange; + config: BuilderChartConfigWithDateRange; dateRange: [Date, Date]; }): string | null { if (!source?.id) { @@ -1143,20 +1147,20 @@ export function buildTableRowSearchUrl({ } export function convertToNumberChartConfig( - config: ChartConfigWithDateRange, -): ChartConfigWithOptTimestamp { + config: BuilderChartConfigWithDateRange, +): BuilderChartConfigWithOptTimestamp { return omit(config, ['granularity', 'groupBy']); } export function convertToPieChartConfig( - config: ChartConfigWithOptTimestamp, -): ChartConfigWithOptTimestamp { + config: BuilderChartConfigWithOptTimestamp, +): BuilderChartConfigWithOptTimestamp { return omit(config, ['granularity']); } export function convertToTableChartConfig( - config: ChartConfigWithOptTimestamp, -): ChartConfigWithOptTimestamp { + config: BuilderChartConfigWithOptTimestamp, +): BuilderChartConfigWithOptTimestamp { const convertedConfig = structuredClone(omit(config, ['granularity'])); // Set a default limit if not already set diff --git a/packages/app/src/DBChartPage.tsx b/packages/app/src/DBChartPage.tsx index bf918f37..045ed411 100644 --- a/packages/app/src/DBChartPage.tsx +++ b/packages/app/src/DBChartPage.tsx @@ -226,6 +226,7 @@ function DBChartExplorerPage() { parseAsJson().withDefault({ ...DEFAULT_CHART_CONFIG, source: sources?.[0]?.id ?? '', + connection: sources?.[0]?.connection, }), ); diff --git a/packages/app/src/DBDashboardImportPage.tsx b/packages/app/src/DBDashboardImportPage.tsx index 2e46a6b7..a6512a9c 100644 --- a/packages/app/src/DBDashboardImportPage.tsx +++ b/packages/app/src/DBDashboardImportPage.tsx @@ -8,6 +8,7 @@ import { StringParam, useQueryParam } from 'use-query-params'; import { z } from 'zod'; import { zodResolver } from '@hookform/resolvers/zod'; import { convertToDashboardDocument } from '@hyperdx/common-utils/dist/core/utils'; +import { isRawSqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards'; import { DashboardTemplateSchema } from '@hyperdx/common-utils/dist/types'; import { Button, @@ -207,9 +208,13 @@ function Mapping({ input }: { input: Input }) { const sourceMappings = input.tiles.map(tile => { // find matching source name + const configSource = !isRawSqlSavedChartConfig(tile.config) + ? tile.config.source + : undefined; const match = sources.find( source => - source.name.toLowerCase() === tile.config.source.toLowerCase(), + configSource && + source.name.toLowerCase() === configSource.toLowerCase(), ); return match?.id || ''; }); @@ -230,6 +235,7 @@ function Mapping({ input }: { input: Input }) { const sourceMappings = useWatch({ control, name: 'sourceMappings' }); const prevSourceMappingsRef = useRef(sourceMappings); + // HDX-3583: Extend this to support connection matching for Raw SQL-based charts. useEffect(() => { if (isUpdatingRef.current) return; if (!sourceMappings || !input.tiles) return; @@ -245,15 +251,22 @@ function Mapping({ input }: { input: Input }) { const inputTile = input.tiles[changedIdx]; if (!inputTile) return; const sourceId = sourceMappings[changedIdx] ?? ''; + const inputTileSource = !isRawSqlSavedChartConfig(inputTile.config) + ? inputTile.config.source + : undefined; const keysForTilesWithMatchingSource = input.tiles .map((tile, index) => ({ ...tile, index })) - .filter(tile => tile.config.source === inputTile.config.source) + .filter( + tile => + !isRawSqlSavedChartConfig(tile.config) && + tile.config.source === inputTileSource, + ) .map(({ index }) => `sourceMappings.${index}` as const); const keysForFiltersWithMatchingSource = input.filters ?.map((filter, index) => ({ ...filter, index })) - .filter(f => f.source === inputTile.config.source) + .filter(f => f.source === inputTileSource) .map(({ index }) => `filterSourceMappings.${index}` as const) ?? []; isUpdatingRef.current = true; @@ -362,7 +375,11 @@ function Mapping({ input }: { input: Input }) { {input.tiles.map((tile, i) => ( {tile.config.name} - {tile.config.source} + + {!isRawSqlSavedChartConfig(tile.config) + ? tile.config.source + : ''} + (undefined); const { data: source } = useSource({ - id: chart.config.source, + id: isBuilderSavedChartConfig(chart.config) + ? chart.config.source + : undefined, }); useEffect(() => { - if (source != null) { + if (isRawSqlSavedChartConfig(chart.config)) { + setQueriedConfig({ ...chart.config, dateRange, granularity }); + return; + } + + if (source != null && isBuilderSavedChartConfig(chart.config)) { const isMetricSource = source.kind === SourceKind.Metric; // TODO: will need to update this when we allow for multiple metrics per chart @@ -223,7 +234,9 @@ const Tile = forwardRef( const [hovered, setHovered] = useState(false); - const alert = chart.config.alert; + const alert = isBuilderSavedChartConfig(chart.config) + ? chart.config.alert + : undefined; const alertIndicatorColor = useMemo(() => { if (!alert) { return 'transparent'; @@ -349,6 +362,9 @@ const Tile = forwardRef( const toolbar = hideToolbar ? [] : [hoverToolbar]; const keyPrefix = isFullscreenView ? 'fullscreen' : 'tile'; + // Markdown charts may not have queriedConfig, if config.source is not set + const effectiveMarkdownConfig = queriedConfig ?? chart.config; + return ( {(queriedConfig?.displayType === DisplayType.Line || - queriedConfig?.displayType === DisplayType.StackedBar) && ( - { - onUpdateChart?.({ - ...chart, - config: { - ...chart.config, - displayType, - }, - }); - }} - /> - )} + queriedConfig?.displayType === DisplayType.StackedBar) && + isBuilderChartConfig(queriedConfig) && + isBuilderSavedChartConfig(chart.config) && ( + { + onUpdateChart?.({ + ...chart, + config: { + ...chart.config, + displayType, + }, + }); + }} + /> + )} {queriedConfig?.displayType === DisplayType.Table && ( - buildTableRowSearchUrl({ - row, - source, - config: queriedConfig, - dateRange: dateRange, - }) + getRowSearchLink={ + isBuilderChartConfig(queriedConfig) + ? row => + buildTableRowSearchUrl({ + row, + source, + config: queriedConfig, + dateRange: dateRange, + }) + : undefined } /> )} - {queriedConfig?.displayType === DisplayType.Number && ( - - )} - {queriedConfig?.displayType === DisplayType.Pie && ( - - )} - {/* Markdown charts may not have queriedConfig, if source is not set */} - {(queriedConfig?.displayType === DisplayType.Markdown || - (!queriedConfig && - chart.config.displayType === DisplayType.Markdown)) && ( - - )} - {queriedConfig?.displayType === DisplayType.Search && ( - - - - )} + )} + {queriedConfig?.displayType === DisplayType.Pie && + isBuilderChartConfig(queriedConfig) && ( + + )} + {effectiveMarkdownConfig?.displayType === DisplayType.Markdown && + 'markdown' in effectiveMarkdownConfig && ( + + )} + {queriedConfig?.displayType === DisplayType.Search && + isBuilderChartConfig(queriedConfig) && + isBuilderSavedChartConfig(chart.config) && ( + + + + )} ); }, @@ -717,6 +740,7 @@ function DBDashboardPage({ presetConfig }: { presetConfig?: Dashboard }) { const tc: TableConnection[] = []; for (const { config } of dashboard.tiles) { + if (!isBuilderSavedChartConfig(config)) continue; const source = sources?.find(v => v.id === config.source); if (!source) continue; // TODO: will need to update this when we allow for multiple metrics per chart diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index 9bf77774..1c443f43 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -32,6 +32,7 @@ import { splitAndTrimWithBracket, } from '@hyperdx/common-utils/dist/core/utils'; import { + BuilderChartConfigWithDateRange, ChartConfigWithDateRange, DisplayType, Filter, @@ -1465,7 +1466,7 @@ function DBSearchPage() { [onTimeRangeSelect, setIsLive], ); - const filtersChartConfig = useMemo(() => { + const filtersChartConfig = useMemo(() => { const overrides = { orderBy: undefined, dateRange: searchedTimeRange, diff --git a/packages/app/src/ServicesDashboardPage.tsx b/packages/app/src/ServicesDashboardPage.tsx index b1b6d480..a971ee06 100644 --- a/packages/app/src/ServicesDashboardPage.tsx +++ b/packages/app/src/ServicesDashboardPage.tsx @@ -11,8 +11,7 @@ import { UseControllerProps, useForm, useWatch } from 'react-hook-form'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; import { convertDateRangeToGranularityString } from '@hyperdx/common-utils/dist/core/utils'; import { - ChartConfigWithDateRange, - ChartConfigWithOptDateRange, + BuilderChartConfigWithDateRange, CteChartConfig, DisplayType, Filter, @@ -370,7 +369,7 @@ function HttpTab({ }, []); const requestErrorRateConfig = - useMemo(() => { + useMemo(() => { if (!source || !expressions) return null; if (reqChartType === 'overall') { return { @@ -407,7 +406,7 @@ function HttpTab({ numberFormat: ERROR_RATE_PERCENTAGE_NUMBER_FORMAT, filters: getScopedFilters({ appliedConfig, expressions }), dateRange: searchedTimeRange, - } satisfies ChartConfigWithDateRange; + } satisfies BuilderChartConfigWithDateRange; } return { timestampValueExpression: 'series_time_bucket', @@ -465,7 +464,7 @@ function HttpTab({ dateRange: searchedTimeRange, granularity: convertDateRangeToGranularityString(searchedTimeRange), - } as ChartConfigWithOptDateRange, + } as CteChartConfig, isSubquery: true, }, // Select the top N series from the search as we don't want to crash the browser. @@ -539,7 +538,7 @@ function HttpTab({ displayType: DisplayType.Line, numberFormat: ERROR_RATE_PERCENTAGE_NUMBER_FORMAT, groupBy: 'zipped, endpoint', - } satisfies ChartConfigWithDateRange; + } satisfies BuilderChartConfigWithDateRange; }, [source, searchedTimeRange, appliedConfig, expressions, reqChartType]); return ( @@ -872,7 +871,7 @@ function DatabaseTab({ }, []); const totalTimePerQueryConfig = - useMemo(() => { + useMemo(() => { if (!source || !expressions) return null; return { @@ -991,11 +990,11 @@ function DatabaseTab({ timestampValueExpression: 'series_time_bucket', connection: source.connection, source: source.id, - } satisfies ChartConfigWithDateRange; + } satisfies BuilderChartConfigWithDateRange; }, [appliedConfig, expressions, searchedTimeRange, source]); const totalThroughputPerQueryConfig = - useMemo(() => { + useMemo(() => { if (!source || !expressions) return null; return { @@ -1112,7 +1111,7 @@ function DatabaseTab({ timestampValueExpression: 'series_time_bucket', connection: source.connection, source: source.id, - } satisfies ChartConfigWithDateRange; + } satisfies BuilderChartConfigWithDateRange; }, [appliedConfig, expressions, searchedTimeRange, source]); const displaySwitcher = ( diff --git a/packages/app/src/__tests__/ChartUtils.test.ts b/packages/app/src/__tests__/ChartUtils.test.ts index 4180c0cf..cce61988 100644 --- a/packages/app/src/__tests__/ChartUtils.test.ts +++ b/packages/app/src/__tests__/ChartUtils.test.ts @@ -1,5 +1,5 @@ import { - ChartConfigWithDateRange, + BuilderChartConfigWithDateRange, SourceKind, TSource, } from '@hyperdx/common-utils/dist/types'; @@ -640,7 +640,7 @@ describe('ChartUtils', () => { new Date('2025-11-26T00:00:00Z'), new Date('2025-11-27T00:00:00Z'), ], - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; const granularityFromFunction = convertToTimeChartConfig(config).granularity; @@ -654,7 +654,7 @@ describe('ChartUtils', () => { new Date('2025-11-26T00:00:00Z'), new Date('2025-11-27T00:00:00Z'), ], - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; const granularityFromFunction = convertToTimeChartConfig(config).granularity; @@ -669,7 +669,7 @@ describe('ChartUtils', () => { new Date('2025-11-26T00:00:00Z'), new Date('2025-11-27T00:00:00Z'), ], - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; const granularityFromFunction = convertToTimeChartConfig(config).granularity; @@ -687,7 +687,7 @@ describe('ChartUtils', () => { new Date('2025-11-26T00:00:00Z'), new Date('2025-11-27T00:00:00Z'), ], - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; const convertedConfig = convertToNumberChartConfig(config); @@ -704,7 +704,7 @@ describe('ChartUtils', () => { new Date('2025-11-26T00:00:00Z'), new Date('2025-11-27T00:00:00Z'), ], - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; const convertedConfig = convertToTableChartConfig(config); @@ -718,7 +718,7 @@ describe('ChartUtils', () => { new Date('2025-11-26T00:00:00Z'), new Date('2025-11-27T00:00:00Z'), ], - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; const convertedConfig = convertToTableChartConfig(config); @@ -732,7 +732,7 @@ describe('ChartUtils', () => { new Date('2025-11-26T00:00:00Z'), new Date('2025-11-27T00:00:00Z'), ], - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; const convertedConfig = convertToTableChartConfig(config); diff --git a/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx b/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx new file mode 100644 index 00000000..57121dd6 --- /dev/null +++ b/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx @@ -0,0 +1,54 @@ +import { Control } from 'react-hook-form'; +import { Box, Button, Group, Stack, Text } from '@mantine/core'; + +import useResizable from '@/hooks/useResizable'; + +import { ConnectionSelectControlled } from '../ConnectionSelect'; +import { SQLEditorControlled } from '../SQLEditor'; + +import { ChartEditorFormState } from './types'; + +import resizeStyles from '@/../styles/ResizablePanel.module.scss'; + +export default function RawSqlChartEditor({ + control, + onOpenDisplaySettings, +}: { + control: Control; + onOpenDisplaySettings: () => void; +}) { + const { size, startResize } = useResizable(20, 'bottom'); + + return ( + + + + Connection + + + + + +
+ + + + + + ); +} diff --git a/packages/app/src/components/ChartEditor/__tests__/utils.test.ts b/packages/app/src/components/ChartEditor/__tests__/utils.test.ts new file mode 100644 index 00000000..75ee5b9b --- /dev/null +++ b/packages/app/src/components/ChartEditor/__tests__/utils.test.ts @@ -0,0 +1,405 @@ +import type { + BuilderChartConfig, + BuilderSavedChartConfig, + RawSqlSavedChartConfig, + TSource, +} from '@hyperdx/common-utils/dist/types'; +import { DisplayType, SourceKind } from '@hyperdx/common-utils/dist/types'; + +import { DEFAULT_CHART_CONFIG } from '@/ChartUtils'; + +import type { ChartEditorFormState } from '../types'; +import { + convertFormStateToChartConfig, + convertFormStateToSavedChartConfig, + convertSavedChartConfigToFormState, +} from '../utils'; + +jest.mock('../../SearchInput', () => ({ + getStoredLanguage: jest.fn().mockReturnValue('lucene'), +})); + +const dateRange: [Date, Date] = [ + new Date('2024-01-01'), + new Date('2024-01-02'), +]; + +const logSource: TSource = { + id: 'source-log', + name: 'Log Source', + kind: SourceKind.Log, + connection: 'conn-1', + from: { databaseName: 'db', tableName: 'logs' }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: 'Body, SeverityText', + implicitColumnExpression: 'Body', +}; + +const metricSource: TSource = { + id: 'source-metric', + name: 'Metric Source', + kind: SourceKind.Metric, + connection: 'conn-1', + from: { databaseName: 'db', tableName: '' }, + timestampValueExpression: 'TimeUnix', + metricTables: { gauge: 'gauge_table' } as TSource['metricTables'], + resourceAttributesExpression: 'ResourceAttributes', +}; + +const seriesItem = { + aggFn: 'count' as const, + valueExpression: '*', + aggCondition: '', + aggConditionLanguage: 'lucene' as const, +}; + +describe('convertFormStateToSavedChartConfig', () => { + it('returns undefined when no source and configType is not sql', () => { + const form: ChartEditorFormState = { + displayType: DisplayType.Line, + series: [seriesItem], + }; + expect(convertFormStateToSavedChartConfig(form, undefined)).toBeUndefined(); + }); + + it('returns RawSqlSavedChartConfig for sql+table config', () => { + const form: ChartEditorFormState = { + configType: 'sql', + displayType: DisplayType.Table, + sqlTemplate: 'SELECT 1', + connection: 'conn-1', + name: 'My Chart', + series: [], + }; + const result = convertFormStateToSavedChartConfig(form, undefined); + expect(result).toEqual({ + configType: 'sql', + displayType: DisplayType.Table, + sqlTemplate: 'SELECT 1', + connection: 'conn-1', + name: 'My Chart', + }); + }); + + it('returns undefined for sql config without Table displayType', () => { + const form: ChartEditorFormState = { + configType: 'sql', + displayType: DisplayType.Line, + sqlTemplate: 'SELECT 1', + connection: 'conn-1', + series: [], + }; + expect(convertFormStateToSavedChartConfig(form, undefined)).toBeUndefined(); + }); + + it('uses sqlTemplate empty string as default when undefined', () => { + const form: ChartEditorFormState = { + configType: 'sql', + displayType: DisplayType.Table, + series: [], + }; + const result = convertFormStateToSavedChartConfig( + form, + undefined, + ) as RawSqlSavedChartConfig; + expect(result.sqlTemplate).toBe(''); + expect(result.connection).toBe(''); + }); + + it('maps series to select for builder config', () => { + const form: ChartEditorFormState = { + displayType: DisplayType.Line, + where: 'status = 200', + series: [seriesItem], + }; + const result = convertFormStateToSavedChartConfig( + form, + logSource, + ) as BuilderSavedChartConfig; + expect(result.select).toEqual([seriesItem]); + expect('series' in result).toBe(false); + expect(result.source).toBe('source-log'); + }); + + it('uses form.select string for Search displayType', () => { + const form: ChartEditorFormState = { + displayType: DisplayType.Search, + select: 'Body, SeverityText', + series: [seriesItem], + }; + const result = convertFormStateToSavedChartConfig( + form, + logSource, + ) as BuilderSavedChartConfig; + expect(result.select).toBe('Body, SeverityText'); + expect('series' in result).toBe(false); + }); + + it('uses empty string for Search displayType when select is not a string', () => { + const form: ChartEditorFormState = { + displayType: DisplayType.Search, + select: [seriesItem], + series: [], + }; + const result = convertFormStateToSavedChartConfig( + form, + logSource, + ) as BuilderSavedChartConfig; + expect(result.select).toBe(''); + expect('series' in result).toBe(false); + }); + + it('strips metricName and metricType from select for non-metric source', () => { + const form: ChartEditorFormState = { + displayType: DisplayType.Line, + series: [ + { + ...seriesItem, + metricName: 'cpu.usage', + metricType: 'gauge' as any, + }, + ], + }; + const result = convertFormStateToSavedChartConfig( + form, + logSource, + ) as BuilderSavedChartConfig; + const select = result.select as (typeof seriesItem)[]; + expect(select[0]).not.toHaveProperty('metricName'); + expect(select[0]).not.toHaveProperty('metricType'); + }); + + it('preserves form metricTables for metric source', () => { + const formMetricTables = { + gauge: 'gauge_table', + } as BuilderSavedChartConfig['metricTables']; + const form: ChartEditorFormState = { + displayType: DisplayType.Line, + series: [seriesItem], + metricTables: formMetricTables, + }; + const result = convertFormStateToSavedChartConfig( + form, + metricSource, + ) as BuilderSavedChartConfig; + expect(result.metricTables).toEqual(formMetricTables); + }); + + it('strips metricTables for non-metric source', () => { + const form: ChartEditorFormState = { + displayType: DisplayType.Line, + series: [seriesItem], + metricTables: { gauge: 'gauge_table' } as any, + }; + const result = convertFormStateToSavedChartConfig( + form, + logSource, + ) as BuilderSavedChartConfig; + expect(result.metricTables).toBeUndefined(); + }); + + it('preserves having and orderBy only for Table displayType', () => { + const having = 'count > 5'; + const orderBy = [ + { + aggFn: 'count', + valueExpression: '*', + aggCondition: '', + ordering: 'DESC' as const, + }, + ]; + const form: ChartEditorFormState = { + displayType: DisplayType.Table, + series: [seriesItem], + having, + orderBy, + }; + const tableResult = convertFormStateToSavedChartConfig( + form, + logSource, + ) as BuilderSavedChartConfig; + expect(tableResult.having).toBe(having); + expect(tableResult.orderBy).toEqual(orderBy); + + const lineResult = convertFormStateToSavedChartConfig( + { ...form, displayType: DisplayType.Line }, + logSource, + ) as BuilderSavedChartConfig; + expect(lineResult.having).toBeUndefined(); + expect(lineResult.orderBy).toBeUndefined(); + }); + + it('defaults where to empty string when undefined', () => { + const form: ChartEditorFormState = { + displayType: DisplayType.Line, + series: [seriesItem], + }; + const result = convertFormStateToSavedChartConfig( + form, + logSource, + ) as BuilderSavedChartConfig; + expect(result.where).toBe(''); + }); +}); + +describe('convertFormStateToChartConfig', () => { + it('returns undefined when no source and configType is not sql', () => { + const form: ChartEditorFormState = { + displayType: DisplayType.Line, + series: [seriesItem], + }; + expect( + convertFormStateToChartConfig(form, dateRange, undefined), + ).toBeUndefined(); + }); + + it('returns RawSqlChartConfig with dateRange for sql config', () => { + const form: ChartEditorFormState = { + configType: 'sql', + displayType: DisplayType.Table, + sqlTemplate: 'SELECT now()', + connection: 'conn-1', + series: [], + }; + const result = convertFormStateToChartConfig(form, dateRange, undefined); + expect(result).toMatchObject({ + configType: 'sql', + sqlTemplate: 'SELECT now()', + connection: 'conn-1', + displayType: DisplayType.Table, + dateRange, + }); + }); + + it('returns builder config with source fields merged', () => { + const form: ChartEditorFormState = { + displayType: DisplayType.Line, + where: 'status = 200', + series: [seriesItem], + }; + const result = convertFormStateToChartConfig(form, dateRange, logSource); + expect(result).toMatchObject({ + from: logSource.from, + timestampValueExpression: logSource.timestampValueExpression, + connection: logSource.connection, + implicitColumnExpression: logSource.implicitColumnExpression, + dateRange, + where: 'status = 200', + }); + }); + + it('falls back to defaultTableSelectExpression when series is empty', () => { + const form: ChartEditorFormState = { + displayType: DisplayType.Line, + series: [], + }; + const result = convertFormStateToChartConfig( + form, + dateRange, + logSource, + ) as BuilderChartConfig; + expect(result?.select).toBe(logSource.defaultTableSelectExpression); + }); + + it('uses series as select for non-Search displayType', () => { + const form: ChartEditorFormState = { + displayType: DisplayType.Line, + series: [seriesItem], + }; + const result = convertFormStateToChartConfig( + form, + dateRange, + logSource, + ) as BuilderChartConfig; + expect(result?.select).toEqual([seriesItem]); + }); + + it('uses form.select for Search displayType', () => { + const form: ChartEditorFormState = { + displayType: DisplayType.Search, + select: 'Body', + series: [], + }; + const result = convertFormStateToChartConfig( + form, + dateRange, + logSource, + ) as BuilderChartConfig; + expect(result?.select).toBe('Body'); + }); +}); + +describe('convertSavedChartConfigToFormState', () => { + it('sets configType to sql for RawSqlSavedChartConfig', () => { + const config: RawSqlSavedChartConfig = { + configType: 'sql', + displayType: DisplayType.Table, + sqlTemplate: 'SELECT 1', + connection: 'conn-1', + }; + const result = convertSavedChartConfigToFormState(config); + expect(result.configType).toBe('sql'); + expect(result.series).toEqual([]); + }); + + it('sets configType to builder for BuilderSavedChartConfig', () => { + const config: BuilderSavedChartConfig = { + source: 'source-1', + displayType: DisplayType.Line, + select: [seriesItem], + where: '', + }; + const result = convertSavedChartConfigToFormState(config); + expect(result.configType).toBe('builder'); + }); + + it('maps array select to series with aggConditionLanguage defaulted', () => { + const selectItem = { + aggFn: 'count' as const, + valueExpression: '*', + aggCondition: '', + }; + const config: BuilderSavedChartConfig = { + source: 'source-1', + select: [selectItem], + where: '', + }; + const result = convertSavedChartConfigToFormState(config); + expect(result.series).toHaveLength(1); + expect(result.series[0].aggConditionLanguage).toBe('lucene'); + }); + + it('preserves existing aggConditionLanguage when already set', () => { + const config: BuilderSavedChartConfig = { + source: 'source-1', + select: [{ ...seriesItem, aggConditionLanguage: 'sql' as const }], + where: '', + }; + const result = convertSavedChartConfigToFormState(config); + expect(result.series[0].aggConditionLanguage).toBe('sql'); + }); + + it('sets series to empty array when select is a string', () => { + const config: BuilderSavedChartConfig = { + source: 'source-1', + select: 'Body, SeverityText', + where: '', + }; + const result = convertSavedChartConfigToFormState(config); + expect(result.series).toEqual([]); + }); + + it('preserves other config fields in the form state', () => { + const config: BuilderSavedChartConfig = { + source: 'source-1', + name: 'My Chart', + displayType: DisplayType.Table, + select: [seriesItem], + where: 'status = 200', + }; + const result = convertSavedChartConfigToFormState(config); + expect(result.name).toBe('My Chart'); + expect(result.displayType).toBe(DisplayType.Table); + expect(result.where).toBe('status = 200'); + }); +}); diff --git a/packages/app/src/components/ChartEditor/types.ts b/packages/app/src/components/ChartEditor/types.ts new file mode 100644 index 00000000..80312856 --- /dev/null +++ b/packages/app/src/components/ChartEditor/types.ts @@ -0,0 +1,29 @@ +import { + BuilderSavedChartConfig, + RawSqlSavedChartConfig, +} from '@hyperdx/common-utils/dist/types'; + +export type SavedChartConfigWithSelectArray = Omit< + BuilderSavedChartConfig, + 'select' +> & { + select: NonNullable>; +}; + +/** + * A type that flattens the SavedChartConfig union so that the form can include + * properties from both BuilderChartConfig and RawSqlSavedChartConfig without + * type errors. + * + * All fields are optional since the form may be in either builder or raw SQL + * mode at any given time. `configType?: 'sql'` is the discriminator. + * + * Additionally, 'series' is added as a separate field that is always an array, + * to work around the fact that useFieldArray only works with fields which are *always* + * arrays. `series` stores the array `select` data for the form. + **/ +export type ChartEditorFormState = Partial & + Partial> & { + series: SavedChartConfigWithSelectArray['select']; + configType?: 'sql' | 'builder'; + }; diff --git a/packages/app/src/components/ChartEditor/utils.ts b/packages/app/src/components/ChartEditor/utils.ts new file mode 100644 index 00000000..817c5e3f --- /dev/null +++ b/packages/app/src/components/ChartEditor/utils.ts @@ -0,0 +1,148 @@ +import { omit, pick } from 'lodash'; +import { + isBuilderSavedChartConfig, + isRawSqlSavedChartConfig, +} from '@hyperdx/common-utils/dist/guards'; +import { + BuilderSavedChartConfig, + ChartConfigWithDateRange, + DisplayType, + RawSqlChartConfig, + RawSqlSavedChartConfig, + SavedChartConfig, + SourceKind, + TSource, +} from '@hyperdx/common-utils/dist/types'; + +import { getStoredLanguage } from '../SearchInput'; + +import { ChartEditorFormState } from './types'; + +function normalizeChartConfig< + C extends Pick< + BuilderSavedChartConfig, + 'select' | 'having' | 'orderBy' | 'displayType' | 'metricTables' + >, +>(config: C, source: TSource): C { + const isMetricSource = source.kind === SourceKind.Metric; + return { + ...config, + // Strip out metric-specific fields for non-metric sources + select: + !isMetricSource && Array.isArray(config.select) + ? config.select.map(s => omit(s, ['metricName', 'metricType'])) + : config.select, + metricTables: isMetricSource ? config.metricTables : undefined, + // Order By and Having can only be set by the user for table charts + having: + config.displayType === DisplayType.Table ? config.having : undefined, + orderBy: + config.displayType === DisplayType.Table ? config.orderBy : undefined, + }; +} + +export function convertFormStateToSavedChartConfig( + form: ChartEditorFormState, + source: TSource | undefined, +): SavedChartConfig | undefined { + if (form.configType === 'sql' && form.displayType === DisplayType.Table) { + const rawSqlConfig: RawSqlSavedChartConfig = { + configType: 'sql', + ...pick(form, [ + 'name', + 'displayType', + 'numberFormat', + 'granularity', + 'compareToPreviousPeriod', + 'fillNulls', + 'alignDateRangeToGranularity', + ]), + sqlTemplate: form.sqlTemplate ?? '', + connection: form.connection ?? '', + }; + return rawSqlConfig; + } + + if (source) { + // Merge the series and select fields back together, and prevent the series field from being submitted + const config: BuilderSavedChartConfig = { + ...omit(form, ['series', 'configType', 'sqlTemplate']), + // If the chart type is search, we need to ensure the select is a string + select: + form.displayType === DisplayType.Search + ? typeof form.select === 'string' + ? form.select + : '' + : form.series, + where: form.where ?? '', + source: source.id, + }; + + return normalizeChartConfig(config, source); + } +} + +export function convertFormStateToChartConfig( + form: ChartEditorFormState, + dateRange: ChartConfigWithDateRange['dateRange'], + source: TSource | undefined, +): ChartConfigWithDateRange | undefined { + if (form.configType === 'sql' && form.displayType === DisplayType.Table) { + const rawSqlConfig: RawSqlChartConfig = { + configType: 'sql', + ...pick(form, [ + 'name', + 'displayType', + 'numberFormat', + 'granularity', + 'compareToPreviousPeriod', + 'fillNulls', + 'alignDateRangeToGranularity', + ]), + sqlTemplate: form.sqlTemplate ?? '', + connection: form.connection ?? '', + }; + + return { ...rawSqlConfig, dateRange }; + } + + if (source) { + // Merge the series and select fields back together, and prevent the series field from being submitted + const mergedSelect = + form.displayType === DisplayType.Search ? form.select : form.series; + const isSelectEmpty = !mergedSelect || mergedSelect.length === 0; + + const newConfig: ChartConfigWithDateRange = { + ...omit(form, ['series', 'configType', 'sqlTemplate']), + from: source.from, + timestampValueExpression: source.timestampValueExpression, + dateRange, + connection: source.connection, + implicitColumnExpression: source.implicitColumnExpression, + metricTables: source.metricTables, + where: form.where ?? '', + select: isSelectEmpty + ? source.defaultTableSelectExpression || '' + : mergedSelect, + }; + + return structuredClone(normalizeChartConfig(newConfig, source)); + } +} + +export function convertSavedChartConfigToFormState( + config: SavedChartConfig, +): ChartEditorFormState { + return { + ...config, + configType: isRawSqlSavedChartConfig(config) ? 'sql' : 'builder', + series: + isBuilderSavedChartConfig(config) && Array.isArray(config.select) + ? config.select.map(s => ({ + ...s, + aggConditionLanguage: + s.aggConditionLanguage ?? getStoredLanguage() ?? 'lucene', + })) + : [], + }; +} diff --git a/packages/app/src/components/ContextSidePanel.tsx b/packages/app/src/components/ContextSidePanel.tsx index efc59957..375b2043 100644 --- a/packages/app/src/components/ContextSidePanel.tsx +++ b/packages/app/src/components/ContextSidePanel.tsx @@ -5,7 +5,7 @@ import { useQueryState } from 'nuqs'; import { useForm, useWatch } from 'react-hook-form'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; import { - ChartConfigWithDateRange, + BuilderChartConfigWithDateRange, TSource, } from '@hyperdx/common-utils/dist/types'; import { Badge, Flex, Group, SegmentedControl } from '@mantine/core'; @@ -38,7 +38,7 @@ enum ContextBy { interface ContextSubpanelProps { source: TSource; - dbSqlRowTableConfig: ChartConfigWithDateRange | undefined; + dbSqlRowTableConfig: BuilderChartConfigWithDateRange | undefined; rowData: Record; rowId: string | undefined; breadcrumbPath?: BreadcrumbPath; diff --git a/packages/app/src/components/DBDeltaChart.tsx b/packages/app/src/components/DBDeltaChart.tsx index d38d44ff..9ff7f9f2 100644 --- a/packages/app/src/components/DBDeltaChart.tsx +++ b/packages/app/src/components/DBDeltaChart.tsx @@ -1,8 +1,7 @@ import { useEffect, useMemo, useState } from 'react'; import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse'; import { - ChartConfigWithDateRange, - ChartConfigWithOptDateRange, + BuilderChartConfigWithDateRange, Filter, } from '@hyperdx/common-utils/dist/types'; import { @@ -46,7 +45,7 @@ export default function DBDeltaChart({ yMax, spanIdExpression, }: { - config: ChartConfigWithDateRange; + config: BuilderChartConfigWithDateRange; valueExpr: string; xMin: number; xMax: number; @@ -102,7 +101,7 @@ export default function DBDeltaChart({ // Helper to build WITH clauses for a query (outlier or inlier) const buildWithClauses = ( isOutlier: boolean, - ): NonNullable => { + ): NonNullable => { const aggregatedTimestampsCTE = buildAggregatedTimestampsCTE(); // Build the SQL condition for filtering diff --git a/packages/app/src/components/DBEditTimeChartForm.tsx b/packages/app/src/components/DBEditTimeChartForm.tsx index ba5d9d7c..a5955a1b 100644 --- a/packages/app/src/components/DBEditTimeChartForm.tsx +++ b/packages/app/src/components/DBEditTimeChartForm.tsx @@ -1,5 +1,4 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; -import { omit } from 'lodash'; import { Control, Controller, @@ -15,6 +14,11 @@ import { NativeSelect, NumberInput } from 'react-hook-form-mantine'; import z from 'zod'; import { zodResolver } from '@hookform/resolvers/zod'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; +import { + isBuilderChartConfig, + isRawSqlChartConfig, + isRawSqlSavedChartConfig, +} from '@hyperdx/common-utils/dist/guards'; import { ChartAlertBaseSchema, ChartConfigWithDateRange, @@ -39,6 +43,7 @@ import { Group, Menu, Paper, + SegmentedControl, Stack, Switch, Tabs, @@ -83,7 +88,7 @@ import SearchWhereInput, { } from '@/components/SearchInput/SearchWhereInput'; import { SQLInlineEditorControlled } from '@/components/SearchInput/SQLInlineEditor'; import { TimePicker } from '@/components/TimePicker'; -import { IS_LOCAL_MODE } from '@/config'; +import { IS_LOCAL_MODE, IS_SQL_CHARTS_ENABLED } from '@/config'; import { GranularityPickerControlled } from '@/GranularityPicker'; import { useFetchMetricMetadata } from '@/hooks/useFetchMetricMetadata'; import { @@ -108,6 +113,16 @@ import { import HDXMarkdownChart from '../HDXMarkdownChart'; +import RawSqlChartEditor from './ChartEditor/RawSqlChartEditor'; +import { + ChartEditorFormState, + SavedChartConfigWithSelectArray, +} from './ChartEditor/types'; +import { + convertFormStateToChartConfig, + convertFormStateToSavedChartConfig, + convertSavedChartConfigToFormState, +} from './ChartEditor/utils'; import { ErrorBoundary } from './Error/ErrorBoundary'; import MVOptimizationIndicator from './MaterializedViews/MVOptimizationIndicator'; import { AggFnSelectControlled } from './AggFnSelect'; @@ -128,13 +143,20 @@ import SaveToDashboardModal from './SaveToDashboardModal'; import SourceSchemaPreview from './SourceSchemaPreview'; import { SourceSelectControlled } from './SourceSelect'; -const isQueryReady = (queriedConfig: ChartConfigWithDateRange | undefined) => - ((queriedConfig?.select?.length ?? 0) > 0 || - typeof queriedConfig?.select === 'string') && - queriedConfig?.from?.databaseName && - // tableName is empty for metric sources - (queriedConfig?.from?.tableName || queriedConfig?.metricTables) && - queriedConfig?.timestampValueExpression; +const isQueryReady = (queriedConfig: ChartConfigWithDateRange | undefined) => { + if (!queriedConfig) return false; + if (isRawSqlChartConfig(queriedConfig)) { + return !!(queriedConfig.sqlTemplate && queriedConfig.connection); + } + return ( + ((queriedConfig.select?.length ?? 0) > 0 || + typeof queriedConfig.select === 'string') && + queriedConfig.from?.databaseName && + // tableName is empty for metric sources + (queriedConfig.from?.tableName || queriedConfig.metricTables) && + queriedConfig.timestampValueExpression + ); +}; const MINIMUM_THRESHOLD_VALUE = 0.0000000001; // to make alert input > 0 @@ -142,39 +164,17 @@ const MINIMUM_THRESHOLD_VALUE = 0.0000000001; // to make alert input > 0 const getSeriesFieldPath = ( namePrefix: string, fieldName: string, -): FieldPath => { - return `${namePrefix}${fieldName}` as FieldPath; +): FieldPath => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + return `${namePrefix}${fieldName}` as FieldPath; }; -export function normalizeChartConfig< - C extends Pick< - SavedChartConfig, - 'select' | 'having' | 'orderBy' | 'displayType' | 'metricTables' - >, ->(config: C, source: TSource): C { - const isMetricSource = source.kind === SourceKind.Metric; - return { - ...config, - // Strip out metric-specific fields for non-metric sources - select: - !isMetricSource && Array.isArray(config.select) - ? config.select.map(s => omit(s, ['metricName', 'metricType'])) - : config.select, - metricTables: isMetricSource ? config.metricTables : undefined, - // Order By and Having can only be set by the user for table charts - having: - config.displayType === DisplayType.Table ? config.having : undefined, - orderBy: - config.displayType === DisplayType.Table ? config.orderBy : undefined, - }; -} - // Helper function to validate metric names for metric sources const validateMetricNames = ( tableSource: TSource | undefined, series: SavedChartConfigWithSelectArray['select'] | undefined, setError: ( - name: FieldPath, + name: FieldPath, error: { type: string; message: string }, ) => void, ): boolean => { @@ -235,7 +235,7 @@ function ChartSeriesEditorComponent({ length: number; tableSource?: TSource; errors?: FieldErrors; - clearErrors: UseFormClearErrors; + clearErrors: UseFormClearErrors; }) { const aggFn = useWatch({ control, name: `${namePrefix}aggFn` }); const aggConditionLanguage = useWatch({ @@ -537,17 +537,6 @@ const zSavedChartConfig = z }) .passthrough(); -export type SavedChartConfigWithSelectArray = Omit< - SavedChartConfig, - 'select' -> & { - select: Exclude; -}; - -type SavedChartConfigWithSeries = SavedChartConfig & { - series: SavedChartConfigWithSelectArray['select']; -}; - export default function EditTimeChartForm({ dashboardId, chartConfig, @@ -579,19 +568,8 @@ export default function EditTimeChartForm({ 'data-testid'?: string; submitRef?: React.MutableRefObject<(() => void) | undefined>; }) { - // useFieldArray only supports array type fields, and select can be either a string or array. - // To solve for this, we maintain an extra form field called 'series' which is always an array. - const configWithSeries: SavedChartConfigWithSeries = useMemo( - () => ({ - ...chartConfig, - series: Array.isArray(chartConfig.select) - ? chartConfig.select.map(s => ({ - ...s, - aggConditionLanguage: - s.aggConditionLanguage ?? getStoredLanguage() ?? 'lucene', - })) - : [], - }), + const formValue: ChartEditorFormState = useMemo( + () => convertSavedChartConfigToFormState(chartConfig), [chartConfig], ); @@ -603,9 +581,9 @@ export default function EditTimeChartForm({ setError, clearErrors, formState: { errors, isDirty }, - } = useForm({ - defaultValues: configWithSeries, - values: configWithSeries, + } = useForm({ + defaultValues: formValue, + values: formValue, resolver: zodResolver(zSavedChartConfig), }); @@ -615,7 +593,7 @@ export default function EditTimeChartForm({ remove: removeSeries, swap: swapSeries, } = useFieldArray({ - control: control as Control, + control, name: 'series', }); @@ -635,6 +613,10 @@ export default function EditTimeChartForm({ const markdown = useWatch({ control, name: 'markdown' }); const alertChannelType = useWatch({ control, name: 'alert.channel.type' }); const granularity = useWatch({ control, name: 'granularity' }); + const configType = useWatch({ control, name: 'configType' }); + + const isRawSqlInput = + configType === 'sql' && displayType === DisplayType.Table; const { data: tableSource } = useSource({ id: sourceId }); const databaseName = tableSource?.from.databaseName; @@ -668,8 +650,10 @@ export default function EditTimeChartForm({ const showGeneratedSql = ['table', 'time', 'number', 'pie'].includes( activeTab, - ); // Whether to show the generated SQL preview - const showSampleEvents = tableSource?.kind !== SourceKind.Metric; + ); + + const showSampleEvents = + tableSource?.kind !== SourceKind.Metric && !isRawSqlInput; const [ alignDateRangeToGranularity, @@ -717,7 +701,7 @@ export default function EditTimeChartForm({ ); const setQueriedConfigAndSource = useCallback( - (config: ChartConfigWithDateRange, source: TSource) => { + (config: ChartConfigWithDateRange, source: TSource | undefined) => { setQueriedConfig(config); setQueriedSource(source); }, @@ -725,7 +709,7 @@ export default function EditTimeChartForm({ ); const dbTimeChartConfig = useMemo(() => { - if (!queriedConfig) { + if (!queriedConfig || !isBuilderChartConfig(queriedConfig)) { return undefined; } @@ -745,40 +729,28 @@ export default function EditTimeChartForm({ const onSubmit = useCallback(() => { handleSubmit(form => { - // Validate metric sources have metric names selected - if (validateMetricNames(tableSource, form.series, setError)) { + const isRawSqlChart = + form.configType === 'sql' && form.displayType === DisplayType.Table; + + if ( + !isRawSqlChart && + validateMetricNames(tableSource, form.series, setError) + ) { return; } - // Merge the series and select fields back together, and prevent the series field from being submitted - const config = { - ...omit(form, ['series']), - select: - form.displayType === DisplayType.Search ? form.select : form.series, - }; + const savedConfig = convertFormStateToSavedChartConfig(form, tableSource); + const queriedConfig = convertFormStateToChartConfig( + form, + dateRange, + tableSource, + ); - setChartConfig?.(config); - if (tableSource != null) { - const isSelectEmpty = !config.select || config.select.length === 0; // select is string or array - const newConfig = { - ...config, - from: tableSource.from, - timestampValueExpression: tableSource.timestampValueExpression, - dateRange, - connection: tableSource.connection, - implicitColumnExpression: tableSource.implicitColumnExpression, - metricTables: tableSource.metricTables, - select: isSelectEmpty - ? tableSource.defaultTableSelectExpression || '' - : config.select, - }; + if (savedConfig && queriedConfig) { + setChartConfig?.(savedConfig); setQueriedConfigAndSource( - // WARNING: DON'T JUST ASSIGN OBJECTS OR DO SPREAD OPERATOR STUFF WHEN - // YOUR STATE IS AN OBJECT. YOU'RE COPYING BY REFERENCE WHICH MIGHT - // ACCIDENTALLY CAUSE A useQuery SOMEWHERE TO FIRE A REQUEST EVERY TIME - // AN INPUT CHANGES. USE structuredClone TO PERFORM A DEEP COPY INSTEAD - structuredClone(normalizeChartConfig(newConfig, tableSource)), - tableSource, + queriedConfig, + isRawSqlChart ? undefined : tableSource, ); } })(); @@ -801,7 +773,10 @@ export default function EditTimeChartForm({ const tableSortState = useMemo( () => - queriedConfig?.orderBy && typeof queriedConfig.orderBy === 'string' + queriedConfig != null && + isBuilderChartConfig(queriedConfig) && + queriedConfig.orderBy && + typeof queriedConfig.orderBy === 'string' ? orderByStringToSortingState(queriedConfig.orderBy) : undefined, [queriedConfig], @@ -814,38 +789,32 @@ export default function EditTimeChartForm({ }, [onSubmit, submitRef]); const handleSave = useCallback( - (v: SavedChartConfigWithSeries) => { - if (tableSource != null) { - // Validate metric sources have metric names selected - if (validateMetricNames(tableSource, v.series, setError)) { - return; - } + (form: ChartEditorFormState) => { + const isRawSqlChart = + form.configType === 'sql' && form.displayType === DisplayType.Table; - // If the chart type is search, we need to ensure the select is a string - if ( - displayType === DisplayType.Search && - typeof v.select !== 'string' - ) { - v.select = ''; - } else if (displayType !== DisplayType.Search) { - v.select = v.series; - } - - const normalizedChartConfig = normalizeChartConfig( - // Avoid saving the series field. Series should be persisted in the select field. - omit(v, ['series']), - tableSource, - ); - - onSave?.(normalizedChartConfig); + // Validate metric sources have metric names selected + if ( + !isRawSqlChart && + validateMetricNames(tableSource, form.series, setError) + ) { + return; } + + const savedChartConfig = convertFormStateToSavedChartConfig( + form, + tableSource, + ); + + if (savedChartConfig) onSave?.(savedChartConfig); }, - [onSave, displayType, tableSource, setError], + [onSave, tableSource, setError], ); // Track previous values for detecting changes const prevGranularityRef = useRef(granularity); const prevDisplayTypeRef = useRef(displayType); + const prevConfigTypeRef = useRef(configType); useEffect(() => { // Emulate the granularity picker auto-searching similar to dashboards @@ -856,14 +825,18 @@ export default function EditTimeChartForm({ }, [granularity, onSubmit]); useEffect(() => { - if (displayType !== prevDisplayTypeRef.current) { + if ( + displayType !== prevDisplayTypeRef.current || + configType !== prevConfigTypeRef.current + ) { prevDisplayTypeRef.current = displayType; + prevConfigTypeRef.current = configType; if (displayType === DisplayType.Search && typeof select !== 'string') { setValue('select', ''); setValue('series', []); } - if (displayType !== DisplayType.Search && typeof select === 'string') { + if (displayType !== DisplayType.Search && !Array.isArray(select)) { const defaultSeries: SavedChartConfigWithSelectArray['select'] = [ { aggFn: 'count', @@ -878,7 +851,7 @@ export default function EditTimeChartForm({ } onSubmit(); } - }, [displayType, select, setValue, onSubmit]); + }, [displayType, select, setValue, onSubmit, configType]); // Emulate the date range picker auto-searching similar to dashboards useEffect(() => { @@ -900,11 +873,17 @@ export default function EditTimeChartForm({ // and explaining whether a MV can be used. const chartConfigForExplanations: ChartConfigWithOptTimestamp | undefined = useMemo(() => { + if (queriedConfig && isRawSqlChartConfig(queriedConfig)) + return { ...queriedConfig, dateRange }; + + if (chartConfig && isRawSqlSavedChartConfig(chartConfig)) + return { ...chartConfig, dateRange }; + const userHasSubmittedQuery = !!queriedConfig; const queriedSourceMatchesSelectedSource = queriedSource?.id === tableSource?.id; const urlParamsSourceMatchesSelectedSource = - chartConfig?.source === tableSource?.id; + chartConfig.source === tableSource?.id; const effectiveQueriedConfig = activeTab === 'time' ? dbTimeChartConfig : queriedConfig; @@ -912,7 +891,7 @@ export default function EditTimeChartForm({ const config = userHasSubmittedQuery && queriedSourceMatchesSelectedSource ? effectiveQueriedConfig - : chartConfig && urlParamsSourceMatchesSelectedSource + : chartConfig && urlParamsSourceMatchesSelectedSource && tableSource ? { ...chartConfig, dateRange, @@ -922,7 +901,7 @@ export default function EditTimeChartForm({ } : undefined; - if (!config) { + if (!config || isRawSqlChartConfig(config)) { return undefined; } @@ -954,7 +933,10 @@ export default function EditTimeChartForm({ const sampleEventsConfig = useMemo( () => - tableSource != null && queriedConfig != null && queryReady + tableSource != null && + queriedConfig != null && + isBuilderChartConfig(queriedConfig) && + queryReady ? { ...queriedConfig, orderBy: [ @@ -1067,11 +1049,27 @@ export default function EditTimeChartForm({ + {IS_SQL_CHARTS_ENABLED && displayType === DisplayType.Table && ( + ( + + )} + /> + )} {activeTab === 'markdown' ? ( @@ -1095,9 +1093,14 @@ export default function EditTimeChartForm({ />
+ ) : isRawSqlInput ? ( + ) : ( <> - + Data Source @@ -1112,14 +1115,18 @@ export default function EditTimeChartForm({ } /> - {tableSource && activeTab !== 'search' && ( - - )} + + {tableSource && + activeTab !== 'search' && + chartConfigForExplanations && + isBuilderChartConfig(chartConfigForExplanations) && ( + + )} + - {displayType !== DisplayType.Search && Array.isArray(select) ? ( <> {fields.map((field, index) => ( @@ -1393,7 +1400,7 @@ export default function EditTimeChartForm({ )} - {activeTab === 'table' && ( + {activeTab === 'table' && !isRawSqlInput && (
- buildTableRowSearchUrl({ - row, - source: tableSource, - config: queriedConfig, - dateRange: queriedConfig.dateRange, - }) + getRowSearchLink={ + isBuilderChartConfig(queriedConfig) + ? row => + buildTableRowSearchUrl({ + row, + source: tableSource, + config: queriedConfig, + dateRange: queriedConfig.dateRange, + }) + : undefined } onSortingChange={onTableSortingChange} sort={tableSortState} @@ -1514,24 +1524,31 @@ export default function EditTimeChartForm({ />
)} - {queryReady && queriedConfig != null && activeTab === 'number' && ( -
- -
- )} + {queryReady && + queriedConfig != null && + isBuilderChartConfig(queriedConfig) && + activeTab === 'number' && ( +
+ +
+ )} {queryReady && tableSource && queriedConfig != null && + isBuilderChartConfig(queriedConfig) && activeTab === 'search' && (
- {sampleEventsConfig != null && ( + {sampleEventsConfig != null && tableSource && (
void; queryKeyPrefix?: string; enabled?: boolean; diff --git a/packages/app/src/components/DBListBarChart.tsx b/packages/app/src/components/DBListBarChart.tsx index 25d322f4..b33caae1 100644 --- a/packages/app/src/components/DBListBarChart.tsx +++ b/packages/app/src/components/DBListBarChart.tsx @@ -1,7 +1,7 @@ import { useMemo } from 'react'; import Link from 'next/link'; import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse'; -import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import type { FloatingPosition } from '@mantine/core'; import { Box, Code, Flex, HoverCard, Text } from '@mantine/core'; @@ -185,7 +185,7 @@ export default function DBListBarChart({ toolbarItems, showMVOptimizationIndicator = true, }: { - config: ChartConfigWithDateRange; + config: BuilderChartConfigWithDateRange; onSettled?: () => void; getRowSearchLink?: (row: any) => string; hoverCardPosition?: FloatingPosition; diff --git a/packages/app/src/components/DBNumberChart.tsx b/packages/app/src/components/DBNumberChart.tsx index 9c752b1a..b179deb7 100644 --- a/packages/app/src/components/DBNumberChart.tsx +++ b/packages/app/src/components/DBNumberChart.tsx @@ -1,6 +1,6 @@ import { useMemo } from 'react'; import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse'; -import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import { Box, Code, Flex, Text } from '@mantine/core'; import { @@ -25,7 +25,7 @@ export default function DBNumberChart({ toolbarSuffix, showMVOptimizationIndicator = true, }: { - config: ChartConfigWithDateRange; + config: BuilderChartConfigWithDateRange; queryKeyPrefix?: string; enabled?: boolean; title?: React.ReactNode; diff --git a/packages/app/src/components/DBPieChart.tsx b/packages/app/src/components/DBPieChart.tsx index ac7869a1..22832020 100644 --- a/packages/app/src/components/DBPieChart.tsx +++ b/packages/app/src/components/DBPieChart.tsx @@ -1,7 +1,7 @@ import { memo, useMemo } from 'react'; import { Cell, Pie, PieChart, ResponsiveContainer, Tooltip } from 'recharts'; import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse'; -import { ChartConfigWithOptTimestamp } from '@hyperdx/common-utils/dist/types'; +import { BuilderChartConfigWithOptTimestamp } from '@hyperdx/common-utils/dist/types'; import { Box, Code, Flex, Text } from '@mantine/core'; import { @@ -55,7 +55,7 @@ export const DBPieChart = ({ toolbarPrefix, toolbarSuffix, }: { - config: ChartConfigWithOptTimestamp; + config: BuilderChartConfigWithOptTimestamp; title?: React.ReactNode; enabled?: boolean; queryKeyPrefix?: string; diff --git a/packages/app/src/components/DBRowSidePanel.tsx b/packages/app/src/components/DBRowSidePanel.tsx index c0a31401..8de925a0 100644 --- a/packages/app/src/components/DBRowSidePanel.tsx +++ b/packages/app/src/components/DBRowSidePanel.tsx @@ -13,7 +13,7 @@ import { parseAsStringEnum, useQueryState } from 'nuqs'; import { ErrorBoundary } from 'react-error-boundary'; import { useHotkeys } from 'react-hotkeys-hook'; import { SourceKind, TSource } from '@hyperdx/common-utils/dist/types'; -import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import { Box, Drawer, Flex, Stack } from '@mantine/core'; import DBRowSidePanelHeader, { @@ -63,7 +63,7 @@ export type RowSidePanelContextProps = { displayedColumns?: string[]; toggleColumn?: (column: string) => void; shareUrl?: string; - dbSqlRowTableConfig?: ChartConfigWithDateRange; + dbSqlRowTableConfig?: BuilderChartConfigWithDateRange; isChildModalOpen?: boolean; setChildModalOpen?: (open: boolean) => void; source?: TSource; diff --git a/packages/app/src/components/DBRowTable.tsx b/packages/app/src/components/DBRowTable.tsx index 612e93e6..02d7f7b8 100644 --- a/packages/app/src/components/DBRowTable.tsx +++ b/packages/app/src/components/DBRowTable.tsx @@ -32,7 +32,7 @@ import { } from '@hyperdx/common-utils/dist/clickhouse'; import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils'; import { - ChartConfigWithDateRange, + BuilderChartConfigWithDateRange, SelectList, TSource, } from '@hyperdx/common-utils/dist/types'; @@ -271,7 +271,7 @@ const SqlModal = ({ }: { opened: boolean; onClose: () => void; - config: ChartConfigWithDateRange; + config: BuilderChartConfigWithDateRange; }) => { const { data: sql, isLoading: isLoadingSql } = useRenderedSqlChartConfig( config, @@ -368,7 +368,7 @@ export const RawLogTable = memo( error?: ClickHouseQueryError | Error; dateRange?: [Date, Date]; loadingDate?: Date; - config?: ChartConfigWithDateRange; + config?: BuilderChartConfigWithDateRange; onChildModalOpen?: (open: boolean) => void; source?: TSource; onExpandedRowsChange?: (hasExpandedRows: boolean) => void; @@ -1290,7 +1290,7 @@ function getSelectLength(select: SelectList): number { } export function useConfigWithPrimaryAndPartitionKey( - config: ChartConfigWithDateRange, + config: BuilderChartConfigWithDateRange, ) { const { data: tableMetadata } = useTableMetadata({ databaseName: config.from.databaseName, @@ -1365,7 +1365,7 @@ function DBSqlRowTableComponent({ initialSortBy, variant = 'default', }: { - config: ChartConfigWithDateRange; + config: BuilderChartConfigWithDateRange; sourceId?: string; onRowDetailsClick?: (rowWhere: RowWhereResult) => void; highlightedLineId?: string; diff --git a/packages/app/src/components/DBSearchPageFilters.tsx b/packages/app/src/components/DBSearchPageFilters.tsx index 890d3e56..dc64d103 100644 --- a/packages/app/src/components/DBSearchPageFilters.tsx +++ b/packages/app/src/components/DBSearchPageFilters.tsx @@ -5,7 +5,7 @@ import { tcFromSource, } from '@hyperdx/common-utils/dist/core/metadata'; import { - ChartConfigWithDateRange, + BuilderChartConfigWithDateRange, SourceKind, } from '@hyperdx/common-utils/dist/types'; import { @@ -346,7 +346,7 @@ export type FilterGroupProps = { hasLoadedMore: boolean; isDefaultExpanded?: boolean; 'data-testid'?: string; - chartConfig: ChartConfigWithDateRange; + chartConfig: BuilderChartConfigWithDateRange; isLive?: boolean; onRangeChange?: (range: { min: number; max: number }) => void; distributionKey?: string; // Optional key to use for distribution queries, defaults to name @@ -848,7 +848,7 @@ const DBSearchPageFiltersComponent = ({ analysisMode: 'results' | 'delta' | 'pattern'; setAnalysisMode: (mode: 'results' | 'delta' | 'pattern') => void; isLive: boolean; - chartConfig: ChartConfigWithDateRange; + chartConfig: BuilderChartConfigWithDateRange; sourceId?: string; showDelta: boolean; denoiseResults: boolean; diff --git a/packages/app/src/components/DBSqlRowTableWithSidebar.tsx b/packages/app/src/components/DBSqlRowTableWithSidebar.tsx index bd24a643..89744266 100644 --- a/packages/app/src/components/DBSqlRowTableWithSidebar.tsx +++ b/packages/app/src/components/DBSqlRowTableWithSidebar.tsx @@ -2,7 +2,7 @@ import { useCallback, useState } from 'react'; import { useQueryState } from 'nuqs'; import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse'; import { - ChartConfigWithDateRange, + BuilderChartConfigWithDateRange, TSource, } from '@hyperdx/common-utils/dist/types'; import { SortingState } from '@tanstack/react-table'; @@ -25,7 +25,7 @@ import { DBRowTableVariant, DBSqlRowTable } from './DBRowTable'; interface Props { sourceId: string; - config: ChartConfigWithDateRange; + config: BuilderChartConfigWithDateRange; onError?: (error: Error | ClickHouseQueryError) => void; onScroll?: (scrollTop: number) => void; onSidebarOpen?: (rowId: string) => void; diff --git a/packages/app/src/components/DBTableChart.tsx b/packages/app/src/components/DBTableChart.tsx index d56f540e..53d6723a 100644 --- a/packages/app/src/components/DBTableChart.tsx +++ b/packages/app/src/components/DBTableChart.tsx @@ -1,5 +1,9 @@ import { useCallback, useMemo, useState } from 'react'; import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse'; +import { + isBuilderChartConfig, + isRawSqlChartConfig, +} from '@hyperdx/common-utils/dist/guards'; import { ChartConfigWithOptTimestamp } from '@hyperdx/common-utils/dist/types'; import { Box, Code, Text } from '@mantine/core'; import { SortingState } from '@tanstack/react-table'; @@ -48,7 +52,9 @@ export default function DBTableChart({ }) { const [sort, setSort] = useState([]); - const { data: source } = useSource({ id: config.source }); + const { data: source } = useSource({ + id: isBuilderChartConfig(config) ? config.source : undefined, + }); const effectiveSort = useMemo( () => controlledSort || sort, @@ -66,6 +72,8 @@ export default function DBTableChart({ ); const queriedConfig = useMemo(() => { + if (isRawSqlChartConfig(config)) return config; + const _config = convertToTableChartConfig(config); if (effectiveSort.length) { @@ -79,8 +87,9 @@ export default function DBTableChart({ return _config; }, [config, effectiveSort]); - const { data: mvOptimizationData } = - useMVOptimizationExplanation(queriedConfig); + const { data: mvOptimizationData } = useMVOptimizationExplanation( + isBuilderChartConfig(queriedConfig) ? queriedConfig : undefined, + ); const { data, fetchNextPage, hasNextPage, isLoading, isError, error } = useOffsetPaginatedQuery(queriedConfig, { @@ -91,6 +100,10 @@ export default function DBTableChart({ // Returns an array of aliases, so we can check if something is using an alias const aliasMap = useMemo(() => { + if (isRawSqlChartConfig(config)) { + return []; + } + // If the config.select is a string, we can't infer this. // One day, we could potentially run this through chSqlToAliasMap but AST parsing // doesn't work for most DBTableChart queries. @@ -103,7 +116,8 @@ export default function DBTableChart({ } return acc; }, [] as string[]); - }, [config.select]); + }, [config]); + const columns = useMemo(() => { const rows = data?.data ?? []; if (rows.length === 0) { @@ -111,7 +125,11 @@ export default function DBTableChart({ } let groupByKeys: string[] = []; - if (queriedConfig.groupBy && typeof queriedConfig.groupBy === 'string') { + if ( + isBuilderChartConfig(queriedConfig) && + queriedConfig.groupBy && + typeof queriedConfig.groupBy === 'string' + ) { groupByKeys = queriedConfig.groupBy.split(',').map(v => v.trim()); } @@ -126,13 +144,7 @@ export default function DBTableChart({ ? undefined : config.numberFormat, })); - }, [ - config.numberFormat, - aliasMap, - queriedConfig.groupBy, - data, - hiddenColumns, - ]); + }, [config.numberFormat, aliasMap, queriedConfig, data, hiddenColumns]); const toolbarItemsMemo = useMemo(() => { const allToolbarItems = []; @@ -141,7 +153,11 @@ export default function DBTableChart({ allToolbarItems.push(...toolbarPrefix); } - if (source && showMVOptimizationIndicator) { + if ( + source && + showMVOptimizationIndicator && + isBuilderChartConfig(queriedConfig) + ) { allToolbarItems.push( ( whereLanguage, }: { dateRange: DateRange['dateRange']; - extraSelects?: ChartConfigWithDateRange['select']; + extraSelects?: BuilderChartConfigWithDateRange['select']; limit?: number; logSource: TSource; order: 'asc' | 'desc'; diff --git a/packages/app/src/components/KubernetesFilters.tsx b/packages/app/src/components/KubernetesFilters.tsx index 521d0cc3..34900108 100644 --- a/packages/app/src/components/KubernetesFilters.tsx +++ b/packages/app/src/components/KubernetesFilters.tsx @@ -2,7 +2,7 @@ import React, { useEffect, useMemo, useRef, useState } from 'react'; import { useForm, useWatch } from 'react-hook-form'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; import { - ChartConfigWithDateRange, + BuilderChartConfigWithDateRange, TSource, } from '@hyperdx/common-utils/dist/types'; import { Box, Group, Select } from '@mantine/core'; @@ -23,7 +23,7 @@ type FilterSelectProps = { fieldName: string; value: string | null; onChange: (value: string | null) => void; - chartConfig: ChartConfigWithDateRange; + chartConfig: BuilderChartConfigWithDateRange; dataTestId?: string; }; @@ -157,7 +157,7 @@ export const KubernetesFilters: React.FC = ({ }, [searchQuery, metricSource.resourceAttributesExpression]); // Create chart config for fetching key values - const chartConfig: ChartConfigWithDateRange = { + const chartConfig: BuilderChartConfigWithDateRange = { from: { databaseName: metricSource.from.databaseName, tableName: metricSource.metricTables?.gauge || '', diff --git a/packages/app/src/components/MaterializedViews/MVOptimizationIndicator.tsx b/packages/app/src/components/MaterializedViews/MVOptimizationIndicator.tsx index 022d333b..dfb3beaa 100644 --- a/packages/app/src/components/MaterializedViews/MVOptimizationIndicator.tsx +++ b/packages/app/src/components/MaterializedViews/MVOptimizationIndicator.tsx @@ -1,6 +1,6 @@ import { useState } from 'react'; import { - ChartConfigWithOptDateRange, + BuilderChartConfigWithOptDateRange, TSource, } from '@hyperdx/common-utils/dist/types'; import { ActionIcon, Badge, Tooltip } from '@mantine/core'; @@ -59,7 +59,7 @@ export default function MVOptimizationIndicator({ variant = 'badge', }: { source: TSource; - config: ChartConfigWithOptDateRange | undefined; + config: BuilderChartConfigWithOptDateRange | undefined; variant?: 'badge' | 'icon'; }) { const [modalOpen, setModalOpen] = useState(false); diff --git a/packages/app/src/components/PatternTable.tsx b/packages/app/src/components/PatternTable.tsx index 1feadf63..7f9931d6 100644 --- a/packages/app/src/components/PatternTable.tsx +++ b/packages/app/src/components/PatternTable.tsx @@ -1,6 +1,6 @@ import { useMemo, useState } from 'react'; import { - ChartConfigWithDateRange, + BuilderChartConfigWithDateRange, TSource, } from '@hyperdx/common-utils/dist/types'; @@ -19,8 +19,8 @@ export default function PatternTable({ bodyValueExpression, source, }: { - config: ChartConfigWithDateRange; - totalCountConfig: ChartConfigWithDateRange; + config: BuilderChartConfigWithDateRange; + totalCountConfig: BuilderChartConfigWithDateRange; bodyValueExpression: string; totalCountQueryKeyPrefix: string; source?: TSource; diff --git a/packages/app/src/components/SQLEditor.tsx b/packages/app/src/components/SQLEditor.tsx index 54e9a616..2c06b133 100644 --- a/packages/app/src/components/SQLEditor.tsx +++ b/packages/app/src/components/SQLEditor.tsx @@ -1,13 +1,11 @@ -import { useCallback, useEffect, useMemo, useRef } from 'react'; +import { useRef } from 'react'; import { useController, UseControllerProps } from 'react-hook-form'; -import { acceptCompletion, startCompletion } from '@codemirror/autocomplete'; -import { sql, SQLDialect } from '@codemirror/lang-sql'; -import { Flex, Group, Paper, Text, useMantineColorScheme } from '@mantine/core'; +import { startCompletion } from '@codemirror/autocomplete'; +import { sql } from '@codemirror/lang-sql'; +import { Paper, useMantineColorScheme } from '@mantine/core'; import CodeMirror, { Compartment, EditorView, - keymap, - Prec, ReactCodeMirrorRef, } from '@uiw/react-codemirror'; @@ -15,6 +13,8 @@ type SQLInlineEditorProps = { value: string; onChange: (value: string) => void; placeholder?: string; + height?: string; + enableLineWrapping?: boolean; }; const styleTheme = EditorView.baseTheme({ @@ -33,6 +33,8 @@ export default function SQLEditor({ onChange, placeholder, value, + height, + enableLineWrapping = false, }: SQLInlineEditorProps) { const { colorScheme } = useMantineColorScheme(); const ref = useRef(null); @@ -57,6 +59,7 @@ export default function SQLEditor({ value={value} onChange={onChange} theme={colorScheme === 'dark' ? 'dark' : 'light'} + height={height} minHeight={'100px'} extensions={[ styleTheme, @@ -66,6 +69,7 @@ export default function SQLEditor({ upperCaseKeywords: true, }), ), + ...(enableLineWrapping ? [EditorView.lineWrapping] : []), ]} onUpdate={update => { // Always open completion window as much as possible @@ -92,6 +96,7 @@ export default function SQLEditor({ export function SQLEditorControlled({ placeholder, + height, ...props }: Omit & UseControllerProps) { const { field } = useController(props); @@ -101,6 +106,7 @@ export function SQLEditorControlled({ onChange={field.onChange} placeholder={placeholder} value={field.value} + height={height} {...props} /> ); diff --git a/packages/app/src/components/Search/DBSearchHeatmapChart.tsx b/packages/app/src/components/Search/DBSearchHeatmapChart.tsx index fea4425c..e027199d 100644 --- a/packages/app/src/components/Search/DBSearchHeatmapChart.tsx +++ b/packages/app/src/components/Search/DBSearchHeatmapChart.tsx @@ -8,7 +8,7 @@ import { tcFromSource, } from '@hyperdx/common-utils/dist/core/metadata'; import { - ChartConfigWithDateRange, + BuilderChartConfigWithDateRange, DisplayType, TSource, } from '@hyperdx/common-utils/dist/types'; @@ -34,7 +34,7 @@ export function DBSearchHeatmapChart({ source, isReady, }: { - chartConfig: ChartConfigWithDateRange; + chartConfig: BuilderChartConfigWithDateRange; source: TSource; isReady: boolean; }) { diff --git a/packages/app/src/components/SearchTotalCountChart.tsx b/packages/app/src/components/SearchTotalCountChart.tsx index f7604b7a..ce2fde71 100644 --- a/packages/app/src/components/SearchTotalCountChart.tsx +++ b/packages/app/src/components/SearchTotalCountChart.tsx @@ -4,7 +4,7 @@ import { JSDataType, ResponseJSON, } from '@hyperdx/common-utils/dist/clickhouse'; -import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import { Text } from '@mantine/core'; import { keepPreviousData } from '@tanstack/react-query'; @@ -25,7 +25,7 @@ function inferCountColumn(meta: ResponseJSON['meta'] | undefined): string { } export function useSearchTotalCount( - config: ChartConfigWithDateRange, + config: BuilderChartConfigWithDateRange, queryKeyPrefix: string, { disableQueryChunking, @@ -88,7 +88,7 @@ export default function SearchTotalCountChart({ disableQueryChunking, enableParallelQueries, }: { - config: ChartConfigWithDateRange; + config: BuilderChartConfigWithDateRange; queryKeyPrefix: string; disableQueryChunking?: boolean; enableParallelQueries?: boolean; diff --git a/packages/app/src/config.ts b/packages/app/src/config.ts index 055897d7..191d7117 100644 --- a/packages/app/src/config.ts +++ b/packages/app/src/config.ts @@ -37,3 +37,5 @@ export const IS_K8S_DASHBOARD_ENABLED = true; export const IS_METRICS_ENABLED = true; export const IS_MTVIEWS_ENABLED = false; export const IS_SESSIONS_ENABLED = true; +export const IS_SQL_CHARTS_ENABLED = + process.env.NEXT_PUBLIC_IS_SQL_CHARTS_ENABLED === 'true'; diff --git a/packages/app/src/defaults.ts b/packages/app/src/defaults.ts index 7b202101..0b37eac3 100644 --- a/packages/app/src/defaults.ts +++ b/packages/app/src/defaults.ts @@ -1,4 +1,4 @@ -import type { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import type { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; // Limit defaults export const DEFAULT_SEARCH_ROW_LIMIT = 200; @@ -6,7 +6,7 @@ export const DEFAULT_QUERY_TIMEOUT = 60; // max_execution_time, seconds export function searchChartConfigDefaults( team: any | undefined | null, -): Partial { +): Partial { return { limit: { limit: team?.searchRowLimit ?? DEFAULT_SEARCH_ROW_LIMIT, diff --git a/packages/app/src/hdxMTViews.ts b/packages/app/src/hdxMTViews.ts index 5b3d00f4..220770a6 100644 --- a/packages/app/src/hdxMTViews.ts +++ b/packages/app/src/hdxMTViews.ts @@ -13,7 +13,7 @@ import { } from '@hyperdx/common-utils/dist/core/renderChartConfig'; import { AggregateFunction, - ChartConfigWithOptDateRange, + BuilderChartConfigWithOptDateRange, DerivedColumn, QuerySettings, SQLInterval, @@ -58,7 +58,7 @@ const getAggFn = ( const buildMTViewDataTableDDL = ( table: string, - chartConfig: ChartConfigWithOptDateRange, + chartConfig: BuilderChartConfigWithOptDateRange, ) => { if (!Array.isArray(chartConfig.select)) { throw new Error('Only array select is supported'); @@ -96,7 +96,7 @@ const buildMTViewDDL = (name: string, table: string, query: ChSql) => { }; export const buildMTViewSelectQuery = async ( - chartConfig: ChartConfigWithOptDateRange, + chartConfig: BuilderChartConfigWithOptDateRange, metadata: Metadata, querySettings: QuerySettings | undefined, customGranularity?: SQLInterval, diff --git a/packages/app/src/hooks/__tests__/useChartConfig.test.tsx b/packages/app/src/hooks/__tests__/useChartConfig.test.tsx index a7bd7732..11b07c0a 100644 --- a/packages/app/src/hooks/__tests__/useChartConfig.test.tsx +++ b/packages/app/src/hooks/__tests__/useChartConfig.test.tsx @@ -1,6 +1,7 @@ import React from 'react'; import { ResponseJSON } from '@hyperdx/common-utils/dist/clickhouse'; import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/browser'; +import { isBuilderChartConfig } from '@hyperdx/common-utils/dist/guards'; import { ChartConfigWithDateRange, ChartConfigWithOptDateRange, @@ -1398,6 +1399,9 @@ describe('useChartConfig', () => { // Verify the query used the optimized config (materialized view) const queryCall = mockClickhouseClient.queryChartConfig.mock.calls[0][0]; + if (!isBuilderChartConfig(queryCall.config)) { + throw new Error('Expected a BuilderChartConfig'); + } expect(queryCall.config.from.tableName).toBe('metrics_rollup_1h'); expect(result2.current.data?.data).toBeDefined(); diff --git a/packages/app/src/hooks/__tests__/useMetadata.test.tsx b/packages/app/src/hooks/__tests__/useMetadata.test.tsx index 5ad7e902..6a55f3cd 100644 --- a/packages/app/src/hooks/__tests__/useMetadata.test.tsx +++ b/packages/app/src/hooks/__tests__/useMetadata.test.tsx @@ -5,7 +5,7 @@ import { Metadata, MetadataCache, } from '@hyperdx/common-utils/dist/core/metadata'; -import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import { renderHook, waitFor } from '@testing-library/react'; @@ -19,8 +19,8 @@ import { // Create a mock ChartConfig based on the Zod schema const createMockChartConfig = ( - overrides: Partial = {}, -): ChartConfigWithDateRange => + overrides: Partial = {}, +): BuilderChartConfigWithDateRange => ({ timestampValueExpression: '', connection: 'foo', @@ -29,7 +29,7 @@ const createMockChartConfig = ( tableName: 'traces', }, ...overrides, - }) as ChartConfigWithDateRange; + }) as BuilderChartConfigWithDateRange; jest.mock('@/source', () => ({ useSources: jest.fn().mockReturnValue({ diff --git a/packages/app/src/hooks/__tests__/useOffsetPaginatedQuery.test.tsx b/packages/app/src/hooks/__tests__/useOffsetPaginatedQuery.test.tsx index 157b2c37..947ef8e5 100644 --- a/packages/app/src/hooks/__tests__/useOffsetPaginatedQuery.test.tsx +++ b/packages/app/src/hooks/__tests__/useOffsetPaginatedQuery.test.tsx @@ -49,6 +49,7 @@ jest.mock('@hyperdx/common-utils/dist/core/renderChartConfig', () => ({ import { getClickhouseClient } from '@hyperdx/app/src/clickhouse'; import { MVOptimizationExplanation } from '@hyperdx/common-utils/dist/core/materializedViews'; import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig'; +import { isBuilderChartConfig } from '@hyperdx/common-utils/dist/guards'; import { MVOptimizationExplanationResult, @@ -763,6 +764,57 @@ describe('useOffsetPaginatedQuery', () => { }); }); + describe('RawSqlChartConfig', () => { + it('should execute raw SQL query without time windowing and not paginate', async () => { + const rawSqlConfig = { + configType: 'sql' as const, + sqlTemplate: 'SELECT status, count() FROM logs GROUP BY status', + connection: 'conn-1', + displayType: undefined, + dateRange: [ + new Date('2024-01-01T00:00:00Z'), + new Date('2024-01-02T00:00:00Z'), + ] as [Date, Date], + }; + + mockReader.read + .mockResolvedValueOnce({ + done: false, + value: [ + { json: () => ['status', 'count()'] }, + { json: () => ['String', 'UInt64'] }, + { json: () => ['error', 42] }, + { json: () => ['info', 100] }, + ], + }) + .mockResolvedValueOnce({ done: true }); + + const { result } = renderHook( + () => useOffsetPaginatedQuery(rawSqlConfig), + { wrapper }, + ); + + await waitFor(() => expect(result.current.isLoading).toBe(false)); + + // Raw SQL config should be passed through to renderChartConfig unchanged + expect(renderChartConfig).toHaveBeenCalledTimes(1); + expect(jest.mocked(renderChartConfig).mock.calls[0][0]).toMatchObject({ + configType: 'sql', + sqlTemplate: 'SELECT status, count() FROM logs GROUP BY status', + }); + + // Should have data + expect(result.current.data?.data).toHaveLength(2); + expect(result.current.data?.data[0]).toEqual({ + status: 'error', + 'count()': 42, + }); + + // Pagination is disabled for raw SQL + expect(result.current.hasNextPage).toBe(false); + }); + }); + describe('MV Optimization Integration', () => { it('should optimize queries using MVs when possible', async () => { const config = createMockChartConfig({ @@ -826,7 +878,9 @@ describe('useOffsetPaginatedQuery', () => { jest .mocked(renderChartConfig) .mock.calls.every( - call => call[0].from.tableName === 'metrics_rollup_1m', + call => + isBuilderChartConfig(call[0]) && + call[0].from.tableName === 'metrics_rollup_1m', ), ).toBeTruthy(); @@ -919,7 +973,9 @@ describe('useOffsetPaginatedQuery', () => { jest .mocked(renderChartConfig) .mock.calls.every( - call => call[0].from.tableName === 'metrics_rollup_1m', + call => + isBuilderChartConfig(call[0]) && + call[0].from.tableName === 'metrics_rollup_1m', ), ).toBeTruthy(); diff --git a/packages/app/src/hooks/useAutoCompleteOptions.tsx b/packages/app/src/hooks/useAutoCompleteOptions.tsx index ed4f6688..2973fa9e 100644 --- a/packages/app/src/hooks/useAutoCompleteOptions.tsx +++ b/packages/app/src/hooks/useAutoCompleteOptions.tsx @@ -3,7 +3,7 @@ import { Field, TableConnection, } from '@hyperdx/common-utils/dist/core/metadata'; -import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import { deduplicate2dArray, @@ -103,21 +103,21 @@ export function useAutoCompleteOptions( ); // hooks to get key values - const chartConfigs: ChartConfigWithDateRange[] = toArray(tableConnection).map( - ({ databaseName, tableName, connectionId }) => ({ - connection: connectionId, - from: { - databaseName, - tableName, - }, - timestampValueExpression: '', - select: '', - where: '', - // TODO: Pull in date for query as arg - // just assuming 1/2 day is okay to query over right now - dateRange: [new Date(NOW - (86400 * 1000) / 2), new Date(NOW)], - }), - ); + const chartConfigs: BuilderChartConfigWithDateRange[] = toArray( + tableConnection, + ).map(({ databaseName, tableName, connectionId }) => ({ + connection: connectionId, + from: { + databaseName, + tableName, + }, + timestampValueExpression: '', + select: '', + where: '', + // TODO: Pull in date for query as arg + // just assuming 1/2 day is okay to query over right now + dateRange: [new Date(NOW - (86400 * 1000) / 2), new Date(NOW)], + })); const { data: keyVals } = useMultipleGetKeyValues({ chartConfigs, keys: searchKeys, diff --git a/packages/app/src/hooks/useChartConfig.tsx b/packages/app/src/hooks/useChartConfig.tsx index a0d9838b..5e8e9aa8 100644 --- a/packages/app/src/hooks/useChartConfig.tsx +++ b/packages/app/src/hooks/useChartConfig.tsx @@ -12,8 +12,13 @@ import { renderChartConfig, } from '@hyperdx/common-utils/dist/core/renderChartConfig'; import { convertDateRangeToGranularityString } from '@hyperdx/common-utils/dist/core/utils'; +import { + isBuilderChartConfig, + isRawSqlChartConfig, +} from '@hyperdx/common-utils/dist/guards'; import { format } from '@hyperdx/common-utils/dist/sqlFormatter'; import { + BuilderChartConfigWithOptDateRange, ChartConfigWithDateRange, ChartConfigWithOptDateRange, QuerySettings, @@ -65,6 +70,9 @@ const shouldUseChunking = ( ): config is ChartConfigWithDateRange & { granularity: string; } => { + // Avoid chunking for raw SQL charts since they can include arbitrary window functions, etc. + if (isRawSqlChartConfig(config)) return false; + // Granularity is required for chunking, otherwise we could break other group-bys. if (!isUsingGranularity(config)) return false; @@ -143,7 +151,7 @@ async function* fetchDataInChunks({ ? getGranularityAlignedTimeWindows(config) : [undefined]; - if (IS_MTVIEWS_ENABLED) { + if (IS_MTVIEWS_ENABLED && isBuilderChartConfig(config)) { const { dataTableDDL, mtViewDDL, renderMTViewConfig } = await buildMTViewSelectQuery(config, metadata, querySettings); // TODO: show the DDLs in the UI so users can run commands manually @@ -154,6 +162,11 @@ async function* fetchDataInChunks({ await renderMTViewConfig(); } + // Readonly = 2 means the query is readonly but can still specify query settings. + const clickHouseSettings = isRawSqlChartConfig(config) + ? { readonly: '2' } + : {}; + if (enableParallelQueries) { // fetch in parallel const promises = windows.map(async (w, index) => { @@ -168,6 +181,7 @@ async function* fetchDataInChunks({ metadata, opts: { abort_signal: signal, + clickhouse_settings: clickHouseSettings, }, querySettings, }), @@ -258,14 +272,15 @@ export function useQueriedChartConfig( const queryClient = useQueryClient(); const metadata = useMetadataWithSettings(); + const builderConfig = isBuilderChartConfig(config) ? config : undefined; const { data: mvOptimizationData, isLoading: isLoadingMVOptimization } = - useMVOptimizationExplanation(config, { - enabled: !!enabled, + useMVOptimizationExplanation(builderConfig, { + enabled: !!enabled && !!builderConfig, placeholderData: undefined, }); const { data: source, isLoading: isSourceLoading } = useSource({ - id: config.source, + id: builderConfig?.source, }); const query = useQuery({ @@ -351,14 +366,15 @@ export function useRenderedSqlChartConfig( const metadata = useMetadataWithSettings(); + const builderConfig = isBuilderChartConfig(config) ? config : undefined; const { data: mvOptimizationData, isLoading: isLoadingMVOptimization } = - useMVOptimizationExplanation(config, { - enabled: !!enabled, + useMVOptimizationExplanation(builderConfig, { + enabled: !!enabled && !!builderConfig, placeholderData: undefined, }); const { data: source, isLoading: isSourceLoading } = useSource({ - id: config.source, + id: builderConfig?.source, }); const query = useQuery({ @@ -383,7 +399,7 @@ export function useRenderedSqlChartConfig( } export function useAliasMapFromChartConfig( - config: ChartConfigWithOptDateRange | undefined, + config: BuilderChartConfigWithOptDateRange | undefined, options?: UseQueryOptions>, ) { // For granularity: 'auto', the bucket size depends on dateRange duration (not absolute times). diff --git a/packages/app/src/hooks/useDashboardFilterValues.tsx b/packages/app/src/hooks/useDashboardFilterValues.tsx index 3464f542..ddddcb66 100644 --- a/packages/app/src/hooks/useDashboardFilterValues.tsx +++ b/packages/app/src/hooks/useDashboardFilterValues.tsx @@ -5,7 +5,7 @@ import { optimizeGetKeyValuesCalls, } from '@hyperdx/common-utils/dist/core/materializedViews'; import { - ChartConfigWithDateRange, + BuilderChartConfigWithDateRange, DashboardFilter, } from '@hyperdx/common-utils/dist/types'; import { @@ -57,53 +57,54 @@ function useOptimizedKeyValuesCalls({ return filtersBySourceIdAndMetric; }, [filters]); - const results: UseQueryResult[]>[] = - useQueries({ - queries: Array.from(filtersBySourceIdAndMetric.entries()) - .filter(([key]) => - sources?.some(s => s.id === filterFromKey(key).sourceId), - ) - .map(([key, filters]) => { - const { sourceId, metricType } = filterFromKey(key); - const source = sources!.find(s => s.id === sourceId)!; - const keys = filters.map(f => f.expression); - const tableName = getMetricTableName(source, metricType) ?? ''; + const results: UseQueryResult< + GetKeyValueCall[] + >[] = useQueries({ + queries: Array.from(filtersBySourceIdAndMetric.entries()) + .filter(([key]) => + sources?.some(s => s.id === filterFromKey(key).sourceId), + ) + .map(([key, filters]) => { + const { sourceId, metricType } = filterFromKey(key); + const source = sources!.find(s => s.id === sourceId)!; + const keys = filters.map(f => f.expression); + const tableName = getMetricTableName(source, metricType) ?? ''; - const chartConfig: ChartConfigWithDateRange = { - ...pick(source, ['timestampValueExpression', 'connection']), - from: { - databaseName: source.from.databaseName, - tableName, - }, + const chartConfig: BuilderChartConfigWithDateRange = { + ...pick(source, ['timestampValueExpression', 'connection']), + from: { + databaseName: source.from.databaseName, + tableName, + }, + dateRange, + source: source.id, + where: '', + whereLanguage: 'sql', + select: '', + }; + + return { + queryKey: [ + 'dashboard-filters-key-value-calls', + sourceId, + metricType, dateRange, - source: source.id, - where: '', - whereLanguage: 'sql', - select: '', - }; - - return { - queryKey: [ - 'dashboard-filters-key-value-calls', - sourceId, - metricType, - dateRange, + keys, + ], + enabled: !isLoadingSources, + staleTime: 1000 * 60 * 5, // Cache every 5 min + queryFn: async ({ signal }) => + await optimizeGetKeyValuesCalls({ + chartConfig, + source, + clickhouseClient, + metadata, keys, - ], - enabled: !isLoadingSources, - staleTime: 1000 * 60 * 5, // Cache every 5 min - queryFn: async ({ signal }) => - await optimizeGetKeyValuesCalls({ - chartConfig, - source, - clickhouseClient, - metadata, - keys, - signal, - }), - }; - }), - }); + signal, + }), + }; + }), + }); return { data: results.map(r => r.data ?? []).flat(), diff --git a/packages/app/src/hooks/useExplainQuery.tsx b/packages/app/src/hooks/useExplainQuery.tsx index 81e0c181..1dfe8e3a 100644 --- a/packages/app/src/hooks/useExplainQuery.tsx +++ b/packages/app/src/hooks/useExplainQuery.tsx @@ -1,4 +1,5 @@ import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig'; +import { isBuilderChartConfig } from '@hyperdx/common-utils/dist/guards'; import { ChartConfigWithOptDateRange } from '@hyperdx/common-utils/dist/types'; import { useQuery, UseQueryOptions } from '@tanstack/react-query'; @@ -20,7 +21,7 @@ export function useExplainQuery( const metadata = useMetadataWithSettings(); const { data: source, isLoading: isSourceLoading } = useSource({ - id: config?.source, + id: isBuilderChartConfig(config) ? config.source : undefined, }); return useQuery({ diff --git a/packages/app/src/hooks/useMVOptimizationExplanation.tsx b/packages/app/src/hooks/useMVOptimizationExplanation.tsx index 311de77f..a5c2455e 100644 --- a/packages/app/src/hooks/useMVOptimizationExplanation.tsx +++ b/packages/app/src/hooks/useMVOptimizationExplanation.tsx @@ -2,7 +2,7 @@ import { MVOptimizationExplanation, tryOptimizeConfigWithMaterializedViewWithExplanations, } from '@hyperdx/common-utils/dist/core/materializedViews'; -import { ChartConfigWithOptDateRange } from '@hyperdx/common-utils/dist/types'; +import { BuilderChartConfigWithOptDateRange } from '@hyperdx/common-utils/dist/types'; import { keepPreviousData, useQuery, @@ -15,14 +15,15 @@ import { useSource } from '@/source'; import { useMetadataWithSettings } from './useMetadata'; export interface MVOptimizationExplanationResult< - C extends ChartConfigWithOptDateRange = ChartConfigWithOptDateRange, + C extends + BuilderChartConfigWithOptDateRange = BuilderChartConfigWithOptDateRange, > { optimizedConfig?: C; explanations: MVOptimizationExplanation[]; } export function useMVOptimizationExplanation< - C extends ChartConfigWithOptDateRange, + C extends BuilderChartConfigWithOptDateRange, >( config: C | undefined, options?: Partial>>, diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx index 05e253a9..11e004b6 100644 --- a/packages/app/src/hooks/useMetadata.tsx +++ b/packages/app/src/hooks/useMetadata.tsx @@ -10,10 +10,7 @@ import { TableConnection, TableMetadata, } from '@hyperdx/common-utils/dist/core/metadata'; -import { - ChartConfigWithDateRange, - TSource, -} from '@hyperdx/common-utils/dist/types'; +import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import { keepPreviousData, useQuery, @@ -204,7 +201,9 @@ export function useMultipleGetKeyValues( limit, disableRowLimit, }: { - chartConfigs: ChartConfigWithDateRange | ChartConfigWithDateRange[]; + chartConfigs: + | BuilderChartConfigWithDateRange + | BuilderChartConfigWithDateRange[]; keys: string[]; limit?: number; disableRowLimit?: boolean; @@ -261,7 +260,7 @@ export function useGetValuesDistribution( key, limit, }: { - chartConfig: ChartConfigWithDateRange; + chartConfig: BuilderChartConfigWithDateRange; key: string; limit: number; }, @@ -297,7 +296,7 @@ export function useGetKeyValues( limit, disableRowLimit, }: { - chartConfig?: ChartConfigWithDateRange; + chartConfig?: BuilderChartConfigWithDateRange; keys: string[]; limit?: number; disableRowLimit?: boolean; diff --git a/packages/app/src/hooks/useOffsetPaginatedQuery.tsx b/packages/app/src/hooks/useOffsetPaginatedQuery.tsx index 217ad509..ea042862 100644 --- a/packages/app/src/hooks/useOffsetPaginatedQuery.tsx +++ b/packages/app/src/hooks/useOffsetPaginatedQuery.tsx @@ -12,6 +12,10 @@ import { isFirstOrderByAscending, isTimestampExpressionInFirstOrderBy, } from '@hyperdx/common-utils/dist/core/utils'; +import { + isBuilderChartConfig, + isRawSqlChartConfig, +} from '@hyperdx/common-utils/dist/guards'; import { ChartConfigWithOptTimestamp, TSource, @@ -72,6 +76,7 @@ type QueryMeta = { metadata: Metadata; optimizedConfig?: ChartConfigWithOptTimestamp; source: TSource | undefined; + readonly: boolean; }; // Get time window from page param @@ -80,9 +85,10 @@ function getTimeWindowFromPageParam( pageParam: TPageParam, ): TimeWindow { const [startDate, endDate] = config.dateRange; - const windows = isFirstOrderByAscending(config.orderBy) - ? generateTimeWindowsAscending(startDate, endDate) - : generateTimeWindowsDescending(startDate, endDate); + const windows = + isBuilderChartConfig(config) && isFirstOrderByAscending(config.orderBy) + ? generateTimeWindowsAscending(startDate, endDate) + : generateTimeWindowsDescending(startDate, endDate); const window = windows[pageParam.windowIndex]; if (window == null) { throw new Error('Invalid time window for page param'); @@ -96,7 +102,8 @@ function getNextPageParam( allPages: TQueryFnData[], config: ChartConfigWithOptTimestamp, ): TPageParam | undefined { - if (lastPage == null) { + // Pagination is not supported for raw SQL tables since they may not be ordered at all. + if (lastPage == null || isRawSqlChartConfig(config)) { return undefined; } @@ -124,7 +131,8 @@ function getNextPageParam( } // If no more results in current window, move to next window (if windowing is being used) - const shouldUseWindowing = isTimestampExpressionInFirstOrderBy(config); + const shouldUseWindowing = + isBuilderChartConfig(config) && isTimestampExpressionInFirstOrderBy(config); const nextWindowIndex = currentWindow.windowIndex + 1; if (shouldUseWindowing && nextWindowIndex < windows.length) { return { @@ -146,8 +154,14 @@ const queryFn: QueryFunction = async ({ throw new Error('Query missing client meta'); } - const { queryClient, metadata, hasPreviousQueries, optimizedConfig, source } = - meta as QueryMeta; + const { + queryClient, + metadata, + hasPreviousQueries, + optimizedConfig, + source, + readonly, + } = meta as QueryMeta; // Only stream incrementally if this is a fresh query with no previous // response or if it's a paginated query @@ -162,7 +176,8 @@ const queryFn: QueryFunction = async ({ const config = optimizedConfig ?? rawConfig; // Get the time window for this page - const shouldUseWindowing = isTimestampExpressionInFirstOrderBy(config); + const shouldUseWindowing = + isBuilderChartConfig(config) && isTimestampExpressionInFirstOrderBy(config); const timeWindow = shouldUseWindowing ? getTimeWindowFromPageParam(config, pageParam) : { @@ -173,14 +188,16 @@ const queryFn: QueryFunction = async ({ }; // Create config with windowed date range - const windowedConfig = { - ...config, - dateRange: [timeWindow.startTime, timeWindow.endTime] as [Date, Date], - limit: { - limit: config.limit?.limit, - offset: pageParam.offset, - }, - }; + const windowedConfig = isBuilderChartConfig(config) + ? { + ...config, + dateRange: [timeWindow.startTime, timeWindow.endTime] as [Date, Date], + limit: { + limit: config.limit?.limit, + offset: pageParam.offset, + }, + } + : config; const query = await renderChartConfig( windowedConfig, @@ -194,6 +211,8 @@ const queryFn: QueryFunction = async ({ setTimeout(() => abortController.abort(), queryTimeout * 1000); } + // Readonly = 2 means the query is readonly but can still specify query settings. + const clickHouseSettings = readonly ? { readonly: '2' } : {}; const resultSet = await clickhouseClient.query<'JSONCompactEachRowWithNamesAndTypes'>({ query: query.sql, @@ -201,6 +220,7 @@ const queryFn: QueryFunction = async ({ format: 'JSONCompactEachRowWithNamesAndTypes', abort_signal: abortController?.signal || signal, connectionId: config.connection, + clickhouse_settings: clickHouseSettings, }); const stream = resultSet.stream(); @@ -402,14 +422,15 @@ export default function useOffsetPaginatedQuery( const hasPreviousQueries = matchedQueries.filter(([_, data]) => data != null).length > 0; + const builderConfig = isBuilderChartConfig(config) ? config : undefined; const { data: mvOptimizationData, isLoading: isLoadingMVOptimization } = - useMVOptimizationExplanation(config, { - enabled: !!enabled, + useMVOptimizationExplanation(builderConfig, { + enabled: !!enabled && !!builderConfig, placeholderData: undefined, }); const { data: source, isLoading: isSourceLoading } = useSource({ - id: config?.source, + id: builderConfig?.source, }); const { @@ -445,6 +466,8 @@ export default function useOffsetPaginatedQuery( metadata, optimizedConfig: mvOptimizationData?.optimizedConfig, source, + // Additional readonly protection when the user is running a raw SQL query + readonly: isRawSqlChartConfig(config), } satisfies QueryMeta, queryFn, gcTime: isLive ? ms('30s') : ms('5m'), // more aggressive gc for live data, since it can end up holding lots of data diff --git a/packages/app/src/hooks/usePatterns.tsx b/packages/app/src/hooks/usePatterns.tsx index 10647074..03c53d0d 100644 --- a/packages/app/src/hooks/usePatterns.tsx +++ b/packages/app/src/hooks/usePatterns.tsx @@ -1,14 +1,11 @@ import { useMemo } from 'react'; import stripAnsi from 'strip-ansi'; import { convertDateRangeToGranularityString } from '@hyperdx/common-utils/dist/core/utils'; -import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import { useQuery } from '@tanstack/react-query'; import { timeBucketByGranularity, toStartOfInterval } from '@/ChartUtils'; -import { - selectColumnMapWithoutAdditionalKeys, - useConfigWithPrimaryAndPartitionKey, -} from '@/components/DBRowTable'; +import { useConfigWithPrimaryAndPartitionKey } from '@/components/DBRowTable'; import { useQueriedChartConfig } from '@/hooks/useChartConfig'; import { getFirstTimestampValueExpression } from '@/source'; @@ -130,7 +127,7 @@ function usePatterns({ statusCodeExpression, enabled = true, }: { - config: ChartConfigWithDateRange; + config: BuilderChartConfigWithDateRange; samples: number; bodyValueExpression: string; severityTextExpression?: string; @@ -220,7 +217,7 @@ export function useGroupedPatterns({ totalCount, enabled = true, }: { - config: ChartConfigWithDateRange; + config: BuilderChartConfigWithDateRange; samples: number; bodyValueExpression: string; severityTextExpression?: string; diff --git a/packages/app/src/hooks/useRowWhere.tsx b/packages/app/src/hooks/useRowWhere.tsx index a2bf2aba..cc2bb810 100644 --- a/packages/app/src/hooks/useRowWhere.tsx +++ b/packages/app/src/hooks/useRowWhere.tsx @@ -7,12 +7,12 @@ import { JSDataType, } from '@hyperdx/common-utils/dist/clickhouse'; import { aliasMapToWithClauses } from '@hyperdx/common-utils/dist/core/utils'; -import { ChartConfig } from '@hyperdx/common-utils/dist/types'; +import { BuilderChartConfig } from '@hyperdx/common-utils/dist/types'; const MAX_STRING_LENGTH = 512; // Type for WITH clause entries, derived from ChartConfig's with property -export type WithClause = NonNullable[number]; +export type WithClause = NonNullable[number]; // Internal row field names used by the table component for row tracking export const INTERNAL_ROW_FIELDS = { diff --git a/packages/app/src/types.ts b/packages/app/src/types.ts index 77deb603..16194ac9 100644 --- a/packages/app/src/types.ts +++ b/packages/app/src/types.ts @@ -2,7 +2,7 @@ import { z } from 'zod'; import { Alert, AlertHistory, - ChartConfig, + BuilderChartConfig, DashboardSchema, Filter, NumberFormat as _NumberFormat, @@ -63,8 +63,8 @@ export type SavedSearchWithEnhancedAlerts = Omit & { export type SearchConfig = { select?: string | null; source?: string | null; - where?: ChartConfig['where'] | null; - whereLanguage?: ChartConfig['whereLanguage'] | null; + where?: BuilderChartConfig['where'] | null; + whereLanguage?: BuilderChartConfig['whereLanguage'] | null; filters?: Filter[] | null; orderBy?: string | null; }; diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index a5f2f9fd..19d8eea0 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1,7 +1,8 @@ import { ClickhouseClient } from '../clickhouse/node'; import { Metadata, MetadataCache } from '../core/metadata'; import * as renderChartConfigModule from '../core/renderChartConfig'; -import { ChartConfigWithDateRange, TSource } from '../types'; +import { isBuilderChartConfig } from '../guards'; +import { BuilderChartConfigWithDateRange, TSource } from '../types'; // Mock ClickhouseClient const mockClickhouseClient = { @@ -232,7 +233,7 @@ describe('Metadata', () => { }); describe('getKeyValues', () => { - const mockChartConfig: ChartConfigWithDateRange = { + const mockChartConfig: BuilderChartConfigWithDateRange = { from: { databaseName: 'test_db', tableName: 'test_table', @@ -372,7 +373,7 @@ describe('Metadata', () => { }); describe('getValuesDistribution', () => { - const mockChartConfig: ChartConfigWithDateRange = { + const mockChartConfig: BuilderChartConfigWithDateRange = { from: { databaseName: 'test_db', tableName: 'test_table', @@ -462,6 +463,8 @@ describe('Metadata', () => { }); const actualConfig = renderChartConfigSpy.mock.calls[0][0]; + if (!isBuilderChartConfig(actualConfig)) + throw new Error('Expected builder config'); expect(actualConfig.with).toContainEqual({ name: 'service', sql: { @@ -480,7 +483,7 @@ describe('Metadata', () => { }); it('should include filters from the config in the query', async () => { - const configWithFilters: ChartConfigWithDateRange = { + const configWithFilters: BuilderChartConfigWithDateRange = { ...mockChartConfig, filters: [ { @@ -502,6 +505,8 @@ describe('Metadata', () => { }); const actualConfig = renderChartConfigSpy.mock.calls[0][0]; + if (!isBuilderChartConfig(actualConfig)) + throw new Error('Expected builder config'); expect(actualConfig.filters).toContainEqual({ type: 'sql', condition: "ServiceName IN ('clickhouse')", diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index fe6bf728..06af9605 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -1427,4 +1427,21 @@ describe('renderChartConfig', () => { expect(actual).toMatchSnapshot(); }); }); + + it('returns sqlTemplate verbatim for raw sql config', async () => { + const rawSqlConfig: ChartConfigWithOptDateRangeEx = { + configType: 'sql', + sqlTemplate: 'SELECT count() FROM logs WHERE level = {level:String}', + connection: 'conn-1', + }; + const result = await renderChartConfig( + rawSqlConfig, + mockMetadata, + undefined, + ); + expect(result.sql).toBe( + 'SELECT count() FROM logs WHERE level = {level:String}', + ); + expect(result.params).toEqual({}); + }); }); diff --git a/packages/common-utils/src/__tests__/utils.test.ts b/packages/common-utils/src/__tests__/utils.test.ts index 16925394..07eef790 100644 --- a/packages/common-utils/src/__tests__/utils.test.ts +++ b/packages/common-utils/src/__tests__/utils.test.ts @@ -1,7 +1,8 @@ import { z } from 'zod'; +import { isBuilderSavedChartConfig } from '@/guards'; import { - ChartConfigWithDateRange, + BuilderChartConfigWithDateRange, DashboardSchema, MetricsDataType, SourceKind, @@ -272,7 +273,10 @@ describe('utils', () => { }); it('should return the first column name for an array of objects input', () => { - const orderBy: Exclude = [ + const orderBy: Exclude< + BuilderChartConfigWithDateRange['orderBy'], + string + > = [ { valueExpression: 'column1', ordering: 'ASC' }, { valueExpression: 'column2', ordering: 'ASC' }, ]; @@ -288,7 +292,7 @@ describe('utils', () => { const config = { timestampValueExpression: 'Timestamp', orderBy: undefined, - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; expect(isTimestampExpressionInFirstOrderBy(config)).toBe(false); }); @@ -297,7 +301,7 @@ describe('utils', () => { const config = { timestampValueExpression: 'Timestamp', orderBy: '', - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; expect(isTimestampExpressionInFirstOrderBy(config)).toBe(false); }); @@ -306,7 +310,7 @@ describe('utils', () => { const config = { timestampValueExpression: 'Timestamp', orderBy: 'ServiceName', - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; expect(isTimestampExpressionInFirstOrderBy(config)).toBe(false); }); @@ -315,7 +319,7 @@ describe('utils', () => { const config = { timestampValueExpression: 'Timestamp', orderBy: 'ServiceName ASC, Timestamp', - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; expect(isTimestampExpressionInFirstOrderBy(config)).toBe(false); }); @@ -324,7 +328,7 @@ describe('utils', () => { const config = { timestampValueExpression: 'Timestamp', orderBy: 'Timestamp', - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; expect(isTimestampExpressionInFirstOrderBy(config)).toBe(true); }); @@ -333,7 +337,7 @@ describe('utils', () => { const config = { timestampValueExpression: 'Timestamp', orderBy: 'Timestamp DESC, ServiceName', - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; expect(isTimestampExpressionInFirstOrderBy(config)).toBe(true); }); @@ -342,7 +346,7 @@ describe('utils', () => { const config = { timestampValueExpression: 'Timestamp', orderBy: 'Timestamp desc, ServiceName', - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; expect(isTimestampExpressionInFirstOrderBy(config)).toBe(true); }); @@ -354,7 +358,7 @@ describe('utils', () => { { valueExpression: 'Timestamp', ordering: 'ASC' }, { valueExpression: 'ServiceName', ordering: 'ASC' }, ], - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; expect(isTimestampExpressionInFirstOrderBy(config)).toBe(true); }); @@ -363,7 +367,7 @@ describe('utils', () => { const config = { timestampValueExpression: 'toStartOfDay(Timestamp), Timestamp', orderBy: '(toStartOfDay(Timestamp)) DESC, Timestamp', - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; expect(isTimestampExpressionInFirstOrderBy(config)).toBe(true); }); @@ -372,7 +376,7 @@ describe('utils', () => { const config = { timestampValueExpression: 'toStartOfDay(Timestamp), Timestamp', orderBy: '(toStartOfHour(TimestampTime), TimestampTime) DESC', - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; expect(isTimestampExpressionInFirstOrderBy(config)).toBe(true); }); @@ -382,7 +386,7 @@ describe('utils', () => { timestampValueExpression: 'toStartOfInterval(TimestampTime, INTERVAL 1 DAY)', orderBy: 'toStartOfInterval(TimestampTime, INTERVAL 1 DAY) DESC', - } as ChartConfigWithDateRange; + } as BuilderChartConfigWithDateRange; expect(isTimestampExpressionInFirstOrderBy(config)).toBe(true); }); @@ -412,7 +416,10 @@ describe('utils', () => { }); it('should return true for ascending order in object input', () => { - const orderBy: Exclude = [ + const orderBy: Exclude< + BuilderChartConfigWithDateRange['orderBy'], + string + > = [ { valueExpression: 'column1', ordering: 'ASC' }, { valueExpression: 'column2', ordering: 'DESC' }, ]; @@ -420,7 +427,10 @@ describe('utils', () => { }); it('should return false for descending order in object input', () => { - const orderBy: Exclude = [ + const orderBy: Exclude< + BuilderChartConfigWithDateRange['orderBy'], + string + > = [ { valueExpression: 'column1', ordering: 'DESC' }, { valueExpression: 'column2', ordering: 'ASC' }, ]; @@ -731,7 +741,10 @@ describe('utils', () => { ]; const template = convertToDashboardTemplate(dashboard, sources); - const selectList = template.tiles[0].config.select; + const tileConfig = template.tiles[0].config; + if (!isBuilderSavedChartConfig(tileConfig)) + throw new Error('Expected builder config'); + const selectList = tileConfig.select; expect(Array.isArray(selectList)).toBe(true); expect((selectList as any[])[0]).toMatchObject({ aggFn: 'quantile', diff --git a/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts b/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts index 574b020e..696b494a 100644 --- a/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts +++ b/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts @@ -6,10 +6,10 @@ import { tryOptimizeConfigWithMaterializedViewWithExplanations, } from '@/core/materializedViews'; import { Metadata } from '@/core/metadata'; +import { isBuilderChartConfig } from '@/guards'; import { ChartConfigWithOptDateRange, MaterializedViewConfiguration, - QuerySettings, TSource, } from '@/types'; @@ -1713,7 +1713,10 @@ describe('materializedViews', () => { it('should optimize a config with the MV that will scan the fewest rows, if multiple MVs could be used', async () => { mockClickHouseClient.testChartConfigValidity.mockImplementation( ({ config }) => { - if (config.from.tableName === 'db_statement_rollup_1s') { + if ( + isBuilderChartConfig(config) && + config.from.tableName === 'db_statement_rollup_1s' + ) { return Promise.resolve({ isValid: true, rowEstimate: 1000, @@ -1964,7 +1967,10 @@ describe('materializedViews', () => { Promise.resolve({ isValid: true, rowEstimate: - config.from.tableName === 'logs_rollup_1h' ? 500 : 1000, + isBuilderChartConfig(config) && + config.from.tableName === 'logs_rollup_1h' + ? 500 + : 1000, }), ); @@ -2124,7 +2130,10 @@ describe('materializedViews', () => { Promise.resolve({ isValid: true, rowEstimate: - config.from.tableName === 'logs_rollup_1h' ? 500 : 1000, + isBuilderChartConfig(config) && + config.from.tableName === 'logs_rollup_1h' + ? 500 + : 1000, }), ); diff --git a/packages/common-utils/src/clickhouse/index.ts b/packages/common-utils/src/clickhouse/index.ts index cc3fa209..9e30f6f2 100644 --- a/packages/common-utils/src/clickhouse/index.ts +++ b/packages/common-utils/src/clickhouse/index.ts @@ -23,6 +23,7 @@ import { replaceJsonExpressions, splitAndTrimWithBracket, } from '@/core/utils'; +import { isBuilderChartConfig } from '@/guards'; import { ChartConfigWithOptDateRange, QuerySettings } from '@/types'; // export @clickhouse/client-common types @@ -625,7 +626,9 @@ export abstract class BaseClickhouseClient { }; querySettings: QuerySettings | undefined; }): Promise>> { - config = setChartSelectsAlias(config); + config = isBuilderChartConfig(config) + ? setChartSelectsAlias(config) + : config; const queries: ChSql[] = await Promise.all( splitChartConfigs(config).map(c => renderChartConfig(c, metadata, querySettings), @@ -652,7 +655,7 @@ export abstract class BaseClickhouseClient { return resultSets[0]; } // metrics -> join resultSets - else if (resultSets.length > 1) { + else if (isBuilderChartConfig(config) && resultSets.length > 1) { const metaSet = new Map(); const tsBucketMap = new Map>(); for (const resultSet of resultSets) { diff --git a/packages/common-utils/src/core/histogram.ts b/packages/common-utils/src/core/histogram.ts index b64c2577..d84453e8 100644 --- a/packages/common-utils/src/core/histogram.ts +++ b/packages/common-utils/src/core/histogram.ts @@ -1,14 +1,14 @@ import { ChSql, chSql } from '@/clickhouse'; -import { ChartConfig } from '@/types'; +import { BuilderChartConfig } from '@/types'; -type WithClauses = ChartConfig['with']; +type WithClauses = BuilderChartConfig['with']; type TemplatedInput = ChSql | string; export const translateHistogram = ({ select, ...rest }: { - select: Exclude[number]; + select: Exclude[number]; timeBucketSelect: TemplatedInput; groupBy?: TemplatedInput; from: TemplatedInput; diff --git a/packages/common-utils/src/core/materializedViews.ts b/packages/common-utils/src/core/materializedViews.ts index 6f16fb40..38ad542f 100644 --- a/packages/common-utils/src/core/materializedViews.ts +++ b/packages/common-utils/src/core/materializedViews.ts @@ -2,7 +2,7 @@ import { differenceInSeconds } from 'date-fns'; import { BaseClickhouseClient } from '@/clickhouse'; import { - ChartConfigWithOptDateRange, + BuilderChartConfigWithOptDateRange, CteChartConfig, InternalAggregateFunction, InternalAggregateFunctionSchema, @@ -19,7 +19,7 @@ import { } from './utils'; type SelectItem = Exclude< - ChartConfigWithOptDateRange['select'], + BuilderChartConfigWithOptDateRange['select'], string >[number]; @@ -123,7 +123,7 @@ function getAggregatedColumnConfig( **/ function mvConfigSupportsGranularity( mvConfig: MaterializedViewConfiguration, - chartConfig: ChartConfigWithOptDateRange, + chartConfig: BuilderChartConfigWithOptDateRange, ): boolean { if (!chartConfig.granularity && !chartConfig.dateRange) { return true; @@ -171,7 +171,7 @@ function countIntervalsInDateRange( function mvConfigSupportsDateRange( mvConfig: MaterializedViewConfiguration, - chartConfig: ChartConfigWithOptDateRange, + chartConfig: BuilderChartConfigWithOptDateRange, ) { if (mvConfig.minDate && !chartConfig.dateRange) { return false; @@ -287,7 +287,7 @@ export type MVOptimizationExplanation = { }; export async function tryConvertConfigToMaterializedViewSelect< - C extends ChartConfigWithOptDateRange | CteChartConfig, + C extends BuilderChartConfigWithOptDateRange | CteChartConfig, >( chartConfig: C, mvConfig: MaterializedViewConfiguration, @@ -377,7 +377,7 @@ export async function tryConvertConfigToMaterializedViewSelect< } /** Attempts to optimize a config with a single MV Config */ -async function tryOptimizeConfig( +async function tryOptimizeConfig( config: C, metadata: Metadata, clickhouseClient: BaseClickhouseClient, @@ -481,7 +481,7 @@ async function tryOptimizeConfig( /** Attempts to optimize a config with each of the provided MV Configs */ export async function tryOptimizeConfigWithMaterializedViewWithExplanations< - C extends ChartConfigWithOptDateRange, + C extends BuilderChartConfigWithOptDateRange, >( config: C, metadata: Metadata, @@ -535,7 +535,7 @@ export async function tryOptimizeConfigWithMaterializedViewWithExplanations< } export async function tryOptimizeConfigWithMaterializedView< - C extends ChartConfigWithOptDateRange, + C extends BuilderChartConfigWithOptDateRange, >( config: C, metadata: Metadata, @@ -580,13 +580,13 @@ function toMvId( return `${mv.databaseName}.${mv.tableName}`; } -export interface GetKeyValueCall { +export interface GetKeyValueCall { chartConfig: C; keys: string[]; } export async function optimizeGetKeyValuesCalls< - C extends ChartConfigWithOptDateRange, + C extends BuilderChartConfigWithOptDateRange, >({ chartConfig, keys, diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 6a239361..64db7aec 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -13,7 +13,11 @@ import { tableExpr, } from '@/clickhouse'; import { renderChartConfig } from '@/core/renderChartConfig'; -import type { ChartConfig, ChartConfigWithDateRange, TSource } from '@/types'; +import type { + BuilderChartConfig, + BuilderChartConfigWithDateRange, + TSource, +} from '@/types'; import { optimizeGetKeyValuesCalls } from './materializedViews'; import { objectHash } from './utils'; @@ -923,7 +927,7 @@ export class Metadata { limit = 100, source, }: { - chartConfig: ChartConfigWithDateRange; + chartConfig: BuilderChartConfigWithDateRange; key: string; samples?: number; limit?: number; @@ -940,7 +944,7 @@ export class Metadata { return this.cache.getOrFetch( `${objectHash(cacheKeyConfig)}.${key}.valuesDistribution`, async () => { - const config: ChartConfigWithDateRange = { + const config: BuilderChartConfigWithDateRange = { ...chartConfig, with: [ ...(chartConfig.with || []), @@ -1013,7 +1017,7 @@ export class Metadata { signal, source, }: { - chartConfig: ChartConfigWithDateRange; + chartConfig: BuilderChartConfigWithDateRange; keys: string[]; limit?: number; disableRowLimit?: boolean; @@ -1132,7 +1136,7 @@ export class Metadata { disableRowLimit, signal, }: { - chartConfig: ChartConfigWithDateRange; + chartConfig: BuilderChartConfigWithDateRange; keys: string[]; source: TSource | undefined; limit?: number; @@ -1210,7 +1214,9 @@ export type TableConnectionChoice = tableConnections?: never; }; -export function tcFromChartConfig(config?: ChartConfig): TableConnection { +export function tcFromChartConfig( + config?: BuilderChartConfig, +): TableConnection { return { databaseName: config?.from?.databaseName ?? '', tableName: config?.from?.tableName ?? '', diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts index 2cdd6fff..ca1ed94c 100644 --- a/packages/common-utils/src/core/renderChartConfig.ts +++ b/packages/common-utils/src/core/renderChartConfig.ts @@ -16,18 +16,22 @@ import { parseToStartOfFunction, splitAndTrimWithBracket, } from '@/core/utils'; +import { isBuilderChartConfig, isRawSqlChartConfig } from '@/guards'; import { CustomSchemaSQLSerializerV2, SearchQueryBuilder } from '@/queryParser'; import { AggregateFunction, AggregateFunctionWithCombinators, + BuilderChartConfigWithDateRange, + BuilderChartConfigWithOptDateRange, ChartConfig, ChartConfigSchema, - ChartConfigWithDateRange, ChartConfigWithOptDateRange, ChSqlSchema, CteChartConfig, + DateRange, MetricsDataType, QuerySettings, + RawSqlChartConfig, SearchCondition, SearchConditionLanguage, SelectList, @@ -71,23 +75,23 @@ const DEFAULT_METRIC_TABLE_TIME_COLUMN = 'TimeUnix'; export const FIXED_TIME_BUCKET_EXPR_ALIAS = '__hdx_time_bucket'; export function isUsingGroupBy( - chartConfig: ChartConfigWithOptDateRange, -): chartConfig is Omit & { - groupBy: NonNullable; + chartConfig: BuilderChartConfigWithOptDateRange, +): chartConfig is Omit & { + groupBy: NonNullable; } { return chartConfig.groupBy != null && chartConfig.groupBy.length > 0; } export function isUsingGranularity( - chartConfig: ChartConfigWithOptDateRange, + chartConfig: BuilderChartConfigWithOptDateRange, ): chartConfig is Omit< - Omit, 'dateRange'>, + Omit, 'dateRange'>, 'timestampValueExpression' > & { - granularity: NonNullable; - dateRange: NonNullable; + granularity: NonNullable; + dateRange: NonNullable; timestampValueExpression: NonNullable< - ChartConfigWithDateRange['timestampValueExpression'] + BuilderChartConfigWithDateRange['timestampValueExpression'] >; } { return ( @@ -97,13 +101,17 @@ export function isUsingGranularity( } export const isMetricChartConfig = ( - chartConfig: ChartConfigWithOptDateRange, -) => { + chartConfig: BuilderChartConfigWithOptDateRange, +): chartConfig is BuilderChartConfigWithOptDateRange & { + metricTables: NonNullable; +} => { return chartConfig.metricTables != null; }; // TODO: apply this to all chart configs -export const setChartSelectsAlias = (config: ChartConfigWithOptDateRange) => { +export const setChartSelectsAlias = ( + config: BuilderChartConfigWithOptDateRange, +) => { if (Array.isArray(config.select) && isMetricChartConfig(config)) { return { ...config, @@ -120,10 +128,16 @@ export const setChartSelectsAlias = (config: ChartConfigWithOptDateRange) => { return config; }; -export const splitChartConfigs = (config: ChartConfigWithOptDateRange) => { +export const splitChartConfigs = ( + config: ChartConfigWithOptDateRange, +): ChartConfigWithOptDateRangeEx[] => { // only split metric queries for now - if (isMetricChartConfig(config) && Array.isArray(config.select)) { - const _configs: ChartConfigWithOptDateRange[] = []; + if ( + isBuilderChartConfig(config) && + isMetricChartConfig(config) && + Array.isArray(config.select) + ) { + const _configs: BuilderChartConfigWithOptDateRange[] = []; // split the query into multiple queries for (const select of config.select) { _configs.push({ @@ -133,7 +147,12 @@ export const splitChartConfigs = (config: ChartConfigWithOptDateRange) => { } return _configs; } - return [config]; + + if (isRawSqlChartConfig(config) || isBuilderChartConfig(config)) { + return [config]; // narrowed to BuilderChartConfig or RawSqlChartConfig, assignable to RawSqlChartConfigEx + } + + throw new Error(`Unexpected chart config type: ${JSON.stringify(config)}`); }; const INVERSE_OPERATOR_MAP = { @@ -391,7 +410,7 @@ const aggFnExpr = ({ async function renderSelectList( selectList: SelectList, - chartConfig: ChartConfigWithOptDateRangeEx, + chartConfig: BuilderChartConfigWithOptDateRangeEx, metadata: Metadata, ) { if (typeof selectList === 'string') { @@ -546,7 +565,7 @@ export async function timeFilterExpr({ metadata: Metadata; tableName: string; timestampValueExpression: string; - with?: ChartConfigWithDateRange['with']; + with?: BuilderChartConfigWithDateRange['with']; }) { const startTime = dateRange[0].getTime(); const endTime = dateRange[1].getTime(); @@ -634,7 +653,7 @@ export async function timeFilterExpr({ } async function renderSelect( - chartConfig: ChartConfigWithOptDateRangeEx, + chartConfig: BuilderChartConfigWithOptDateRangeEx, metadata: Metadata, ): Promise { /** @@ -666,7 +685,7 @@ async function renderSelect( function renderFrom({ from, }: { - from: ChartConfigWithDateRange['from']; + from: BuilderChartConfigWithDateRange['from']; }): ChSql { return concatChSql( '.', @@ -689,10 +708,10 @@ async function renderWhereExpression({ condition: SearchCondition; language: SearchConditionLanguage; metadata: Metadata; - from: ChartConfigWithDateRange['from']; + from: BuilderChartConfigWithDateRange['from']; implicitColumnExpression?: string; connectionId: string; - with?: ChartConfigWithDateRange['with']; + with?: BuilderChartConfigWithDateRange['with']; }): Promise { let _condition = condition; if (language === 'lucene') { @@ -740,7 +759,7 @@ async function renderWhereExpression({ } async function renderWhere( - chartConfig: ChartConfigWithOptDateRangeEx, + chartConfig: BuilderChartConfigWithOptDateRangeEx, metadata: Metadata, ): Promise { let whereSearchCondition: ChSql | [] = []; @@ -847,7 +866,7 @@ async function renderWhere( } async function renderGroupBy( - chartConfig: ChartConfigWithOptDateRange, + chartConfig: BuilderChartConfigWithOptDateRange, metadata: Metadata, ): Promise { return concatChSql( @@ -866,7 +885,7 @@ async function renderGroupBy( } async function renderHaving( - chartConfig: ChartConfigWithOptDateRangeEx, + chartConfig: BuilderChartConfigWithOptDateRangeEx, metadata: Metadata, ): Promise { if (!isNonEmptyWhereExpr(chartConfig.having)) { @@ -885,7 +904,7 @@ async function renderHaving( } function renderOrderBy( - chartConfig: ChartConfigWithOptDateRange, + chartConfig: BuilderChartConfigWithOptDateRange, ): ChSql | undefined { const isIncludingTimeBucket = isUsingGranularity(chartConfig); @@ -909,7 +928,7 @@ function renderOrderBy( } function renderLimit( - chartConfig: ChartConfigWithOptDateRange, + chartConfig: BuilderChartConfigWithOptDateRange, ): ChSql | undefined { if (chartConfig.limit == null || chartConfig.limit.limit == null) { return undefined; @@ -924,7 +943,7 @@ function renderLimit( } function renderSettings( - chartConfig: ChartConfigWithOptDateRangeEx, + chartConfig: BuilderChartConfigWithOptDateRangeEx, querySettings: QuerySettings | undefined, ) { const querySettingsJoined = joinQuerySettings(querySettings); @@ -937,13 +956,24 @@ function renderSettings( // includedDataInterval isn't exported at this time. It's only used internally // for metric SQL generation. -export type ChartConfigWithOptDateRangeEx = ChartConfigWithOptDateRange & { +type InternalChartFields = { includedDataInterval?: string; settings?: ChSql; }; +type BuilderChartConfigWithOptDateRangeEx = BuilderChartConfigWithOptDateRange & + InternalChartFields; + +type RawSqlChartConfigEx = RawSqlChartConfig & + Partial & + InternalChartFields; + +export type ChartConfigWithOptDateRangeEx = + | BuilderChartConfigWithOptDateRangeEx + | RawSqlChartConfigEx; + async function renderWith( - chartConfig: ChartConfigWithOptDateRangeEx, + chartConfig: BuilderChartConfigWithOptDateRangeEx, metadata: Metadata, querySettings: QuerySettings | undefined, ): Promise { @@ -1031,7 +1061,7 @@ function intervalToSeconds(interval: SQLInterval): number { } function renderFill( - chartConfig: ChartConfigWithOptDateRangeEx, + chartConfig: BuilderChartConfigWithOptDateRangeEx, ): ChSql | undefined { const { granularity, dateRange } = chartConfig; if (dateRange && granularity && granularity !== 'auto') { @@ -1049,7 +1079,7 @@ function renderFill( } function renderDeltaExpression( - chartConfig: ChartConfigWithOptDateRange, + chartConfig: BuilderChartConfigWithOptDateRange, valueExpression: string, ) { const interval = @@ -1067,9 +1097,9 @@ function renderDeltaExpression( } async function translateMetricChartConfig( - chartConfig: ChartConfigWithOptDateRange, + chartConfig: BuilderChartConfigWithOptDateRangeEx, metadata: Metadata, -): Promise { +): Promise { const metricTables = chartConfig.metricTables; if (!metricTables) { return chartConfig; @@ -1318,7 +1348,7 @@ async function translateMetricChartConfig( Array.isArray(chartConfig.dateRange) ? convertDateRangeToGranularityString(chartConfig.dateRange) : chartConfig.granularity, - } as ChartConfigWithOptDateRangeEx; + } as BuilderChartConfigWithOptDateRangeEx; const timeBucketSelect = isUsingGranularity(cteChartConfig) ? timeBucketExpr({ @@ -1377,6 +1407,10 @@ export async function renderChartConfig( metadata: Metadata, querySettings: QuerySettings | undefined, ): Promise { + if (isRawSqlChartConfig(rawChartConfig)) { + return chSql`${{ UNSAFE_RAW_SQL: rawChartConfig.sqlTemplate ?? '' }}`; + } + // metric types require more rewriting since we know more about the schema // but goes through the same generation process const chartConfig = isMetricChartConfig(rawChartConfig) diff --git a/packages/common-utils/src/core/utils.ts b/packages/common-utils/src/core/utils.ts index 906bdc69..b8c3f3a6 100644 --- a/packages/common-utils/src/core/utils.ts +++ b/packages/common-utils/src/core/utils.ts @@ -5,10 +5,11 @@ import { z } from 'zod'; export { default as objectHash } from 'object-hash'; +import { isBuilderSavedChartConfig } from '@/guards'; import { - ChartConfig, - ChartConfigWithDateRange, - ChartConfigWithOptTimestamp, + BuilderChartConfig, + BuilderChartConfigWithDateRange, + BuilderChartConfigWithOptTimestamp, DashboardFilter, DashboardFilterSchema, DashboardSchema, @@ -472,9 +473,13 @@ export function convertToDashboardTemplate( ): TileTemplate => { const tile = TileTemplateSchema.strip().parse(structuredClone(input)); // Extract name from source or default to '' if not found - tile.config.source = ( - sources.find(source => source.id === tile.config.source) ?? { name: '' } - ).name; + // Raw SQL configs don't have a source field, so only update builder configs + const tileConfig = tile.config; + if (isBuilderSavedChartConfig(tileConfig)) { + tileConfig.source = ( + sources.find(source => source.id === tileConfig.source) ?? { name: '' } + ).name; + } return tile; }; @@ -539,7 +544,7 @@ export function convertToDashboardDocument( } export const getFirstOrderingItem = ( - orderBy: ChartConfigWithDateRange['orderBy'], + orderBy: BuilderChartConfigWithDateRange['orderBy'], ) => { if (!orderBy || orderBy.length === 0) return undefined; @@ -560,7 +565,7 @@ export const removeTrailingDirection = (s: string) => { }; export const isTimestampExpressionInFirstOrderBy = ( - config: ChartConfigWithOptTimestamp, + config: BuilderChartConfigWithOptTimestamp, ) => { const firstOrderingItem = getFirstOrderingItem(config.orderBy); if (!firstOrderingItem || config.timestampValueExpression == null) @@ -581,7 +586,7 @@ export const isTimestampExpressionInFirstOrderBy = ( }; export const isFirstOrderByAscending = ( - orderBy: ChartConfigWithDateRange['orderBy'], + orderBy: BuilderChartConfigWithDateRange['orderBy'], ): boolean => { const primaryOrderingItem = getFirstOrderingItem(orderBy); @@ -936,7 +941,7 @@ export function parseTokenizerFromTextIndex({ */ export function aliasMapToWithClauses( aliasMap: Record | undefined, -): ChartConfig['with'] { +): BuilderChartConfig['with'] { if (!aliasMap) { return undefined; } diff --git a/packages/common-utils/src/guards.ts b/packages/common-utils/src/guards.ts new file mode 100644 index 00000000..62621db5 --- /dev/null +++ b/packages/common-utils/src/guards.ts @@ -0,0 +1,33 @@ +import { + BuilderChartConfig, + BuilderSavedChartConfig, + ChartConfig, + ChartConfigWithOptDateRange, + RawSqlChartConfig, + RawSqlSavedChartConfig, + SavedChartConfig, +} from './types'; + +export function isRawSqlChartConfig( + chartConfig: ChartConfig | ChartConfigWithOptDateRange, +): chartConfig is RawSqlChartConfig { + return 'configType' in chartConfig && chartConfig.configType === 'sql'; +} + +export function isBuilderChartConfig( + chartConfig: ChartConfig | ChartConfigWithOptDateRange, +): chartConfig is BuilderChartConfig { + return !isRawSqlChartConfig(chartConfig); +} + +export function isRawSqlSavedChartConfig( + chartConfig: SavedChartConfig, +): chartConfig is RawSqlSavedChartConfig { + return 'configType' in chartConfig && chartConfig.configType === 'sql'; +} + +export function isBuilderSavedChartConfig( + chartConfig: SavedChartConfig, +): chartConfig is BuilderSavedChartConfig { + return !isRawSqlSavedChartConfig(chartConfig); +} diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index d829eabb..662f4650 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -416,29 +416,37 @@ export type NumberFormat = z.infer; // When making changes here, consider if they need to be made to the external API // schema as well (packages/api/src/utils/zod.ts). -export const _ChartConfigSchema = z.object({ + +/** + * Schema describing display settings which are shared between Raw SQL + * chart configs and Structured ChartBuilder chart configs + **/ +const SharedChartDisplaySettingsSchema = z.object({ displayType: z.nativeEnum(DisplayType).optional(), numberFormat: NumberFormatSchema.optional(), + granularity: z.union([SQLIntervalSchema, z.literal('auto')]).optional(), + compareToPreviousPeriod: z.boolean().optional(), + fillNulls: z.union([z.number(), z.literal(false)]).optional(), + alignDateRangeToGranularity: z.boolean().optional(), +}); + +export const _ChartConfigSchema = SharedChartDisplaySettingsSchema.extend({ timestampValueExpression: z.string(), implicitColumnExpression: z.string().optional(), - granularity: z.union([SQLIntervalSchema, z.literal('auto')]).optional(), markdown: z.string().optional(), filtersLogicalOperator: z.enum(['AND', 'OR']).optional(), filters: z.array(FilterSchema).optional(), connection: z.string(), - fillNulls: z.union([z.number(), z.literal(false)]).optional(), selectGroupBy: z.boolean().optional(), metricTables: MetricTableSchema.optional(), seriesReturnType: z.enum(['ratio', 'column']).optional(), // Used to preserve original table select string when chart overrides it (e.g., histograms) eventTableSelect: z.string().optional(), - compareToPreviousPeriod: z.boolean().optional(), source: z.string().optional(), - alignDateRangeToGranularity: z.boolean().optional(), }); // This is a ChartConfig type without the `with` CTE clause included. -// It needs to be a separate, named schema to avoid use ot z.lazy(...), +// It needs to be a separate, named schema to avoid use of z.lazy(...), // use of which allows for type mistakes to make it past linting. export const CteChartConfigSchema = z.intersection( _ChartConfigSchema.partial({ timestampValueExpression: true }), @@ -451,7 +459,7 @@ export type CteChartConfig = z.infer; // non-recursive chart config so that it can reference a complete chart config // schema. This structure does mean that we cannot nest `with` clauses but does // ensure the type system can catch more issues in the build pipeline. -export const ChartConfigSchema = z.intersection( +const BuilderChartConfigSchema = z.intersection( z.intersection(_ChartConfigSchema, SelectSQLStatementSchema), z .object({ @@ -477,6 +485,22 @@ export const ChartConfigSchema = z.intersection( .partial(), ); +export type BuilderChartConfig = z.infer; + +/** Schema describing Raw SQL chart configs */ +const RawSqlChartConfigSchema = SharedChartDisplaySettingsSchema.extend({ + configType: z.literal('sql'), + sqlTemplate: z.string(), + connection: z.string(), +}); + +export type RawSqlChartConfig = z.infer; + +export const ChartConfigSchema = z.union([ + BuilderChartConfigSchema, + RawSqlChartConfigSchema, +]); + export type ChartConfig = z.infer; export type DateRange = { @@ -486,31 +510,38 @@ export type DateRange = { }; export type ChartConfigWithDateRange = ChartConfig & DateRange; +export type BuilderChartConfigWithDateRange = BuilderChartConfig & DateRange; +export type RawSqlConfigWithDateRange = RawSqlChartConfig & DateRange; -export type ChartConfigWithOptTimestamp = Omit< - ChartConfigWithDateRange, +export type BuilderChartConfigWithOptTimestamp = Omit< + BuilderChartConfigWithDateRange, 'timestampValueExpression' > & { timestampValueExpression?: string; }; + +export type ChartConfigWithOptTimestamp = + | BuilderChartConfigWithOptTimestamp + | RawSqlConfigWithDateRange; + // For non-time-based searches (ex. grab 1 row) -export type ChartConfigWithOptDateRange = Omit< - ChartConfig, +export type BuilderChartConfigWithOptDateRange = Omit< + BuilderChartConfig, 'timestampValueExpression' > & { timestampValueExpression?: string; } & Partial; +export type ChartConfigWithOptDateRange = + | BuilderChartConfigWithOptDateRange + | (RawSqlChartConfig & Partial); + // When making changes here, consider if they need to be made to the external API // schema as well (packages/api/src/utils/zod.ts). -export const SavedChartConfigSchema = z +const BuilderSavedChartConfigWithoutAlertSchema = z .object({ name: z.string().optional(), source: z.string(), - alert: z.union([ - AlertBaseSchema.optional(), - ChartAlertBaseSchema.optional(), - ]), }) .extend( _ChartConfigSchema.omit({ @@ -525,6 +556,31 @@ export const SavedChartConfigSchema = z }).shape, ); +const BuilderSavedChartConfigSchema = + BuilderSavedChartConfigWithoutAlertSchema.extend({ + alert: z.union([ + AlertBaseSchema.optional(), + ChartAlertBaseSchema.optional(), + ]), + }); + +export type BuilderSavedChartConfig = z.infer< + typeof BuilderSavedChartConfigSchema +>; + +const RawSqlSavedChartConfigSchema = RawSqlChartConfigSchema.extend({ + name: z.string().optional(), +}); + +export const SavedChartConfigSchema = z.union([ + BuilderSavedChartConfigSchema, + RawSqlSavedChartConfigSchema, +]); + +export type RawSqlSavedChartConfig = z.infer< + typeof RawSqlSavedChartConfigSchema +>; + export type SavedChartConfig = z.infer; export const TileSchema = z.object({ @@ -535,8 +591,12 @@ export const TileSchema = z.object({ h: z.number(), config: SavedChartConfigSchema, }); + export const TileTemplateSchema = TileSchema.extend({ - config: TileSchema.shape.config.omit({ alert: true }), + config: z.union([ + BuilderSavedChartConfigWithoutAlertSchema, + RawSqlSavedChartConfigSchema, + ]), }); export type Tile = z.infer;