diff --git a/.changeset/odd-hats-wash.md b/.changeset/odd-hats-wash.md new file mode 100644 index 00000000..99aa0e27 --- /dev/null +++ b/.changeset/odd-hats-wash.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/common-utils": minor +--- + +Add chart support for querying OTEL histogram metric table diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index 304b6bd1..6bb9f542 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -60,8 +60,12 @@ describe('renderChartConfig', () => { expect(actual).toBe( 'SELECT quantile(0.95)(toFloat64OrNull(toString(Value))),toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS `__hdx_time_bucket`' + ' FROM default.otel_metrics_gauge WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1765670400000)) AND' + - " (MetricName = 'nodejs.event_loop.utilization') GROUP BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS `__hdx_time_bucket` " + - 'ORDER BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS `__hdx_time_bucket` LIMIT 10', + " (MetricName = 'nodejs.event_loop.utilization') GROUP BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS `__hdx_time_bucket`" + + ' ORDER BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS `__hdx_time_bucket`' + + ' WITH FILL FROM toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(1739318400000), INTERVAL 1 minute))\n' + + ' TO toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(1765670400000), INTERVAL 1 minute))\n' + + ' STEP 60' + + ' LIMIT 10', ); }); @@ -93,7 +97,7 @@ describe('renderChartConfig', () => { whereLanguage: 'sql', timestampValueExpression: 'TimeUnix', dateRange: [new Date('2025-02-12'), new Date('2025-12-14')], - granularity: '5 minutes', + granularity: '5 minute', limit: { limit: 10 }, }; @@ -112,12 +116,84 @@ describe('renderChartConfig', () => { ' FROM default.otel_metrics_sum\n' + " WHERE MetricName = 'db.client.connections.usage'\n" + ' ORDER BY AttributesHash, TimeUnix ASC\n' + - ' ) )SELECT avg(\n' + + ' ) ) SELECT avg(\n' + ' toFloat64OrNull(toString(Rate))\n' + - ' ),toStartOfInterval(toDateTime(TimeUnix), INTERVAL 5 minutes) AS `__hdx_time_bucket` ' + - 'FROM RawSum WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1765670400000)) ' + - "AND (MetricName = 'db.client.connections.usage') GROUP BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 5 minutes) AS `__hdx_time_bucket` " + - 'ORDER BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 5 minutes) AS `__hdx_time_bucket` LIMIT 10', + ' ),toStartOfInterval(toDateTime(TimeUnix), INTERVAL 5 minute) AS `__hdx_time_bucket`' + + ' FROM RawSum WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1765670400000))' + + ' GROUP BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 5 minute) AS `__hdx_time_bucket`' + + ' ORDER BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 5 minute) AS `__hdx_time_bucket`' + + ' WITH FILL FROM toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(1739318400000), INTERVAL 5 minute))\n' + + ' TO toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(1765670400000), INTERVAL 5 minute))\n' + + ' STEP 300' + + ' LIMIT 10', + ); + }); + + it('should generate sql for a single histogram metric', async () => { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + // metricTables is added from the Source object via spread operator + metricTables: { + gauge: 'otel_metrics_gauge', + histogram: 'otel_metrics_histogram', + sum: 'otel_metrics_sum', + }, + from: { + databaseName: 'default', + tableName: '', + }, + select: [ + { + aggFn: 'quantile', + level: 0.5, + valueExpression: 'Value', + metricName: 'http.server.duration', + metricType: MetricsDataType.Histogram, + }, + ], + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'TimeUnix', + dateRange: [new Date('2025-02-12'), new Date('2025-12-14')], + limit: { limit: 10 }, + }; + + const generatedSql = await renderChartConfig(config, mockMetadata); + const actual = parameterizedQueryToSql(generatedSql); + expect(actual).toBe( + 'WITH HistRate AS (SELECT *, any(BucketCounts) OVER (ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS PrevBucketCounts,\n' + + ' any(CountLength) OVER (ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS PrevCountLength,\n' + + ' any(AttributesHash) OVER (ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS PrevAttributesHash,\n' + + ' IF(AggregationTemporality = 1,\n' + + ' BucketCounts,\n' + + ' IF(AttributesHash = PrevAttributesHash AND CountLength = PrevCountLength,\n' + + ' arrayMap((prev, curr) -> IF(curr < prev, curr, toUInt64(toInt64(curr) - toInt64(prev))), PrevBucketCounts, BucketCounts),\n' + + ' BucketCounts)) as BucketRates\n' + + ' FROM (\n' + + ' SELECT *, cityHash64(mapConcat(ScopeAttributes, ResourceAttributes, Attributes)) AS AttributesHash,\n' + + ' length(BucketCounts) as CountLength\n' + + ' FROM default.otel_metrics_histogram)\n' + + " WHERE MetricName = 'http.server.duration'\n " + + ' ORDER BY Attributes, TimeUnix ASC\n' + + ' ),RawHist AS (\n' + + ' SELECT *, toUInt64( 0.5 * arraySum(BucketRates)) AS Rank,\n' + + ' arrayCumSum(BucketRates) as CumRates,\n' + + ' arrayFirstIndex(x -> if(x > Rank, 1, 0), CumRates) AS BucketLowIdx,\n' + + ' IF(BucketLowIdx = length(BucketRates),\n' + + ' ExplicitBounds[length(ExplicitBounds)], -- if the low bound is the last bucket, use the last bound value\n' + + ' IF(BucketLowIdx > 1, -- indexes are 1-based\n' + + ' ExplicitBounds[BucketLowIdx] + (ExplicitBounds[BucketLowIdx + 1] - ExplicitBounds[BucketLowIdx]) *\n' + + ' intDivOrZero(\n' + + ' Rank - CumRates[BucketLowIdx - 1],\n' + + ' CumRates[BucketLowIdx] - CumRates[BucketLowIdx - 1]),\n' + + ' arrayElement(ExplicitBounds, BucketLowIdx + 1) * intDivOrZero(Rank, CumRates[BucketLowIdx]))) as Rate\n' + + ' FROM HistRate) SELECT sum(\n' + + ' toFloat64OrNull(toString(Rate))\n' + + ' )' + + ' FROM RawHist' + + ' WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1765670400000))' + + ' LIMIT 10', ); }); }); diff --git a/packages/common-utils/src/clickhouse.ts b/packages/common-utils/src/clickhouse.ts index 820d56f5..46c3194b 100644 --- a/packages/common-utils/src/clickhouse.ts +++ b/packages/common-utils/src/clickhouse.ts @@ -186,7 +186,11 @@ export const concatChSql = (sep: string, ...args: (ChSql | ChSql[])[]) => { } acc.sql += - (acc.sql.length > 0 ? sep : '') + arg.map(a => a.sql).join(sep); + (acc.sql.length > 0 ? sep : '') + + arg + .map(a => a.sql) + .filter(Boolean) // skip empty string expressions + .join(sep); acc.params = arg.reduce((acc, a) => { Object.assign(acc, a.params); return acc; diff --git a/packages/common-utils/src/renderChartConfig.ts b/packages/common-utils/src/renderChartConfig.ts index 765cd8a9..bcc2857b 100644 --- a/packages/common-utils/src/renderChartConfig.ts +++ b/packages/common-utils/src/renderChartConfig.ts @@ -292,7 +292,7 @@ const aggFnExpr = ({ async function renderSelectList( selectList: SelectList, - chartConfig: ChartConfigWithOptDateRateAndCte, + chartConfig: ChartConfigWithOptDateRangeEx, metadata: Metadata, ) { if (typeof selectList === 'string') { @@ -467,7 +467,7 @@ async function timeFilterExpr({ } async function renderSelect( - chartConfig: ChartConfigWithOptDateRateAndCte, + chartConfig: ChartConfigWithOptDateRangeEx, metadata: Metadata, ): Promise { /** @@ -565,7 +565,7 @@ async function renderWhereExpression({ } async function renderWhere( - chartConfig: ChartConfigWithOptDateRateAndCte, + chartConfig: ChartConfigWithOptDateRangeEx, metadata: Metadata, ): Promise { let whereSearchCondition: ChSql | [] = []; @@ -729,28 +729,64 @@ function renderLimit( // CTE (Common Table Expressions) isn't exported at this time. It's only used internally // for metric SQL generation. -type ChartConfigWithOptDateRateAndCte = ChartConfigWithOptDateRange & { +type ChartConfigWithOptDateRangeEx = ChartConfigWithOptDateRange & { with?: { name: string; sql: ChSql }[]; }; function renderWith( - chartConfig: ChartConfigWithOptDateRateAndCte, + chartConfig: ChartConfigWithOptDateRangeEx, metadata: Metadata, ): ChSql | undefined { const { with: withClauses } = chartConfig; if (withClauses) { return concatChSql( - '', - withClauses.map(clause => chSql`WITH ${clause.name} AS (${clause.sql})`), + ',', + withClauses.map(clause => chSql`${clause.name} AS (${clause.sql})`), ); } return undefined; } +function intervalToSeconds(interval: SQLInterval): number { + // Parse interval string like "15 second" into number of seconds + const [amount, unit] = interval.split(' '); + const value = parseInt(amount, 10); + switch (unit) { + case 'second': + return value; + case 'minute': + return value * 60; + case 'hour': + return value * 60 * 60; + case 'day': + return value * 24 * 60 * 60; + default: + throw new Error(`Invalid interval unit ${unit} in interval ${interval}`); + } +} + +function renderFill( + chartConfig: ChartConfigWithOptDateRangeEx, +): ChSql | undefined { + const { granularity, dateRange } = chartConfig; + if (dateRange && granularity && granularity !== 'auto') { + const [start, end] = dateRange; + const step = intervalToSeconds(granularity); + + return concatChSql(' ', [ + chSql`FROM toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(${{ Int64: start.getTime() }}), INTERVAL ${granularity})) + TO toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(${{ Int64: end.getTime() }}), INTERVAL ${granularity})) + STEP ${{ Int32: step }}`, + ]); + } + + return undefined; +} + function translateMetricChartConfig( chartConfig: ChartConfigWithOptDateRange, -): ChartConfigWithOptDateRateAndCte { +): ChartConfigWithOptDateRangeEx { const metricTables = chartConfig.metricTables; if (!metricTables) { return chartConfig; @@ -810,8 +846,67 @@ function translateMetricChartConfig( databaseName: '', tableName: 'RawSum', }, - where: `MetricName = '${metricName}'`, - whereLanguage: 'sql', + }; + } else if (metricType === MetricsDataType.Histogram && metricName) { + // histograms are only valid for quantile selections + const { aggFn, level, ..._selectRest } = _select as { + aggFn: string; + level?: number; + }; + + if (aggFn !== 'quantile' || level == null) { + throw new Error('quantile must be specified for histogram metrics'); + } + + return { + ...restChartConfig, + with: [ + { + name: 'HistRate', + sql: chSql`SELECT *, any(BucketCounts) OVER (ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS PrevBucketCounts, + any(CountLength) OVER (ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS PrevCountLength, + any(AttributesHash) OVER (ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS PrevAttributesHash, + IF(AggregationTemporality = 1, + BucketCounts, + IF(AttributesHash = PrevAttributesHash AND CountLength = PrevCountLength, + arrayMap((prev, curr) -> IF(curr < prev, curr, toUInt64(toInt64(curr) - toInt64(prev))), PrevBucketCounts, BucketCounts), + BucketCounts)) as BucketRates + FROM ( + SELECT *, cityHash64(mapConcat(ScopeAttributes, ResourceAttributes, Attributes)) AS AttributesHash, + length(BucketCounts) as CountLength + FROM ${renderFrom({ from: { ...from, tableName: metricTables[MetricsDataType.Histogram] } })}) + WHERE MetricName = '${metricName}' + ORDER BY Attributes, TimeUnix ASC + `, + }, + { + name: 'RawHist', + sql: chSql` + SELECT *, toUInt64( ${{ Float64: level }} * arraySum(BucketRates)) AS Rank, + arrayCumSum(BucketRates) as CumRates, + arrayFirstIndex(x -> if(x > Rank, 1, 0), CumRates) AS BucketLowIdx, + IF(BucketLowIdx = length(BucketRates), + ExplicitBounds[length(ExplicitBounds)], -- if the low bound is the last bucket, use the last bound value + IF(BucketLowIdx > 1, -- indexes are 1-based + ExplicitBounds[BucketLowIdx] + (ExplicitBounds[BucketLowIdx + 1] - ExplicitBounds[BucketLowIdx]) * + intDivOrZero( + Rank - CumRates[BucketLowIdx - 1], + CumRates[BucketLowIdx] - CumRates[BucketLowIdx - 1]), + arrayElement(ExplicitBounds, BucketLowIdx + 1) * intDivOrZero(Rank, CumRates[BucketLowIdx]))) as Rate + FROM HistRate`, + }, + ], + select: [ + { + ..._selectRest, + aggFn: 'sum', + valueExpression: 'Rate', + }, + ], + from: { + databaseName: '', + tableName: 'RawHist', + }, }; } @@ -835,15 +930,19 @@ export async function renderChartConfig( const where = await renderWhere(chartConfig, metadata); const groupBy = await renderGroupBy(chartConfig, metadata); const orderBy = renderOrderBy(chartConfig); + const fill = renderFill(chartConfig); const limit = renderLimit(chartConfig); - return chSql`${ - withClauses?.sql ? chSql`${withClauses}` : '' - }SELECT ${select} FROM ${from} ${where?.sql ? chSql`WHERE ${where}` : ''} ${ - groupBy?.sql ? chSql`GROUP BY ${groupBy}` : '' - } ${orderBy?.sql ? chSql`ORDER BY ${orderBy}` : ''} ${ - limit?.sql ? chSql`LIMIT ${limit}` : '' - }`; + return concatChSql(' ', [ + chSql`${withClauses?.sql ? chSql`WITH ${withClauses}` : ''}`, + chSql`SELECT ${select}`, + chSql`FROM ${from}`, + chSql`${where.sql ? chSql`WHERE ${where}` : ''}`, + chSql`${groupBy?.sql ? chSql`GROUP BY ${groupBy}` : ''}`, + chSql`${orderBy?.sql ? chSql`ORDER BY ${orderBy}` : ''}`, + chSql`${fill?.sql ? chSql`WITH FILL ${fill}` : ''}`, + chSql`${limit?.sql ? chSql`LIMIT ${limit}` : ''}`, + ]); } // EditForm -> translateToQueriedChartConfig -> QueriedChartConfig