Elgato_dark/hyperdx
1
0
Fork 0
mirror of https://github.com/hyperdxio/hyperdx synced 2026-04-21 13:37:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

feat: supporting quantile histogram metrics (#635)

Browse source 
Additional `renderChartConfig` support to transform a histogram select into the correct SQL syntax to generate a chart. For parity with v1, this query only handles quantile queries.

<img width="1939" alt="Screenshot 2025-02-26 at 12 58 55 PM" src="https://github.com/user-attachments/assets/1126ac6c-c431-4d89-92d7-9df1e49e25cf" />

<img width="1960" alt="Screenshot 2025-02-26 at 3 11 07 PM" src="https://github.com/user-attachments/assets/e4fa09bf-1e27-4a90-ad25-6c6cb2890877" />

Ref: HDX-1339
This commit is contained in:
Dan Hable 2025-02-27 10:55:36 -06:00 committed by GitHub
parent 521793df2d
commit e80630c107
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 210 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.changeset/odd-hats-wash.md Normal file
View file

@ -0,0 +1,5 @@
---
"@hyperdx/common-utils": minor
---
Add chart support for querying OTEL histogram metric table

92
packages/common-utils/src/__tests__/renderChartConfig.test.ts
View file

@ -60,8 +60,12 @@ describe('renderChartConfig', () => {
expect(actual).toBe(
'SELECT quantile(0.95)(toFloat64OrNull(toString(Value))),toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS `__hdx_time_bucket`' +
' FROM default.otel_metrics_gauge WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1765670400000)) AND' +
" (MetricName = 'nodejs.event_loop.utilization') GROUP BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS `__hdx_time_bucket` " +
'ORDER BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS `__hdx_time_bucket` LIMIT 10',
" (MetricName = 'nodejs.event_loop.utilization') GROUP BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS `__hdx_time_bucket`" +
' ORDER BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS `__hdx_time_bucket`' +
' WITH FILL FROM toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(1739318400000), INTERVAL 1 minute))\n' +
' TO toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(1765670400000), INTERVAL 1 minute))\n' +
' STEP 60' +
' LIMIT 10',
);
});
@ -93,7 +97,7 @@ describe('renderChartConfig', () => {
whereLanguage: 'sql',
timestampValueExpression: 'TimeUnix',
dateRange: [new Date('2025-02-12'), new Date('2025-12-14')],
granularity: '5 minutes',
granularity: '5 minute',
limit: { limit: 10 },
};
@ -112,12 +116,84 @@ describe('renderChartConfig', () => {
' FROM default.otel_metrics_sum\n' +
" WHERE MetricName = 'db.client.connections.usage'\n" +
' ORDER BY AttributesHash, TimeUnix ASC\n' +
' ) )SELECT avg(\n' +
' ) ) SELECT avg(\n' +
' toFloat64OrNull(toString(Rate))\n' +
' ),toStartOfInterval(toDateTime(TimeUnix), INTERVAL 5 minutes) AS `__hdx_time_bucket` ' +
'FROM RawSum WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1765670400000)) ' +
"AND (MetricName = 'db.client.connections.usage') GROUP BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 5 minutes) AS `__hdx_time_bucket` " +
'ORDER BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 5 minutes) AS `__hdx_time_bucket` LIMIT 10',
' ),toStartOfInterval(toDateTime(TimeUnix), INTERVAL 5 minute) AS `__hdx_time_bucket`' +
' FROM RawSum WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1765670400000))' +
' GROUP BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 5 minute) AS `__hdx_time_bucket`' +
' ORDER BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 5 minute) AS `__hdx_time_bucket`' +
' WITH FILL FROM toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(1739318400000), INTERVAL 5 minute))\n' +
' TO toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(1765670400000), INTERVAL 5 minute))\n' +
' STEP 300' +
' LIMIT 10',
);
});
it('should generate sql for a single histogram metric', async () => {
const config: ChartConfigWithOptDateRange = {
displayType: DisplayType.Line,
connection: 'test-connection',
// metricTables is added from the Source object via spread operator
metricTables: {
gauge: 'otel_metrics_gauge',
histogram: 'otel_metrics_histogram',
sum: 'otel_metrics_sum',
},
from: {
databaseName: 'default',
tableName: '',
},
select: [
{
aggFn: 'quantile',
level: 0.5,
valueExpression: 'Value',
metricName: 'http.server.duration',
metricType: MetricsDataType.Histogram,
},
],
where: '',
whereLanguage: 'sql',
timestampValueExpression: 'TimeUnix',
dateRange: [new Date('2025-02-12'), new Date('2025-12-14')],
limit: { limit: 10 },
};
const generatedSql = await renderChartConfig(config, mockMetadata);
const actual = parameterizedQueryToSql(generatedSql);
expect(actual).toBe(
'WITH HistRate AS (SELECT *, any(BucketCounts) OVER (ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS PrevBucketCounts,\n' +
' any(CountLength) OVER (ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS PrevCountLength,\n' +
' any(AttributesHash) OVER (ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS PrevAttributesHash,\n' +
' IF(AggregationTemporality = 1,\n' +
' BucketCounts,\n' +
' IF(AttributesHash = PrevAttributesHash AND CountLength = PrevCountLength,\n' +
' arrayMap((prev, curr) -> IF(curr < prev, curr, toUInt64(toInt64(curr) - toInt64(prev))), PrevBucketCounts, BucketCounts),\n' +
' BucketCounts)) as BucketRates\n' +
' FROM (\n' +
' SELECT *, cityHash64(mapConcat(ScopeAttributes, ResourceAttributes, Attributes)) AS AttributesHash,\n' +
' length(BucketCounts) as CountLength\n' +
' FROM default.otel_metrics_histogram)\n' +
" WHERE MetricName = 'http.server.duration'\n " +
' ORDER BY Attributes, TimeUnix ASC\n' +
' ),RawHist AS (\n' +
' SELECT *, toUInt64( 0.5 * arraySum(BucketRates)) AS Rank,\n' +
' arrayCumSum(BucketRates) as CumRates,\n' +
' arrayFirstIndex(x -> if(x > Rank, 1, 0), CumRates) AS BucketLowIdx,\n' +
' IF(BucketLowIdx = length(BucketRates),\n' +
' ExplicitBounds[length(ExplicitBounds)], -- if the low bound is the last bucket, use the last bound value\n' +
' IF(BucketLowIdx > 1, -- indexes are 1-based\n' +
' ExplicitBounds[BucketLowIdx] + (ExplicitBounds[BucketLowIdx + 1] - ExplicitBounds[BucketLowIdx]) *\n' +
' intDivOrZero(\n' +
' Rank - CumRates[BucketLowIdx - 1],\n' +
' CumRates[BucketLowIdx] - CumRates[BucketLowIdx - 1]),\n' +
' arrayElement(ExplicitBounds, BucketLowIdx + 1) * intDivOrZero(Rank, CumRates[BucketLowIdx]))) as Rate\n' +
' FROM HistRate) SELECT sum(\n' +
' toFloat64OrNull(toString(Rate))\n' +
' )' +
' FROM RawHist' +
' WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1765670400000))' +
' LIMIT 10',
);
});
});

6
packages/common-utils/src/clickhouse.ts
View file

@ -186,7 +186,11 @@ export const concatChSql = (sep: string, ...args: (ChSql | ChSql[])[]) => {
}
acc.sql +=
(acc.sql.length > 0 ? sep : '') + arg.map(a => a.sql).join(sep);
(acc.sql.length > 0 ? sep : '') +
arg
.map(a => a.sql)
.filter(Boolean) // skip empty string expressions
.join(sep);
acc.params = arg.reduce((acc, a) => {
Object.assign(acc, a.params);
return acc;

133
packages/common-utils/src/renderChartConfig.ts
View file

@ -292,7 +292,7 @@ const aggFnExpr = ({
async function renderSelectList(
selectList: SelectList,
chartConfig: ChartConfigWithOptDateRateAndCte,
chartConfig: ChartConfigWithOptDateRangeEx,
metadata: Metadata,
) {
if (typeof selectList === 'string') {
@ -467,7 +467,7 @@ async function timeFilterExpr({
}
async function renderSelect(
chartConfig: ChartConfigWithOptDateRateAndCte,
chartConfig: ChartConfigWithOptDateRangeEx,
metadata: Metadata,
): Promise<ChSql> {
/**
@ -565,7 +565,7 @@ async function renderWhereExpression({
}
async function renderWhere(
chartConfig: ChartConfigWithOptDateRateAndCte,
chartConfig: ChartConfigWithOptDateRangeEx,
metadata: Metadata,
): Promise<ChSql> {
let whereSearchCondition: ChSql | [] = [];
@ -729,28 +729,64 @@ function renderLimit(
// CTE (Common Table Expressions) isn't exported at this time. It's only used internally
// for metric SQL generation.
type ChartConfigWithOptDateRateAndCte = ChartConfigWithOptDateRange & {
type ChartConfigWithOptDateRangeEx = ChartConfigWithOptDateRange & {
with?: { name: string; sql: ChSql }[];
};
function renderWith(
chartConfig: ChartConfigWithOptDateRateAndCte,
chartConfig: ChartConfigWithOptDateRangeEx,
metadata: Metadata,
): ChSql | undefined {
const { with: withClauses } = chartConfig;
if (withClauses) {
return concatChSql(
'',
withClauses.map(clause => chSql`WITH ${clause.name} AS (${clause.sql})`),
',',
withClauses.map(clause => chSql`${clause.name} AS (${clause.sql})`),
);
}
return undefined;
}
function intervalToSeconds(interval: SQLInterval): number {
// Parse interval string like "15 second" into number of seconds
const [amount, unit] = interval.split(' ');
const value = parseInt(amount, 10);
switch (unit) {
case 'second':
return value;
case 'minute':
return value * 60;
case 'hour':
return value * 60 * 60;
case 'day':
return value * 24 * 60 * 60;
default:
throw new Error(`Invalid interval unit ${unit} in interval ${interval}`);
}
}
function renderFill(
chartConfig: ChartConfigWithOptDateRangeEx,
): ChSql | undefined {
const { granularity, dateRange } = chartConfig;
if (dateRange && granularity && granularity !== 'auto') {
const [start, end] = dateRange;
const step = intervalToSeconds(granularity);
return concatChSql(' ', [
chSql`FROM toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(${{ Int64: start.getTime() }}), INTERVAL ${granularity}))
TO toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(${{ Int64: end.getTime() }}), INTERVAL ${granularity}))
STEP ${{ Int32: step }}`,
]);
}
return undefined;
}
function translateMetricChartConfig(
chartConfig: ChartConfigWithOptDateRange,
): ChartConfigWithOptDateRateAndCte {
): ChartConfigWithOptDateRangeEx {
const metricTables = chartConfig.metricTables;
if (!metricTables) {
return chartConfig;
@ -810,8 +846,67 @@ function translateMetricChartConfig(
databaseName: '',
tableName: 'RawSum',
},
where: `MetricName = '${metricName}'`,
whereLanguage: 'sql',
};
} else if (metricType === MetricsDataType.Histogram && metricName) {
// histograms are only valid for quantile selections
const { aggFn, level, ..._selectRest } = _select as {
aggFn: string;
level?: number;
};
if (aggFn !== 'quantile' || level == null) {
throw new Error('quantile must be specified for histogram metrics');
}
return {
...restChartConfig,
with: [
{
name: 'HistRate',
sql: chSql`SELECT *, any(BucketCounts) OVER (ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS PrevBucketCounts,
any(CountLength) OVER (ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS PrevCountLength,
any(AttributesHash) OVER (ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS PrevAttributesHash,
IF(AggregationTemporality = 1,
BucketCounts,
IF(AttributesHash = PrevAttributesHash AND CountLength = PrevCountLength,
arrayMap((prev, curr) -> IF(curr < prev, curr, toUInt64(toInt64(curr) - toInt64(prev))), PrevBucketCounts, BucketCounts),
BucketCounts)) as BucketRates
FROM (
SELECT *, cityHash64(mapConcat(ScopeAttributes, ResourceAttributes, Attributes)) AS AttributesHash,
length(BucketCounts) as CountLength
FROM ${renderFrom({ from: { ...from, tableName: metricTables[MetricsDataType.Histogram] } })})
WHERE MetricName = '${metricName}'
ORDER BY Attributes, TimeUnix ASC
`,
},
{
name: 'RawHist',
sql: chSql`
SELECT *, toUInt64( ${{ Float64: level }} * arraySum(BucketRates)) AS Rank,
arrayCumSum(BucketRates) as CumRates,
arrayFirstIndex(x -> if(x > Rank, 1, 0), CumRates) AS BucketLowIdx,
IF(BucketLowIdx = length(BucketRates),
ExplicitBounds[length(ExplicitBounds)], -- if the low bound is the last bucket, use the last bound value
IF(BucketLowIdx > 1, -- indexes are 1-based
ExplicitBounds[BucketLowIdx] + (ExplicitBounds[BucketLowIdx + 1] - ExplicitBounds[BucketLowIdx]) *
intDivOrZero(
Rank - CumRates[BucketLowIdx - 1],
CumRates[BucketLowIdx] - CumRates[BucketLowIdx - 1]),
arrayElement(ExplicitBounds, BucketLowIdx + 1) * intDivOrZero(Rank, CumRates[BucketLowIdx]))) as Rate
FROM HistRate`,
},
],
select: [
{
..._selectRest,
aggFn: 'sum',
valueExpression: 'Rate',
},
],
from: {
databaseName: '',
tableName: 'RawHist',
},
};
}
@ -835,15 +930,19 @@ export async function renderChartConfig(
const where = await renderWhere(chartConfig, metadata);
const groupBy = await renderGroupBy(chartConfig, metadata);
const orderBy = renderOrderBy(chartConfig);
const fill = renderFill(chartConfig);
const limit = renderLimit(chartConfig);
return chSql`${
withClauses?.sql ? chSql`${withClauses}` : ''
}SELECT ${select} FROM ${from} ${where?.sql ? chSql`WHERE ${where}` : ''} ${
groupBy?.sql ? chSql`GROUP BY ${groupBy}` : ''
} ${orderBy?.sql ? chSql`ORDER BY ${orderBy}` : ''} ${
limit?.sql ? chSql`LIMIT ${limit}` : ''
}`;
return concatChSql(' ', [
chSql`${withClauses?.sql ? chSql`WITH ${withClauses}` : ''}`,
chSql`SELECT ${select}`,
chSql`FROM ${from}`,
chSql`${where.sql ? chSql`WHERE ${where}` : ''}`,
chSql`${groupBy?.sql ? chSql`GROUP BY ${groupBy}` : ''}`,
chSql`${orderBy?.sql ? chSql`ORDER BY ${orderBy}` : ''}`,
chSql`${fill?.sql ? chSql`WITH FILL ${fill}` : ''}`,
chSql`${limit?.sql ? chSql`LIMIT ${limit}` : ''}`,
]);
}
// EditForm -> translateToQueriedChartConfig -> QueriedChartConfig