Elgato_dark/hyperdx
1
0
Fork 0
mirror of https://github.com/hyperdxio/hyperdx synced 2026-04-21 13:37:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

feat: Add delta() function for gauge metrics (#1147)

Browse source
This commit is contained in:
Drew Davis 2025-09-11 17:10:43 -04:00 committed by GitHub
parent 5d567b9975
commit fa45875d38
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 327 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.changeset/honest-pens-bathe.md Normal file
View file

@ -0,0 +1,6 @@
---
"@hyperdx/common-utils": minor
"@hyperdx/app": minor
---
Add delta() function for gauge metrics

38
packages/api/src/clickhouse/__tests__/__snapshots__/renderChartConfig.test.ts.snap
View file

@ -59,6 +59,44 @@ Array [
]
`;
exports[`renderChartConfig Query Metrics - Gauge single max gauge with delta 1`] = `
Array [
Object {
"__hdx_time_bucket": "2022-01-05T00:00:00Z",
"max(toFloat64OrDefault(toString(LastValue)))": 5,
},
Object {
"__hdx_time_bucket": "2022-01-05T00:05:00Z",
"max(toFloat64OrDefault(toString(LastValue)))": -1.6666666666666667,
},
]
`;
exports[`renderChartConfig Query Metrics - Gauge single max gauge with delta and group by 1`] = `
Array [
Object {
"__hdx_time_bucket": "2022-01-05T00:00:00Z",
"arrayElement(ResourceAttributes, 'host')": "host2",
"max(toFloat64OrDefault(toString(LastValue)))": 5,
},
Object {
"__hdx_time_bucket": "2022-01-05T00:00:00Z",
"arrayElement(ResourceAttributes, 'host')": "host1",
"max(toFloat64OrDefault(toString(LastValue)))": -72.91666666666667,
},
Object {
"__hdx_time_bucket": "2022-01-05T00:05:00Z",
"arrayElement(ResourceAttributes, 'host')": "host2",
"max(toFloat64OrDefault(toString(LastValue)))": -1.6666666666666667,
},
Object {
"__hdx_time_bucket": "2022-01-05T00:05:00Z",
"arrayElement(ResourceAttributes, 'host')": "host1",
"max(toFloat64OrDefault(toString(LastValue)))": -33.333333333333336,
},
]
`;
exports[`renderChartConfig Query Metrics - Gauge single max/avg/sum gauge 1`] = `
Array [
Object {

51
packages/api/src/clickhouse/__tests__/renderChartConfig.test.ts
View file

@ -438,6 +438,57 @@ describe('renderChartConfig', () => {
);
expect(await queryData(query)).toMatchSnapshot();
});
it('single max gauge with delta', async () => {
const query = await renderChartConfig(
{
select: [
{
aggFn: 'max',
metricName: 'test.cpu',
metricType: MetricsDataType.Gauge,
valueExpression: 'Value',
isDelta: true,
},
],
from: metricSource.from,
where: '',
metricTables: TEST_METRIC_TABLES,
dateRange: [new Date(now), new Date(now + ms('10m'))],
granularity: '5 minute',
timestampValueExpression: metricSource.timestampValueExpression,
connection: connection.id,
},
metadata,
);
expect(await queryData(query)).toMatchSnapshot();
});
it('single max gauge with delta and group by', async () => {
const query = await renderChartConfig(
{
select: [
{
aggFn: 'max',
metricName: 'test.cpu',
metricType: MetricsDataType.Gauge,
valueExpression: 'Value',
isDelta: true,
},
],
from: metricSource.from,
where: '',
metricTables: TEST_METRIC_TABLES,
dateRange: [new Date(now), new Date(now + ms('10m'))],
granularity: '5 minute',
groupBy: `ResourceAttributes['host']`,
timestampValueExpression: metricSource.timestampValueExpression,
connection: connection.id,
},
metadata,
);
expect(await queryData(query)).toMatchSnapshot();
});
});
describe('Query Metrics - Sum', () => {

123
packages/app/src/components/DBEditTimeChartForm.tsx
View file

@ -75,7 +75,11 @@ import HDXMarkdownChart from '../HDXMarkdownChart';
import { AggFnSelectControlled } from './AggFnSelect';
import DBNumberChart from './DBNumberChart';
import { InputControlled, TextInputControlled } from './InputControlled';
import {
CheckBoxControlled,
InputControlled,
TextInputControlled,
} from './InputControlled';
import { MetricNameSelect } from './MetricNameSelect';
import { NumberFormatInput } from './NumberFormat';
import { SourceSelectControlled } from './SourceSelect';
@ -202,7 +206,7 @@ function ChartSeriesEditorComponent({
mb={8}
mt="sm"
/>
<Flex gap="sm" mt="xs" align="center">
<Flex gap="sm" mt="xs" align="start">
<div
style={{
minWidth: 200,
@ -216,17 +220,32 @@ function ChartSeriesEditorComponent({
/>
</div>
{tableSource?.kind === SourceKind.Metric && (
<MetricNameSelect
metricName={metricName}
dateRange={dateRange}
metricType={metricType}
setMetricName={value => {
setValue(`${namePrefix}metricName`, value);
setValue(`${namePrefix}valueExpression`, 'Value');
}}
setMetricType={value => setValue(`${namePrefix}metricType`, value)}
metricSource={tableSource}
/>
<div style={{ minWidth: 220 }}>
<MetricNameSelect
metricName={metricName}
dateRange={dateRange}
metricType={metricType}
setMetricName={value => {
setValue(`${namePrefix}metricName`, value);
setValue(`${namePrefix}valueExpression`, 'Value');
}}
setMetricType={value =>
setValue(`${namePrefix}metricType`, value)
}
metricSource={tableSource}
/>
{metricType === 'gauge' && (
<Flex justify="end">
<CheckBoxControlled
control={control}
name={`${namePrefix}isDelta`}
label="Delta"
size="xs"
className="mt-2"
/>
</Flex>
)}
</div>
)}
{tableSource?.kind !== SourceKind.Metric && aggFn !== 'count' && (
<div style={{ minWidth: 220 }}>
@ -243,44 +262,46 @@ function ChartSeriesEditorComponent({
/>
</div>
)}
<Text size="sm">Where</Text>
{aggConditionLanguage === 'sql' ? (
<SQLInlineEditorControlled
tableConnections={{
databaseName,
tableName: tableName ?? '',
connectionId: connectionId ?? '',
}}
control={control}
name={`${namePrefix}aggCondition`}
placeholder="SQL WHERE clause (ex. column = 'foo')"
onLanguageChange={lang =>
setValue(`${namePrefix}aggConditionLanguage`, lang)
}
additionalSuggestions={attributeKeys}
language="sql"
onSubmit={onSubmit}
/>
) : (
<SearchInputV2
tableConnections={{
connectionId: connectionId ?? '',
databaseName: databaseName ?? '',
tableName: tableName ?? '',
}}
control={control}
name={`${namePrefix}aggCondition`}
onLanguageChange={lang =>
setValue(`${namePrefix}aggConditionLanguage`, lang)
}
language="lucene"
placeholder="Search your events w/ Lucene ex. column:foo"
onSubmit={onSubmit}
additionalSuggestions={attributeKeys}
/>
)}
<Flex align={'center'} gap={'xs'} className="flex-grow-1">
<Text size="sm">Where</Text>
{aggConditionLanguage === 'sql' ? (
<SQLInlineEditorControlled
tableConnections={{
databaseName,
tableName: tableName ?? '',
connectionId: connectionId ?? '',
}}
control={control}
name={`${namePrefix}aggCondition`}
placeholder="SQL WHERE clause (ex. column = 'foo')"
onLanguageChange={lang =>
setValue(`${namePrefix}aggConditionLanguage`, lang)
}
additionalSuggestions={attributeKeys}
language="sql"
onSubmit={onSubmit}
/>
) : (
<SearchInputV2
tableConnections={{
connectionId: connectionId ?? '',
databaseName: databaseName ?? '',
tableName: tableName ?? '',
}}
control={control}
name={`${namePrefix}aggCondition`}
onLanguageChange={lang =>
setValue(`${namePrefix}aggConditionLanguage`, lang)
}
language="lucene"
placeholder="Search your events w/ Lucene ex. column:foo"
onSubmit={onSubmit}
additionalSuggestions={attributeKeys}
/>
)}
</Flex>
{showGroupBy && (
<>
<Flex align={'center'} gap={'xs'}>
<Text size="sm" style={{ whiteSpace: 'nowrap' }}>
Group By
</Text>
@ -303,7 +324,7 @@ function ChartSeriesEditorComponent({
onSubmit={onSubmit}
/>
</div>
</>
</Flex>
)}
</Flex>
</>

36
packages/app/src/components/InputControlled.tsx
View file

@ -1,6 +1,8 @@
import React from 'react';
import { Control, Controller, FieldValues, Path } from 'react-hook-form';
import {
Checkbox,
CheckboxProps,
Input,
InputProps,
PasswordInput,
@ -33,6 +35,17 @@ interface TextInputControlledProps<T extends FieldValues>
rules?: Parameters<Control<T>['register']>[1];
}
interface CheckboxControlledProps<T extends FieldValues>
extends Omit<CheckboxProps, 'name' | 'style'>,
Omit<
React.InputHTMLAttributes<HTMLInputElement>,
'name' | 'size' | 'color'
> {
name: Path<T>;
control: Control<T>;
rules?: Parameters<Control<T>['register']>[1];
}
export function TextInputControlled<T extends FieldValues>({
name,
control,
@ -86,3 +99,26 @@ export function PasswordInputControlled<T extends FieldValues>({
/>
);
}
export function CheckBoxControlled<T extends FieldValues>({
name,
control,
rules,
...props
}: CheckboxControlledProps<T>) {
return (
<Controller
name={name}
control={control}
rules={rules}
render={({ field: { value, ...field }, fieldState: { error } }) => (
<Checkbox
{...props}
{...field}
checked={value}
error={error?.message}
/>
)}
/>
);
}

35
packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap
View file

@ -284,7 +284,40 @@ exports[`renderChartConfig should generate sql for a single gauge metric 1`] = `
FROM Source
GROUP BY AttributesHash, __hdx_time_bucket2
ORDER BY AttributesHash, __hdx_time_bucket2
) SELECT quantile(0.95)(toFloat64OrDefault(toString(LastValue))),toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (__hdx_time_bucket2 >= fromUnixTimestamp64Milli(1739318400000) AND __hdx_time_bucket2 <= fromUnixTimestamp64Milli(1765670400000)) GROUP BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` LIMIT 10"
) SELECT quantile(0.95)(toFloat64OrDefault(toString(LastValue))),toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (__hdx_time_bucket2 >= fromUnixTimestamp64Milli(1739318400000) AND __hdx_time_bucket2 <= fromUnixTimestamp64Milli(1765670400000)) GROUP BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable'"
`;
exports[`renderChartConfig should generate sql for a single gauge metric with a delta() function applied 1`] = `
"WITH Source AS (
SELECT
*,
cityHash64(mapConcat(ScopeAttributes, ResourceAttributes, Attributes)) AS AttributesHash
FROM default.otel_metrics_gauge
WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1765670400000)) AND ((MetricName = 'nodejs.event_loop.utilization'))
),Bucketed AS (
SELECT
toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS \`__hdx_time_bucket2\`,
AttributesHash,
IF(date_diff('second', min(toDateTime(TimeUnix)), max(toDateTime(TimeUnix))) > 0, (argMax(Value, TimeUnix) - argMin(Value, TimeUnix)) * 60 / date_diff('second', min(toDateTime(TimeUnix)), max(toDateTime(TimeUnix))), 0) AS LastValue,
any(ScopeAttributes) AS ScopeAttributes,
any(ResourceAttributes) AS ResourceAttributes,
any(Attributes) AS Attributes,
any(ResourceSchemaUrl) AS ResourceSchemaUrl,
any(ScopeName) AS ScopeName,
any(ScopeVersion) AS ScopeVersion,
any(ScopeDroppedAttrCount) AS ScopeDroppedAttrCount,
any(ScopeSchemaUrl) AS ScopeSchemaUrl,
any(ServiceName) AS ServiceName,
any(MetricDescription) AS MetricDescription,
any(MetricUnit) AS MetricUnit,
any(StartTimeUnix) AS StartTimeUnix,
any(Flags) AS Flags
FROM Source
GROUP BY AttributesHash, __hdx_time_bucket2
ORDER BY AttributesHash, __hdx_time_bucket2
) SELECT max(
toFloat64OrDefault(toString(LastValue))
),toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (__hdx_time_bucket2 >= fromUnixTimestamp64Milli(1739318400000) AND __hdx_time_bucket2 <= fromUnixTimestamp64Milli(1765670400000)) GROUP BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable'"
`;
exports[`renderChartConfig should generate sql for a single sum metric 1`] = `

91
packages/common-utils/src/__tests__/renderChartConfig.test.ts
View file

@ -22,42 +22,65 @@ describe('renderChartConfig', () => {
} as unknown as Metadata;
});
it('should generate sql for a single gauge metric', async () => {
const config: ChartConfigWithOptDateRange = {
displayType: DisplayType.Line,
connection: 'test-connection',
// metricTables is added from the Source object via spread operator
metricTables: {
gauge: 'otel_metrics_gauge',
histogram: 'otel_metrics_histogram',
sum: 'otel_metrics_sum',
summary: 'otel_metrics_summary',
'exponential histogram': 'otel_metrics_exponential_histogram',
const gaugeConfiguration: ChartConfigWithOptDateRange = {
displayType: DisplayType.Line,
connection: 'test-connection',
// metricTables is added from the Source object via spread operator
metricTables: {
gauge: 'otel_metrics_gauge',
histogram: 'otel_metrics_histogram',
sum: 'otel_metrics_sum',
summary: 'otel_metrics_summary',
'exponential histogram': 'otel_metrics_exponential_histogram',
},
from: {
databaseName: 'default',
tableName: '',
},
select: [
{
aggFn: 'quantile',
aggCondition: '',
aggConditionLanguage: 'lucene',
valueExpression: 'Value',
level: 0.95,
metricName: 'nodejs.event_loop.utilization',
metricType: MetricsDataType.Gauge,
},
from: {
databaseName: 'default',
tableName: '',
},
select: [
{
aggFn: 'quantile',
aggCondition: '',
aggConditionLanguage: 'lucene',
valueExpression: 'Value',
level: 0.95,
metricName: 'nodejs.event_loop.utilization',
metricType: MetricsDataType.Gauge,
},
],
where: '',
whereLanguage: 'lucene',
timestampValueExpression: 'TimeUnix',
dateRange: [new Date('2025-02-12'), new Date('2025-12-14')],
granularity: '1 minute',
limit: { limit: 10 },
};
],
where: '',
whereLanguage: 'lucene',
timestampValueExpression: 'TimeUnix',
dateRange: [new Date('2025-02-12'), new Date('2025-12-14')],
granularity: '1 minute',
limit: { limit: 10 },
};
const generatedSql = await renderChartConfig(config, mockMetadata);
it('should generate sql for a single gauge metric', async () => {
const generatedSql = await renderChartConfig(
gaugeConfiguration,
mockMetadata,
);
const actual = parameterizedQueryToSql(generatedSql);
expect(actual).toMatchSnapshot();
});
it('should generate sql for a single gauge metric with a delta() function applied', async () => {
const generatedSql = await renderChartConfig(
{
...gaugeConfiguration,
select: [
{
aggFn: 'max',
valueExpression: 'Value',
metricName: 'nodejs.event_loop.utilization',
metricType: MetricsDataType.Gauge,
isDelta: true,
},
],
},
mockMetadata,
);
const actual = parameterizedQueryToSql(generatedSql);
expect(actual).toMatchSnapshot();
});

32
packages/common-utils/src/renderChartConfig.ts
View file

@ -24,6 +24,7 @@ import {
} from '@/types';
import {
convertDateRangeToGranularityString,
convertGranularityToSeconds,
getFirstTimestampValueExpression,
splitAndTrimWithBracket,
} from '@/utils';
@ -85,7 +86,11 @@ export const setChartSelectsAlias = (config: ChartConfigWithOptDateRange) => {
...config,
select: config.select.map(s => ({
...s,
alias: s.alias || `${s.aggFn}(${s.metricName})`, // use an alias if one isn't already set
alias:
s.alias ||
(s.isDelta
? `${s.aggFn}(delta(${s.metricName}))`
: `${s.aggFn}(${s.metricName})`), // use an alias if one isn't already set
})),
};
}
@ -893,6 +898,24 @@ function renderFill(
return undefined;
}
function renderDeltaExpression(
chartConfig: ChartConfigWithOptDateRange,
valueExpression: string,
) {
const interval =
chartConfig.granularity === 'auto' && Array.isArray(chartConfig.dateRange)
? convertDateRangeToGranularityString(chartConfig.dateRange, 60)
: chartConfig.granularity;
const intervalInSeconds = convertGranularityToSeconds(interval ?? '');
const valueDiff = `(argMax(${valueExpression}, ${chartConfig.timestampValueExpression}) - argMin(${valueExpression}, ${chartConfig.timestampValueExpression}))`;
const timeDiffInSeconds = `date_diff('second', min(toDateTime(${chartConfig.timestampValueExpression})), max(toDateTime(${chartConfig.timestampValueExpression})))`;
// Prevent division by zero, if timeDiffInSeconds is 0, return 0
// The delta is extrapolated to the bucket interval, to match prometheus delta() behavior
return `IF(${timeDiffInSeconds} > 0, ${valueDiff} * ${intervalInSeconds} / ${timeDiffInSeconds}, 0)`;
}
async function translateMetricChartConfig(
chartConfig: ChartConfigWithOptDateRange,
metadata: Metadata,
@ -938,6 +961,10 @@ async function translateMetricChartConfig(
metadata,
);
const bucketValueExpr = _select.isDelta
? renderDeltaExpression(chartConfig, _select.valueExpression)
: `last_value(${_select.valueExpression})`;
return {
...restChartConfig,
with: [
@ -957,7 +984,7 @@ async function translateMetricChartConfig(
SELECT
${timeExpr},
AttributesHash,
last_value(Value) AS LastValue,
${bucketValueExpr} AS LastValue,
any(ScopeAttributes) AS ScopeAttributes,
any(ResourceAttributes) AS ResourceAttributes,
any(Attributes) AS Attributes,
@ -990,6 +1017,7 @@ async function translateMetricChartConfig(
},
where: '', // clear up the condition since the where clause is already applied at the upstream CTE
timestampValueExpression: timeBucketCol,
settings: chSql`short_circuit_function_evaluation = 'force_enable'`,
};
} else if (metricType === MetricsDataType.Sum && metricName) {
const timeBucketCol = '__hdx_time_bucket2';

3
packages/common-utils/src/types.ts
View file

@ -76,6 +76,7 @@ export const RootValueExpressionSchema = z
aggCondition: SearchConditionSchema,
aggConditionLanguage: SearchConditionLanguageSchema,
valueExpression: z.string(),
isDelta: z.boolean().optional(),
})
.or(
z.object({
@ -84,6 +85,7 @@ export const RootValueExpressionSchema = z
aggCondition: SearchConditionSchema,
aggConditionLanguage: SearchConditionLanguageSchema,
valueExpression: z.string(),
isDelta: z.boolean().optional(),
}),
)
.or(
@ -93,6 +95,7 @@ export const RootValueExpressionSchema = z
aggConditionLanguage: SearchConditionLanguageSchema,
valueExpression: z.string(),
metricType: z.nativeEnum(MetricsDataType).optional(),
isDelta: z.boolean().optional(),
}),
);
export const DerivedColumnSchema = z.intersection(