mirror of
https://github.com/hyperdxio/hyperdx
synced 2026-04-21 13:37:15 +00:00
feat: add last_value agg function (#202)
This commit is contained in:
Warren
GitHub
parent
1b4607ba7d
commit
7bc4cd3678
3 changed files with 197 additions and 134 deletions
6
.changeset/bright-turkeys-remember.md
Normal file
6
.changeset/bright-turkeys-remember.md
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
---
|
||||
'@hyperdx/api': patch
|
||||
'@hyperdx/app': patch
|
||||
---
|
||||
|
||||
feat: add last_value agg function
|
||||
|
|
@ -4,6 +4,7 @@ import ms from 'ms';
|
|||
import {
|
||||
buildEvent,
|
||||
buildMetricSeries,
|
||||
clearClickhouseTables,
|
||||
closeDB,
|
||||
getServer,
|
||||
mockLogsPropertyTypeMappingsModel,
|
||||
|
|
@ -25,7 +26,8 @@ describe('clickhouse', () => {
|
|||
await closeDB();
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
afterEach(async () => {
|
||||
await clearClickhouseTables();
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
|
|
@ -869,154 +871,204 @@ Array [
|
|||
`);
|
||||
});
|
||||
|
||||
it('fetches multi series metric table chart correctly', async () => {
|
||||
describe('fetches multi series metric table chart correctly', () => {
|
||||
const now = new Date('2022-01-05').getTime();
|
||||
const runId = Math.random().toString(); // dedup watch mode runs
|
||||
const teamId = `test`;
|
||||
|
||||
// Rate: 8, 1, 8, 25
|
||||
await clickhouse.bulkInsertTeamMetricStream(
|
||||
buildMetricSeries({
|
||||
name: 'test.users',
|
||||
tags: { host: 'test1', runId },
|
||||
data_type: clickhouse.MetricsDataType.Sum,
|
||||
is_monotonic: true,
|
||||
is_delta: true,
|
||||
unit: 'Users',
|
||||
points: [
|
||||
{ value: 0, timestamp: now - ms('1m') }, // 0
|
||||
{ value: 1, timestamp: now },
|
||||
{ value: 8, timestamp: now + ms('4m') }, // 8
|
||||
{ value: 8, timestamp: now + ms('6m') },
|
||||
{ value: 9, timestamp: now + ms('9m') }, // 9
|
||||
{ value: 15, timestamp: now + ms('11m') },
|
||||
{ value: 17, timestamp: now + ms('14m') }, // 17
|
||||
{ value: 32, timestamp: now + ms('16m') },
|
||||
{ value: 42, timestamp: now + ms('19m') }, // 42
|
||||
],
|
||||
}),
|
||||
);
|
||||
beforeEach(async () => {
|
||||
// Rate: 8, 1, 8, 25
|
||||
await clickhouse.bulkInsertTeamMetricStream(
|
||||
buildMetricSeries({
|
||||
name: 'test.users',
|
||||
tags: { host: 'test1', runId },
|
||||
data_type: clickhouse.MetricsDataType.Sum,
|
||||
is_monotonic: true,
|
||||
is_delta: true,
|
||||
unit: 'Users',
|
||||
points: [
|
||||
{ value: 0, timestamp: now - ms('1m') }, // 0
|
||||
{ value: 1, timestamp: now },
|
||||
{ value: 8, timestamp: now + ms('4m') }, // 8
|
||||
{ value: 8, timestamp: now + ms('6m') },
|
||||
{ value: 9, timestamp: now + ms('9m') }, // 9
|
||||
{ value: 15, timestamp: now + ms('11m') },
|
||||
{ value: 17, timestamp: now + ms('14m') }, // 17
|
||||
{ value: 32, timestamp: now + ms('16m') },
|
||||
{ value: 42, timestamp: now + ms('19m') }, // 42
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
// Rate: 11, 78, 5805, 78729
|
||||
// Sum: 12, 79, 5813, 78754
|
||||
await clickhouse.bulkInsertTeamMetricStream(
|
||||
buildMetricSeries({
|
||||
name: 'test.users',
|
||||
tags: { host: 'test2', runId },
|
||||
data_type: clickhouse.MetricsDataType.Sum,
|
||||
is_monotonic: true,
|
||||
is_delta: true,
|
||||
unit: 'Users',
|
||||
points: [
|
||||
{ value: 3, timestamp: now - ms('1m') }, // 3
|
||||
{ value: 3, timestamp: now },
|
||||
{ value: 14, timestamp: now + ms('4m') }, // 14
|
||||
{ value: 15, timestamp: now + ms('6m') },
|
||||
{ value: 92, timestamp: now + ms('9m') }, // 92
|
||||
{ value: 653, timestamp: now + ms('11m') },
|
||||
{ value: 5897, timestamp: now + ms('14m') }, // 5897
|
||||
{ value: 9323, timestamp: now + ms('16m') },
|
||||
{ value: 84626, timestamp: now + ms('19m') }, // 84626
|
||||
],
|
||||
}),
|
||||
);
|
||||
// Rate: 11, 78, 5805, 78729
|
||||
// Sum: 12, 79, 5813, 78754
|
||||
await clickhouse.bulkInsertTeamMetricStream(
|
||||
buildMetricSeries({
|
||||
name: 'test.users',
|
||||
tags: { host: 'test2', runId },
|
||||
data_type: clickhouse.MetricsDataType.Sum,
|
||||
is_monotonic: true,
|
||||
is_delta: true,
|
||||
unit: 'Users',
|
||||
points: [
|
||||
{ value: 3, timestamp: now - ms('1m') }, // 3
|
||||
{ value: 3, timestamp: now },
|
||||
{ value: 14, timestamp: now + ms('4m') }, // 14
|
||||
{ value: 15, timestamp: now + ms('6m') },
|
||||
{ value: 92, timestamp: now + ms('9m') }, // 92
|
||||
{ value: 653, timestamp: now + ms('11m') },
|
||||
{ value: 5897, timestamp: now + ms('14m') }, // 5897
|
||||
{ value: 9323, timestamp: now + ms('16m') },
|
||||
{ value: 84626, timestamp: now + ms('19m') }, // 84626
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
await clickhouse.bulkInsertTeamMetricStream(
|
||||
buildMetricSeries({
|
||||
name: 'test.cpu',
|
||||
tags: { host: 'test1', runId },
|
||||
data_type: clickhouse.MetricsDataType.Gauge,
|
||||
is_monotonic: false,
|
||||
is_delta: false,
|
||||
unit: 'Percent',
|
||||
points: [
|
||||
{ value: 50, timestamp: now },
|
||||
{ value: 25, timestamp: now + ms('1m') },
|
||||
{ value: 12.5, timestamp: now + ms('2m') },
|
||||
{ value: 6.25, timestamp: now + ms('3m') }, // Last 5min
|
||||
{ value: 100, timestamp: now + ms('6m') },
|
||||
{ value: 75, timestamp: now + ms('7m') },
|
||||
{ value: 10, timestamp: now + ms('8m') },
|
||||
{ value: 80, timestamp: now + ms('9m') }, // Last 5min
|
||||
],
|
||||
}),
|
||||
);
|
||||
await clickhouse.bulkInsertTeamMetricStream(
|
||||
buildMetricSeries({
|
||||
name: 'test.cpu',
|
||||
tags: { host: 'test1', runId },
|
||||
data_type: clickhouse.MetricsDataType.Gauge,
|
||||
is_monotonic: false,
|
||||
is_delta: false,
|
||||
unit: 'Percent',
|
||||
points: [
|
||||
{ value: 50, timestamp: now },
|
||||
{ value: 25, timestamp: now + ms('1m') },
|
||||
{ value: 12.5, timestamp: now + ms('2m') },
|
||||
{ value: 6.25, timestamp: now + ms('3m') }, // Last 5min
|
||||
{ value: 100, timestamp: now + ms('6m') },
|
||||
{ value: 75, timestamp: now + ms('7m') },
|
||||
{ value: 10, timestamp: now + ms('8m') },
|
||||
{ value: 80, timestamp: now + ms('9m') }, // Last 5min
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
await clickhouse.bulkInsertTeamMetricStream(
|
||||
buildMetricSeries({
|
||||
name: 'test.cpu',
|
||||
tags: { host: 'test2', runId },
|
||||
data_type: clickhouse.MetricsDataType.Gauge,
|
||||
is_monotonic: false,
|
||||
is_delta: false,
|
||||
unit: 'Percent',
|
||||
points: [
|
||||
{ value: 1, timestamp: now },
|
||||
{ value: 2, timestamp: now + ms('1m') },
|
||||
{ value: 3, timestamp: now + ms('2m') },
|
||||
{ value: 4, timestamp: now + ms('3m') }, // Last 5min
|
||||
{ value: 5, timestamp: now + ms('6m') },
|
||||
{ value: 6, timestamp: now + ms('7m') },
|
||||
{ value: 5, timestamp: now + ms('8m') },
|
||||
{ value: 4, timestamp: now + ms('9m') }, // Last 5min
|
||||
],
|
||||
}),
|
||||
);
|
||||
await clickhouse.bulkInsertTeamMetricStream(
|
||||
buildMetricSeries({
|
||||
name: 'test.cpu',
|
||||
tags: { host: 'test2', runId },
|
||||
data_type: clickhouse.MetricsDataType.Gauge,
|
||||
is_monotonic: false,
|
||||
is_delta: false,
|
||||
unit: 'Percent',
|
||||
points: [
|
||||
{ value: 1, timestamp: now },
|
||||
{ value: 2, timestamp: now + ms('1m') },
|
||||
{ value: 3, timestamp: now + ms('2m') },
|
||||
{ value: 4, timestamp: now + ms('3m') }, // Last 5min
|
||||
{ value: 5, timestamp: now + ms('6m') },
|
||||
{ value: 6, timestamp: now + ms('7m') },
|
||||
{ value: 5, timestamp: now + ms('8m') },
|
||||
{ value: 4, timestamp: now + ms('9m') }, // Last 5min
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
mockSpyMetricPropertyTypeMappingsModel({
|
||||
runId: 'string',
|
||||
host: 'string',
|
||||
mockSpyMetricPropertyTypeMappingsModel({
|
||||
runId: 'string',
|
||||
host: 'string',
|
||||
});
|
||||
});
|
||||
|
||||
const singleSumSeriesData = (
|
||||
await clickhouse.getMultiSeriesChart({
|
||||
series: [
|
||||
{
|
||||
type: 'table',
|
||||
table: 'metrics',
|
||||
aggFn: clickhouse.AggFn.SumRate,
|
||||
field: 'test.users',
|
||||
where: `runId:${runId}`,
|
||||
groupBy: [],
|
||||
metricDataType: clickhouse.MetricsDataType.Sum,
|
||||
},
|
||||
{
|
||||
type: 'time',
|
||||
table: 'metrics',
|
||||
aggFn: clickhouse.AggFn.Avg,
|
||||
field: 'test.cpu',
|
||||
where: `runId:${runId}`,
|
||||
groupBy: [],
|
||||
metricDataType: clickhouse.MetricsDataType.Gauge,
|
||||
},
|
||||
],
|
||||
tableVersion: undefined,
|
||||
teamId,
|
||||
startTime: now,
|
||||
endTime: now + ms('20m'),
|
||||
granularity: undefined,
|
||||
maxNumGroups: 20,
|
||||
seriesReturnType: clickhouse.SeriesReturnType.Column,
|
||||
})
|
||||
).data.map(d => {
|
||||
return _.pick(d, [
|
||||
'group',
|
||||
'series_0.data',
|
||||
'series_1.data',
|
||||
'ts_bucket',
|
||||
]);
|
||||
});
|
||||
it('gauge (last value)', async () => {
|
||||
const data = (
|
||||
await clickhouse.getMultiSeriesChart({
|
||||
series: [
|
||||
{
|
||||
type: 'time',
|
||||
table: 'metrics',
|
||||
aggFn: clickhouse.AggFn.LastValue,
|
||||
field: 'test.cpu',
|
||||
where: `runId:${runId}`,
|
||||
groupBy: ['host'],
|
||||
metricDataType: clickhouse.MetricsDataType.Gauge,
|
||||
},
|
||||
],
|
||||
tableVersion: undefined,
|
||||
teamId,
|
||||
startTime: now,
|
||||
endTime: now + ms('20m'),
|
||||
granularity: undefined,
|
||||
maxNumGroups: 20,
|
||||
seriesReturnType: clickhouse.SeriesReturnType.Column,
|
||||
})
|
||||
).data.map(d => {
|
||||
return _.pick(d, ['group', 'series_0.data', 'ts_bucket']);
|
||||
});
|
||||
|
||||
expect(singleSumSeriesData).toMatchInlineSnapshot(`
|
||||
expect(data).toMatchInlineSnapshot(`
|
||||
Array [
|
||||
Object {
|
||||
"group": Array [],
|
||||
"series_0.data": 84665,
|
||||
"series_1.data": 42,
|
||||
"group": Array [
|
||||
"test1",
|
||||
],
|
||||
"series_0.data": 80,
|
||||
"ts_bucket": "0",
|
||||
},
|
||||
Object {
|
||||
"group": Array [
|
||||
"test2",
|
||||
],
|
||||
"series_0.data": 4,
|
||||
"ts_bucket": "0",
|
||||
},
|
||||
]
|
||||
`);
|
||||
});
|
||||
|
||||
it('sum (rate) + gauge (avg)', async () => {
|
||||
const data = (
|
||||
await clickhouse.getMultiSeriesChart({
|
||||
series: [
|
||||
{
|
||||
type: 'table',
|
||||
table: 'metrics',
|
||||
aggFn: clickhouse.AggFn.SumRate,
|
||||
field: 'test.users',
|
||||
where: `runId:${runId}`,
|
||||
groupBy: [],
|
||||
metricDataType: clickhouse.MetricsDataType.Sum,
|
||||
},
|
||||
{
|
||||
type: 'time',
|
||||
table: 'metrics',
|
||||
aggFn: clickhouse.AggFn.Avg,
|
||||
field: 'test.cpu',
|
||||
where: `runId:${runId}`,
|
||||
groupBy: [],
|
||||
metricDataType: clickhouse.MetricsDataType.Gauge,
|
||||
},
|
||||
],
|
||||
tableVersion: undefined,
|
||||
teamId,
|
||||
startTime: now,
|
||||
endTime: now + ms('20m'),
|
||||
granularity: undefined,
|
||||
maxNumGroups: 20,
|
||||
seriesReturnType: clickhouse.SeriesReturnType.Column,
|
||||
})
|
||||
).data.map(d => {
|
||||
return _.pick(d, [
|
||||
'group',
|
||||
'series_0.data',
|
||||
'series_1.data',
|
||||
'ts_bucket',
|
||||
]);
|
||||
});
|
||||
|
||||
expect(data).toMatchInlineSnapshot(`
|
||||
Array [
|
||||
Object {
|
||||
"group": Array [],
|
||||
"series_0.data": 84665,
|
||||
"series_1.data": 42,
|
||||
"ts_bucket": "0",
|
||||
},
|
||||
]
|
||||
`);
|
||||
});
|
||||
});
|
||||
|
||||
it('limits groups and sorts multi series charts properly', async () => {
|
||||
|
|
|
|||
|
|
@ -61,9 +61,10 @@ export enum AggFn {
|
|||
AvgRate = 'avg_rate',
|
||||
Count = 'count',
|
||||
CountDistinct = 'count_distinct',
|
||||
CountPerSec = 'count_per_sec',
|
||||
CountPerMin = 'count_per_min',
|
||||
CountPerHour = 'count_per_hour',
|
||||
CountPerMin = 'count_per_min',
|
||||
CountPerSec = 'count_per_sec',
|
||||
LastValue = 'last_value',
|
||||
Max = 'max',
|
||||
MaxRate = 'max_rate',
|
||||
Min = 'min',
|
||||
|
|
@ -925,6 +926,8 @@ export const buildMetricSeriesQuery = async ({
|
|||
selectClause.push(
|
||||
aggFn === AggFn.Count
|
||||
? 'COUNT(value) as data'
|
||||
: aggFn === AggFn.LastValue
|
||||
? 'LAST_VALUE(value) as data'
|
||||
: aggFn === AggFn.Sum
|
||||
? `SUM(value) as data`
|
||||
: aggFn === AggFn.Avg
|
||||
|
|
@ -1167,6 +1170,8 @@ const buildEventSeriesQuery = async ({
|
|||
"divide(count(), age('hh', toDateTime(?), toDateTime(?))) as data",
|
||||
[startTime / 1000, endTime / 1000],
|
||||
)
|
||||
: aggFn === AggFn.LastValue
|
||||
? `toFloat64(last_value(${selectField})) as data`
|
||||
: aggFn === AggFn.Sum
|
||||
? `toFloat64(sum(${selectField})) as data`
|
||||
: aggFn === AggFn.Avg
|
||||
|
|
|
|||
Loading…
Reference in a new issue