mirror of
https://github.com/hyperdxio/hyperdx
synced 2026-04-21 13:37:15 +00:00
fix: aggFn use default instead of null (#782)
Switched to the `-OrDefault` version of the float conversion when combined with the aggregate functions to prevent emitting null. Ref: HDX-1379
This commit is contained in:
Dan Hable
GitHub
parent
293a2affc8
commit
79fe30f503
6 changed files with 162 additions and 32 deletions
6
.changeset/curly-ducks-jump.md
Normal file
6
.changeset/curly-ducks-jump.md
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
---
|
||||
"@hyperdx/common-utils": minor
|
||||
"@hyperdx/api": minor
|
||||
---
|
||||
|
||||
Queries depending on numeric aggregates now use the type's default value (e.g. 0) instead of null when dealing with non-numeric data.
|
||||
|
|
@ -64,7 +64,7 @@ exports[`renderChartConfig Query Metrics should include multiple data points in
|
|||
Array [
|
||||
Object {
|
||||
"__hdx_time_bucket": "2022-01-05T00:00:00Z",
|
||||
"any(toFloat64OrNull(toString(Rate)))": 2.9,
|
||||
"any(toFloat64OrDefault(toString(Rate)))": 2.9,
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
|
@ -74,22 +74,22 @@ Array [
|
|||
Object {
|
||||
"__hdx_time_bucket": "2022-01-05T00:00:00Z",
|
||||
"arrayElement(ResourceAttributes, 'host')": "host2",
|
||||
"avg(toFloat64OrNull(toString(LastValue)))": 4,
|
||||
"avg(toFloat64OrDefault(toString(LastValue)))": 4,
|
||||
},
|
||||
Object {
|
||||
"__hdx_time_bucket": "2022-01-05T00:00:00Z",
|
||||
"arrayElement(ResourceAttributes, 'host')": "host1",
|
||||
"avg(toFloat64OrNull(toString(LastValue)))": 6.25,
|
||||
"avg(toFloat64OrDefault(toString(LastValue)))": 6.25,
|
||||
},
|
||||
Object {
|
||||
"__hdx_time_bucket": "2022-01-05T00:05:00Z",
|
||||
"arrayElement(ResourceAttributes, 'host')": "host2",
|
||||
"avg(toFloat64OrNull(toString(LastValue)))": 4,
|
||||
"avg(toFloat64OrDefault(toString(LastValue)))": 4,
|
||||
},
|
||||
Object {
|
||||
"__hdx_time_bucket": "2022-01-05T00:05:00Z",
|
||||
"arrayElement(ResourceAttributes, 'host')": "host1",
|
||||
"avg(toFloat64OrNull(toString(LastValue)))": 80,
|
||||
"avg(toFloat64OrDefault(toString(LastValue)))": 80,
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
|
@ -98,11 +98,11 @@ exports[`renderChartConfig Query Metrics single avg gauge with where 1`] = `
|
|||
Array [
|
||||
Object {
|
||||
"__hdx_time_bucket": "2022-01-05T00:00:00Z",
|
||||
"avg(toFloat64OrNull(toString(LastValue)))": 6.25,
|
||||
"avg(toFloat64OrDefault(toString(LastValue)))": 6.25,
|
||||
},
|
||||
Object {
|
||||
"__hdx_time_bucket": "2022-01-05T00:05:00Z",
|
||||
"avg(toFloat64OrNull(toString(LastValue)))": 80,
|
||||
"avg(toFloat64OrDefault(toString(LastValue)))": 80,
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
|
@ -111,11 +111,11 @@ exports[`renderChartConfig Query Metrics single max/avg/sum gauge 1`] = `
|
|||
Array [
|
||||
Object {
|
||||
"__hdx_time_bucket": "2022-01-05T00:00:00Z",
|
||||
"avg(toFloat64OrNull(toString(LastValue)))": 5.125,
|
||||
"avg(toFloat64OrDefault(toString(LastValue)))": 5.125,
|
||||
},
|
||||
Object {
|
||||
"__hdx_time_bucket": "2022-01-05T00:05:00Z",
|
||||
"avg(toFloat64OrNull(toString(LastValue)))": 42,
|
||||
"avg(toFloat64OrDefault(toString(LastValue)))": 42,
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
|
@ -124,11 +124,11 @@ exports[`renderChartConfig Query Metrics single max/avg/sum gauge 2`] = `
|
|||
Array [
|
||||
Object {
|
||||
"__hdx_time_bucket": "2022-01-05T00:00:00Z",
|
||||
"max(toFloat64OrNull(toString(LastValue)))": 6.25,
|
||||
"max(toFloat64OrDefault(toString(LastValue)))": 6.25,
|
||||
},
|
||||
Object {
|
||||
"__hdx_time_bucket": "2022-01-05T00:05:00Z",
|
||||
"max(toFloat64OrNull(toString(LastValue)))": 80,
|
||||
"max(toFloat64OrDefault(toString(LastValue)))": 80,
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
|
@ -137,11 +137,11 @@ exports[`renderChartConfig Query Metrics single max/avg/sum gauge 3`] = `
|
|||
Array [
|
||||
Object {
|
||||
"__hdx_time_bucket": "2022-01-05T00:00:00Z",
|
||||
"sum(toFloat64OrNull(toString(LastValue)))": 10.25,
|
||||
"sum(toFloat64OrDefault(toString(LastValue)))": 10.25,
|
||||
},
|
||||
Object {
|
||||
"__hdx_time_bucket": "2022-01-05T00:05:00Z",
|
||||
"sum(toFloat64OrNull(toString(LastValue)))": 84,
|
||||
"sum(toFloat64OrDefault(toString(LastValue)))": 84,
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
|
@ -201,11 +201,11 @@ exports[`renderChartConfig Query Metrics two_timestamps_bounded histogram (p25)
|
|||
Array [
|
||||
Object {
|
||||
"__hdx_time_bucket": "2022-01-05T00:00:00Z",
|
||||
"any(toFloat64OrNull(toString(Rate)))": 0,
|
||||
"any(toFloat64OrDefault(toString(Rate)))": 0,
|
||||
},
|
||||
Object {
|
||||
"__hdx_time_bucket": "2022-01-05T00:01:00Z",
|
||||
"any(toFloat64OrNull(toString(Rate)))": 10,
|
||||
"any(toFloat64OrDefault(toString(Rate)))": 10,
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
|
@ -214,11 +214,11 @@ exports[`renderChartConfig Query Metrics two_timestamps_bounded histogram (p50)
|
|||
Array [
|
||||
Object {
|
||||
"__hdx_time_bucket": "2022-01-05T00:00:00Z",
|
||||
"any(toFloat64OrNull(toString(Rate)))": 0,
|
||||
"any(toFloat64OrDefault(toString(Rate)))": 0,
|
||||
},
|
||||
Object {
|
||||
"__hdx_time_bucket": "2022-01-05T00:01:00Z",
|
||||
"any(toFloat64OrNull(toString(Rate)))": 13.333333333333332,
|
||||
"any(toFloat64OrDefault(toString(Rate)))": 13.333333333333332,
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
|
@ -227,11 +227,11 @@ exports[`renderChartConfig Query Metrics two_timestamps_bounded histogram (p90)
|
|||
Array [
|
||||
Object {
|
||||
"__hdx_time_bucket": "2022-01-05T00:00:00Z",
|
||||
"any(toFloat64OrNull(toString(Rate)))": 0,
|
||||
"any(toFloat64OrDefault(toString(Rate)))": 0,
|
||||
},
|
||||
Object {
|
||||
"__hdx_time_bucket": "2022-01-05T00:01:00Z",
|
||||
"any(toFloat64OrNull(toString(Rate)))": 30,
|
||||
"any(toFloat64OrDefault(toString(Rate)))": 30,
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
|
@ -240,11 +240,11 @@ exports[`renderChartConfig Query Metrics two_timestamps_lower_bound_inf histogra
|
|||
Array [
|
||||
Object {
|
||||
"__hdx_time_bucket": "2022-01-05T00:00:00Z",
|
||||
"any(toFloat64OrNull(toString(Rate)))": 0,
|
||||
"any(toFloat64OrDefault(toString(Rate)))": 0,
|
||||
},
|
||||
Object {
|
||||
"__hdx_time_bucket": "2022-01-05T00:01:00Z",
|
||||
"any(toFloat64OrNull(toString(Rate)))": 1,
|
||||
"any(toFloat64OrDefault(toString(Rate)))": 1,
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
|
@ -253,11 +253,35 @@ exports[`renderChartConfig Query Metrics two_timestamps_upper_bound_inf histogra
|
|||
Array [
|
||||
Object {
|
||||
"__hdx_time_bucket": "2022-01-05T00:00:00Z",
|
||||
"any(toFloat64OrNull(toString(Rate)))": 0,
|
||||
"any(toFloat64OrDefault(toString(Rate)))": 0,
|
||||
},
|
||||
Object {
|
||||
"__hdx_time_bucket": "2022-01-05T00:01:00Z",
|
||||
"any(toFloat64OrNull(toString(Rate)))": 30,
|
||||
"any(toFloat64OrDefault(toString(Rate)))": 30,
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
||||
exports[`renderChartConfig aggFn numeric agg functions should handle numeric values as strings 1`] = `
|
||||
Array [
|
||||
Object {
|
||||
"AVG(toFloat64OrDefault(toString(strVal)))": 0.5,
|
||||
"MAX(toFloat64OrDefault(toString(strVal)))": 3,
|
||||
"MIN(toFloat64OrDefault(toString(strVal)))": -1.1,
|
||||
"SUM(toFloat64OrDefault(toString(strVal)))": 2,
|
||||
"quantile(0.5)(toFloat64OrDefault(toString(strVal)))": 0.050000000000000044,
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
||||
exports[`renderChartConfig aggFn numeric agg functions should use default values for other types 1`] = `
|
||||
Array [
|
||||
Object {
|
||||
"AVG(toFloat64OrDefault(toString(strVal)))": 0,
|
||||
"MAX(toFloat64OrDefault(toString(strVal)))": 0,
|
||||
"MIN(toFloat64OrDefault(toString(strVal)))": 0,
|
||||
"SUM(toFloat64OrDefault(toString(strVal)))": 0,
|
||||
"quantile(0.5)(toFloat64OrDefault(toString(strVal)))": 0,
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
|
|
|||
|
|
@ -6,7 +6,11 @@ import { renderChartConfig } from '@hyperdx/common-utils/dist/renderChartConfig'
|
|||
import _ from 'lodash';
|
||||
import ms from 'ms';
|
||||
|
||||
import { MetricsDataType } from '@/../../common-utils/dist/types';
|
||||
import {
|
||||
AggregateFunctionSchema,
|
||||
DerivedColumn,
|
||||
MetricsDataType,
|
||||
} from '@/../../common-utils/dist/types';
|
||||
import * as config from '@/config';
|
||||
import { createTeam } from '@/controllers/team';
|
||||
import {
|
||||
|
|
@ -17,6 +21,7 @@ import {
|
|||
DEFAULT_DATABASE,
|
||||
DEFAULT_LOGS_TABLE,
|
||||
DEFAULT_METRICS_TABLE,
|
||||
executeSqlCommand,
|
||||
getServer,
|
||||
} from '@/fixtures';
|
||||
import Connection from '@/models/connection';
|
||||
|
|
@ -103,6 +108,90 @@ describe('renderChartConfig', () => {
|
|||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('aggFn', () => {
|
||||
afterAll(async () => {
|
||||
await executeSqlCommand('DROP TABLE IF EXISTS agg_fn_str_test');
|
||||
await executeSqlCommand('DROP TABLE IF EXISTS agg_fn_default_test');
|
||||
});
|
||||
|
||||
it('numeric agg functions should handle numeric values as strings', async () => {
|
||||
await executeSqlCommand(`
|
||||
CREATE TABLE agg_fn_str_test(
|
||||
ts UInt64,
|
||||
strVal String
|
||||
) ENGINE = MergeTree
|
||||
ORDER BY ts
|
||||
`);
|
||||
await executeSqlCommand(`
|
||||
INSERT INTO agg_fn_str_test(ts, strVal) VALUES
|
||||
(fromUnixTimestamp64Milli(1519211811570), '3'),
|
||||
(fromUnixTimestamp64Milli(1519211811770), '-1'),
|
||||
(fromUnixTimestamp64Milli(1519211811870), '1.1'),
|
||||
(fromUnixTimestamp64Milli(1519211811970), '-1.1'),
|
||||
`);
|
||||
|
||||
const query = await renderChartConfig(
|
||||
{
|
||||
select: [
|
||||
{ aggFn: 'avg', valueExpression: 'strVal' },
|
||||
{ aggFn: 'max', valueExpression: 'strVal' },
|
||||
{ aggFn: 'min', valueExpression: 'strVal' },
|
||||
{ aggFn: 'quantile', level: 0.5, valueExpression: 'strVal' },
|
||||
{ aggFn: 'sum', valueExpression: 'strVal' },
|
||||
],
|
||||
from: {
|
||||
databaseName: '',
|
||||
tableName: `agg_fn_str_test`,
|
||||
},
|
||||
where: '',
|
||||
connection: connection.id,
|
||||
timestampValueExpression: 'ts',
|
||||
},
|
||||
metadata,
|
||||
);
|
||||
const res = await queryData(query);
|
||||
expect(res).toMatchSnapshot();
|
||||
});
|
||||
|
||||
it('numeric agg functions should use default values for other types', async () => {
|
||||
await executeSqlCommand(`
|
||||
CREATE TABLE agg_fn_default_test(
|
||||
ts UInt64,
|
||||
strVal String,
|
||||
boolVal Bool,
|
||||
enumVal Enum('a' = 1, 'b' = 2),
|
||||
nullVal Nullable(String),
|
||||
) ENGINE = MergeTree
|
||||
ORDER BY ts
|
||||
`);
|
||||
await executeSqlCommand(`
|
||||
INSERT INTO agg_fn_default_test(ts, strVal, boolVal, enumVal, nullVal) VALUES
|
||||
(fromUnixTimestamp64Milli(1519211811570), 'a', false, 'b', NULL)
|
||||
`);
|
||||
const query = await renderChartConfig(
|
||||
{
|
||||
select: [
|
||||
{ aggFn: 'avg', valueExpression: 'strVal' },
|
||||
{ aggFn: 'max', valueExpression: 'strVal' },
|
||||
{ aggFn: 'min', valueExpression: 'strVal' },
|
||||
{ aggFn: 'quantile', level: 0.5, valueExpression: 'strVal' },
|
||||
{ aggFn: 'sum', valueExpression: 'strVal' },
|
||||
],
|
||||
from: {
|
||||
databaseName: '',
|
||||
tableName: `agg_fn_default_test`,
|
||||
},
|
||||
where: '',
|
||||
connection: connection.id,
|
||||
timestampValueExpression: 'ts',
|
||||
},
|
||||
metadata,
|
||||
);
|
||||
const res = await queryData(query);
|
||||
expect(res).toMatchSnapshot();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Query Events', () => {
|
||||
it('simple select + where query logs', async () => {
|
||||
const now = new Date('2023-11-16T22:12:00.000Z');
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ const connectClickhouse = async () => {
|
|||
PRIMARY KEY (ServiceName, TimestampTime)
|
||||
ORDER BY (ServiceName, TimestampTime, Timestamp)
|
||||
TTL TimestampTime + toIntervalDay(3)
|
||||
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
|
||||
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
|
||||
`,
|
||||
// Recommended for cluster usage to avoid situations
|
||||
// where a query processing error occurred after the response code
|
||||
|
|
@ -111,7 +111,7 @@ const connectClickhouse = async () => {
|
|||
PARTITION BY toDate(TimeUnix)
|
||||
ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))
|
||||
TTL toDateTime(TimeUnix) + toIntervalDay(3)
|
||||
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
|
||||
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
|
||||
`,
|
||||
// Recommended for cluster usage to avoid situations
|
||||
// where a query processing error occurred after the response code
|
||||
|
|
@ -155,7 +155,7 @@ const connectClickhouse = async () => {
|
|||
PARTITION BY toDate(TimeUnix)
|
||||
ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))
|
||||
TTL toDateTime(TimeUnix) + toIntervalDay(15)
|
||||
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
|
||||
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
|
||||
`,
|
||||
// Recommended for cluster usage to avoid situations
|
||||
// where a query processing error occurred after the response code
|
||||
|
|
@ -203,7 +203,7 @@ const connectClickhouse = async () => {
|
|||
PARTITION BY toDate(TimeUnix)
|
||||
ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))
|
||||
TTL toDateTime(TimeUnix) + toIntervalDay(3)
|
||||
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
|
||||
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
|
||||
`,
|
||||
// Recommended for cluster usage to avoid situations
|
||||
// where a query processing error occurred after the response code
|
||||
|
|
@ -340,6 +340,15 @@ export const clearRedis = async () => {
|
|||
// ------------------------------------------------
|
||||
// ------------------ Clickhouse ------------------
|
||||
// ------------------------------------------------
|
||||
export const executeSqlCommand = async (sql: string) => {
|
||||
return await clickhouse.client.command({
|
||||
query: sql,
|
||||
clickhouse_settings: {
|
||||
wait_end_of_query: 1,
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
export const clearClickhouseTables = async () => {
|
||||
if (!config.IS_CI) {
|
||||
throw new Error('ONLY execute this in CI env 😈 !!!');
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ exports[`renderChartConfig should generate sql for a single gauge metric 1`] = `
|
|||
FROM Source
|
||||
GROUP BY AttributesHash, __hdx_time_bucket2
|
||||
ORDER BY AttributesHash, __hdx_time_bucket2
|
||||
) SELECT quantile(0.95)(toFloat64OrNull(toString(LastValue))),toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (__hdx_time_bucket2 >= fromUnixTimestamp64Milli(1739318400000) AND __hdx_time_bucket2 <= fromUnixTimestamp64Milli(1765670400000)) GROUP BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` LIMIT 10"
|
||||
) SELECT quantile(0.95)(toFloat64OrDefault(toString(LastValue))),toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (__hdx_time_bucket2 >= fromUnixTimestamp64Milli(1739318400000) AND __hdx_time_bucket2 <= fromUnixTimestamp64Milli(1765670400000)) GROUP BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` LIMIT 10"
|
||||
`;
|
||||
|
||||
exports[`renderChartConfig should generate sql for a single histogram metric 1`] = `
|
||||
|
|
@ -75,7 +75,7 @@ exports[`renderChartConfig should generate sql for a single histogram metric 1`]
|
|||
FROM Bucketed
|
||||
GROUP BY \`__hdx_time_bucket\`
|
||||
) SELECT any(
|
||||
toFloat64OrNull(toString(Rate))
|
||||
toFloat64OrDefault(toString(Rate))
|
||||
) FROM Rates WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10"
|
||||
`;
|
||||
|
||||
|
|
@ -118,6 +118,6 @@ exports[`renderChartConfig should generate sql for a single sum metric 1`] = `
|
|||
GROUP BY AttributesHash, \`__hdx_time_bucket2\`
|
||||
ORDER BY AttributesHash, \`__hdx_time_bucket2\`
|
||||
) SELECT avg(
|
||||
toFloat64OrNull(toString(Rate))
|
||||
toFloat64OrDefault(toString(Rate))
|
||||
) AS \\"Value\\",toStartOfInterval(toDateTime(\`__hdx_time_bucket2\`), INTERVAL 5 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (\`__hdx_time_bucket2\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket2\` <= fromUnixTimestamp64Milli(1765670400000)) GROUP BY toStartOfInterval(toDateTime(\`__hdx_time_bucket2\`), INTERVAL 5 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(\`__hdx_time_bucket2\`), INTERVAL 5 minute) AS \`__hdx_time_bucket\` LIMIT 10"
|
||||
`;
|
||||
|
|
|
|||
|
|
@ -270,7 +270,9 @@ const aggFnExpr = ({
|
|||
const isCount = fn.startsWith('count');
|
||||
const isWhereUsed = isNonEmptyWhereExpr(where);
|
||||
// Cast to float64 because the expr might not be a number
|
||||
const unsafeExpr = { UNSAFE_RAW_SQL: `toFloat64OrNull(toString(${expr}))` };
|
||||
const unsafeExpr = {
|
||||
UNSAFE_RAW_SQL: `toFloat64OrDefault(toString(${expr}))`,
|
||||
};
|
||||
const whereWithExtraNullCheck = `${where} AND ${unsafeExpr.UNSAFE_RAW_SQL} IS NOT NULL`;
|
||||
|
||||
if (fn.endsWith('Merge')) {
|
||||
|
|
|
|||
Loading…
Reference in a new issue