Elgato_dark/hyperdx
1
0
Fork 0
mirror of https://github.com/hyperdxio/hyperdx synced 2026-04-21 13:37:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

refactor: rename bulkInsertTeamLogStream to bulkInsertLogStream (#281)

Browse source
This commit is contained in:
Warren 2024-01-25 21:08:39 -08:00 committed by GitHub
parent d3abeff863
commit 4a6db4048a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 200 additions and 157 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.changeset/smooth-months-appear.md Normal file
View file

@ -0,0 +1,6 @@
---
'@hyperdx/api': patch
'@hyperdx/app': patch
---
refactor: rename bulkInsertTeamLogStream to bulkInsertLogStream

104
packages/api/src/clickhouse/__tests__/clickhouse.test.ts
View file

@ -2,10 +2,10 @@ import _ from 'lodash';
import ms from 'ms';
import {
buildEvent,
buildMetricSeries,
clearClickhouseTables,
closeDB,
generateBuildTeamEventFn,
getServer,
mockLogsPropertyTypeMappingsModel,
mockSpyMetricPropertyTypeMappingsModel,
@ -36,37 +36,34 @@ describe('clickhouse', () => {
const runId = Math.random().toString(); // dedup watch mode runs
const teamId = `test`;
await clickhouse.bulkInsertTeamLogStream(undefined, teamId, [
const buildEvent = generateBuildTeamEventFn(teamId, {
platform: LogPlatform.NodeJS,
source: 'test',
runId,
});
await clickhouse.bulkInsertLogStream([
buildEvent({
source: 'test',
timestamp: now,
platform: LogPlatform.NodeJS,
type: LogType.Log,
test: 'test1',
runId,
}),
buildEvent({
source: 'test',
timestamp: now + 1,
platform: LogPlatform.NodeJS,
type: LogType.Log,
test: 'test2',
justanumber: 777,
runId,
}),
buildEvent({
source: 'test',
timestamp: now + 1,
platform: LogPlatform.NodeJS,
type: LogType.Log,
justanumber: 777,
runId,
}),
]);
const data = (
await clickhouse.getLogBatch({
tableVersion: undefined,
tableVersion: 1,
teamId,
q: `runId:${runId} test:*`,
limit: 20,
@ -111,25 +108,26 @@ Array [
const runId = Math.random().toString(); // dedup watch mode runs
const teamId = `test`;
await clickhouse.bulkInsertTeamLogStream(undefined, teamId, [
const buildEvent = generateBuildTeamEventFn(teamId, {
runId,
});
await clickhouse.bulkInsertLogStream([
// Group 1, sum: 77, avg:25.666666667
buildEvent({
timestamp: now,
runId,
testGroup: 'group1',
testOtherGroup: 'otherGroup1',
awesomeNumber: 1,
}),
buildEvent({
timestamp: now + ms('1m'),
runId,
testGroup: 'group1',
testOtherGroup: 'otherGroup1',
awesomeNumber: 15,
}),
buildEvent({
timestamp: now + ms('2m'),
runId,
testGroup: 'group1',
testOtherGroup: 'otherGroup2',
awesomeNumber: 61,
@ -137,21 +135,18 @@ Array [
// Group 1, sum: 7, avg: 2.3333333
buildEvent({
timestamp: now + ms('6m'),
runId,
testGroup: 'group1',
testOtherGroup: 'otherGroup2',
awesomeNumber: 4,
}),
buildEvent({
timestamp: now + ms('7m'),
runId,
testGroup: 'group1',
testOtherGroup: 'otherGroup2',
awesomeNumber: 2,
}),
buildEvent({
timestamp: now + ms('8m'),
runId,
testGroup: 'group1',
testOtherGroup: 'otherGroup3',
awesomeNumber: 1,
@ -159,21 +154,18 @@ Array [
// Group 2, sum: 777, avg: 259
buildEvent({
timestamp: now,
runId,
testGroup: 'group2',
testOtherGroup: 'otherGroup1',
awesomeNumber: 70,
}),
buildEvent({
timestamp: now + ms('4m'),
runId,
testGroup: 'group2',
testOtherGroup: 'otherGroup1',
awesomeNumber: 700,
}),
buildEvent({
timestamp: now + ms('1m'),
runId,
testGroup: 'group2',
testOtherGroup: 'otherGroup1',
awesomeNumber: 7,
@ -207,7 +199,7 @@ Array [
groupBy: ['testGroup'],
},
],
tableVersion: undefined,
tableVersion: 1,
teamId,
startTime: now,
endTime: now + ms('10m'),
@ -265,7 +257,7 @@ Array [
groupBy: ['testGroup', 'testOtherGroup'],
},
],
tableVersion: undefined,
tableVersion: 1,
teamId,
startTime: now,
endTime: now + ms('10m'),
@ -339,7 +331,7 @@ Array [
groupBy: ['testGroup', 'testOtherGroup'],
},
],
tableVersion: undefined,
tableVersion: 1,
teamId,
startTime: now,
endTime: now + ms('10m'),
@ -421,7 +413,7 @@ Array [
groupBy: ['testGroup'],
},
],
tableVersion: undefined,
tableVersion: 1,
teamId,
startTime: now,
endTime: now + ms('10m'),
@ -471,7 +463,7 @@ Array [
groupBy: ['testGroup'],
},
],
tableVersion: undefined,
tableVersion: 1,
teamId,
startTime: now,
endTime: now + ms('10m'),
@ -531,6 +523,7 @@ Array [
{ value: 32, timestamp: now + ms('16m') },
{ value: 42, timestamp: now + ms('19m') }, // 42
],
team_id: teamId,
}),
);
@ -555,6 +548,7 @@ Array [
{ value: 9323, timestamp: now + ms('16m') },
{ value: 84626, timestamp: now + ms('19m') }, // 84626
],
team_id: teamId,
}),
);
@ -576,6 +570,7 @@ Array [
{ value: 10, timestamp: now + ms('8m') },
{ value: 80, timestamp: now + ms('9m') }, // Last 5min
],
team_id: teamId,
}),
);
@ -597,6 +592,7 @@ Array [
{ value: 5, timestamp: now + ms('8m') },
{ value: 4, timestamp: now + ms('9m') }, // Last 5min
],
team_id: teamId,
}),
);
@ -618,7 +614,7 @@ Array [
metricDataType: clickhouse.MetricsDataType.Sum,
},
],
tableVersion: undefined,
tableVersion: 1,
teamId,
startTime: now,
endTime: now + ms('20m'),
@ -668,7 +664,7 @@ Array [
metricDataType: clickhouse.MetricsDataType.Gauge,
},
],
tableVersion: undefined,
tableVersion: 1,
teamId,
startTime: now,
endTime: now + ms('10m'),
@ -726,7 +722,7 @@ Array [
metricDataType: clickhouse.MetricsDataType.Gauge,
},
],
tableVersion: undefined,
tableVersion: 1,
teamId,
startTime: now,
endTime: now + ms('10m'),
@ -766,7 +762,7 @@ Array [
metricDataType: clickhouse.MetricsDataType.Gauge,
},
],
tableVersion: undefined,
tableVersion: 1,
teamId,
startTime: now,
endTime: now + ms('10m'),
@ -815,7 +811,7 @@ Array [
metricDataType: clickhouse.MetricsDataType.Sum,
},
],
tableVersion: undefined,
tableVersion: 1,
teamId,
startTime: now,
endTime: now + ms('20m'),
@ -876,9 +872,11 @@ Array [
const runId = Math.random().toString(); // dedup watch mode runs
const teamId = `test`;
await clickhouse.bulkInsertTeamLogStream(
undefined,
teamId,
const buildEvent = generateBuildTeamEventFn(teamId, {
runId,
});
await clickhouse.bulkInsertLogStream(
Array(10)
.fill(0)
.flatMap((_, i) => [
@ -926,7 +924,7 @@ Array [
sortOrder: 'asc',
},
],
tableVersion: undefined,
tableVersion: 1,
teamId,
startTime: now,
endTime: now + ms('10m'),
@ -1013,7 +1011,7 @@ Array [
sortOrder: 'desc',
},
],
tableVersion: undefined,
tableVersion: 1,
teamId,
startTime: now,
endTime: now + ms('10m'),
@ -1100,7 +1098,7 @@ Array [
sortOrder: 'desc',
},
],
tableVersion: undefined,
tableVersion: 1,
teamId,
startTime: now,
endTime: now + ms('5m'),
@ -1147,61 +1145,56 @@ Array [
const runId = Math.random().toString(); // dedup watch mode runs
const teamId = `test`;
await clickhouse.bulkInsertTeamLogStream(undefined, teamId, [
const buildEvent = generateBuildTeamEventFn(teamId, {
runId,
});
await clickhouse.bulkInsertLogStream([
// Group 1, sum: 77, avg:25.666666667
buildEvent({
timestamp: now,
runId,
testGroup: 'group1',
awesomeNumber: 1,
}),
buildEvent({
timestamp: now + ms('1m'),
runId,
testGroup: 'group1',
awesomeNumber: 15,
}),
buildEvent({
timestamp: now + ms('2m'),
runId,
testGroup: 'group1',
awesomeNumber: 61,
}),
// Group 1, sum: 7, avg: 2.3333333
buildEvent({
timestamp: now + ms('6m'),
runId,
testGroup: 'group1',
awesomeNumber: 4,
}),
buildEvent({
timestamp: now + ms('7m'),
runId,
testGroup: 'group1',
awesomeNumber: 2,
}),
buildEvent({
timestamp: now + ms('8m'),
runId,
testGroup: 'group1',
awesomeNumber: 1,
}),
// Group 2, sum: 777, avg: 259
buildEvent({
timestamp: now,
runId,
testGroup: 'group2',
awesomeNumber: 70,
}),
buildEvent({
timestamp: now + ms('4m'),
runId,
testGroup: 'group2',
awesomeNumber: 700,
}),
buildEvent({
timestamp: now + ms('1m'),
runId,
testGroup: 'group2',
awesomeNumber: 7,
}),
@ -1225,7 +1218,7 @@ Array [
groupBy: ['testGroup'],
},
],
tableVersion: undefined,
tableVersion: 1,
teamId,
startTime: now,
endTime: now + ms('10m'),
@ -1240,7 +1233,7 @@ Array [
field: 'awesomeNumber',
q: `runId:${runId}`,
groupBy: 'testGroup',
tableVersion: undefined,
tableVersion: 1,
teamId,
startTime: now,
endTime: now + ms('10m'),
@ -1317,11 +1310,13 @@ Array [
expect.assertions(2);
});
// TODO: Test this with real data and new chart fn
it.skip('getMetricsChart avoids sending NaN to frontend', async () => {
jest
.spyOn(clickhouse.client, 'query')
.mockResolvedValueOnce({ json: () => Promise.resolve({}) } as any);
jest.spyOn(clickhouse.client, 'query').mockResolvedValueOnce({
json: () =>
Promise.resolve({
data: [],
}),
} as any);
await clickhouse.getMetricsChart({
aggFn: clickhouse.AggFn.AvgRate,
@ -1334,7 +1329,6 @@ Array [
teamId: 'test',
});
expect(clickhouse.client.query).toHaveBeenCalledTimes(2);
expect(clickhouse.client.query).toHaveBeenCalledWith(
expect.objectContaining({
format: 'JSON',

17
packages/api/src/clickhouse/__tests__/clickhouseGetMultiSeriesChart.test.ts
View file

@ -50,6 +50,7 @@ describe('clickhouse - getMultiSeriesChart', () => {
{ value: 32, timestamp: now + ms('16m') },
{ value: 42, timestamp: now + ms('19m') }, // 42
],
team_id: teamId,
}),
),
// Rate: 11, 78, 5805, 78729
@ -73,6 +74,7 @@ describe('clickhouse - getMultiSeriesChart', () => {
{ value: 9323, timestamp: now + ms('16m') },
{ value: 84626, timestamp: now + ms('19m') }, // 84626
],
team_id: teamId,
}),
),
clickhouse.bulkInsertTeamMetricStream(
@ -93,6 +95,7 @@ describe('clickhouse - getMultiSeriesChart', () => {
{ value: 10, timestamp: now + ms('8m') },
{ value: 80, timestamp: now + ms('9m') }, // Last 5min
],
team_id: teamId,
}),
),
clickhouse.bulkInsertTeamMetricStream(
@ -113,6 +116,7 @@ describe('clickhouse - getMultiSeriesChart', () => {
{ value: 5, timestamp: now + ms('8m') },
{ value: 4, timestamp: now + ms('9m') }, // Last 5min
],
team_id: teamId,
}),
),
clickhouse.bulkInsertTeamMetricStream(
@ -132,6 +136,7 @@ describe('clickhouse - getMultiSeriesChart', () => {
{ value: 10, timestamp: now + ms('1m'), le: '30' },
{ value: 10, timestamp: now + ms('1m'), le: '+Inf' },
],
team_id: teamId,
}),
),
clickhouse.bulkInsertTeamMetricStream(
@ -151,6 +156,7 @@ describe('clickhouse - getMultiSeriesChart', () => {
{ value: 10, timestamp: now + ms('1m'), le: '30' },
{ value: 10, timestamp: now + ms('1m'), le: '+Inf' },
],
team_id: teamId,
}),
),
clickhouse.bulkInsertTeamMetricStream(
@ -170,6 +176,7 @@ describe('clickhouse - getMultiSeriesChart', () => {
{ value: 0, timestamp: now + ms('1m'), le: '30' },
{ value: 10, timestamp: now + ms('1m'), le: '50' },
],
team_id: teamId,
}),
),
clickhouse.bulkInsertTeamMetricStream(
@ -189,6 +196,7 @@ describe('clickhouse - getMultiSeriesChart', () => {
{ value: 0, timestamp: now + ms('1m'), le: '30' },
{ value: 10, timestamp: now + ms('1m'), le: '+Inf' },
],
team_id: teamId,
}),
),
clickhouse.bulkInsertTeamMetricStream(
@ -208,6 +216,7 @@ describe('clickhouse - getMultiSeriesChart', () => {
{ value: 20, timestamp: now + ms('1m'), le: '30' },
{ value: 50, timestamp: now + ms('1m'), le: '50' },
],
team_id: teamId,
}),
),
clickhouse.bulkInsertTeamMetricStream(
@ -227,6 +236,7 @@ describe('clickhouse - getMultiSeriesChart', () => {
{ value: 20, timestamp: now + ms('1m'), le: '30' },
{ value: 50, timestamp: now + ms('1m'), le: '50' },
],
team_id: teamId,
}),
),
clickhouse.bulkInsertTeamMetricStream(
@ -250,6 +260,7 @@ describe('clickhouse - getMultiSeriesChart', () => {
{ value: 20, timestamp: now + ms('2m'), le: '30' },
{ value: 50, timestamp: now + ms('2m'), le: '50' },
],
team_id: teamId,
}),
),
clickhouse.bulkInsertTeamMetricStream(
@ -273,6 +284,7 @@ describe('clickhouse - getMultiSeriesChart', () => {
{ value: 40, timestamp: now + ms('2m'), le: '30' },
{ value: 80, timestamp: now + ms('2m'), le: '50' },
],
team_id: teamId,
}),
),
clickhouse.bulkInsertTeamMetricStream(
@ -296,6 +308,7 @@ describe('clickhouse - getMultiSeriesChart', () => {
{ value: 20, timestamp: now + ms('2m'), le: '30' },
{ value: 50, timestamp: now + ms('2m'), le: '50' },
],
team_id: teamId,
}),
),
clickhouse.bulkInsertTeamMetricStream(
@ -322,6 +335,7 @@ describe('clickhouse - getMultiSeriesChart', () => {
{ value: 20, timestamp: now + ms('2m'), le: '30' },
{ value: 50, timestamp: now + ms('2m'), le: '50' },
],
team_id: teamId,
}),
),
clickhouse.bulkInsertTeamMetricStream(
@ -352,6 +366,7 @@ describe('clickhouse - getMultiSeriesChart', () => {
{ value: 20, timestamp: now + ms('4m'), le: '30' },
{ value: 50, timestamp: now + ms('4m'), le: '50' },
],
team_id: teamId,
}),
),
clickhouse.bulkInsertTeamMetricStream([
@ -383,6 +398,7 @@ describe('clickhouse - getMultiSeriesChart', () => {
{ value: 20, timestamp: now + ms('4m'), le: '30' },
{ value: 50, timestamp: now + ms('4m'), le: '50' },
],
team_id: teamId,
}),
...buildMetricSeries({
name: 'test.counter_might_reset',
@ -412,6 +428,7 @@ describe('clickhouse - getMultiSeriesChart', () => {
{ value: 50, timestamp: now + ms('4m'), le: '30' },
{ value: 80, timestamp: now + ms('4m'), le: '50' },
],
team_id: teamId,
}),
]),
]);

8
packages/api/src/clickhouse/index.ts
View file

@ -423,12 +423,8 @@ export const bulkInsertRrwebEvents = async (events: RrwebEventModel[]) => {
});
};
export const bulkInsertTeamLogStream = async (
version: number | undefined | null,
teamId: string,
logs: LogStreamModel[],
) => {
const tableName = getLogStreamTableName(version, teamId);
export const bulkInsertLogStream = async (logs: LogStreamModel[]) => {
const tableName = `default.${TableName.LogStream}`;
await clientInsertWithRetries<LogStreamModel>({
table: tableName,
values: logs,

33
packages/api/src/fixtures.ts
View file

@ -141,6 +141,31 @@ export const getLoggedInAgent = async (server: MockServer) => {
};
};
type BaseEvent = {
level?: string;
source?: string;
timestamp?: number; // ms timestamp
platform?: LogPlatform;
type?: LogType;
end_timestamp?: number; //ms timestamp
span_name?: string;
} & {
[key: string]: number | string | boolean;
};
export function generateBuildTeamEventFn(
teamId: string,
commonAttributes: Partial<BaseEvent>,
) {
return (attributes: Partial<BaseEvent>) => {
return buildEvent({
...commonAttributes,
...attributes,
team_id: teamId,
});
};
}
// ------------------------------------------------
// ------------------ Redis -----------------------
// ------------------------------------------------
@ -172,7 +197,7 @@ export const clearClickhouseTables = async () => {
await Promise.all(promises);
};
export function buildEvent({
function buildEvent({
level,
source = 'test',
timestamp,
@ -180,12 +205,14 @@ export function buildEvent({
type = LogType.Log,
end_timestamp = 0,
span_name,
team_id,
service = 'test-service',
...properties
}: {
level?: string;
source?: string;
timestamp?: number; // ms timestamp
team_id: string;
platform?: LogPlatform;
type?: LogType;
end_timestamp?: number; //ms timestamp
@ -228,6 +255,7 @@ export function buildEvent({
severity_text: level,
// @ts-ignore
end_timestamp: `${end_timestamp}000000`,
team_id,
type,
span_name,
'bool.names': boolNames,
@ -247,6 +275,7 @@ export function buildMetricSeries({
is_delta,
is_monotonic,
unit,
team_id,
}: {
tags: Record<string, string>;
name: string;
@ -255,6 +284,7 @@ export function buildMetricSeries({
is_monotonic: boolean;
is_delta: boolean;
unit: string;
team_id: string;
}): MetricModel[] {
// @ts-ignore TODO: Fix Timestamp types
return points.map(({ value, timestamp, le }) => ({
@ -266,6 +296,7 @@ export function buildMetricSeries({
is_monotonic,
is_delta,
unit,
team_id,
}));
}

10
packages/api/src/routers/aggregator/root.ts
View file

@ -2,8 +2,8 @@ import express from 'express';
import groupBy from 'lodash/groupBy';
import {
bulkInsertLogStream,
bulkInsertRrwebEvents,
bulkInsertTeamLogStream,
bulkInsertTeamMetricStream,
} from '@/clickhouse';
import * as config from '@/config';
@ -42,13 +42,7 @@ const bulkInsert = async (
logs.push(log);
}
}
const promises = [
bulkInsertTeamLogStream(
team.logStreamTableVersion,
team._id.toString(),
vectorLogParser.parse(logs),
),
];
const promises = [bulkInsertLogStream(vectorLogParser.parse(logs))];
if (rrwebEvents.length > 0) {
promises.push(
bulkInsertRrwebEvents(vectorRrwebParser.parse(rrwebEvents)),

128
packages/api/src/routers/api/__tests__/chart.test.ts
View file

@ -2,11 +2,11 @@ import ms from 'ms';
import * as clickhouse from '@/clickhouse';
import {
buildEvent,
clearClickhouseTables,
clearDBCollections,
clearRedis,
closeDB,
generateBuildTeamEventFn,
getLoggedInAgent,
getServer,
mockLogsPropertyTypeMappingsModel,
@ -34,36 +34,34 @@ describe('charts router', () => {
const now = Date.now();
const { agent, team } = await getLoggedInAgent(server);
await clickhouse.bulkInsertTeamLogStream(
team.logStreamTableVersion,
team.id,
[
buildEvent({
'k8s.namespace.name': 'namespace1',
'k8s.pod.name': 'pod1',
'k8s.pod.uid': 'uid1',
level: 'ok',
service: 'service1',
timestamp: now,
}),
buildEvent({
'k8s.namespace.name': 'namespace1',
'k8s.pod.name': 'pod2',
'k8s.pod.uid': 'uid2',
level: 'ok',
service: 'service1',
timestamp: now,
}),
buildEvent({
'k8s.namespace.name': 'namespace2',
'k8s.pod.name': 'pod3',
'k8s.pod.uid': 'uid3',
level: 'ok',
service: 'service2',
timestamp: now - ms('1d'),
}),
],
);
const buildEvent = generateBuildTeamEventFn(team.id, {});
await clickhouse.bulkInsertLogStream([
buildEvent({
'k8s.namespace.name': 'namespace1',
'k8s.pod.name': 'pod1',
'k8s.pod.uid': 'uid1',
level: 'ok',
service: 'service1',
timestamp: now,
}),
buildEvent({
'k8s.namespace.name': 'namespace1',
'k8s.pod.name': 'pod2',
'k8s.pod.uid': 'uid2',
level: 'ok',
service: 'service1',
timestamp: now,
}),
buildEvent({
'k8s.namespace.name': 'namespace2',
'k8s.pod.name': 'pod3',
'k8s.pod.uid': 'uid3',
level: 'ok',
service: 'service2',
timestamp: now - ms('1d'),
}),
]);
const results = await agent.get('/chart/services').expect(200);
expect(results.body.data).toMatchInlineSnapshot(`
@ -95,24 +93,22 @@ Object {
const now = Date.now();
const { agent, team } = await getLoggedInAgent(server);
await clickhouse.bulkInsertTeamLogStream(
team.logStreamTableVersion,
team.id,
[
buildEvent({
timestamp: now,
service: 'service1',
}),
buildEvent({
timestamp: now,
service: 'service1',
}),
buildEvent({
timestamp: now - ms('1d'),
service: 'service2',
}),
],
);
const buildEvent = generateBuildTeamEventFn(team.id, {});
await clickhouse.bulkInsertLogStream([
buildEvent({
timestamp: now,
service: 'service1',
}),
buildEvent({
timestamp: now,
service: 'service1',
}),
buildEvent({
timestamp: now - ms('1d'),
service: 'service2',
}),
]);
const results = await agent.get('/chart/services').expect(200);
expect(results.body.data).toMatchInlineSnapshot(`
@ -127,24 +123,22 @@ Object {
const now = Date.now();
const { agent, team } = await getLoggedInAgent(server);
await clickhouse.bulkInsertTeamLogStream(
team.logStreamTableVersion,
team.id,
[
buildEvent({
timestamp: now,
service: 'service1',
}),
buildEvent({
timestamp: now,
service: 'service1',
}),
buildEvent({
timestamp: now - ms('1d'),
service: 'service2',
}),
],
);
const buildEvent = generateBuildTeamEventFn(team.id, {});
await clickhouse.bulkInsertLogStream([
buildEvent({
timestamp: now,
service: 'service1',
}),
buildEvent({
timestamp: now,
service: 'service1',
}),
buildEvent({
timestamp: now - ms('1d'),
service: 'service2',
}),
]);
mockLogsPropertyTypeMappingsModel({
service: 'string',

5
packages/api/src/routers/api/__tests__/metrics.test.ts
View file

@ -28,6 +28,8 @@ describe('metrics router', () => {
});
it('GET /metrics/tags', async () => {
const { agent, team } = await getLoggedInAgent(server);
const now = Date.now();
await clickhouse.bulkInsertTeamMetricStream(
buildMetricSeries({
@ -38,6 +40,7 @@ describe('metrics router', () => {
is_delta: false,
unit: 'Percent',
points: [{ value: 1, timestamp: now }],
team_id: team.id,
}),
);
await clickhouse.bulkInsertTeamMetricStream(
@ -49,10 +52,10 @@ describe('metrics router', () => {
is_delta: false,
unit: 'Percent',
points: [{ value: 1, timestamp: now }],
team_id: team.id,
}),
);
const { agent } = await getLoggedInAgent(server);
const results = await agent.get('/metrics/tags').expect(200);
expect(results.body.data).toEqual([
{

6
packages/api/src/routers/external-api/__tests__/charts.test.ts
View file

@ -3,9 +3,9 @@ import ms from 'ms';
import * as clickhouse from '@/clickhouse';
import {
buildEvent,
clearDBCollections,
closeDB,
generateBuildTeamEventFn,
getLoggedInAgent,
getServer,
mockLogsPropertyTypeMappingsModel,
@ -35,7 +35,9 @@ describe('/api/v1/charts/series', () => {
const runId = Math.random().toString(); // dedup watch mode runs
const teamId = `test`;
await clickhouse.bulkInsertTeamLogStream(undefined, teamId, [
const buildEvent = generateBuildTeamEventFn(teamId, {});
await clickhouse.bulkInsertLogStream([
// Group 1, sum: 77, avg:25.666666667
buildEvent({
timestamp: now,

40
packages/api/src/tasks/__tests__/checkAlerts.test.ts
View file

@ -1,10 +1,10 @@
import ms from 'ms';
import {
buildEvent,
buildMetricSeries,
clearDBCollections,
closeDB,
generateBuildTeamEventFn,
getServer,
mockLogsPropertyTypeMappingsModel,
mockSpyMetricPropertyTypeMappingsModel,
@ -27,6 +27,10 @@ import {
} from '../checkAlerts';
describe('checkAlerts', () => {
afterAll(async () => {
await clickhouse.client.close();
});
it('roundDownToXMinutes', () => {
// 1 min
const roundDownTo1Minute = roundDownToXMinutes(1);
@ -122,12 +126,9 @@ describe('checkAlerts', () => {
await server.start();
});
beforeEach(() => {
jest.resetAllMocks();
});
afterEach(async () => {
await clearDBCollections();
jest.clearAllMocks();
});
afterAll(async () => {
@ -270,32 +271,32 @@ describe('checkAlerts', () => {
runId: 'string',
});
const team = await createTeam({ name: 'My Team' });
const runId = Math.random().toString(); // dedup watch mode runs
const teamId = 'test';
const teamId = team._id.toString();
const now = new Date('2023-11-16T22:12:00.000Z');
// Send events in the last alert window 22:05 - 22:10
const eventMs = now.getTime() - ms('5m');
await clickhouse.bulkInsertTeamLogStream(undefined, teamId, [
const buildEvent = generateBuildTeamEventFn(teamId, {
runId,
span_name: 'HyperDX',
type: LogType.Span,
level: 'error',
});
await clickhouse.bulkInsertLogStream([
buildEvent({
span_name: 'HyperDX',
level: 'error',
timestamp: eventMs,
runId,
end_timestamp: eventMs + 100,
type: LogType.Span,
}),
buildEvent({
span_name: 'HyperDX',
level: 'error',
timestamp: eventMs + 5,
runId,
end_timestamp: eventMs + 7,
type: LogType.Span,
}),
]);
const team = await createTeam({ name: 'My Team' });
const webhook = await new Webhook({
team: team._id,
service: 'slack',
@ -427,7 +428,10 @@ describe('checkAlerts', () => {
});
it('CHART alert (metrics table series)', async () => {
const team = await createTeam({ name: 'My Team' });
const runId = Math.random().toString(); // dedup watch mode runs
const teamId = team._id.toString();
jest
.spyOn(slack, 'postMessageToWebhook')
@ -470,6 +474,7 @@ describe('checkAlerts', () => {
{ value: 35, timestamp: metricNowTs + ms('10m') },
{ value: 36, timestamp: metricNowTs + ms('11m') },
],
team_id: teamId,
}),
);
@ -500,6 +505,7 @@ describe('checkAlerts', () => {
{ value: 30005, timestamp: metricNowTs + ms('10m') },
{ value: 30006, timestamp: metricNowTs + ms('11m') },
],
team_id: teamId,
}),
);
@ -525,10 +531,10 @@ describe('checkAlerts', () => {
{ value: 22, timestamp: metricNowTs + ms('10m') },
{ value: 23, timestamp: metricNowTs + ms('11m') },
],
team_id: teamId,
}),
);
const team = await createTeam({ name: 'My Team' });
const webhook = await new Webhook({
team: team._id,
service: 'slack',