Elgato_dark/hyperdx
1
0
Fork 0
mirror of https://github.com/hyperdxio/hyperdx synced 2026-04-21 13:37:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

Multi series support for event/metric line charts and alerts (#171)

Browse source 
<img width="1123" alt="image" src="https://github.com/hyperdxio/hyperdx/assets/2781687/30d8d1f7-092a-4401-84b9-faab676b14e3">

- Introduces a new `/chart/series` endpoint that will take an array of ChartSeries (logs or metrics) and return them back in `series_${i}.data` columns. Can also return in `ratio` mode if `seriesReturnType` is specified and 2 series are submitted.
- Creates a new `HDXMultiSeriesTimeChart` to render a chart with multiple series. I have a draft of mostly-done `HDXMultiSeriesTableChart` as well but I figured I'd spare the PR from blowing up further.
- Introduces `ChartSeriesFormCompact`, a hacky-ish UI to fit multiple series in an edit form easily (part of `EditMultiSeriesChartForm` which will be reused for tables). 
- Modifies checkAlert to operate on multiple series, mostly through `getMultiSeriesChartLegacyFormat` which will flatten a multi-series query back into the `{ data:.., ts_bucket:..., group:... }` format.
- Adds real CH-backed tests to `checkAlert.test.ts` and `clickhouse.test.ts` with some util functions.
This commit is contained in:
Mike Shi 2024-01-02 10:30:05 -08:00 committed by GitHub
parent 70f5fc4c9a
commit 3b8effea7d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
25 changed files with 4261 additions and 512 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.changeset/new-months-invite.md Normal file
View file

@ -0,0 +1,7 @@
---
'@hyperdx/api': minor
'@hyperdx/app': minor
---
Add specifying multiple series of charts for time/line charts and tables in
dashboard (ex. min, max, avg all in one chart).

3
.vscode/settings.json vendored
View file

@ -1,8 +1,5 @@
{
"editor.formatOnSave": true,
"editor.codeActionsOnSave": {
"source.fixAll": "explicit"
},
"[typescript]": {
"editor.defaultFormatter": "esbenp.prettier-vscode"
},

1
packages/api/src/api-app.ts
View file

@ -103,6 +103,7 @@ app.use('/metrics', isUserAuthenticated, routers.metricsRouter);
app.use('/sessions', isUserAuthenticated, routers.sessionsRouter);
app.use('/team', isUserAuthenticated, routers.teamRouter);
app.use('/webhooks', isUserAuthenticated, routers.webhooksRouter);
app.use('/chart', isUserAuthenticated, routers.chartRouter);
// ---------------------------------------------------------------------
// TODO: Separate external API routers from internal routers

1173
packages/api/src/clickhouse/__tests__/clickhouse.test.ts
View file

File diff suppressed because it is too large Load diff

638
packages/api/src/clickhouse/index.ts
View file

@ -14,6 +14,7 @@ import ms from 'ms';
import { serializeError } from 'serialize-error';
import SqlString from 'sqlstring';
import { Readable } from 'stream';
import { z } from 'zod';
import * as config from '@/config';
import { sleep } from '@/utils/common';
@ -23,6 +24,7 @@ import type {
MetricModel,
RrwebEventModel,
} from '@/utils/logParser';
import { chartSeriesSchema } from '@/utils/zod';
import { redisClient } from '../utils/redis';
import {
@ -43,6 +45,11 @@ const tracer = opentelemetry.trace.getTracer(__filename);
export type SortOrder = 'asc' | 'desc' | null;
export enum SeriesReturnType {
Ratio = 'ratio',
Column = 'column',
}
export enum MetricsDataType {
Gauge = 'Gauge',
Histogram = 'Histogram',
@ -833,18 +840,19 @@ export const getMetricsChart = async ({
const gaugeMetricSource = SqlString.format(
`
SELECT
timestamp,
SELECT
toStartOfInterval(timestamp, INTERVAL ?) as timestamp,
name,
value,
last_value(value) as value,
_string_attributes
FROM ??
WHERE name = ?
AND data_type = ?
AND (?)
ORDER BY _timestamp_sort_key ASC
GROUP BY name, _string_attributes, timestamp
ORDER BY timestamp ASC
`.trim(),
[tableName, name, dataType, SqlString.raw(whereClause)],
[granularity, tableName, name, dataType, SqlString.raw(whereClause)],
);
const query = SqlString.format(
@ -904,6 +912,626 @@ export const getMetricsChart = async ({
return result;
};
export const buildMetricSeriesQuery = async ({
aggFn,
dataType,
endTime,
granularity,
groupBy,
name,
q,
startTime,
teamId,
sortOrder,
}: {
aggFn: AggFn;
dataType: MetricsDataType;
endTime: number; // unix in ms,
granularity?: Granularity | string;
groupBy?: string;
name: string;
q: string;
startTime: number; // unix in ms
teamId: string;
sortOrder?: 'asc' | 'desc';
}) => {
const tableName = `default.${TableName.Metric}`;
const propertyTypeMappingsModel = await buildMetricsPropertyTypeMappingsModel(
undefined, // default version
teamId,
);
const isRate = isRateAggFn(aggFn);
const shouldModifyStartTime = isRate;
// If it's a rate function, then we'll need to look 1 window back to calculate
// the initial rate value.
// We'll filter this extra bucket out later
const modifiedStartTime = shouldModifyStartTime
? // If granularity is not defined (tables), we'll just look behind 5min
startTime - ms(granularity ?? '5 minute')
: startTime;
const whereClause = await buildSearchQueryWhereCondition({
endTime,
propertyTypeMappingsModel,
query: q,
startTime: modifiedStartTime,
});
const selectClause = [
granularity != null
? SqlString.format(
'toUnixTimestamp(toStartOfInterval(timestamp, INTERVAL ?)) AS ts_bucket',
[granularity],
)
: "'0' as ts_bucket",
groupBy
? SqlString.format(`_string_attributes[?] AS group`, [groupBy])
: "'' AS group",
];
const hasGroupBy = groupBy != '' && groupBy != null;
if (dataType === MetricsDataType.Gauge || dataType === MetricsDataType.Sum) {
selectClause.push(
aggFn === AggFn.Count
? 'COUNT(value) as data'
: aggFn === AggFn.Sum
? `SUM(value) as data`
: aggFn === AggFn.Avg
? `AVG(value) as data`
: aggFn === AggFn.Max
? `MAX(value) as data`
: aggFn === AggFn.Min
? `MIN(value) as data`
: aggFn === AggFn.SumRate
? `SUM(rate) as data`
: aggFn === AggFn.AvgRate
? `AVG(rate) as data`
: aggFn === AggFn.MaxRate
? `MAX(rate) as data`
: aggFn === AggFn.MinRate
? `MIN(rate) as data`
: `quantile(${
aggFn === AggFn.P50 || aggFn === AggFn.P50Rate
? '0.5'
: aggFn === AggFn.P90 || aggFn === AggFn.P90Rate
? '0.90'
: aggFn === AggFn.P95 || aggFn === AggFn.P95Rate
? '0.95'
: '0.99'
})(${isRate ? 'rate' : 'value'}) as data`,
);
} else {
logger.error(`Unsupported data type: ${dataType}`);
}
const startTimeUnixTs = Math.floor(startTime / 1000);
// TODO: Can remove the ORDER BY _string_attributes for Gauge metrics
// since they don't get subjected to runningDifference afterwards
const gaugeMetricSource = SqlString.format(
`
SELECT
?,
name,
last_value(value) as value,
_string_attributes
FROM ??
WHERE name = ?
AND data_type = ?
AND (?)
GROUP BY name, _string_attributes, timestamp
ORDER BY _string_attributes, timestamp ASC
`.trim(),
[
SqlString.raw(
granularity != null
? `toStartOfInterval(timestamp, INTERVAL ${SqlString.format(
granularity,
)}) as timestamp`
: modifiedStartTime
? // Manually create the time buckets if we're including the prev time range
`if(timestamp < fromUnixTimestamp(${startTimeUnixTs}), 0, ${startTimeUnixTs}) as timestamp`
: // Otherwise lump everything into one bucket
'0 as timestamp',
),
tableName,
name,
dataType,
SqlString.raw(whereClause),
],
);
const rateMetricSource = SqlString.format(
`
SELECT
if(
runningDifference(value) < 0
OR neighbor(_string_attributes, -1, _string_attributes) != _string_attributes,
nan,
runningDifference(value)
) AS rate,
timestamp,
_string_attributes,
name
FROM (?)
WHERE isNaN(rate) = 0
${shouldModifyStartTime ? 'AND timestamp >= fromUnixTimestamp(?)' : ''}
`.trim(),
[
SqlString.raw(gaugeMetricSource),
...(shouldModifyStartTime ? [Math.floor(startTime / 1000)] : []),
],
);
const query = SqlString.format(
`
WITH metrics AS (?)
SELECT ?
FROM metrics
GROUP BY group, ts_bucket
ORDER BY ts_bucket ASC
${
granularity != null
? `WITH FILL
FROM toUnixTimestamp(toStartOfInterval(toDateTime(?), INTERVAL ?))
TO toUnixTimestamp(toStartOfInterval(toDateTime(?), INTERVAL ?))
STEP ?`
: ''
}
`,
[
SqlString.raw(isRate ? rateMetricSource : gaugeMetricSource),
SqlString.raw(selectClause.join(',')),
...(granularity != null
? [
startTime / 1000,
granularity,
endTime / 1000,
granularity,
ms(granularity) / 1000,
]
: []),
],
);
return {
query,
hasGroupBy,
sortOrder,
};
};
const buildEventSeriesQuery = async ({
aggFn,
endTime,
field,
granularity,
groupBy,
propertyTypeMappingsModel,
q,
sortOrder,
startTime,
tableVersion,
teamId,
}: {
aggFn: AggFn;
endTime: number; // unix in ms,
field?: string;
granularity: string | undefined; // can be undefined in the number chart
groupBy: string;
propertyTypeMappingsModel: LogsPropertyTypeMappingsModel;
q: string;
sortOrder?: 'asc' | 'desc';
startTime: number; // unix in ms
tableVersion: number | undefined;
teamId: string;
}) => {
if (isRateAggFn(aggFn)) {
throw new Error('Rate is not supported in logs chart');
}
const tableName = getLogStreamTableName(tableVersion, teamId);
const whereClause = await buildSearchQueryWhereCondition({
endTime,
propertyTypeMappingsModel,
query: q,
startTime,
});
if (field == null && aggFn !== AggFn.Count) {
throw new Error(
'Field is required for all aggregation functions except Count',
);
}
const selectField =
field != null
? buildSearchColumnName(propertyTypeMappingsModel.get(field), field)
: '';
const hasGroupBy = groupBy != '' && groupBy != null;
const isCountFn = aggFn === AggFn.Count;
const groupByField =
hasGroupBy &&
buildSearchColumnName(propertyTypeMappingsModel.get(groupBy), groupBy);
const serializer = new SQLSerializer(propertyTypeMappingsModel);
const label = SqlString.escape(`${aggFn}(${field})`);
const selectClause = [
isCountFn
? 'toFloat64(count()) as data'
: aggFn === AggFn.Sum
? `toFloat64(sum(${selectField})) as data`
: aggFn === AggFn.Avg
? `toFloat64(avg(${selectField})) as data`
: aggFn === AggFn.Max
? `toFloat64(max(${selectField})) as data`
: aggFn === AggFn.Min
? `toFloat64(min(${selectField})) as data`
: aggFn === AggFn.CountDistinct
? `toFloat64(count(distinct ${selectField})) as data`
: `toFloat64(quantile(${
aggFn === AggFn.P50
? '0.5'
: aggFn === AggFn.P90
? '0.90'
: aggFn === AggFn.P95
? '0.95'
: '0.99'
})(${selectField})) as data`,
granularity != null
? `toUnixTimestamp(toStartOfInterval(timestamp, INTERVAL ${granularity})) as ts_bucket`
: "'0' as ts_bucket",
groupByField ? `${groupByField} as group` : `'' as group`, // FIXME: should we fallback to use aggFn as group
`${label} as label`,
].join(',');
const groupByClause = `ts_bucket ${groupByField ? `, ${groupByField}` : ''}`;
const query = SqlString.format(
`
SELECT ?
FROM ??
WHERE ? AND (?) ? ?
GROUP BY ?
ORDER BY ts_bucket ASC
${
granularity != null
? `WITH FILL
FROM toUnixTimestamp(toStartOfInterval(toDateTime(?), INTERVAL ?))
TO toUnixTimestamp(toStartOfInterval(toDateTime(?), INTERVAL ?))
STEP ?`
: ''
}${
sortOrder === 'asc' || sortOrder === 'desc' ? `, data ${sortOrder}` : ''
}
`,
[
SqlString.raw(selectClause),
tableName,
buildTeamLogStreamWhereCondition(tableVersion, teamId),
SqlString.raw(whereClause),
SqlString.raw(
!isCountFn && field != null
? ` AND (${await serializer.isNotNull(field, false)})`
: '',
),
SqlString.raw(
hasGroupBy
? ` AND (${await serializer.isNotNull(groupBy, false)})`
: '',
),
SqlString.raw(groupByClause),
...(granularity != null
? [
startTime / 1000,
granularity,
endTime / 1000,
granularity,
ms(granularity) / 1000,
]
: []),
],
);
return {
query,
hasGroupBy,
sortOrder,
};
};
export const queryMultiSeriesChart = async ({
maxNumGroups,
tableVersion,
teamId,
seriesReturnType = SeriesReturnType.Column,
queries,
}: {
maxNumGroups: number;
tableVersion: number | undefined;
teamId: string;
seriesReturnType?: SeriesReturnType;
queries: { query: string; hasGroupBy: boolean; sortOrder?: 'desc' | 'asc' }[];
}) => {
// For now only supports same-table series with the same groupBy
const seriesCTEs = queries
.map((q, i) => `series_${i} AS (${q.query})`)
.join(',\n');
// Only join on group bys if all queries have group bys
// TODO: This will not work for an array of group by fields
const allQueiesHaveGroupBy = queries.every(q => q.hasGroupBy);
let seriesIndexWithSorting = -1;
let sortOrder: 'asc' | 'desc' = 'desc';
for (let i = 0; i < queries.length; i++) {
const q = queries[i];
if (q.sortOrder === 'asc' || q.sortOrder === 'desc') {
seriesIndexWithSorting = i;
sortOrder = q.sortOrder;
break;
}
}
let leftJoin = '';
// Join every series after the first one
for (let i = 1; i < queries.length; i++) {
leftJoin += `LEFT JOIN series_${i} ON series_${i}.ts_bucket=series_0.ts_bucket${
allQueiesHaveGroupBy ? ` AND series_${i}.group = series_0.group` : ''
}\n`;
}
const select =
seriesReturnType === 'column'
? queries
.map((_, i) => {
return `series_${i}.data as "series_${i}.data"`;
})
.join(',\n')
: 'series_0.data / series_1.data as "series_0.data"';
// Return each series data as a separate column
const query = SqlString.format(
`WITH ?
,raw_groups AS (
SELECT
?,
series_0.ts_bucket as ts_bucket,
series_0.group as group
FROM series_0 AS series_0
?
), groups AS (
SELECT *, ?(?) OVER (PARTITION BY group) as rank_order_by_value
FROM raw_groups
), final AS (
SELECT *, DENSE_RANK() OVER (ORDER BY rank_order_by_value ?) as rank
FROM groups
)
SELECT *
FROM final
WHERE rank <= ?
ORDER BY ts_bucket ASC
?
`,
[
SqlString.raw(seriesCTEs),
SqlString.raw(select),
SqlString.raw(leftJoin),
// Setting rank_order_by_value
SqlString.raw(sortOrder === 'asc' ? 'MIN' : 'MAX'),
SqlString.raw(
// If ratio, we judge on series_0
seriesReturnType === 'ratio'
? 'series_0.data'
: // If the user specified a sorting order, we use that
seriesIndexWithSorting > -1
? `series_${seriesIndexWithSorting}.data`
: // Otherwise we just grab the greatest value
`greatest(${queries.map((_, i) => `series_${i}.data`).join(', ')})`,
),
// ORDER BY rank_order_by_value ....
SqlString.raw(sortOrder === 'asc' ? 'ASC' : 'DESC'),
maxNumGroups,
// Final row sort ordering
SqlString.raw(
sortOrder === 'asc' || sortOrder === 'desc'
? `, series_${
seriesIndexWithSorting > -1 ? seriesIndexWithSorting : 0
}.data ${sortOrder}`
: '',
),
],
);
const rows = await client.query({
query,
format: 'JSON',
clickhouse_settings: {
additional_table_filters: buildLogStreamAdditionalFilters(
tableVersion,
teamId,
),
},
});
const result = await rows.json<
ResponseJSON<{
ts_bucket: number;
group: string;
[series_data: `series_${number}.data`]: number;
}>
>();
return result;
};
export const getMultiSeriesChart = async ({
series,
endTime,
granularity,
maxNumGroups,
propertyTypeMappingsModel,
startTime,
tableVersion,
teamId,
seriesReturnType = SeriesReturnType.Column,
}: {
series: z.infer<typeof chartSeriesSchema>[];
endTime: number; // unix in ms,
startTime: number; // unix in ms
granularity: string | undefined; // can be undefined in the number chart
maxNumGroups: number;
propertyTypeMappingsModel?: LogsPropertyTypeMappingsModel;
tableVersion: number | undefined;
teamId: string;
seriesReturnType?: SeriesReturnType;
}) => {
let queries: {
query: string;
hasGroupBy: boolean;
sortOrder?: 'desc' | 'asc';
}[] = [];
if (
// Default table is logs
('table' in series[0] &&
(series[0].table === 'logs' || series[0].table == null)) ||
!('table' in series[0])
) {
if (propertyTypeMappingsModel == null) {
throw new Error('propertyTypeMappingsModel is required for logs chart');
}
queries = await Promise.all(
series.map(s => {
if (s.type != 'time' && s.type != 'table') {
throw new Error(`Unsupported series type: ${s.type}`);
}
if (s.table != 'logs' && s.table != null) {
throw new Error(`All series must have the same table`);
}
return buildEventSeriesQuery({
aggFn: s.aggFn,
endTime,
field: s.field,
granularity,
groupBy: s.groupBy[0],
propertyTypeMappingsModel,
q: s.where,
sortOrder: s.type === 'table' ? s.sortOrder : undefined,
startTime,
tableVersion,
teamId,
});
}),
);
} else if ('table' in series[0] && series[0].table === 'metrics') {
queries = await Promise.all(
series.map(s => {
if (s.type != 'time' && s.type != 'table') {
throw new Error(`Unsupported series type: ${s.type}`);
}
if (s.table != 'metrics') {
throw new Error(`All series must have the same table`);
}
if (s.field == null) {
throw new Error('Metric name is required');
}
if (s.metricDataType == null) {
throw new Error('Metric data type is required');
}
return buildMetricSeriesQuery({
aggFn: s.aggFn,
endTime,
name: s.field,
granularity,
groupBy: s.groupBy[0],
sortOrder: s.type === 'table' ? s.sortOrder : undefined,
q: s.where,
startTime,
teamId,
dataType: s.metricDataType,
});
}),
);
}
return queryMultiSeriesChart({
maxNumGroups,
tableVersion,
teamId,
seriesReturnType,
queries,
});
};
export const getMultiSeriesChartLegacyFormat = async ({
series,
endTime,
granularity,
maxNumGroups,
propertyTypeMappingsModel,
startTime,
tableVersion,
teamId,
seriesReturnType,
}: {
series: z.infer<typeof chartSeriesSchema>[];
endTime: number; // unix in ms,
startTime: number; // unix in ms
granularity: string | undefined; // can be undefined in the number chart
maxNumGroups: number;
propertyTypeMappingsModel?: LogsPropertyTypeMappingsModel;
tableVersion: number | undefined;
teamId: string;
seriesReturnType?: SeriesReturnType;
}) => {
const result = await getMultiSeriesChart({
series,
endTime,
granularity,
maxNumGroups,
propertyTypeMappingsModel,
startTime,
tableVersion,
teamId,
seriesReturnType,
});
const flatData = result.data.flatMap(row => {
if (seriesReturnType === 'column') {
return series.map((_, i) => {
return {
ts_bucket: row.ts_bucket,
group: row.group,
data: row[`series_${i}.data`],
};
});
}
// Ratio only has 1 series
return [
{
ts_bucket: row.ts_bucket,
group: row.group,
data: row['series_0.data'],
},
];
});
return {
rows: flatData.length,
data: flatData,
};
};
export const getLogsChart = async ({
aggFn,
endTime,

146
packages/api/src/fixtures.ts
View file

@ -1,6 +1,18 @@
import mongoose from 'mongoose';
import request from 'supertest';
import * as clickhouse from '@/clickhouse';
import {
LogsPropertyTypeMappingsModel,
MetricsPropertyTypeMappingsModel,
} from '@/clickhouse/propertyTypeMappingsModel';
import {
LogPlatform,
LogStreamModel,
LogType,
MetricModel,
} from '@/utils/logParser';
import * as config from './config';
import { createTeam, getTeam } from './controllers/team';
import { findUserByEmail } from './controllers/user';
@ -105,3 +117,137 @@ export const getLoggedInAgent = async (server: MockServer) => {
user,
};
};
export function buildEvent({
level,
source = 'test',
timestamp,
platform = LogPlatform.NodeJS,
type = LogType.Log,
end_timestamp = 0,
span_name,
...properties
}: {
level?: string;
source?: string;
timestamp?: number; // ms timestamp
platform?: LogPlatform;
type?: LogType;
end_timestamp?: number; //ms timestamp
span_name?: string;
} & {
[key: string]: number | string | boolean;
}): LogStreamModel {
const ts = timestamp ?? Date.now();
const boolNames: string[] = [];
const boolValues: number[] = [];
const numberNames: string[] = [];
const numberValues: number[] = [];
const stringNames: string[] = [];
const stringValues: string[] = [];
Object.entries(properties).forEach(([key, value]) => {
if (typeof value === 'boolean') {
boolNames.push(key);
boolValues.push(value ? 1 : 0);
} else if (typeof value === 'number') {
numberNames.push(key);
numberValues.push(value);
} else if (typeof value === 'string') {
stringNames.push(key);
stringValues.push(value);
}
});
return {
// TODO: Fix Timestamp Types
// @ts-ignore
timestamp: `${ts}000000`,
// @ts-ignore
observed_timestamp: `${ts}000000`,
_source: source,
_platform: platform,
severity_text: level,
// @ts-ignore
end_timestamp: `${end_timestamp}000000`,
type,
span_name,
'bool.names': boolNames,
'bool.values': boolValues,
'number.names': numberNames,
'number.values': numberValues,
'string.names': stringNames,
'string.values': stringValues,
};
}
export function buildMetricSeries({
tags,
name,
points,
data_type,
is_delta,
is_monotonic,
unit,
}: {
tags: Record<string, string>;
name: string;
points: { value: number; timestamp: number }[];
data_type: clickhouse.MetricsDataType;
is_monotonic: boolean;
is_delta: boolean;
unit: string;
}): MetricModel[] {
// @ts-ignore TODO: Fix Timestamp types
return points.map(({ value, timestamp }) => ({
_string_attributes: tags,
name,
value,
timestamp: `${timestamp}000000`,
data_type,
is_monotonic,
is_delta,
unit,
}));
}
export function mockLogsPropertyTypeMappingsModel(propertyMap: {
[property: string]: 'bool' | 'number' | 'string';
}) {
const propertyTypesMappingsModel = new LogsPropertyTypeMappingsModel(
1,
'fake team id',
() => Promise.resolve({}),
);
jest
.spyOn(propertyTypesMappingsModel, 'get')
.mockImplementation((property: string) => {
// eslint-disable-next-line security/detect-object-injection
return propertyMap[property];
});
jest
.spyOn(clickhouse, 'buildLogsPropertyTypeMappingsModel')
.mockImplementation(() => Promise.resolve(propertyTypesMappingsModel));
return propertyTypesMappingsModel;
}
export function mockSpyMetricPropertyTypeMappingsModel(propertyMap: {
[property: string]: 'bool' | 'number' | 'string';
}) {
const model = new MetricsPropertyTypeMappingsModel(1, 'fake', () =>
Promise.resolve({}),
);
jest.spyOn(model, 'get').mockImplementation((property: string) => {
// eslint-disable-next-line security/detect-object-injection
return propertyMap[property];
});
jest
.spyOn(clickhouse, 'buildMetricsPropertyTypeMappingsModel')
.mockImplementation(() => Promise.resolve(model));
return model;
}

13
packages/api/src/models/dashboard.ts
View file

@ -1,6 +1,8 @@
import mongoose, { Schema } from 'mongoose';
import { AggFn } from '../clickhouse';
import { SourceTable } from '@/utils/zod';
import { AggFn, SeriesReturnType } from '../clickhouse';
import type { ObjectId } from '.';
// Based on numbro.js format
@ -25,7 +27,7 @@ type Chart = {
h: number;
series: (
| {
table: string;
table: SourceTable;
type: 'time';
aggFn: AggFn; // TODO: Type
field: string | undefined;
@ -34,7 +36,7 @@ type Chart = {
numberFormat?: NumberFormat;
}
| {
table: string;
table: SourceTable;
type: 'histogram';
field: string | undefined;
where: string;
@ -46,7 +48,7 @@ type Chart = {
}
| {
type: 'number';
table: string;
table: SourceTable;
aggFn: AggFn;
field: string | undefined;
where: string;
@ -54,7 +56,7 @@ type Chart = {
}
| {
type: 'table';
table: string;
table: SourceTable;
aggFn: AggFn;
field: string | undefined;
where: string;
@ -67,6 +69,7 @@ type Chart = {
content: string;
}
)[];
seriesReturnType?: SeriesReturnType;
};
export interface IDashboard {

2
packages/api/src/routers/api/alerts.ts
View file

@ -39,7 +39,7 @@ const zAlert = z
.object({
channel: zChannel,
interval: z.enum(['1m', '5m', '15m', '30m', '1h', '6h', '12h', '1d']),
threshold: z.number().min(1),
threshold: z.number().min(0),
type: z.enum(['presence', 'absence']),
source: z.enum(['LOG', 'CHART']).default('LOG'),
})

105
packages/api/src/routers/api/chart.ts Normal file
View file

@ -0,0 +1,105 @@
import opentelemetry, { SpanStatusCode } from '@opentelemetry/api';
import express from 'express';
import { isNumber, parseInt } from 'lodash';
import { z } from 'zod';
import { validateRequest } from 'zod-express-middleware';
import * as clickhouse from '@/clickhouse';
import { getTeam } from '@/controllers/team';
import logger from '@/utils/logger';
import { chartSeriesSchema } from '@/utils/zod';
const router = express.Router();
router.post(
'/series',
validateRequest({
body: z.object({
series: z.array(chartSeriesSchema),
endTime: z.number(),
granularity: z.nativeEnum(clickhouse.Granularity).optional(),
startTime: z.number(),
seriesReturnType: z.optional(z.nativeEnum(clickhouse.SeriesReturnType)),
}),
}),
async (req, res, next) => {
try {
const teamId = req.user?.team;
const { endTime, granularity, startTime, seriesReturnType, series } =
req.body;
if (teamId == null) {
return res.sendStatus(403);
}
if (!isNumber(startTime) || !isNumber(endTime)) {
return res.sendStatus(400);
}
const team = await getTeam(teamId);
if (team == null) {
return res.sendStatus(403);
}
const propertyTypeMappingsModel =
await clickhouse.buildLogsPropertyTypeMappingsModel(
team.logStreamTableVersion,
teamId.toString(),
startTime,
endTime,
);
const propertySet = new Set<string>();
series.map(s => {
if ('field' in s && s.field != null) {
propertySet.add(s.field);
}
if ('groupBy' in s && s.groupBy.length > 0) {
s.groupBy.map(g => propertySet.add(g));
}
});
// Hack to refresh property cache if needed
const properties = Array.from(propertySet);
if (
properties.some(p => {
return !clickhouse.doesLogsPropertyExist(
p,
propertyTypeMappingsModel,
);
})
) {
logger.warn({
message: `getChart: Property type mappings cache is out of date (${properties.join(
', ',
)})`,
});
await propertyTypeMappingsModel.refresh();
}
// TODO: expose this to the frontend
const MAX_NUM_GROUPS = 20;
res.json(
await clickhouse.getMultiSeriesChart({
series,
endTime: endTime,
granularity,
maxNumGroups: MAX_NUM_GROUPS,
propertyTypeMappingsModel,
startTime: startTime,
tableVersion: team.logStreamTableVersion,
teamId: teamId.toString(),
seriesReturnType,
}),
);
} catch (e) {
const span = opentelemetry.trace.getActiveSpan();
span?.recordException(e as Error);
span?.setStatus({ code: SpanStatusCode.ERROR });
next(e);
}
},
);
export default router;

2
packages/api/src/routers/api/index.ts
View file

@ -1,4 +1,5 @@
import alertsRouter from './alerts';
import chartRouter from './chart';
import dashboardRouter from './dashboards';
import logsRouter from './logs';
import logViewsRouter from './logViews';
@ -20,4 +21,5 @@ export default {
sessionsRouter,
teamRouter,
webhooksRouter,
chartRouter,
};

254
packages/api/src/tasks/__tests__/checkAlerts.test.ts
View file

@ -1,7 +1,19 @@
import ms from 'ms';
import {
buildEvent,
buildMetricSeries,
clearDBCollections,
closeDB,
getServer,
mockLogsPropertyTypeMappingsModel,
mockSpyMetricPropertyTypeMappingsModel,
} from '@/fixtures';
import { LogType } from '@/utils/logParser';
import * as clickhouse from '../../clickhouse';
import { createAlert } from '../../controllers/alerts';
import { createTeam } from '../../controllers/team';
import { clearDBCollections, closeDB, getServer } from '../../fixtures';
import AlertHistory from '../../models/alertHistory';
import Dashboard from '../../models/dashboard';
import LogView from '../../models/logView';
@ -254,25 +266,34 @@ describe('checkAlerts', () => {
jest
.spyOn(slack, 'postMessageToWebhook')
.mockResolvedValueOnce(null as any);
jest
.spyOn(clickhouse, 'getLogsChart')
.mockResolvedValueOnce({
rows: 1,
data: [
{
data: '11',
group: 'HyperDX',
rank: '1',
rank_order_by_value: '11',
ts_bucket: 1700172600,
},
],
} as any)
// no logs found in the next window
.mockResolvedValueOnce({
rows: 0,
data: [],
} as any);
mockLogsPropertyTypeMappingsModel({
runId: 'string',
});
const runId = Math.random().toString(); // dedup watch mode runs
const teamId = 'test';
const now = new Date('2023-11-16T22:12:00.000Z');
// Send events in the last alert window 22:05 - 22:10
const eventMs = now.getTime() - ms('5m');
await clickhouse.bulkInsertTeamLogStream(undefined, teamId, [
buildEvent({
span_name: 'HyperDX',
level: 'error',
timestamp: eventMs,
runId,
end_timestamp: eventMs + 100,
type: LogType.Span,
}),
buildEvent({
span_name: 'HyperDX',
level: 'error',
timestamp: eventMs + 5,
runId,
end_timestamp: eventMs + 7,
type: LogType.Span,
}),
]);
const team = await createTeam({ name: 'My Team' });
const webhook = await new Webhook({
@ -296,12 +317,21 @@ describe('checkAlerts', () => {
{
table: 'logs',
type: 'time',
aggFn: 'max',
aggFn: 'sum',
field: 'duration',
where: 'level:error',
where: `level:error runId:${runId}`,
groupBy: ['span_name'],
},
{
table: 'logs',
type: 'time',
aggFn: 'min',
field: 'duration',
where: `level:error runId:${runId}`,
groupBy: ['span_name'],
},
],
seriesReturnType: 'column',
},
{
id: 'obil1',
@ -336,9 +366,7 @@ describe('checkAlerts', () => {
chartId: '198hki',
});
const now = new Date('2023-11-16T22:12:00.000Z');
// shoud fetch 5m of logs
// should fetch 5m of logs
await processAlert(now, alert);
expect(alert.state).toBe('ALERT');
@ -359,12 +387,13 @@ describe('checkAlerts', () => {
}).sort({
createdAt: 1,
});
expect(alertHistories.length).toBe(2);
const [history1, history2] = alertHistories;
expect(history1.state).toBe('ALERT');
expect(history1.counts).toBe(1);
expect(history1.createdAt).toEqual(new Date('2023-11-16T22:10:00.000Z'));
expect(history1.lastValues.length).toBe(1);
expect(history1.lastValues.length).toBe(2);
expect(history1.lastValues.length).toBeGreaterThan(0);
expect(history1.lastValues[0].count).toBeGreaterThanOrEqual(1);
@ -372,32 +401,19 @@ describe('checkAlerts', () => {
expect(history2.counts).toBe(0);
expect(history2.createdAt).toEqual(new Date('2023-11-16T22:15:00.000Z'));
// check if getLogsChart query + webhook were triggered
expect(clickhouse.getLogsChart).toHaveBeenNthCalledWith(1, {
aggFn: 'max',
endTime: 1700172600000,
field: 'duration',
granularity: '5 minute',
groupBy: 'span_name',
maxNumGroups: 20,
propertyTypeMappingsModel: expect.any(Object),
q: 'level:error',
startTime: 1700172300000,
tableVersion: team.logStreamTableVersion,
teamId: team._id.toString(),
});
// check if webhook was triggered
expect(slack.postMessageToWebhook).toHaveBeenNthCalledWith(
1,
'https://hooks.slack.com/services/123',
{
text: 'Alert for "Max Duration" in "My Dashboard" - 11 exceeds 10',
text: 'Alert for "Max Duration" in "My Dashboard" - 102 exceeds 10',
blocks: [
{
text: {
text: [
`*<http://localhost:9090/dashboards/${dashboard._id}?from=1700170500000&granularity=5+minute&to=1700175000000 | Alert for "Max Duration" in "My Dashboard">*`,
`*<http://localhost:9090/dashboards/${dashboard._id}?from=1700170200000&granularity=5+minute&to=1700174700000 | Alert for "Max Duration" in "My Dashboard">*`,
'Group: "HyperDX"',
'11 exceeds 10',
'102 exceeds 10',
].join('\n'),
type: 'mrkdwn',
},
@ -406,29 +422,111 @@ describe('checkAlerts', () => {
],
},
);
jest.resetAllMocks();
});
it('CHART alert (metrics table series)', async () => {
const runId = Math.random().toString(); // dedup watch mode runs
jest
.spyOn(slack, 'postMessageToWebhook')
.mockResolvedValueOnce(null as any);
jest
.spyOn(clickhouse, 'getMetricsChart')
.mockResolvedValueOnce({
rows: 1,
data: [
{
data: 11,
group: 'HyperDX',
ts_bucket: 1700172600,
},
const now = new Date('2023-11-16T22:12:00.000Z');
// Need data in 22:00 - 22:05 to calculate a rate for 22:05 - 22:10
const metricNowTs = new Date('2023-11-16T22:00:00.000Z').getTime();
mockSpyMetricPropertyTypeMappingsModel({
runId: 'string',
host: 'string',
'cloud.provider': 'string',
});
await clickhouse.bulkInsertTeamMetricStream(
buildMetricSeries({
name: 'redis.memory.rss',
tags: {
host: 'HyperDX',
'cloud.provider': 'aws',
runId,
series: '1',
},
data_type: clickhouse.MetricsDataType.Sum,
is_monotonic: true,
is_delta: true,
unit: 'Bytes',
points: [
{ value: 1, timestamp: metricNowTs },
{ value: 8, timestamp: metricNowTs + ms('1m') },
{ value: 8, timestamp: metricNowTs + ms('2m') },
{ value: 9, timestamp: metricNowTs + ms('3m') },
{ value: 15, timestamp: metricNowTs + ms('4m') }, // 15
{ value: 30, timestamp: metricNowTs + ms('5m') },
{ value: 31, timestamp: metricNowTs + ms('6m') },
{ value: 32, timestamp: metricNowTs + ms('7m') },
{ value: 33, timestamp: metricNowTs + ms('8m') },
{ value: 34, timestamp: metricNowTs + ms('9m') }, // 34
{ value: 35, timestamp: metricNowTs + ms('10m') },
{ value: 36, timestamp: metricNowTs + ms('11m') },
],
} as any)
// no logs found in the next window
.mockResolvedValueOnce({
rows: 0,
data: [],
} as any);
}),
);
await clickhouse.bulkInsertTeamMetricStream(
buildMetricSeries({
name: 'redis.memory.rss',
tags: {
host: 'HyperDX',
'cloud.provider': 'aws',
runId,
series: '2',
},
data_type: clickhouse.MetricsDataType.Sum,
is_monotonic: true,
is_delta: true,
unit: 'Bytes',
points: [
{ value: 1000, timestamp: metricNowTs },
{ value: 8000, timestamp: metricNowTs + ms('1m') },
{ value: 8000, timestamp: metricNowTs + ms('2m') },
{ value: 9000, timestamp: metricNowTs + ms('3m') },
{ value: 15000, timestamp: metricNowTs + ms('4m') }, // 15000
{ value: 30000, timestamp: metricNowTs + ms('5m') },
{ value: 30001, timestamp: metricNowTs + ms('6m') },
{ value: 30002, timestamp: metricNowTs + ms('7m') },
{ value: 30003, timestamp: metricNowTs + ms('8m') },
{ value: 30004, timestamp: metricNowTs + ms('9m') }, // 30004
{ value: 30005, timestamp: metricNowTs + ms('10m') },
{ value: 30006, timestamp: metricNowTs + ms('11m') },
],
}),
);
await clickhouse.bulkInsertTeamMetricStream(
buildMetricSeries({
name: 'redis.memory.rss',
tags: { host: 'test2', 'cloud.provider': 'aws', runId, series: '0' },
data_type: clickhouse.MetricsDataType.Sum,
is_monotonic: true,
is_delta: true,
unit: 'Bytes',
points: [
{ value: 1, timestamp: metricNowTs },
{ value: 8, timestamp: metricNowTs + ms('1m') },
{ value: 8, timestamp: metricNowTs + ms('2m') },
{ value: 9, timestamp: metricNowTs + ms('3m') },
{ value: 15, timestamp: metricNowTs + ms('4m') }, // 15
{ value: 17, timestamp: metricNowTs + ms('5m') },
{ value: 18, timestamp: metricNowTs + ms('6m') },
{ value: 19, timestamp: metricNowTs + ms('7m') },
{ value: 20, timestamp: metricNowTs + ms('8m') },
{ value: 21, timestamp: metricNowTs + ms('9m') }, // 21
{ value: 22, timestamp: metricNowTs + ms('10m') },
{ value: 23, timestamp: metricNowTs + ms('11m') },
],
}),
);
const team = await createTeam({ name: 'My Team' });
const webhook = await new Webhook({
@ -452,12 +550,21 @@ describe('checkAlerts', () => {
{
table: 'metrics',
type: 'time',
aggFn: 'max',
field: 'redis.memory.rss - Gauge',
where: 'cloud.provider:"aws"',
aggFn: 'avg_rate',
field: 'redis.memory.rss - Sum',
where: `cloud.provider:"aws" runId:${runId}`,
groupBy: ['host'],
},
{
table: 'metrics',
type: 'time',
aggFn: 'min_rate',
field: 'redis.memory.rss - Sum',
where: `cloud.provider:"aws" runId:${runId}`,
groupBy: ['host'],
},
],
seriesReturnType: 'ratio',
},
{
id: 'obil1',
@ -492,9 +599,7 @@ describe('checkAlerts', () => {
chartId: '198hki',
});
const now = new Date('2023-11-16T22:12:00.000Z');
// shoud fetch 5m of logs
// shoud fetch 5m of metrics
await processAlert(now, alert);
expect(alert.state).toBe('ALERT');
@ -527,30 +632,19 @@ describe('checkAlerts', () => {
new Date('2023-11-16T22:15:00.000Z'),
);
// check if getLogsChart query + webhook were triggered
expect(clickhouse.getMetricsChart).toHaveBeenNthCalledWith(1, {
aggFn: 'max',
dataType: 'Gauge',
endTime: 1700172600000,
granularity: '5 minute',
groupBy: 'host',
name: 'redis.memory.rss',
q: 'cloud.provider:"aws"',
startTime: 1700172300000,
teamId: team._id.toString(),
});
// check if webhook was triggered
expect(slack.postMessageToWebhook).toHaveBeenNthCalledWith(
1,
'https://hooks.slack.com/services/123',
{
text: 'Alert for "Redis Memory" in "My Dashboard" - 11 exceeds 10',
text: 'Alert for "Redis Memory" in "My Dashboard" - 395.3421052631579 exceeds 10',
blocks: [
{
text: {
text: [
`*<http://localhost:9090/dashboards/${dashboard._id}?from=1700170500000&granularity=5+minute&to=1700175000000 | Alert for "Redis Memory" in "My Dashboard">*`,
`*<http://localhost:9090/dashboards/${dashboard._id}?from=1700170200000&granularity=5+minute&to=1700174700000 | Alert for "Redis Memory" in "My Dashboard">*`,
'Group: "HyperDX"',
'11 exceeds 10',
'395.3421052631579 exceeds 10',
].join('\n'),
type: 'mrkdwn',
},

69
packages/api/src/tasks/checkAlerts.ts
View file

@ -155,7 +155,7 @@ const buildLogEventSlackMessage = async ({
endTime: endTime.getTime(),
limit: 5,
offset: 0,
order: 'desc', // TODO: better to use null
order: 'desc',
q: searchQuery,
startTime: startTime.getTime(),
tableVersion: logView.team.logStreamTableVersion,
@ -336,8 +336,7 @@ export const processAlert = async (now: Date, alert: AlertDocument) => {
// Logs Source
let checksData:
| Awaited<ReturnType<typeof clickhouse.checkAlert>>
| Awaited<ReturnType<typeof clickhouse.getLogsChart>>
| Awaited<ReturnType<typeof clickhouse.getMetricsChart>>
| Awaited<ReturnType<typeof clickhouse.getMultiSeriesChartLegacyFormat>>
| null = null;
let logView: Awaited<ReturnType<typeof getLogViewEnhanced>> | null = null;
let targetDashboard: EnhancedDashboard | null = null;
@ -380,17 +379,19 @@ export const processAlert = async (now: Date, alert: AlertDocument) => {
).populate<{
team: ITeam;
}>('team');
if (
dashboard &&
Array.isArray(dashboard.charts) &&
dashboard.charts.length === 1
) {
const chart = dashboard.charts[0];
// Doesn't work for metric alerts yet
const MAX_NUM_GROUPS = 20;
// TODO: assuming that the chart has only 1 series for now
const series = chart.series[0];
if (series.type === 'time' && series.table === 'logs') {
const firstSeries = chart.series[0];
if (firstSeries.type === 'time' && firstSeries.table === 'logs') {
targetDashboard = dashboard;
const MAX_NUM_GROUPS = 20;
const startTimeMs = fns.getTime(checkStartTime);
const endTimeMs = fns.getTime(checkEndTime);
const propertyTypeMappingsModel =
@ -400,51 +401,49 @@ export const processAlert = async (now: Date, alert: AlertDocument) => {
startTimeMs,
endTimeMs,
);
checksData = await clickhouse.getLogsChart({
aggFn: series.aggFn,
checksData = await clickhouse.getMultiSeriesChartLegacyFormat({
series: chart.series,
endTime: endTimeMs,
// @ts-expect-error
field: series.field,
granularity: `${windowSizeInMins} minute`,
groupBy: series.groupBy[0],
maxNumGroups: MAX_NUM_GROUPS,
propertyTypeMappingsModel,
q: series.where,
startTime: startTimeMs,
tableVersion: dashboard.team.logStreamTableVersion,
teamId: dashboard.team._id.toString(),
seriesReturnType: chart.seriesReturnType,
});
} else if (
series.type === 'time' &&
series.table === 'metrics' &&
series.field
firstSeries.type === 'time' &&
firstSeries.table === 'metrics' &&
firstSeries.field
) {
targetDashboard = dashboard;
let startTimeMs = fns.getTime(checkStartTime);
const startTimeMs = fns.getTime(checkStartTime);
const endTimeMs = fns.getTime(checkEndTime);
const [metricName, rawMetricDataType] = series.field.split(' - ');
const metricDataType = z
.nativeEnum(clickhouse.MetricsDataType)
.parse(rawMetricDataType);
if (
metricDataType === clickhouse.MetricsDataType.Sum &&
clickhouse.isRateAggFn(series.aggFn)
) {
// adjust the time so that we have enough data points to calculate a rate
startTimeMs = fns
.subMinutes(startTimeMs, windowSizeInMins)
.getTime();
}
checksData = await clickhouse.getMetricsChart({
aggFn: series.aggFn,
dataType: metricDataType,
checksData = await clickhouse.getMultiSeriesChartLegacyFormat({
series: chart.series.map(series => {
if ('field' in series && series.field != null) {
const [metricName, rawMetricDataType] =
series.field.split(' - ');
const metricDataType = z
.nativeEnum(clickhouse.MetricsDataType)
.parse(rawMetricDataType);
return {
...series,
metricDataType,
field: metricName,
};
}
return series;
}),
endTime: endTimeMs,
granularity: `${windowSizeInMins} minute`,
groupBy: series.groupBy[0],
name: metricName,
q: series.where,
maxNumGroups: MAX_NUM_GROUPS,
startTime: startTimeMs,
tableVersion: dashboard.team.logStreamTableVersion,
teamId: dashboard.team._id.toString(),
seriesReturnType: chart.seriesReturnType,
});
}
}

83
packages/api/src/utils/zod.ts Normal file
View file

@ -0,0 +1,83 @@
import { z } from 'zod';
import { AggFn, MetricsDataType } from '@/clickhouse';
export const numberFormatSchema = z.object({
output: z
.union([
z.literal('currency'),
z.literal('percent'),
z.literal('byte'),
z.literal('time'),
z.literal('number'),
])
.optional(),
mantissa: z.number().optional(),
thousandSeparated: z.boolean().optional(),
average: z.boolean().optional(),
decimalBytes: z.boolean().optional(),
factor: z.number().optional(),
currencySymbol: z.string().optional(),
unit: z.string().optional(),
});
export const aggFnSchema = z.nativeEnum(AggFn);
export const sourceTableSchema = z.union([
z.literal('logs'),
z.literal('rrweb'),
z.literal('metrics'),
]);
export type SourceTable = z.infer<typeof sourceTableSchema>;
export const timeChartSeriesSchema = z.object({
table: z.optional(sourceTableSchema),
type: z.literal('time'),
aggFn: aggFnSchema,
field: z.union([z.string(), z.undefined()]),
where: z.string(),
groupBy: z.array(z.string()),
numberFormat: numberFormatSchema.optional(),
metricDataType: z.optional(z.nativeEnum(MetricsDataType)),
});
export const tableChartSeriesSchema = z.object({
type: z.literal('table'),
table: z.optional(sourceTableSchema),
aggFn: aggFnSchema,
field: z.optional(z.string()),
where: z.string(),
groupBy: z.array(z.string()),
sortOrder: z.optional(z.union([z.literal('desc'), z.literal('asc')])),
numberFormat: numberFormatSchema.optional(),
metricDataType: z.optional(z.nativeEnum(MetricsDataType)),
});
export const chartSeriesSchema = z.union([
timeChartSeriesSchema,
tableChartSeriesSchema,
z.object({
table: z.optional(sourceTableSchema),
type: z.literal('histogram'),
field: z.union([z.string(), z.undefined()]),
where: z.string(),
}),
z.object({
type: z.literal('search'),
fields: z.array(z.string()),
where: z.string(),
}),
z.object({
type: z.literal('number'),
table: z.optional(sourceTableSchema),
aggFn: aggFnSchema,
field: z.union([z.string(), z.undefined()]),
where: z.string(),
numberFormat: numberFormatSchema.optional(),
}),
z.object({
type: z.literal('markdown'),
content: z.string(),
}),
]);

57
packages/app/src/ChartPage.tsx
View file

@ -8,7 +8,7 @@ import { decodeArray, encodeArray } from 'serialize-query-params';
import { StringParam, useQueryParam, withDefault } from 'use-query-params';
import AppNav from './AppNav';
import { AggFn, ChartSeries, ChartSeriesForm } from './ChartUtils';
import { ChartSeriesForm } from './ChartUtils';
import DSSelect from './DSSelect';
import HDXLineChart from './HDXLineChart';
import { LogTableWithSidePanel } from './LogTableWithSidePanel';
@ -16,6 +16,7 @@ import SearchTimeRangePicker, {
parseTimeRangeInput,
} from './SearchTimeRangePicker';
import { parseTimeQuery, useTimeQuery } from './timeQuery';
import type { AggFn, ChartSeries, SourceTable } from './types';
import { useQueryParam as useHDXQueryParam } from './useQueryParam';
export const ChartSeriesParam: QueryParamConfig<ChartSeries[] | undefined> = {
@ -56,21 +57,12 @@ export default function GraphPage() {
},
);
const setTable = (index: number, table: string) => {
setChartSeries(
produce(chartSeries, series => {
if (series?.[index] != null) {
series[index].table = table;
}
}),
);
};
const setAggFn = (index: number, fn: AggFn) => {
setChartSeries(
produce(chartSeries, series => {
if (series?.[index] != null) {
series[index].aggFn = fn;
const s = series?.[index];
if (s != null && s.type === 'time') {
s.aggFn = fn;
}
}),
);
@ -78,8 +70,9 @@ export default function GraphPage() {
const setField = (index: number, field: string | undefined) => {
setChartSeries(
produce(chartSeries, series => {
if (series?.[index] != null) {
series[index].field = field;
const s = series?.[index];
if (s != null && s.type === 'time') {
s.field = field;
}
}),
);
@ -91,9 +84,10 @@ export default function GraphPage() {
) => {
setChartSeries(
produce(chartSeries, series => {
if (series?.[index] != null) {
series[index].field = field;
series[index].aggFn = fn;
const s = series?.[index];
if (s != null && s.type === 'time') {
s.field = field;
s.aggFn = fn;
}
}),
);
@ -101,8 +95,9 @@ export default function GraphPage() {
const setWhere = (index: number, where: string) => {
setChartSeries(
produce(chartSeries, series => {
if (series?.[index] != null) {
series[index].where = where;
const s = series?.[index];
if (s != null && s.type === 'time') {
s.where = where;
}
}),
);
@ -110,8 +105,9 @@ export default function GraphPage() {
const setGroupBy = (index: number, groupBy: string | undefined) => {
setChartSeries(
produce(chartSeries, series => {
if (series?.[index] != null) {
series[index].groupBy = groupBy != null ? [groupBy] : [];
const s = series?.[index];
if (s != null && s.type === 'time') {
s.groupBy = groupBy != null ? [groupBy] : [];
}
}),
);
@ -148,7 +144,11 @@ export default function GraphPage() {
onSearch(displayedTimeInputValue);
const dateRange = parseTimeRangeInput(displayedTimeInputValue);
if (dateRange[0] != null && dateRange[1] != null) {
if (
dateRange[0] != null &&
dateRange[1] != null &&
chartSeries[0].type === 'time'
) {
setChartConfig({
// TODO: Support multiple series
table: chartSeries[0].table ?? 'logs',
@ -177,6 +177,9 @@ export default function GraphPage() {
<form className="bg-body p-3" onSubmit={e => e.preventDefault()}>
<div className="fs-5 mb-3 fw-500">Create New Chart</div>
{chartSeries.map((series, index) => {
if (series.type !== 'time') {
return null;
}
return (
<ChartSeriesForm
key={index}
@ -191,14 +194,14 @@ export default function GraphPage() {
setTableAndAggFn={(table, aggFn) => {
setChartSeries(
produce(chartSeries, series => {
if (series?.[index] != null) {
series[index].table = table;
series[index].aggFn = aggFn;
const s = series?.[index];
if (s != null && s.type === 'time') {
s.table = table;
s.aggFn = aggFn;
}
}),
);
}}
setTable={table => setTable(index, table)}
setWhere={where => setWhere(index, where)}
setAggFn={fn => setAggFn(index, fn)}
setGroupBy={groupBy => setGroupBy(index, groupBy)}

494
packages/app/src/ChartUtils.tsx
View file

@ -9,12 +9,15 @@ import api from './api';
import Checkbox from './Checkbox';
import MetricTagFilterInput from './MetricTagFilterInput';
import SearchInput from './SearchInput';
import type { AggFn, ChartSeries, SourceTable } from './types';
import { NumberFormat } from './types';
export const SORT_ORDER = [
{ value: 'asc' as const, label: 'Ascending' },
{ value: 'desc' as const, label: 'Descending' },
];
export type SortOrder = (typeof SORT_ORDER)[number]['value'];
import type { NumberFormat } from './types';
export const TABLES = [
{ value: 'logs' as const, label: 'Logs / Spans' },
@ -61,14 +64,94 @@ export type Granularity =
| '7 day'
| '30 day';
export type ChartSeries = {
table: string;
type: 'time';
aggFn: AggFn;
field: string | undefined;
where: string;
groupBy: string[];
};
const seriesDisplayName = (s: ChartSeries) =>
s.type === 'time' || s.type === 'table'
? `${s.aggFn}${
s.aggFn !== 'count'
? `(${
s.table === 'metrics'
? s.field?.split(' - ')?.[0] ?? s.field
: s.field
})`
: '()'
}${s.where ? `{${s.where}}` : ''}`
: '';
export function seriesColumns({
series,
seriesReturnType,
}: {
seriesReturnType: 'ratio' | 'column';
series: ChartSeries[];
}) {
const seriesMeta =
seriesReturnType === 'ratio'
? [
{
dataKey: `series_0.data`,
displayName: `${seriesDisplayName(series[0])}/${seriesDisplayName(
series[1],
)}`,
sortOrder:
'sortOrder' in series[0] ? series[0].sortOrder : undefined,
},
]
: series.map((s, i) => {
return {
dataKey: `series_${i}.data`,
displayName: seriesDisplayName(s),
sortOrder: 'sortOrder' in s ? s.sortOrder : undefined,
};
});
return seriesMeta;
}
export function seriesToSearchQuery({
series,
groupByValue,
}: {
series: ChartSeries[];
groupByValue?: string;
}) {
const queries = series
.map((s, i) => {
if (s.type === 'time' || s.type === 'table' || s.type === 'number') {
const { where, aggFn, field } = s;
return `${where.trim()}${aggFn !== 'count' ? ` ${field}:*` : ''}${
'groupBy' in s && s.groupBy != null && s.groupBy.length > 0
? ` ${s.groupBy}:${groupByValue}`
: ''
}`.trim();
}
})
.filter(q => q != null && q.length > 0);
const q =
queries.length > 1
? queries.map(q => `(${q})`).join(' OR ')
: queries.join('');
return q;
}
export function seriesToUrlSearchQueryParam({
series,
dateRange,
groupByValue = '*',
}: {
series: ChartSeries[];
dateRange: [Date, Date];
groupByValue?: string | undefined;
}) {
const q = seriesToSearchQuery({ series, groupByValue });
return new URLSearchParams({
q,
from: `${dateRange[0].getTime()}`,
to: `${dateRange[1].getTime()}`,
});
}
export function usePropertyOptions(types: ('number' | 'string' | 'bool')[]) {
const { data: propertyTypeMappingsResult } = api.usePropertyTypeMappings();
@ -107,27 +190,38 @@ export function usePropertyOptions(types: ('number' | 'string' | 'bool')[]) {
return propertyOptions;
}
export function MetricTagSelect({
value,
setValue,
metricName,
}: {
value: string | undefined | null;
setValue: (value: string | undefined) => void;
metricName: string | undefined;
}) {
function useMetricTagOptions({ metricNames }: { metricNames?: string[] }) {
const { data: metricTagsData } = api.useMetricsTags();
const options = useMemo(() => {
const tags =
metricTagsData?.data?.filter(metric => metric.name === metricName)?.[0]
?.tags ?? [];
let tagNameSet = new Set<string>();
if (metricNames != null && metricNames.length > 0) {
const firstMetricName = metricNames[0]; // Start the set
const tagNameSet = new Set<string>();
const tags =
metricTagsData?.data?.filter(
metric => metric.name === firstMetricName,
)?.[0]?.tags ?? [];
tags.forEach(tag => {
Object.keys(tag).forEach(tagName => tagNameSet.add(tagName));
});
tags.forEach(tag => {
Object.keys(tag).forEach(tagName => tagNameSet.add(tagName));
});
for (let i = 1; i < metricNames.length; i++) {
const tags =
metricTagsData?.data?.filter(
metric => metric.name === metricNames[i],
)?.[0]?.tags ?? [];
const intersection = new Set<string>();
tags.forEach(tag => {
Object.keys(tag).forEach(tagName => {
if (tagNameSet.has(tagName)) {
intersection.add(tagName);
}
});
});
tagNameSet = intersection;
}
}
return [
{ value: undefined, label: 'None' },
@ -136,7 +230,21 @@ export function MetricTagSelect({
label: tagName,
})),
];
}, [metricTagsData, metricName]);
}, [metricTagsData, metricNames]);
return options;
}
export function MetricTagSelect({
value,
setValue,
metricNames,
}: {
value: string | undefined | null;
setValue: (value: string | undefined) => void;
metricNames?: string[];
}) {
const options = useMetricTagOptions({ metricNames });
return (
<AsyncSelect
@ -198,7 +306,7 @@ export function MetricSelect({
return (
<>
<div className="ms-3 flex-grow-1">
<div className="flex-grow-1">
<MetricNameSelect
isLoading={isLoading}
isError={isError}
@ -214,7 +322,7 @@ export function MetricSelect({
}}
/>
</div>
<div className="ms-3 flex-shrink-1">
<div className="flex-shrink-1 ms-3">
<MetricRateSelect
metricName={metricName}
isRate={isRate}
@ -361,26 +469,6 @@ export function FieldSelect({
);
}
export type AggFn =
| 'avg_rate'
| 'avg'
| 'count_distinct'
| 'count'
| 'max_rate'
| 'max'
| 'min_rate'
| 'min'
| 'p50_rate'
| 'p50'
| 'p90_rate'
| 'p90'
| 'p95_rate'
| 'p95'
| 'p99_rate'
| 'p99'
| 'sum_rate'
| 'sum';
export function ChartSeriesForm({
aggFn,
field,
@ -391,7 +479,6 @@ export function ChartSeriesForm({
setTableAndAggFn,
setGroupBy,
setSortOrder,
setTable,
setWhere,
sortOrder,
table,
@ -405,10 +492,9 @@ export function ChartSeriesForm({
setAggFn: (fn: AggFn) => void;
setField: (field: string | undefined) => void;
setFieldAndAggFn: (field: string | undefined, fn: AggFn) => void;
setTableAndAggFn: (table: string, fn: AggFn) => void;
setTableAndAggFn: (table: SourceTable, fn: AggFn) => void;
setGroupBy: (groupBy: string | undefined) => void;
setSortOrder?: (sortOrder: SortOrder) => void;
setTable: (table: string) => void;
setWhere: (where: string) => void;
sortOrder?: string;
table: string;
@ -495,7 +581,7 @@ export function ChartSeriesForm({
</div>
) : null}
{table === 'metrics' ? (
<div className="d-flex align-items-center align-middle flex-grow-1">
<div className="d-flex align-items-center align-middle flex-grow-1 ms-3">
<MetricSelect
metricName={field}
setMetricName={setField}
@ -576,7 +662,7 @@ export function ChartSeriesForm({
<MetricTagSelect
value={groupBy}
setValue={setGroupBy}
metricName={field}
metricNames={field != null ? [field] : []}
/>
</div>
</div>
@ -629,6 +715,308 @@ export function ChartSeriesForm({
);
}
export function TableSelect({
table,
setTableAndAggFn,
}: {
setTableAndAggFn: (table: SourceTable, fn: AggFn) => void;
table: string;
}) {
return (
<Select
options={TABLES}
className="ds-select w-auto text-nowrap"
value={TABLES.find(v => v.value === table)}
onChange={opt => {
const val = opt?.value ?? 'logs';
if (val === 'logs') {
setTableAndAggFn('logs', 'count');
} else if (val === 'metrics') {
// TODO: This should set rate if metric field is a sum
// or we should just reset the field if changing tables
setTableAndAggFn('metrics', 'max');
}
}}
classNamePrefix="ds-react-select"
/>
);
}
export function GroupBySelect(
props:
| {
fields?: string[];
table: 'metrics';
groupBy?: string | undefined;
setGroupBy: (groupBy: string | undefined) => void;
}
| {
table: 'logs';
groupBy?: string | undefined;
setGroupBy: (groupBy: string | undefined) => void;
}
| { table: 'rrweb' },
) {
return (
<>
{props.table === 'metrics' && (
<MetricTagSelect
value={props.groupBy}
setValue={props.setGroupBy}
metricNames={props.fields}
/>
)}
{props.table === 'logs' && props.setGroupBy != null && (
<FieldSelect
className="w-auto text-nowrap"
value={props.groupBy}
setValue={props.setGroupBy}
types={['number', 'bool', 'string']}
/>
)}
</>
);
}
export function ChartSeriesFormCompact({
aggFn,
field,
groupBy,
setAggFn,
setField,
setFieldAndAggFn,
setTableAndAggFn,
setGroupBy,
setSortOrder,
setWhere,
sortOrder,
table,
where,
numberFormat,
setNumberFormat,
}: {
aggFn: AggFn;
field: string | undefined;
groupBy?: string | undefined;
setAggFn: (fn: AggFn) => void;
setField: (field: string | undefined) => void;
setFieldAndAggFn: (field: string | undefined, fn: AggFn) => void;
setTableAndAggFn?: (table: SourceTable, fn: AggFn) => void;
setGroupBy?: (groupBy: string | undefined) => void;
setSortOrder?: (sortOrder: SortOrder) => void;
setWhere: (where: string) => void;
sortOrder?: string;
table?: string;
where: string;
numberFormat?: NumberFormat;
setNumberFormat?: (format?: NumberFormat) => void;
}) {
const labelWidth = 350;
const searchInputRef = useRef<HTMLInputElement>(null);
const isRate = useMemo(() => {
return aggFn.includes('_rate');
}, [aggFn]);
const _setAggFn = (fn: AggFn, _isRate?: boolean) => {
if (_isRate ?? isRate) {
if (fn.includes('_rate')) {
setAggFn(fn);
} else {
setAggFn(`${fn}_rate` as AggFn);
}
} else {
if (fn.includes('_rate')) {
setAggFn(fn.replace('_rate', '') as AggFn);
} else {
setAggFn(fn);
}
}
};
return (
<div>
<div
className="d-flex align-items-center flex-wrap"
style={{ rowGap: '1rem', columnGap: '1rem' }}
>
{setTableAndAggFn && (
<TableSelect
table={table ?? 'logs'}
setTableAndAggFn={setTableAndAggFn}
/>
)}
<div className="">
{table === 'logs' ? (
<Select
options={AGG_FNS}
className="ds-select w-auto text-nowrap"
value={AGG_FNS.find(v => v.value === aggFn)}
onChange={opt => _setAggFn(opt?.value ?? 'count')}
classNamePrefix="ds-react-select"
/>
) : (
<Select
options={METRIC_AGG_FNS}
className="ds-select w-auto text-nowrap"
value={METRIC_AGG_FNS.find(
v => v.value === aggFn.replace('_rate', ''),
)}
onChange={opt => _setAggFn(opt?.value ?? 'sum')}
classNamePrefix="ds-react-select"
/>
)}
</div>
{table === 'logs' && aggFn != 'count' && aggFn != 'count_distinct' ? (
<div className="flex-grow-1">
<FieldSelect
className="w-auto text-nowrap"
value={field}
setValue={setField}
types={['number']}
/>
</div>
) : null}
{table === 'logs' && aggFn != 'count' && aggFn == 'count_distinct' ? (
<div className="flex-grow-1">
<FieldSelect
className="w-auto text-nowrap"
value={field}
setValue={setField}
types={['string', 'number', 'bool']}
/>
</div>
) : null}
{table === 'logs' && (
<div
className="d-flex flex-grow-1 align-items-center"
style={{
minWidth: where.length > 30 ? '50%' : 'auto',
}}
>
<div className="text-muted">Where</div>
<div className="ms-3 flex-grow-1">
<SearchInput
inputRef={searchInputRef}
placeholder={'Filter results by a search query'}
value={where}
onChange={v => setWhere(v)}
onSearch={() => {}}
showHotkey={false}
/>
</div>
</div>
)}
{table === 'logs' && setGroupBy != null && (
<div className="d-flex align-items-center">
<div className="text-muted">Group By</div>
<div className="ms-3 flex-grow-1">
<GroupBySelect
groupBy={groupBy}
table={table}
setGroupBy={setGroupBy}
/>
</div>
</div>
)}
{table === 'metrics' ? (
<div className="d-flex align-items-center align-middle flex-grow-1">
<MetricSelect
metricName={field}
setMetricName={setField}
isRate={isRate}
setAggFn={setAggFn}
setFieldAndAggFn={setFieldAndAggFn}
aggFn={aggFn}
/>
</div>
) : null}
{table === 'metrics' && (
<>
<div className="d-flex align-items-center flex-grow-1">
<div className="text-muted fw-500">Where</div>
<div className="ms-3 flex-grow-1">
<MetricTagFilterInput
placeholder={
field
? 'Filter metric by tag...'
: 'Select a metric above to start filtering by tag...'
}
inputRef={searchInputRef}
value={where}
onChange={v => setWhere(v)}
metricName={field}
showHotkey={false}
/>
</div>
</div>
{setGroupBy != null && (
<div className="d-flex align-items-center">
<div className="text-muted fw-500">Group By</div>
<div className="ms-3 flex-grow-1">
<GroupBySelect
groupBy={groupBy}
fields={field != null ? [field] : []}
table={table}
setGroupBy={setGroupBy}
/>
{/* <MetricTagSelect
value={groupBy}
setValue={setGroupBy}
metricName={field}
/> */}
</div>
</div>
)}
</>
)}
</div>
{
// TODO: support metrics
sortOrder != null && setSortOrder != null && table === 'logs' && (
<div className="d-flex mt-3 align-items-center">
<div
style={{ width: labelWidth }}
className="text-muted fw-500 ps-2"
>
Sort Order
</div>
<div className="ms-3 flex-grow-1">
<Select
options={SORT_ORDER}
className="ds-select"
value={SORT_ORDER.find(v => v.value === sortOrder)}
onChange={opt => setSortOrder(opt?.value ?? 'desc')}
classNamePrefix="ds-react-select"
/>
</div>
</div>
)
}
{setNumberFormat && (
<div className="ms-2 mt-2 mb-3">
<Divider
label={
<>
<i className="bi bi-gear me-1" />
Chart Settings
</>
}
c="dark.2"
mb={8}
/>
<Group>
<div className="fs-8 text-slate-300">Number Format</div>
<NumberFormatInput
value={numberFormat}
onChange={setNumberFormat}
/>
</Group>
</div>
)}
</div>
);
}
export function convertDateRangeToGranularityString(
dateRange: [Date, Date],
maxNumBuckets: number,

56
packages/app/src/DashboardPage.tsx
View file

@ -27,7 +27,6 @@ import { Badge, Transition } from '@mantine/core';
import api from './api';
import AppNav from './AppNav';
import { convertDateRangeToGranularityString, Granularity } from './ChartUtils';
import type { Chart } from './EditChartForm';
import {
EditHistogramChartForm,
EditLineChartForm,
@ -38,17 +37,17 @@ import {
} from './EditChartForm';
import GranularityPicker from './GranularityPicker';
import HDXHistogramChart from './HDXHistogramChart';
import HDXLineChart from './HDXLineChart';
import HDXMarkdownChart from './HDXMarkdownChart';
import HDXMultiSeriesTableChart from './HDXMultiSeriesTableChart';
import HDXMultiSeriesLineChart from './HDXMultiSeriesTimeChart';
import HDXNumberChart from './HDXNumberChart';
import HDXTableChart from './HDXTableChart';
import { LogTableWithSidePanel } from './LogTableWithSidePanel';
import SearchInput from './SearchInput';
import SearchTimeRangePicker from './SearchTimeRangePicker';
import { FloppyIcon, Histogram } from './SVGIcons';
import TabBar from './TabBar';
import { parseTimeQuery, useNewTimeQuery, useTimeQuery } from './timeQuery';
import type { Alert } from './types';
import { parseTimeQuery, useNewTimeQuery } from './timeQuery';
import type { Alert, Chart } from './types';
import { useConfirm } from './useConfirm';
import { hashCode } from './utils';
import { ZIndexContext } from './zIndex';
@ -272,11 +271,23 @@ const Tile = forwardRef(
className="fs-7 text-muted flex-grow-1 overflow-hidden"
onMouseDown={e => e.stopPropagation()}
>
{config.type === 'time' && (
<HDXLineChart config={config} onSettled={onSettled} />
{chart.series[0].type === 'time' && config.type === 'time' && (
<HDXMultiSeriesLineChart
config={{
...config,
seriesReturnType: chart.seriesReturnType,
series: chart.series,
}}
/>
)}
{config.type === 'table' && (
<HDXTableChart config={config} onSettled={onSettled} />
{chart.series[0].type === 'table' && config.type === 'table' && (
<HDXMultiSeriesTableChart
config={{
...config,
seriesReturnType: chart.seriesReturnType,
series: chart.series,
}}
/>
)}
{config.type === 'histogram' && (
<HDXHistogramChart config={config} onSettled={onSettled} />
@ -322,17 +333,25 @@ const EditChartModal = ({
onClose: () => void;
show: boolean;
}) => {
const [tab, setTab] = useState<
type Tab =
| 'time'
| 'search'
| 'histogram'
| 'markdown'
| 'number'
| 'table'
| undefined
>(undefined);
| undefined;
const [tab, setTab] = useState<Tab>(undefined);
const displayedTab = tab ?? chart?.series?.[0]?.type ?? 'time';
const onTabClick = useCallback(
(newTab: Tab) => {
setTab(newTab);
},
[setTab],
);
return (
<ZIndexContext.Provider value={1055}>
<Modal
@ -397,15 +416,15 @@ const EditChartModal = ({
},
]}
activeItem={displayedTab}
onClick={v => {
setTab(v);
}}
onClick={onTabClick}
/>
{displayedTab === 'time' && chart != null && (
<EditLineChartForm
isLocalDashboard={isLocalDashboard}
chart={produce(chart, draft => {
draft.series[0].type = 'time';
for (const series of draft.series) {
series.type = 'time';
}
})}
alerts={alerts}
onSave={onSave}
@ -416,7 +435,9 @@ const EditChartModal = ({
{displayedTab === 'table' && chart != null && (
<EditTableChartForm
chart={produce(chart, draft => {
draft.series[0].type = 'table';
for (const series of draft.series) {
series.type = 'table';
}
})}
onSave={onSave}
onClose={onClose}
@ -737,6 +758,7 @@ export default function DashboardPage() {
groupBy: [],
},
],
seriesReturnType: 'column',
});
};

638
packages/app/src/EditChartForm.tsx
View file

@ -1,83 +1,33 @@
import { useEffect, useMemo, useState } from 'react';
import { useMemo, useState } from 'react';
import type { Draft } from 'immer';
import produce from 'immer';
import { Button, Form, InputGroup, Modal } from 'react-bootstrap';
import { Button as BSButton, Form, InputGroup } from 'react-bootstrap';
import Select from 'react-select';
import { Divider, Group, Paper } from '@mantine/core';
import { Button, Divider, Flex, Group, Paper, Switch } from '@mantine/core';
import { NumberFormatInput } from './components/NumberFormat';
import { intervalToGranularity } from './Alert';
import {
AGG_FNS,
AggFn,
ChartSeriesForm,
ChartSeriesFormCompact,
convertDateRangeToGranularityString,
FieldSelect,
GroupBySelect,
seriesToSearchQuery,
TableSelect,
} from './ChartUtils';
import Checkbox from './Checkbox';
import * as config from './config';
import { METRIC_ALERTS_ENABLED } from './config';
import EditChartFormAlerts from './EditChartFormAlerts';
import HDXHistogramChart from './HDXHistogramChart';
import HDXLineChart from './HDXLineChart';
import HDXMarkdownChart from './HDXMarkdownChart';
import HDXMultiSeriesTableChart from './HDXMultiSeriesTableChart';
import HDXMultiSeriesLineChart from './HDXMultiSeriesTimeChart';
import HDXNumberChart from './HDXNumberChart';
import HDXTableChart from './HDXTableChart';
import { LogTableWithSidePanel } from './LogTableWithSidePanel';
import type { Alert, NumberFormat } from './types';
import { hashCode, useDebounce } from './utils';
export type Chart = {
id: string;
name: string;
x: number;
y: number;
w: number;
h: number;
series: (
| {
table: string;
type: 'time';
aggFn: AggFn; // TODO: Type
field: string | undefined;
where: string;
groupBy: string[];
numberFormat?: NumberFormat;
}
| {
table: string;
type: 'histogram';
field: string | undefined;
where: string;
}
| {
type: 'search';
fields: string[];
where: string;
}
| {
type: 'number';
table: string;
aggFn: AggFn;
field: string | undefined;
where: string;
numberFormat?: NumberFormat;
}
| {
type: 'table';
table: string;
aggFn: AggFn;
field: string | undefined;
where: string;
groupBy: string[];
sortOrder: 'desc' | 'asc';
numberFormat?: NumberFormat;
}
| {
type: 'markdown';
content: string;
}
)[];
};
import type { Alert, Chart, TimeChartSeries } from './types';
import { useDebounce } from './utils';
const DEFAULT_ALERT: Alert = {
channel: {
@ -169,16 +119,16 @@ export const EditMarkdownChartForm = ({
</div>
</div>
<div className="d-flex justify-content-between my-3 ps-2">
<Button
<BSButton
variant="outline-success"
className="fs-7 text-muted-hover-black"
type="submit"
>
Save
</Button>
<Button onClick={onClose} variant="dark">
</BSButton>
<BSButton onClick={onClose} variant="dark">
Cancel
</Button>
</BSButton>
</div>
<div className="mt-4">
<div className="mb-3 text-muted ps-2 fs-7">Markdown Preview</div>
@ -272,16 +222,16 @@ export const EditSearchChartForm = ({
</div>
</div>
<div className="d-flex justify-content-between my-3 ps-2">
<Button
<BSButton
variant="outline-success"
className="fs-7 text-muted-hover-black"
type="submit"
>
Save
</Button>
<Button onClick={onClose} variant="dark">
</BSButton>
<BSButton onClick={onClose} variant="dark">
Cancel
</Button>
</BSButton>
</div>
<div className="mt-4">
<div className="mb-3 text-muted ps-2 fs-7">Search Preview</div>
@ -465,16 +415,16 @@ export const EditNumberChartForm = ({
</div>
<div className="d-flex justify-content-between my-3 ps-2">
<Button
<BSButton
variant="outline-success"
className="fs-7 text-muted-hover-black"
type="submit"
>
Save
</Button>
<Button onClick={onClose} variant="dark">
</BSButton>
<BSButton onClick={onClose} variant="dark">
Cancel
</Button>
</BSButton>
</div>
<div className="mt-4">
<div className="mb-3 text-muted ps-2 fs-7">Chart Preview</div>
@ -535,6 +485,8 @@ export const EditTableChartForm = ({
granularity: convertDateRangeToGranularityString(dateRange, 60),
dateRange,
numberFormat: editedChart.series[0].numberFormat,
series: editedChart.series,
seriesReturnType: editedChart.seriesReturnType,
}
: null,
[editedChart, dateRange],
@ -573,118 +525,54 @@ export const EditTableChartForm = ({
placeholder="Chart Name"
/>
</div>
<ChartSeriesForm
sortOrder={editedChart.series[0].sortOrder ?? 'desc'}
table={editedChart.series[0].table ?? 'logs'}
aggFn={editedChart.series[0].aggFn}
where={editedChart.series[0].where}
groupBy={editedChart.series[0].groupBy[0]}
field={editedChart.series[0].field ?? ''}
setTable={table =>
setEditedChart(
produce(editedChart, draft => {
if (draft.series[0].type === CHART_TYPE) {
draft.series[0].table = table;
}
}),
)
}
setAggFn={aggFn =>
setEditedChart(
produce(editedChart, draft => {
if (draft.series[0].type === CHART_TYPE) {
draft.series[0].aggFn = aggFn;
}
}),
)
}
setTableAndAggFn={(table, aggFn) => {
setEditedChart(
produce(editedChart, draft => {
if (draft.series[0].type === CHART_TYPE) {
draft.series[0].table = table;
draft.series[0].aggFn = aggFn;
}
}),
);
}}
setWhere={where =>
setEditedChart(
produce(editedChart, draft => {
if (draft.series[0].type === CHART_TYPE) {
draft.series[0].where = where;
}
}),
)
}
setGroupBy={groupBy =>
setEditedChart(
produce(editedChart, draft => {
if (draft.series[0].type === CHART_TYPE) {
if (groupBy != undefined) {
draft.series[0].groupBy[0] = groupBy;
} else {
draft.series[0].groupBy = [];
}
}
}),
)
}
setField={field =>
setEditedChart(
produce(editedChart, draft => {
if (draft.series[0].type === CHART_TYPE) {
draft.series[0].field = field;
}
}),
)
}
setFieldAndAggFn={(field, aggFn) => {
setEditedChart(
produce(editedChart, draft => {
if (draft.series[0].type === CHART_TYPE) {
draft.series[0].field = field;
draft.series[0].aggFn = aggFn;
}
}),
);
}}
setSortOrder={sortOrder =>
setEditedChart(
produce(editedChart, draft => {
if (draft.series[0].type === CHART_TYPE) {
draft.series[0].sortOrder = sortOrder;
}
}),
)
}
numberFormat={editedChart.series[0].numberFormat}
setNumberFormat={numberFormat =>
setEditedChart(
produce(editedChart, draft => {
if (draft.series[0].type === CHART_TYPE) {
draft.series[0].numberFormat = numberFormat;
}
}),
)
}
<EditMultiSeriesChartForm
{...{ editedChart, setEditedChart, CHART_TYPE }}
/>
<div className="d-flex justify-content-between my-3 ps-2">
<Button
<BSButton
variant="outline-success"
className="fs-7 text-muted-hover-black"
type="submit"
>
Save
</Button>
<Button onClick={onClose} variant="dark">
</BSButton>
<BSButton onClick={onClose} variant="dark">
Cancel
</Button>
</BSButton>
</div>
<div className="mt-4">
<div className="mb-3 text-muted ps-2 fs-7">Chart Preview</div>
<div style={{ height: 400 }}>
<HDXTableChart config={previewConfig} />
<HDXMultiSeriesTableChart
config={previewConfig}
onSortClick={seriesIndex => {
setEditedChart(
produce(editedChart, draft => {
// We need to clear out all other series sort orders first
for (let i = 0; i < draft.series.length; i++) {
if (i !== seriesIndex) {
const s = draft.series[i];
if (s.type === CHART_TYPE) {
s.sortOrder = undefined;
}
}
}
const s = draft.series[seriesIndex];
if (s.type === CHART_TYPE) {
s.sortOrder =
s.sortOrder == null
? 'desc'
: s.sortOrder === 'asc'
? 'desc'
: 'asc';
}
return;
}),
);
}}
/>
</div>
</div>
{editedChart.series[0].table === 'logs' ? (
@ -822,16 +710,16 @@ export const EditHistogramChartForm = ({
</div>
</div>
<div className="d-flex justify-content-between my-3 ps-2">
<Button
<BSButton
variant="outline-success"
className="fs-7 text-muted-hover-black"
type="submit"
>
Save
</Button>
<Button onClick={onClose} variant="dark">
</BSButton>
<BSButton onClick={onClose} variant="dark">
Cancel
</Button>
</BSButton>
</div>
<div className="mt-4">
<div className="mb-3 text-muted ps-2 fs-7">Chart Preview</div>
@ -864,6 +752,285 @@ export const EditHistogramChartForm = ({
);
};
function pushNewSeries(draft: Draft<Chart>) {
const firstSeries = draft.series[0] as TimeChartSeries;
const { table, type, groupBy, numberFormat } = firstSeries;
draft.series.push({
table,
type,
aggFn: table === 'logs' ? 'count' : 'avg',
field: '',
where: '',
groupBy,
numberFormat,
});
}
export const EditMultiSeriesChartForm = ({
editedChart,
setEditedChart,
CHART_TYPE,
}: {
editedChart: Chart;
setEditedChart: (chart: Chart) => void;
CHART_TYPE: 'time' | 'table';
}) => {
if (editedChart.series[0].type !== CHART_TYPE) {
return null;
}
return (
<>
{editedChart.series.length > 1 && (
<Flex align="center" gap="md" mb="sm">
<div className="text-muted">
<i className="bi bi-database me-2" />
Data Source
</div>
<div className="flex-grow-1">
<TableSelect
table={editedChart.series[0].table ?? 'logs'}
setTableAndAggFn={(table, aggFn) => {
setEditedChart(
produce(editedChart, draft => {
draft.series.forEach((series, i) => {
if (series.type === CHART_TYPE) {
series.table = table;
series.aggFn = aggFn;
}
});
}),
);
}}
/>
</div>
</Flex>
)}
{editedChart.series.map((series, i) => {
if (series.type !== CHART_TYPE) {
return null;
}
return (
<div className="mb-2" key={i}>
<Divider
label={
<>
{editedChart.series.length > 1 && (
<Button
variant="subtle"
color="gray"
size="xs"
onClick={() => {
setEditedChart(
produce(editedChart, draft => {
draft.series.splice(i, 1);
if (draft.series.length != 2) {
draft.seriesReturnType = 'column';
}
}),
);
}}
>
<i className="bi bi-trash me-2" />
Remove Series
</Button>
)}
</>
}
c="dark.2"
labelPosition="right"
mb={8}
/>
<ChartSeriesFormCompact
table={series.table ?? 'logs'}
aggFn={series.aggFn}
where={series.where}
groupBy={series.groupBy[0]}
field={series.field ?? ''}
numberFormat={series.numberFormat}
setAggFn={aggFn =>
setEditedChart(
produce(editedChart, draft => {
const draftSeries = draft.series[i];
if (draftSeries.type === CHART_TYPE) {
draftSeries.aggFn = aggFn;
}
}),
)
}
setWhere={where =>
setEditedChart(
produce(editedChart, draft => {
const draftSeries = draft.series[i];
if (draftSeries.type === CHART_TYPE) {
draftSeries.where = where;
}
}),
)
}
setGroupBy={
editedChart.series.length === 1
? groupBy =>
setEditedChart(
produce(editedChart, draft => {
const draftSeries = draft.series[i];
if (draftSeries.type === CHART_TYPE) {
if (groupBy != undefined) {
draftSeries.groupBy[0] = groupBy;
} else {
draftSeries.groupBy = [];
}
}
}),
)
: undefined
}
setField={field =>
setEditedChart(
produce(editedChart, draft => {
const draftSeries = draft.series[i];
if (draftSeries.type === CHART_TYPE) {
draftSeries.field = field;
}
}),
)
}
setTableAndAggFn={
editedChart.series.length === 1
? (table, aggFn) => {
setEditedChart(
produce(editedChart, draft => {
const draftSeries = draft.series[i];
if (draftSeries.type === CHART_TYPE) {
draftSeries.table = table;
draftSeries.aggFn = aggFn;
}
}),
);
}
: undefined
}
setFieldAndAggFn={(field, aggFn) => {
setEditedChart(
produce(editedChart, draft => {
const draftSeries = draft.series[i];
if (draftSeries.type === CHART_TYPE) {
draftSeries.field = field;
draftSeries.aggFn = aggFn;
}
}),
);
}}
/>
</div>
);
})}
<Divider my="md" />
{editedChart.series.length > 1 && (
<Flex align="center" gap="md" mb="sm">
<div className="text-muted">Group By</div>
<div className="flex-grow-1">
<GroupBySelect
table={editedChart.series[0].table ?? 'logs'}
groupBy={editedChart.series[0].groupBy[0]}
fields={
editedChart.series
.map(s => (s as TimeChartSeries).field)
.filter(f => f != null) as string[]
}
setGroupBy={groupBy => {
setEditedChart(
produce(editedChart, draft => {
draft.series.forEach((series, i) => {
if (series.type === CHART_TYPE) {
if (groupBy != undefined) {
series.groupBy[0] = groupBy;
} else {
series.groupBy = [];
}
}
});
}),
);
}}
/>
</div>
</Flex>
)}
<Flex justify="space-between">
<Flex gap="md" align="center">
{editedChart.series.length === 1 && (
<Button
mt={4}
variant="subtle"
size="sm"
color="gray"
onClick={() => {
setEditedChart(
produce(editedChart, draft => {
pushNewSeries(draft);
draft.seriesReturnType = 'ratio';
}),
);
}}
>
<i className="bi bi-plus-circle me-2" />
Add Ratio
</Button>
)}
<Button
mt={4}
variant="subtle"
size="sm"
color="gray"
onClick={() => {
setEditedChart(
produce(editedChart, draft => {
pushNewSeries(draft);
draft.seriesReturnType = 'column';
}),
);
}}
>
<i className="bi bi-plus-circle me-2" />
Add Series
</Button>
{editedChart.series.length == 2 && (
<Switch
label="As Ratio"
checked={editedChart.seriesReturnType === 'ratio'}
onChange={event =>
setEditedChart(
produce(editedChart, draft => {
draft.seriesReturnType = event.currentTarget.checked
? 'ratio'
: 'column';
}),
)
}
/>
)}
</Flex>
<NumberFormatInput
value={editedChart.series[0].numberFormat}
onChange={numberFormat => {
setEditedChart(
produce(editedChart, draft => {
draft.series.forEach((series, i) => {
if (series.type === CHART_TYPE) {
series.numberFormat = numberFormat;
}
});
}),
);
}}
/>
</Flex>
</>
);
};
export const EditLineChartForm = ({
isLocalDashboard,
chart,
@ -900,6 +1067,8 @@ export const EditLineChartForm = ({
: convertDateRangeToGranularityString(dateRange, 60),
dateRange,
numberFormat: editedChart.series[0].numberFormat,
series: editedChart.series,
seriesReturnType: editedChart.seriesReturnType,
}
: null,
[editedChart, alertEnabled, editedAlert?.interval, dateRange],
@ -944,95 +1113,12 @@ export const EditLineChartForm = ({
placeholder="Chart Name"
/>
</div>
<ChartSeriesForm
table={editedChart.series[0].table ?? 'logs'}
aggFn={editedChart.series[0].aggFn}
where={editedChart.series[0].where}
groupBy={editedChart.series[0].groupBy[0]}
field={editedChart.series[0].field ?? ''}
numberFormat={editedChart.series[0].numberFormat}
setTable={table =>
setEditedChart(
produce(editedChart, draft => {
if (draft.series[0].type === CHART_TYPE) {
draft.series[0].table = table;
}
}),
)
}
setAggFn={aggFn =>
setEditedChart(
produce(editedChart, draft => {
if (draft.series[0].type === CHART_TYPE) {
draft.series[0].aggFn = aggFn;
}
}),
)
}
setWhere={where =>
setEditedChart(
produce(editedChart, draft => {
if (draft.series[0].type === CHART_TYPE) {
draft.series[0].where = where;
}
}),
)
}
setGroupBy={groupBy =>
setEditedChart(
produce(editedChart, draft => {
if (draft.series[0].type === CHART_TYPE) {
if (groupBy != undefined) {
draft.series[0].groupBy[0] = groupBy;
} else {
draft.series[0].groupBy = [];
}
}
}),
)
}
setField={field =>
setEditedChart(
produce(editedChart, draft => {
if (draft.series[0].type === CHART_TYPE) {
draft.series[0].field = field;
}
}),
)
}
setTableAndAggFn={(table, aggFn) => {
setEditedChart(
produce(editedChart, draft => {
if (draft.series[0].type === CHART_TYPE) {
draft.series[0].table = table;
draft.series[0].aggFn = aggFn;
}
}),
);
}}
setFieldAndAggFn={(field, aggFn) => {
setEditedChart(
produce(editedChart, draft => {
if (draft.series[0].type === CHART_TYPE) {
draft.series[0].field = field;
draft.series[0].aggFn = aggFn;
}
}),
);
}}
setNumberFormat={numberFormat =>
setEditedChart(
produce(editedChart, draft => {
if (draft.series[0].type === CHART_TYPE) {
draft.series[0].numberFormat = numberFormat;
}
}),
)
}
<EditMultiSeriesChartForm
{...{ editedChart, setEditedChart, CHART_TYPE }}
/>
{isChartAlertsFeatureEnabled && (
<Paper bg="dark.7" p="md" py="xs" mt="md" withBorder className="ms-2">
<Paper bg="dark.7" p="md" py="xs" mt="md" withBorder>
{isLocalDashboard ? (
<span className="text-gray-600 fs-8">
Alerts are not available in unsaved dashboards.
@ -1060,22 +1146,22 @@ export const EditLineChartForm = ({
</Paper>
)}
<div className="d-flex justify-content-between my-3 ps-2">
<Button
<div className="d-flex justify-content-between my-3">
<BSButton
variant="outline-success"
className="fs-7 text-muted-hover-black"
type="submit"
>
Save
</Button>
<Button onClick={onClose} variant="dark">
</BSButton>
<BSButton onClick={onClose} variant="dark">
Cancel
</Button>
</BSButton>
</div>
<div className="mt-4">
<div className="mb-3 text-muted ps-2 fs-7">Chart Preview</div>
<div style={{ height: 400 }}>
<HDXLineChart
<HDXMultiSeriesLineChart
config={previewConfig}
{...(alertEnabled && {
alertThreshold: editedAlert?.threshold,
@ -1093,15 +1179,9 @@ export const EditLineChartForm = ({
<LogTableWithSidePanel
config={{
...previewConfig,
where: `${previewConfig.where} ${
previewConfig.aggFn != 'count' && previewConfig.field != ''
? `${previewConfig.field}:*`
: ''
} ${
previewConfig.groupBy != '' && previewConfig.groupBy != null
? `${previewConfig.groupBy}:*`
: ''
}`,
where: `${seriesToSearchQuery({
series: previewConfig.series,
})}`,
}}
isLive={false}
isUTC={false}

4
packages/app/src/HDXLineChart.tsx
View file

@ -20,8 +20,8 @@ import {
import { Tooltip as MTooltip } from '@mantine/core';
import api from './api';
import { AggFn, convertGranularityToSeconds, Granularity } from './ChartUtils';
import type { NumberFormat } from './types';
import { convertGranularityToSeconds, Granularity } from './ChartUtils';
import type { AggFn, NumberFormat } from './types';
import useUserPreferences, { TimeFormat } from './useUserPreferences';
import { formatNumber } from './utils';
import { semanticKeyedColor, TIME_TOKENS, truncateMiddle } from './utils';

363
packages/app/src/HDXMultiSeriesTableChart.tsx Normal file
View file

@ -0,0 +1,363 @@
import { memo, useCallback, useEffect, useMemo, useRef } from 'react';
import Link from 'next/link';
import { useRouter } from 'next/router';
import cx from 'classnames';
import { Anchor, Flex, Text } from '@mantine/core';
import {
flexRender,
getCoreRowModel,
Getter,
Row,
Row as TableRow,
useReactTable,
} from '@tanstack/react-table';
import { ColumnDef } from '@tanstack/react-table';
import { useVirtualizer } from '@tanstack/react-virtual';
import api from './api';
import {
Granularity,
seriesColumns,
seriesToUrlSearchQueryParam,
} from './ChartUtils';
import { UNDEFINED_WIDTH } from './tableUtils';
import type { ChartSeries, NumberFormat } from './types';
import { AggFn } from './types';
import { formatNumber } from './utils';
const Table = ({
data,
groupColumnName,
numberFormat,
columns,
getRowSearchLink,
onSortClick,
}: {
data: any[];
columns: {
dataKey: string;
displayName: string;
sortOrder?: 'asc' | 'desc';
}[];
groupColumnName: string;
numberFormat?: NumberFormat;
getRowSearchLink?: (row: any) => string;
onSortClick?: (columnNumber: number) => void;
}) => {
//we need a reference to the scrolling element for logic down below
const tableContainerRef = useRef<HTMLDivElement>(null);
const tableWidth = tableContainerRef.current?.clientWidth;
const numColumns = columns.length + 1;
const reactTableColumns: ColumnDef<any>[] = [
{
accessorKey: 'group',
header: groupColumnName,
size: tableWidth != null ? tableWidth / numColumns : 200,
},
...columns.map(({ dataKey, displayName }, i) => ({
accessorKey: dataKey,
header: displayName,
accessorFn: (row: any) => row[dataKey],
cell: ({
getValue,
row,
}: {
getValue: Getter<number>;
row: Row<any>;
}) => {
const value = getValue();
let formattedValue: string | number | null = value ?? null;
if (numberFormat) {
formattedValue = formatNumber(value, numberFormat);
}
if (getRowSearchLink == null) {
return formattedValue;
}
return (
<Link href={getRowSearchLink(row.original)} passHref>
<a
className={'align-top overflow-hidden py-1 pe-3'}
style={{
display: 'block',
color: 'inherit',
textDecoration: 'none',
}}
>
{formattedValue}
</a>
</Link>
);
},
size:
i === columns.length - 1
? UNDEFINED_WIDTH
: tableWidth != null
? tableWidth / numColumns
: 200,
enableResizing: i !== columns.length - 1,
})),
];
const table = useReactTable({
data,
columns: reactTableColumns,
getCoreRowModel: getCoreRowModel(),
enableColumnResizing: true,
columnResizeMode: 'onChange',
});
const { rows } = table.getRowModel();
const rowVirtualizer = useVirtualizer({
count: rows.length,
getScrollElement: () => tableContainerRef.current,
estimateSize: useCallback(() => 58, []),
overscan: 10,
paddingEnd: 20,
});
const items = rowVirtualizer.getVirtualItems();
const [paddingTop, paddingBottom] =
items.length > 0
? [
Math.max(0, items[0].start - rowVirtualizer.options.scrollMargin),
Math.max(
0,
rowVirtualizer.getTotalSize() - items[items.length - 1].end,
),
]
: [0, 0];
return (
<div
className="overflow-auto h-100 fs-8 bg-inherit"
ref={tableContainerRef}
// Fixes flickering scroll bar: https://github.com/TanStack/virtual/issues/426#issuecomment-1403438040
// style={{ overflowAnchor: 'none' }}
>
<table className="w-100 bg-inherit" style={{ tableLayout: 'fixed' }}>
<thead
className="bg-inherit"
style={{
background: 'inherit',
position: 'sticky',
top: 0,
}}
>
{table.getHeaderGroups().map(headerGroup => (
<tr key={headerGroup.id}>
{headerGroup.headers.map((header, headerIndex) => {
const sortOrder = columns[headerIndex - 1]?.sortOrder;
return (
<th
className="overflow-hidden text-truncate"
key={header.id}
colSpan={header.colSpan}
style={{
width:
header.getSize() === UNDEFINED_WIDTH
? '100%'
: header.getSize(),
minWidth: 100,
position: 'relative',
}}
>
{header.isPlaceholder ? null : (
<Flex justify="space-between">
<div>
{flexRender(
header.column.columnDef.header,
header.getContext(),
)}
</div>
<Flex gap="sm">
{headerIndex > 0 && onSortClick != null && (
<div
role="button"
onClick={() => onSortClick(headerIndex - 1)}
>
{sortOrder === 'asc' ? (
<Text c="green">
<i className="bi bi-sort-numeric-up-alt"></i>
</Text>
) : sortOrder === 'desc' ? (
<Text c="green">
<i className="bi bi-sort-numeric-down-alt"></i>
</Text>
) : (
<Text c="dark.2">
<i className="bi bi-sort-numeric-down-alt"></i>
</Text>
)}
</div>
)}
{header.column.getCanResize() && (
<div
onMouseDown={header.getResizeHandler()}
onTouchStart={header.getResizeHandler()}
className={`resizer text-gray-600 cursor-grab ${
header.column.getIsResizing()
? 'isResizing'
: ''
}`}
// style={{
// position: 'absolute',
// right: 4,
// top: 0,
// bottom: 0,
// width: 12,
// }}
>
<i className="bi bi-three-dots-vertical" />
</div>
)}
</Flex>
</Flex>
)}
</th>
);
})}
</tr>
))}
</thead>
<tbody>
{paddingTop > 0 && (
<tr>
<td colSpan={99999} style={{ height: `${paddingTop}px` }} />
</tr>
)}
{items.map(virtualRow => {
const row = rows[virtualRow.index] as TableRow<any>;
return (
<tr
key={virtualRow.key}
className="bg-default-dark-grey-hover"
data-index={virtualRow.index}
ref={rowVirtualizer.measureElement}
>
{row.getVisibleCells().map(cell => {
return (
<td key={cell.id}>
{getRowSearchLink == null ? (
<div className="align-top overflow-hidden py-1 pe-3">
{flexRender(
cell.column.columnDef.cell,
cell.getContext(),
)}
</div>
) : (
<Link href={getRowSearchLink(row.original)} passHref>
<a
className="align-top overflow-hidden py-1 pe-3"
style={{
display: 'block',
color: 'inherit',
textDecoration: 'none',
}}
>
{flexRender(
cell.column.columnDef.cell,
cell.getContext(),
)}
</a>
</Link>
)}
</td>
);
})}
</tr>
);
})}
{paddingBottom > 0 && (
<tr>
<td colSpan={99999} style={{ height: `${paddingBottom}px` }} />
</tr>
)}
</tbody>
</table>
</div>
);
};
const HDXMultiSeriesTableChart = memo(
({
config: {
series,
seriesReturnType = 'column',
// numberFormat,
dateRange,
sortOrder,
numberFormat,
},
onSettled,
onSortClick,
}: {
config: {
series: ChartSeries[];
granularity: Granularity;
dateRange: [Date, Date];
seriesReturnType: 'ratio' | 'column';
sortOrder: 'asc' | 'desc';
numberFormat?: NumberFormat;
};
onSettled?: () => void;
onSortClick?: (seriesIndex: number) => void;
}) => {
const { data, isError, isLoading } = api.useMultiSeriesChart({
series,
endDate: dateRange[1] ?? new Date(),
startDate: dateRange[0] ?? new Date(),
seriesReturnType,
});
const seriesMeta = seriesColumns({
series,
seriesReturnType,
});
const getRowSearchLink = useCallback(
(row: { group: string }) => {
return `/search?${seriesToUrlSearchQueryParam({
series,
groupByValue: row.group ? `"${row.group}"` : undefined,
dateRange,
})}`;
},
[series, dateRange],
);
return isLoading ? (
<div className="d-flex h-100 w-100 align-items-center justify-content-center text-muted">
Loading Chart Data...
</div>
) : isError ? (
<div className="d-flex h-100 w-100 align-items-center justify-content-center text-muted">
Error loading chart, please try again or contact support.
</div>
) : data?.data?.length === 0 ? (
<div className="d-flex h-100 w-100 align-items-center justify-content-center text-muted">
No data found within time range.
</div>
) : (
<div className="d-flex align-items-center justify-content-center fs-2 h-100">
<Table
data={data?.data ?? []}
groupColumnName={
series[0].type === 'table'
? series[0].groupBy.join(' ') || 'Group'
: 'Group'
}
columns={seriesMeta}
numberFormat={numberFormat}
getRowSearchLink={getRowSearchLink}
onSortClick={onSortClick}
/>
</div>
);
},
);
export default HDXMultiSeriesTableChart;

508
packages/app/src/HDXMultiSeriesTimeChart.tsx Normal file
View file

@ -0,0 +1,508 @@
import { memo, useCallback, useEffect, useMemo, useRef, useState } from 'react';
import Link from 'next/link';
import cx from 'classnames';
import { add, format } from 'date-fns';
import pick from 'lodash/pick';
import {
Bar,
BarChart,
Label,
Legend,
Line,
LineChart,
ReferenceArea,
ReferenceLine,
ResponsiveContainer,
Tooltip,
XAxis,
YAxis,
} from 'recharts';
import api from './api';
import {
convertGranularityToSeconds,
Granularity,
seriesColumns,
seriesToUrlSearchQueryParam,
} from './ChartUtils';
import type { ChartSeries, NumberFormat } from './types';
import useUserPreferences, { TimeFormat } from './useUserPreferences';
import { formatNumber } from './utils';
import { semanticKeyedColor, TIME_TOKENS, truncateMiddle } from './utils';
function ExpandableLegendItem({ value, entry }: any) {
const [expanded, setExpanded] = useState(false);
const { color } = entry;
return (
<span>
<span
style={{ color }}
role="button"
onClick={() => setExpanded(v => !v)}
title="Click to expand"
>
{expanded ? value : truncateMiddle(`${value}`, 45)}
</span>
</span>
);
}
const legendFormatter = (value: string, entry: any) => (
<ExpandableLegendItem value={value} entry={entry} />
);
const MemoChart = memo(function MemoChart({
graphResults,
setIsClickActive,
isClickActive,
dateRange,
groupKeys,
lineNames,
alertThreshold,
alertThresholdType,
displayType = 'line',
numberFormat,
}: {
graphResults: any[];
setIsClickActive: (v: any) => void;
isClickActive: any;
dateRange: [Date, Date];
groupKeys: string[];
lineNames: string[];
alertThreshold?: number;
alertThresholdType?: 'above' | 'below';
displayType?: 'stacked_bar' | 'line';
numberFormat?: NumberFormat;
}) {
const ChartComponent = displayType === 'stacked_bar' ? BarChart : LineChart;
const lines = useMemo(() => {
return groupKeys.map((key, i) =>
displayType === 'stacked_bar' ? (
<Bar
key={key}
type="monotone"
dataKey={key}
name={lineNames[i] ?? key}
fill={semanticKeyedColor(key)}
stackId="1"
/>
) : (
<Line
key={key}
type="monotone"
dataKey={key}
name={lineNames[i] ?? key}
stroke={semanticKeyedColor(key)}
dot={false}
/>
),
);
}, [groupKeys, displayType, lineNames]);
const sizeRef = useRef<[number, number]>([0, 0]);
const timeFormat: TimeFormat = useUserPreferences().timeFormat;
const tsFormat = TIME_TOKENS[timeFormat];
// Gets the preffered time format from User Preferences, then converts it to a formattable token
const tickFormatter = useCallback(
(value: number) =>
numberFormat
? formatNumber(value, {
...numberFormat,
average: true,
mantissa: 0,
unit: undefined,
})
: new Intl.NumberFormat('en-US', {
notation: 'compact',
compactDisplay: 'short',
}).format(value),
[numberFormat],
);
return (
<ResponsiveContainer
width="100%"
height="100%"
minWidth={0}
onResize={(width, height) => {
sizeRef.current = [width ?? 1, height ?? 1];
}}
>
<ChartComponent
width={500}
height={300}
data={graphResults}
syncId="hdx"
syncMethod="value"
onClick={(state, e) => {
if (
state != null &&
state.chartX != null &&
state.chartY != null &&
state.activeLabel != null
) {
setIsClickActive({
x: state.chartX,
y: state.chartY,
activeLabel: state.activeLabel,
xPerc: state.chartX / sizeRef.current[0],
yPerc: state.chartY / sizeRef.current[1],
});
} else {
// We clicked on the chart but outside of a line
setIsClickActive(undefined);
}
// TODO: Properly detect clicks outside of the fake tooltip
e.stopPropagation();
}}
>
<XAxis
dataKey={'ts_bucket'}
domain={[
dateRange[0].getTime() / 1000,
dateRange[1].getTime() / 1000,
]}
interval="preserveStartEnd"
scale="time"
type="number"
tickFormatter={tick => format(new Date(tick * 1000), tsFormat)}
minTickGap={50}
tick={{ fontSize: 12, fontFamily: 'IBM Plex Mono, monospace' }}
/>
<YAxis
width={40}
minTickGap={25}
tickFormatter={tickFormatter}
tick={{ fontSize: 12, fontFamily: 'IBM Plex Mono, monospace' }}
/>
{lines}
<Tooltip
content={<HDXLineChartTooltip numberFormat={numberFormat} />}
/>
{alertThreshold != null && alertThresholdType === 'below' && (
<ReferenceArea
y1={0}
y2={alertThreshold}
ifOverflow="extendDomain"
strokeWidth={0}
fillOpacity={0.05}
/>
)}
{alertThreshold != null && alertThresholdType === 'above' && (
<ReferenceArea
y1={alertThreshold}
ifOverflow="extendDomain"
strokeWidth={0}
fillOpacity={0.05}
/>
)}
{alertThreshold != null && (
<ReferenceLine
y={alertThreshold}
label={<Label value="Alert Threshold" fill={'white'} />}
stroke="red"
strokeDasharray="3 3"
/>
)}
<Legend
iconSize={10}
verticalAlign="bottom"
formatter={legendFormatter}
/>
{/** Needs to be at the bottom to prevent re-rendering */}
{isClickActive != null ? (
<ReferenceLine x={isClickActive.activeLabel} stroke="#ccc" />
) : null}
</ChartComponent>
</ResponsiveContainer>
);
});
const HDXLineChartTooltip = (props: any) => {
const timeFormat: TimeFormat = useUserPreferences().timeFormat;
const tsFormat = TIME_TOKENS[timeFormat];
const { active, payload, label, numberFormat } = props;
if (active && payload && payload.length) {
return (
<div className="bg-grey px-3 py-2 rounded fs-8">
<div className="mb-2">{format(new Date(label * 1000), tsFormat)}</div>
{payload
.sort((a: any, b: any) => b.value - a.value)
.map((p: any) => (
<div key={p.dataKey} style={{ color: p.color }}>
{p.name ?? p.dataKey}:{' '}
{numberFormat ? formatNumber(p.value, numberFormat) : p.value}
</div>
))}
</div>
);
}
return null;
};
const HDXMultiSeriesLineChart = memo(
({
config: { series, granularity, dateRange, seriesReturnType = 'column' },
onSettled,
alertThreshold,
alertThresholdType,
}: {
config: {
series: ChartSeries[];
granularity: Granularity;
dateRange: [Date, Date];
seriesReturnType: 'ratio' | 'column';
};
onSettled?: () => void;
alertThreshold?: number;
alertThresholdType?: 'above' | 'below';
}) => {
const { data, isError, isLoading } = api.useMultiSeriesChart({
series,
granularity,
endDate: dateRange[1] ?? new Date(),
startDate: dateRange[0] ?? new Date(),
seriesReturnType,
});
const tsBucketMap = new Map();
let graphResults: {
ts_bucket: number;
[key: string]: number | undefined;
}[] = [];
// TODO: FIX THIS COUNTER
let totalGroups = 0;
const groupSet = new Set(); // to count how many unique groups there were
const lineDataMap: {
[seriesGroup: string]: {
dataKey: string;
displayName: string;
};
} = {};
const seriesMeta = seriesColumns({
series,
seriesReturnType,
});
// Each row of data will contain the ts_bucket, group name
// and a data value per series, we just need to turn them all into keys
if (data != null) {
for (const row of data.data) {
groupSet.add(row.group);
const tsBucket = tsBucketMap.get(row.ts_bucket) ?? {};
tsBucketMap.set(row.ts_bucket, {
...tsBucket,
ts_bucket: row.ts_bucket,
...seriesMeta.reduce((acc, meta, i) => {
// We set an arbitrary data key that is unique
// per series/group
const dataKey = `series_${i}.data:::${row.group}`;
const displayName =
series.length === 1
? // If there's only one series, just show the group, unless there is no group
row.group
? `${row.group}`
: meta.displayName
: // Otherwise, show the series and a group if there is any
`${row.group ? `${row.group} • ` : ''}${meta.displayName}`;
acc[dataKey] = row[meta.dataKey];
lineDataMap[dataKey] = {
dataKey,
displayName,
};
return acc;
}, {} as any),
});
}
graphResults = Array.from(tsBucketMap.values()).sort(
(a, b) => a.ts_bucket - b.ts_bucket,
);
totalGroups = groupSet.size;
}
const groupKeys = Object.values(lineDataMap).map(s => s.dataKey);
const lineNames = Object.values(lineDataMap).map(s => s.displayName);
const [activeClickPayload, setActiveClickPayload] = useState<
| {
x: number;
y: number;
activeLabel: string;
xPerc: number;
yPerc: number;
}
| undefined
>(undefined);
useEffect(() => {
const onClickHandler = () => {
if (activeClickPayload) {
setActiveClickPayload(undefined);
}
};
document.addEventListener('click', onClickHandler);
return () => document.removeEventListener('click', onClickHandler);
}, [activeClickPayload]);
const clickedActiveLabelDate =
activeClickPayload?.activeLabel != null
? new Date(Number.parseInt(activeClickPayload.activeLabel) * 1000)
: undefined;
let qparams: URLSearchParams | undefined;
if (clickedActiveLabelDate != null) {
const to = add(clickedActiveLabelDate, {
seconds: convertGranularityToSeconds(granularity),
});
qparams = seriesToUrlSearchQueryParam({
series,
dateRange: [clickedActiveLabelDate, to],
});
}
const numberFormat =
series[0].type === 'time' ? series[0]?.numberFormat : undefined;
const [displayType, setDisplayType] = useState<'stacked_bar' | 'line'>(
'line',
);
return isLoading ? (
<div className="d-flex h-100 w-100 align-items-center justify-content-center text-muted">
Loading Chart Data...
</div>
) : isError ? (
<div className="d-flex h-100 w-100 align-items-center justify-content-center text-muted">
Error loading chart, please try again or contact support.
</div>
) : graphResults.length === 0 ? (
<div className="d-flex h-100 w-100 align-items-center justify-content-center text-muted">
No data found within time range.
</div>
) : (
<div
// Hack, recharts will release real fix soon https://github.com/recharts/recharts/issues/172
style={{
position: 'relative',
width: '100%',
height: '100%',
}}
>
<div
style={{
position: 'absolute',
left: 0,
right: 0,
bottom: 0,
top: 0,
}}
>
{activeClickPayload != null && clickedActiveLabelDate != null ? (
<div
className="bg-grey px-3 py-2 rounded fs-8"
style={{
zIndex: 5,
position: 'absolute',
top: 0,
left: 0,
visibility: 'visible',
transform: `translate(${
activeClickPayload.xPerc > 0.5
? (activeClickPayload?.x ?? 0) - 130
: (activeClickPayload?.x ?? 0) + 4
}px, ${activeClickPayload?.y ?? 0}px)`,
}}
>
<Link href={`/search?${qparams?.toString()}`}>
<a className="text-white-hover text-decoration-none">
<i className="bi bi-search"></i> View Events
</a>
</Link>
</div>
) : null}
{totalGroups > groupKeys.length ? (
<div
className="bg-grey px-3 py-2 rounded fs-8"
style={{
zIndex: 5,
position: 'absolute',
top: 0,
left: 50,
visibility: 'visible',
}}
title={`Only the top ${groupKeys.length} groups are shown, ${
totalGroups - groupKeys.length
} groups are hidden. Try grouping by a different field.`}
>
<span className="text-muted-hover text-decoration-none fs-8">
<i className="bi bi-exclamation-triangle"></i> Only top{' '}
{groupKeys.length} groups shown
</span>
</div>
) : null}
<div
className="bg-grey px-3 py-2 rounded fs-8"
style={{
zIndex: 5,
position: 'absolute',
top: 0,
right: 0,
visibility: 'visible',
}}
title={`Only the top ${groupKeys.length} groups are shown, ${
totalGroups - groupKeys.length
} groups are hidden. Try grouping by a different field.`}
>
<span
className={cx('text-decoration-none fs-7 cursor-pointer me-2', {
'text-success': displayType === 'line',
'text-muted-hover': displayType !== 'line',
})}
role="button"
title="Display as line chart"
onClick={() => setDisplayType('line')}
>
<i className="bi bi-graph-up"></i>
</span>
<span
className={cx('text-decoration-none fs-7 cursor-pointer', {
'text-success': displayType === 'stacked_bar',
'text-muted-hover': displayType !== 'stacked_bar',
})}
role="button"
title="Display as bar chart"
onClick={() => setDisplayType('stacked_bar')}
>
<i className="bi bi-bar-chart"></i>
</span>
</div>
<MemoChart
lineNames={lineNames}
graphResults={graphResults}
groupKeys={groupKeys}
isClickActive={activeClickPayload}
setIsClickActive={setActiveClickPayload}
dateRange={dateRange}
alertThreshold={alertThreshold}
alertThresholdType={alertThresholdType}
displayType={displayType}
numberFormat={numberFormat}
/>
</div>
</div>
);
},
);
export default HDXMultiSeriesLineChart;

3
packages/app/src/HDXNumberChart.tsx
View file

@ -1,8 +1,7 @@
import { memo } from 'react';
import api from './api';
import { AggFn } from './ChartUtils';
import { NumberFormat } from './types';
import type { AggFn, NumberFormat } from './types';
import { formatNumber } from './utils';
const HDXNumberChart = memo(

2
packages/app/src/HDXTableChart.tsx
View file

@ -11,9 +11,9 @@ import { ColumnDef } from '@tanstack/react-table';
import { useVirtualizer } from '@tanstack/react-virtual';
import api from './api';
import { AggFn } from './ChartUtils';
import { UNDEFINED_WIDTH } from './tableUtils';
import type { NumberFormat } from './types';
import { AggFn } from './types';
import { formatNumber } from './utils';
const Table = ({
data,

60
packages/app/src/api.ts
View file

@ -15,6 +15,7 @@ import type {
AlertInterval,
AlertSource,
AlertType,
ChartSeries,
LogView,
Session,
} from './types';
@ -150,6 +151,65 @@ const api = {
...options,
});
},
useMultiSeriesChart(
{
startDate,
series,
sortOrder,
granularity,
endDate,
seriesReturnType,
}: {
series: ChartSeries[];
endDate: Date;
granularity?: string;
startDate: Date;
sortOrder?: 'asc' | 'desc';
seriesReturnType: 'column' | 'ratio';
},
options?: UseQueryOptions<any, Error>,
) {
const enrichedSeries = series.map(s => {
if (s.type != 'search' && s.type != 'markdown' && s.table === 'metrics') {
const [metricName, metricDataType] = (s.field ?? '').split(' - ');
return {
...s,
field: metricName,
metricDataType,
};
}
return s;
});
const startTime = startDate.getTime();
const endTime = endDate.getTime();
return useQuery<any, Error>({
refetchOnWindowFocus: false,
queryKey: [
'chart/series',
enrichedSeries,
endTime,
granularity,
startTime,
sortOrder,
seriesReturnType,
],
queryFn: () =>
server('chart/series', {
method: 'POST',
json: {
series: enrichedSeries,
endTime,
startTime,
granularity,
sortOrder,
seriesReturnType,
},
}).json(),
retry: 1,
...options,
});
},
useLogsChart(
{
aggFn,

81
packages/app/src/types.ts
View file

@ -159,3 +159,84 @@ export type NumberFormat = {
currencySymbol?: string;
unit?: string;
};
export type AggFn =
| 'avg_rate'
| 'avg'
| 'count_distinct'
| 'count'
| 'max_rate'
| 'max'
| 'min_rate'
| 'min'
| 'p50_rate'
| 'p50'
| 'p90_rate'
| 'p90'
| 'p95_rate'
| 'p95'
| 'p99_rate'
| 'p99'
| 'sum_rate'
| 'sum';
export type SourceTable = 'logs' | 'rrweb' | 'metrics';
export type TimeChartSeries = {
table: SourceTable;
type: 'time';
aggFn: AggFn; // TODO: Type
field?: string | undefined;
where: string;
groupBy: string[];
numberFormat?: NumberFormat;
};
export type TableChartSeries = {
type: 'table';
table: SourceTable;
aggFn: AggFn;
field: string | undefined;
where: string;
groupBy: string[];
sortOrder?: 'desc' | 'asc';
numberFormat?: NumberFormat;
};
export type ChartSeries =
| TimeChartSeries
| TableChartSeries
| {
table: SourceTable;
type: 'histogram';
field: string | undefined;
where: string;
}
| {
type: 'search';
fields: string[];
where: string;
}
| {
type: 'number';
table: SourceTable;
aggFn: AggFn;
field: string | undefined;
where: string;
numberFormat?: NumberFormat;
}
| {
type: 'markdown';
content: string;
};
export type Chart = {
id: string;
name: string;
x: number;
y: number;
w: number;
h: number;
series: ChartSeries[];
seriesReturnType: 'ratio' | 'column';
};

11
packages/app/styles/app.scss
View file

@ -630,6 +630,17 @@ div.react-datepicker {
}
.ds-select {
&.w-auto {
.ds-react-select__menu {
width: auto;
}
}
&.text-nowrap {
.ds-react-select__option {
text-wrap: nowrap;
}
}
&.input-bg .ds-react-select__control {
background: $input-bg;
}