diff --git a/.changeset/new-months-invite.md b/.changeset/new-months-invite.md new file mode 100644 index 00000000..a5e7cd71 --- /dev/null +++ b/.changeset/new-months-invite.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/api': minor +'@hyperdx/app': minor +--- + +Add specifying multiple series of charts for time/line charts and tables in +dashboard (ex. min, max, avg all in one chart). diff --git a/.vscode/settings.json b/.vscode/settings.json index 3c4ec5b9..0c1bb29f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,8 +1,5 @@ { "editor.formatOnSave": true, - "editor.codeActionsOnSave": { - "source.fixAll": "explicit" - }, "[typescript]": { "editor.defaultFormatter": "esbenp.prettier-vscode" }, diff --git a/packages/api/src/api-app.ts b/packages/api/src/api-app.ts index a8eca3aa..42b1a018 100644 --- a/packages/api/src/api-app.ts +++ b/packages/api/src/api-app.ts @@ -103,6 +103,7 @@ app.use('/metrics', isUserAuthenticated, routers.metricsRouter); app.use('/sessions', isUserAuthenticated, routers.sessionsRouter); app.use('/team', isUserAuthenticated, routers.teamRouter); app.use('/webhooks', isUserAuthenticated, routers.webhooksRouter); +app.use('/chart', isUserAuthenticated, routers.chartRouter); // --------------------------------------------------------------------- // TODO: Separate external API routers from internal routers diff --git a/packages/api/src/clickhouse/__tests__/clickhouse.test.ts b/packages/api/src/clickhouse/__tests__/clickhouse.test.ts index 1b381de1..9157cf8b 100644 --- a/packages/api/src/clickhouse/__tests__/clickhouse.test.ts +++ b/packages/api/src/clickhouse/__tests__/clickhouse.test.ts @@ -1,4 +1,15 @@ -import { closeDB, getServer } from '@/fixtures'; +import _ from 'lodash'; +import ms from 'ms'; + +import { + buildEvent, + buildMetricSeries, + closeDB, + getServer, + mockLogsPropertyTypeMappingsModel, + mockSpyMetricPropertyTypeMappingsModel, +} from '@/fixtures'; +import { LogPlatform, LogType } from '@/utils/logParser'; import * as clickhouse from '..'; @@ -18,6 +29,1163 @@ describe('clickhouse', () => { jest.clearAllMocks(); }); + it('fetches bulk events correctly', async () => { + const now = new Date('2022-01-05').getTime(); + const runId = Math.random().toString(); // dedup watch mode runs + const teamId = `test`; + + await clickhouse.bulkInsertTeamLogStream(undefined, teamId, [ + buildEvent({ + source: 'test', + timestamp: now, + platform: LogPlatform.NodeJS, + type: LogType.Log, + test: 'test1', + runId, + }), + buildEvent({ + source: 'test', + timestamp: now + 1, + platform: LogPlatform.NodeJS, + type: LogType.Log, + test: 'test2', + justanumber: 777, + runId, + }), + buildEvent({ + source: 'test', + timestamp: now + 1, + platform: LogPlatform.NodeJS, + type: LogType.Log, + justanumber: 777, + runId, + }), + ]); + + const data = ( + await clickhouse.getLogBatch({ + tableVersion: undefined, + teamId, + q: `runId:${runId} test:*`, + limit: 20, + offset: 0, + startTime: now - 1, + endTime: now + 5, + order: 'desc', + }) + ).data.map(({ id, ...d }) => d); // pluck non-deterministic id + + expect(data.length).toEqual(2); + expect(data).toMatchInlineSnapshot(` +Array [ + Object { + "_host": "", + "_platform": "nodejs", + "_service": "", + "body": "", + "duration": -1641340800001, + "severity_text": "", + "sort_key": "1641340800001000000", + "timestamp": "2022-01-05T00:00:00.001000000Z", + "type": "log", + }, + Object { + "_host": "", + "_platform": "nodejs", + "_service": "", + "body": "", + "duration": -1641340800000, + "severity_text": "", + "sort_key": "1641340800000000000", + "timestamp": "2022-01-05T00:00:00.000000000Z", + "type": "log", + }, +] +`); + }); + + it('fetches multi-series event charts correctly', async () => { + const now = new Date('2022-01-05').getTime(); + const runId = Math.random().toString(); // dedup watch mode runs + const teamId = `test`; + + await clickhouse.bulkInsertTeamLogStream(undefined, teamId, [ + // Group 1, sum: 77, avg:25.666666667 + buildEvent({ + timestamp: now, + runId, + testGroup: 'group1', + awesomeNumber: 1, + }), + buildEvent({ + timestamp: now + ms('1m'), + runId, + testGroup: 'group1', + awesomeNumber: 15, + }), + buildEvent({ + timestamp: now + ms('2m'), + runId, + testGroup: 'group1', + awesomeNumber: 61, + }), + // Group 1, sum: 7, avg: 2.3333333 + buildEvent({ + timestamp: now + ms('6m'), + runId, + testGroup: 'group1', + awesomeNumber: 4, + }), + buildEvent({ + timestamp: now + ms('7m'), + runId, + testGroup: 'group1', + awesomeNumber: 2, + }), + buildEvent({ + timestamp: now + ms('8m'), + runId, + testGroup: 'group1', + awesomeNumber: 1, + }), + // Group 2, sum: 777, avg: 259 + buildEvent({ + timestamp: now, + runId, + testGroup: 'group2', + awesomeNumber: 70, + }), + buildEvent({ + timestamp: now + ms('4m'), + runId, + testGroup: 'group2', + awesomeNumber: 700, + }), + buildEvent({ + timestamp: now + ms('1m'), + runId, + testGroup: 'group2', + awesomeNumber: 7, + }), + ]); + + const propertyTypeMappingsModel = mockLogsPropertyTypeMappingsModel({ + testGroup: 'string', + awesomeNumber: 'number', + runId: 'string', + }); + + const data = ( + await clickhouse.getMultiSeriesChart({ + series: [ + { + type: 'time', + table: 'logs', + aggFn: clickhouse.AggFn.Sum, + field: 'awesomeNumber', + where: `runId:${runId}`, + groupBy: ['testGroup'], + }, + { + type: 'time', + table: 'logs', + aggFn: clickhouse.AggFn.Avg, + field: 'awesomeNumber', + where: `runId:${runId}`, + groupBy: ['testGroup'], + }, + ], + tableVersion: undefined, + teamId, + startTime: now, + endTime: now + ms('10m'), + granularity: '5 minute', + maxNumGroups: 20, + seriesReturnType: clickhouse.SeriesReturnType.Column, + propertyTypeMappingsModel, + }) + ).data.map(d => { + return _.pick(d, [ + 'group', + 'series_0.data', + 'series_1.data', + 'ts_bucket', + ]); + }); + + expect(data.length).toEqual(3); + expect(data).toMatchInlineSnapshot(` +Array [ + Object { + "group": "group2", + "series_0.data": 777, + "series_1.data": 259, + "ts_bucket": 1641340800, + }, + Object { + "group": "group1", + "series_0.data": 77, + "series_1.data": 25.666666666666668, + "ts_bucket": 1641340800, + }, + Object { + "group": "group1", + "series_0.data": 7, + "series_1.data": 2.3333333333333335, + "ts_bucket": 1641341100, + }, +] +`); + + const ratioData = ( + await clickhouse.getMultiSeriesChart({ + series: [ + { + type: 'time', + table: 'logs', + aggFn: clickhouse.AggFn.Sum, + field: 'awesomeNumber', + where: `runId:${runId}`, + groupBy: ['testGroup'], + }, + { + type: 'time', + table: 'logs', + aggFn: clickhouse.AggFn.Avg, + field: 'awesomeNumber', + where: `runId:${runId}`, + groupBy: ['testGroup'], + }, + ], + tableVersion: undefined, + teamId, + startTime: now, + endTime: now + ms('10m'), + granularity: '5 minute', + maxNumGroups: 20, + seriesReturnType: clickhouse.SeriesReturnType.Ratio, + propertyTypeMappingsModel, + }) + ).data.map(d => { + return _.pick(d, ['group', 'series_0.data', 'ts_bucket']); + }); + + expect(ratioData.length).toEqual(3); + expect(ratioData).toMatchInlineSnapshot(` +Array [ + Object { + "group": "group1", + "series_0.data": 3, + "ts_bucket": 1641340800, + }, + Object { + "group": "group2", + "series_0.data": 3, + "ts_bucket": 1641340800, + }, + Object { + "group": "group1", + "series_0.data": 3, + "ts_bucket": 1641341100, + }, +] +`); + + jest.clearAllMocks(); + }); + + it('fetches multi-series metric time chart correctly', async () => { + const now = new Date('2022-01-05').getTime(); + const runId = Math.random().toString(); // dedup watch mode runs + const teamId = `test`; + + // Rate: 8, 1, 8, 25 + await clickhouse.bulkInsertTeamMetricStream( + buildMetricSeries({ + name: 'test.users', + tags: { host: 'test1', runId }, + data_type: clickhouse.MetricsDataType.Sum, + is_monotonic: true, + is_delta: true, + unit: 'Users', + points: [ + { value: 0, timestamp: now - ms('1m') }, // 0 + { value: 1, timestamp: now }, + { value: 8, timestamp: now + ms('4m') }, // 8 + { value: 8, timestamp: now + ms('6m') }, + { value: 9, timestamp: now + ms('9m') }, // 9 + { value: 15, timestamp: now + ms('11m') }, + { value: 17, timestamp: now + ms('14m') }, // 17 + { value: 32, timestamp: now + ms('16m') }, + { value: 42, timestamp: now + ms('19m') }, // 42 + ], + }), + ); + + // Rate: 11, 78, 5805, 78729 + // Sum: 12, 79, 5813, 78754 + await clickhouse.bulkInsertTeamMetricStream( + buildMetricSeries({ + name: 'test.users', + tags: { host: 'test2', runId }, + data_type: clickhouse.MetricsDataType.Sum, + is_monotonic: true, + is_delta: true, + unit: 'Users', + points: [ + { value: 3, timestamp: now - ms('1m') }, // 3 + { value: 3, timestamp: now }, + { value: 14, timestamp: now + ms('4m') }, // 14 + { value: 15, timestamp: now + ms('6m') }, + { value: 92, timestamp: now + ms('9m') }, // 92 + { value: 653, timestamp: now + ms('11m') }, + { value: 5897, timestamp: now + ms('14m') }, // 5897 + { value: 9323, timestamp: now + ms('16m') }, + { value: 84626, timestamp: now + ms('19m') }, // 84626 + ], + }), + ); + + await clickhouse.bulkInsertTeamMetricStream( + buildMetricSeries({ + name: 'test.cpu', + tags: { host: 'test1', runId }, + data_type: clickhouse.MetricsDataType.Gauge, + is_monotonic: false, + is_delta: false, + unit: 'Percent', + points: [ + { value: 50, timestamp: now }, + { value: 25, timestamp: now + ms('1m') }, + { value: 12.5, timestamp: now + ms('2m') }, + { value: 6.25, timestamp: now + ms('3m') }, // Last 5min + { value: 100, timestamp: now + ms('6m') }, + { value: 75, timestamp: now + ms('7m') }, + { value: 10, timestamp: now + ms('8m') }, + { value: 80, timestamp: now + ms('9m') }, // Last 5min + ], + }), + ); + + await clickhouse.bulkInsertTeamMetricStream( + buildMetricSeries({ + name: 'test.cpu', + tags: { host: 'test2', runId }, + data_type: clickhouse.MetricsDataType.Gauge, + is_monotonic: false, + is_delta: false, + unit: 'Percent', + points: [ + { value: 1, timestamp: now }, + { value: 2, timestamp: now + ms('1m') }, + { value: 3, timestamp: now + ms('2m') }, + { value: 4, timestamp: now + ms('3m') }, // Last 5min + { value: 5, timestamp: now + ms('6m') }, + { value: 6, timestamp: now + ms('7m') }, + { value: 5, timestamp: now + ms('8m') }, + { value: 4, timestamp: now + ms('9m') }, // Last 5min + ], + }), + ); + + const propertyTypeMappingsModel = mockLogsPropertyTypeMappingsModel({}); + + mockSpyMetricPropertyTypeMappingsModel({ + runId: 'string', + host: 'string', + }); + + const singleSumSeriesData = ( + await clickhouse.getMultiSeriesChart({ + series: [ + { + type: 'time', + table: 'metrics', + aggFn: clickhouse.AggFn.SumRate, + field: 'test.users', + where: `runId:${runId}`, + groupBy: [], + metricDataType: clickhouse.MetricsDataType.Sum, + }, + ], + tableVersion: undefined, + teamId, + startTime: now, + endTime: now + ms('20m'), + granularity: '5 minute', + maxNumGroups: 20, + seriesReturnType: clickhouse.SeriesReturnType.Column, + propertyTypeMappingsModel, + }) + ).data.map(d => { + return _.pick(d, ['group', 'series_0.data', 'ts_bucket']); + }); + + expect(singleSumSeriesData).toMatchInlineSnapshot(` +Array [ + Object { + "group": "", + "series_0.data": 19, + "ts_bucket": 1641340800, + }, + Object { + "group": "", + "series_0.data": 79, + "ts_bucket": 1641341100, + }, + Object { + "group": "", + "series_0.data": 5813, + "ts_bucket": 1641341400, + }, + Object { + "group": "", + "series_0.data": 78754, + "ts_bucket": 1641341700, + }, +] +`); + + const singleGaugeGroupedSeriesData = ( + await clickhouse.getMultiSeriesChart({ + series: [ + { + type: 'time', + table: 'metrics', + aggFn: clickhouse.AggFn.Avg, + field: 'test.cpu', + where: `runId:${runId}`, + groupBy: ['host'], + metricDataType: clickhouse.MetricsDataType.Gauge, + }, + ], + tableVersion: undefined, + teamId, + startTime: now, + endTime: now + ms('10m'), + granularity: '5 minute', + maxNumGroups: 20, + seriesReturnType: clickhouse.SeriesReturnType.Column, + propertyTypeMappingsModel, + }) + ).data.map(d => { + return _.pick(d, ['group', 'series_0.data', 'ts_bucket']); + }); + + expect(singleGaugeGroupedSeriesData).toMatchInlineSnapshot(` +Array [ + Object { + "group": "test1", + "series_0.data": 6.25, + "ts_bucket": 1641340800, + }, + Object { + "group": "test2", + "series_0.data": 4, + "ts_bucket": 1641340800, + }, + Object { + "group": "test1", + "series_0.data": 80, + "ts_bucket": 1641341100, + }, + Object { + "group": "test2", + "series_0.data": 4, + "ts_bucket": 1641341100, + }, +] +`); + + const singleGaugeSeriesData = ( + await clickhouse.getMultiSeriesChart({ + series: [ + { + type: 'time', + table: 'metrics', + aggFn: clickhouse.AggFn.Avg, + field: 'test.cpu', + where: `runId:${runId}`, + groupBy: [], + metricDataType: clickhouse.MetricsDataType.Gauge, + }, + ], + tableVersion: undefined, + teamId, + startTime: now, + endTime: now + ms('10m'), + granularity: '5 minute', + maxNumGroups: 20, + seriesReturnType: clickhouse.SeriesReturnType.Column, + propertyTypeMappingsModel, + }) + ).data.map(d => { + return _.pick(d, ['group', 'series_0.data', 'ts_bucket']); + }); + + expect(singleGaugeSeriesData).toMatchInlineSnapshot(` +Array [ + Object { + "group": "", + "series_0.data": 5.125, + "ts_bucket": 1641340800, + }, + Object { + "group": "", + "series_0.data": 42, + "ts_bucket": 1641341100, + }, +] +`); + + const singleGaugeSeriesSummedData = ( + await clickhouse.getMultiSeriesChart({ + series: [ + { + type: 'time', + table: 'metrics', + aggFn: clickhouse.AggFn.Sum, + field: 'test.cpu', + where: `runId:${runId}`, + groupBy: [], + metricDataType: clickhouse.MetricsDataType.Gauge, + }, + ], + tableVersion: undefined, + teamId, + startTime: now, + endTime: now + ms('10m'), + granularity: '5 minute', + maxNumGroups: 20, + seriesReturnType: clickhouse.SeriesReturnType.Column, + propertyTypeMappingsModel, + }) + ).data.map(d => { + return _.pick(d, ['group', 'series_0.data', 'ts_bucket']); + }); + + expect(singleGaugeSeriesSummedData).toMatchInlineSnapshot(` +Array [ + Object { + "group": "", + "series_0.data": 10.25, + "ts_bucket": 1641340800, + }, + Object { + "group": "", + "series_0.data": 84, + "ts_bucket": 1641341100, + }, +] +`); + + const ratioData = ( + await clickhouse.getMultiSeriesChart({ + series: [ + { + type: 'time', + table: 'metrics', + aggFn: clickhouse.AggFn.Avg, + field: 'test.cpu', + where: `runId:${runId}`, + groupBy: ['host'], + metricDataType: clickhouse.MetricsDataType.Gauge, + }, + { + type: 'time', + table: 'metrics', + aggFn: clickhouse.AggFn.SumRate, + field: 'test.users', + where: `runId:${runId}`, + groupBy: ['host'], + metricDataType: clickhouse.MetricsDataType.Sum, + }, + ], + tableVersion: undefined, + teamId, + startTime: now, + endTime: now + ms('20m'), + granularity: '5 minute', + maxNumGroups: 20, + seriesReturnType: clickhouse.SeriesReturnType.Ratio, + propertyTypeMappingsModel, + }) + ).data.map(d => { + return _.pick(d, ['group', 'series_0.data', 'ts_bucket']); + }); + + expect(ratioData).toMatchInlineSnapshot(` +Array [ + Object { + "group": "test1", + "series_0.data": 0.78125, + "ts_bucket": 1641340800, + }, + Object { + "group": "test2", + "series_0.data": 0.36363636363636365, + "ts_bucket": 1641340800, + }, + Object { + "group": "test1", + "series_0.data": 80, + "ts_bucket": 1641341100, + }, + Object { + "group": "test2", + "series_0.data": 0.05128205128205128, + "ts_bucket": 1641341100, + }, + Object { + "group": "", + "series_0.data": null, + "ts_bucket": 1641341400, + }, + Object { + "group": "", + "series_0.data": null, + "ts_bucket": 1641341700, + }, +] +`); + + jest.clearAllMocks(); + }); + + it('fetches multi series metric table chart correctly', async () => { + const now = new Date('2022-01-05').getTime(); + const runId = Math.random().toString(); // dedup watch mode runs + const teamId = `test`; + + // Rate: 8, 1, 8, 25 + await clickhouse.bulkInsertTeamMetricStream( + buildMetricSeries({ + name: 'test.users', + tags: { host: 'test1', runId }, + data_type: clickhouse.MetricsDataType.Sum, + is_monotonic: true, + is_delta: true, + unit: 'Users', + points: [ + { value: 0, timestamp: now - ms('1m') }, // 0 + { value: 1, timestamp: now }, + { value: 8, timestamp: now + ms('4m') }, // 8 + { value: 8, timestamp: now + ms('6m') }, + { value: 9, timestamp: now + ms('9m') }, // 9 + { value: 15, timestamp: now + ms('11m') }, + { value: 17, timestamp: now + ms('14m') }, // 17 + { value: 32, timestamp: now + ms('16m') }, + { value: 42, timestamp: now + ms('19m') }, // 42 + ], + }), + ); + + // Rate: 11, 78, 5805, 78729 + // Sum: 12, 79, 5813, 78754 + await clickhouse.bulkInsertTeamMetricStream( + buildMetricSeries({ + name: 'test.users', + tags: { host: 'test2', runId }, + data_type: clickhouse.MetricsDataType.Sum, + is_monotonic: true, + is_delta: true, + unit: 'Users', + points: [ + { value: 3, timestamp: now - ms('1m') }, // 3 + { value: 3, timestamp: now }, + { value: 14, timestamp: now + ms('4m') }, // 14 + { value: 15, timestamp: now + ms('6m') }, + { value: 92, timestamp: now + ms('9m') }, // 92 + { value: 653, timestamp: now + ms('11m') }, + { value: 5897, timestamp: now + ms('14m') }, // 5897 + { value: 9323, timestamp: now + ms('16m') }, + { value: 84626, timestamp: now + ms('19m') }, // 84626 + ], + }), + ); + + await clickhouse.bulkInsertTeamMetricStream( + buildMetricSeries({ + name: 'test.cpu', + tags: { host: 'test1', runId }, + data_type: clickhouse.MetricsDataType.Gauge, + is_monotonic: false, + is_delta: false, + unit: 'Percent', + points: [ + { value: 50, timestamp: now }, + { value: 25, timestamp: now + ms('1m') }, + { value: 12.5, timestamp: now + ms('2m') }, + { value: 6.25, timestamp: now + ms('3m') }, // Last 5min + { value: 100, timestamp: now + ms('6m') }, + { value: 75, timestamp: now + ms('7m') }, + { value: 10, timestamp: now + ms('8m') }, + { value: 80, timestamp: now + ms('9m') }, // Last 5min + ], + }), + ); + + await clickhouse.bulkInsertTeamMetricStream( + buildMetricSeries({ + name: 'test.cpu', + tags: { host: 'test2', runId }, + data_type: clickhouse.MetricsDataType.Gauge, + is_monotonic: false, + is_delta: false, + unit: 'Percent', + points: [ + { value: 1, timestamp: now }, + { value: 2, timestamp: now + ms('1m') }, + { value: 3, timestamp: now + ms('2m') }, + { value: 4, timestamp: now + ms('3m') }, // Last 5min + { value: 5, timestamp: now + ms('6m') }, + { value: 6, timestamp: now + ms('7m') }, + { value: 5, timestamp: now + ms('8m') }, + { value: 4, timestamp: now + ms('9m') }, // Last 5min + ], + }), + ); + + const propertyTypeMappingsModel = mockLogsPropertyTypeMappingsModel({}); + + mockSpyMetricPropertyTypeMappingsModel({ + runId: 'string', + host: 'string', + }); + + const singleSumSeriesData = ( + await clickhouse.getMultiSeriesChart({ + series: [ + { + type: 'table', + table: 'metrics', + aggFn: clickhouse.AggFn.SumRate, + field: 'test.users', + where: `runId:${runId}`, + groupBy: [], + metricDataType: clickhouse.MetricsDataType.Sum, + }, + { + type: 'time', + table: 'metrics', + aggFn: clickhouse.AggFn.Avg, + field: 'test.cpu', + where: `runId:${runId}`, + groupBy: [], + metricDataType: clickhouse.MetricsDataType.Gauge, + }, + ], + tableVersion: undefined, + teamId, + startTime: now, + endTime: now + ms('20m'), + granularity: undefined, + maxNumGroups: 20, + seriesReturnType: clickhouse.SeriesReturnType.Column, + propertyTypeMappingsModel, + }) + ).data.map(d => { + return _.pick(d, [ + 'group', + 'series_0.data', + 'series_1.data', + 'ts_bucket', + ]); + }); + + expect(singleSumSeriesData).toMatchInlineSnapshot(` +Array [ + Object { + "group": "", + "series_0.data": 84665, + "series_1.data": 42, + "ts_bucket": "0", + }, +] +`); + + jest.clearAllMocks(); + }); + + it('limits groups and sorts multi series charts properly', async () => { + const now = new Date('2022-01-05').getTime(); + const runId = Math.random().toString(); // dedup watch mode runs + const teamId = `test`; + + await clickhouse.bulkInsertTeamLogStream( + undefined, + teamId, + Array(10) + .fill(0) + .flatMap((_, i) => [ + buildEvent({ + timestamp: now, + runId, + testGroup: `group${i}`, + awesomeNumber: i, + }), + buildEvent({ + timestamp: now + ms('6m'), + runId, + testGroup: `group${i}`, + // Make sure that the asc sort order will choose the min data point + awesomeNumber: i % 2 === 1 ? 20 - i : i, + }), + ]), + ); + + const propertyTypeMappingsModel = mockLogsPropertyTypeMappingsModel({ + testGroup: 'string', + awesomeNumber: 'number', + runId: 'string', + }); + + const ascData = ( + await clickhouse.getMultiSeriesChart({ + series: [ + { + type: 'table', + table: 'logs', + aggFn: clickhouse.AggFn.Count, + field: 'awesomeNumber', + where: `runId:${runId}`, + groupBy: ['testGroup'], + sortOrder: undefined, + }, + { + type: 'table', + table: 'logs', + aggFn: clickhouse.AggFn.Sum, + field: 'awesomeNumber', + where: `runId:${runId}`, + groupBy: ['testGroup'], + sortOrder: 'asc', + }, + ], + tableVersion: undefined, + teamId, + startTime: now, + endTime: now + ms('10m'), + granularity: '5 minute', + maxNumGroups: 3, + seriesReturnType: clickhouse.SeriesReturnType.Column, + propertyTypeMappingsModel, + }) + ).data.map(d => + _.pick(d, ['series_0.data', 'series_1.data', 'group', 'ts_bucket']), + ); + + expect(ascData).toMatchInlineSnapshot(` +Array [ + Object { + "group": "group0", + "series_0.data": 1, + "series_1.data": 0, + "ts_bucket": 1641340800, + }, + Object { + "group": "group1", + "series_0.data": 1, + "series_1.data": 1, + "ts_bucket": 1641340800, + }, + Object { + "group": "group2", + "series_0.data": 1, + "series_1.data": 2, + "ts_bucket": 1641340800, + }, + Object { + "group": "group0", + "series_0.data": 1, + "series_1.data": 0, + "ts_bucket": 1641341100, + }, + Object { + "group": "group2", + "series_0.data": 1, + "series_1.data": 2, + "ts_bucket": 1641341100, + }, + Object { + "group": "group1", + "series_0.data": 1, + "series_1.data": 19, + "ts_bucket": 1641341100, + }, +] +`); + + const descData = ( + await clickhouse.getMultiSeriesChart({ + series: [ + { + type: 'table', + table: 'logs', + aggFn: clickhouse.AggFn.Count, + field: 'awesomeNumber', + where: `runId:${runId}`, + groupBy: ['testGroup'], + sortOrder: undefined, + }, + { + type: 'table', + table: 'logs', + aggFn: clickhouse.AggFn.Sum, + field: 'awesomeNumber', + where: `runId:${runId}`, + groupBy: ['testGroup'], + sortOrder: 'desc', + }, + ], + tableVersion: undefined, + teamId, + startTime: now, + endTime: now + ms('10m'), + granularity: '5 minute', + maxNumGroups: 3, + seriesReturnType: clickhouse.SeriesReturnType.Column, + propertyTypeMappingsModel, + }) + ).data.map(d => + _.pick(d, ['series_0.data', 'series_1.data', 'group', 'ts_bucket']), + ); + + expect(descData).toMatchInlineSnapshot(` +Array [ + Object { + "group": "group5", + "series_0.data": 1, + "series_1.data": 5, + "ts_bucket": 1641340800, + }, + Object { + "group": "group3", + "series_0.data": 1, + "series_1.data": 3, + "ts_bucket": 1641340800, + }, + Object { + "group": "group1", + "series_0.data": 1, + "series_1.data": 1, + "ts_bucket": 1641340800, + }, + Object { + "group": "group1", + "series_0.data": 1, + "series_1.data": 19, + "ts_bucket": 1641341100, + }, + Object { + "group": "group3", + "series_0.data": 1, + "series_1.data": 17, + "ts_bucket": 1641341100, + }, + Object { + "group": "group5", + "series_0.data": 1, + "series_1.data": 15, + "ts_bucket": 1641341100, + }, +] +`); + + const descDataSimple = ( + await clickhouse.getMultiSeriesChart({ + series: [ + { + type: 'table', + table: 'logs', + aggFn: clickhouse.AggFn.Count, + field: 'awesomeNumber', + where: `runId:${runId}`, + groupBy: ['testGroup'], + sortOrder: undefined, + }, + { + type: 'table', + table: 'logs', + aggFn: clickhouse.AggFn.Sum, + field: 'awesomeNumber', + where: `runId:${runId}`, + groupBy: ['testGroup'], + sortOrder: 'desc', + }, + ], + tableVersion: undefined, + teamId, + startTime: now, + endTime: now + ms('5m'), + granularity: '5 minute', + maxNumGroups: 3, + seriesReturnType: clickhouse.SeriesReturnType.Column, + propertyTypeMappingsModel, + }) + ).data.map(d => + _.pick(d, ['series_0.data', 'series_1.data', 'group', 'ts_bucket']), + ); + + expect(descDataSimple).toMatchInlineSnapshot(` +Array [ + Object { + "group": "group9", + "series_0.data": 1, + "series_1.data": 9, + "ts_bucket": 1641340800, + }, + Object { + "group": "group8", + "series_0.data": 1, + "series_1.data": 8, + "ts_bucket": 1641340800, + }, + Object { + "group": "group7", + "series_0.data": 1, + "series_1.data": 7, + "ts_bucket": 1641340800, + }, +] +`); + jest.clearAllMocks(); + }); + + it('fetches legacy format correctly for alerts', async () => { + const now = new Date('2022-01-05').getTime(); + const runId = Math.random().toString(); // dedup watch mode runs + const teamId = `test`; + + await clickhouse.bulkInsertTeamLogStream(undefined, teamId, [ + // Group 1, sum: 77, avg:25.666666667 + buildEvent({ + timestamp: now, + runId, + testGroup: 'group1', + awesomeNumber: 1, + }), + buildEvent({ + timestamp: now + ms('1m'), + runId, + testGroup: 'group1', + awesomeNumber: 15, + }), + buildEvent({ + timestamp: now + ms('2m'), + runId, + testGroup: 'group1', + awesomeNumber: 61, + }), + // Group 1, sum: 7, avg: 2.3333333 + buildEvent({ + timestamp: now + ms('6m'), + runId, + testGroup: 'group1', + awesomeNumber: 4, + }), + buildEvent({ + timestamp: now + ms('7m'), + runId, + testGroup: 'group1', + awesomeNumber: 2, + }), + buildEvent({ + timestamp: now + ms('8m'), + runId, + testGroup: 'group1', + awesomeNumber: 1, + }), + // Group 2, sum: 777, avg: 259 + buildEvent({ + timestamp: now, + runId, + testGroup: 'group2', + awesomeNumber: 70, + }), + buildEvent({ + timestamp: now + ms('4m'), + runId, + testGroup: 'group2', + awesomeNumber: 700, + }), + buildEvent({ + timestamp: now + ms('1m'), + runId, + testGroup: 'group2', + awesomeNumber: 7, + }), + ]); + + const propertyTypeMappingsModel = mockLogsPropertyTypeMappingsModel({ + testGroup: 'string', + awesomeNumber: 'number', + runId: 'string', + }); + + const data = ( + await clickhouse.getMultiSeriesChartLegacyFormat({ + series: [ + { + type: 'time', + table: 'logs', + aggFn: clickhouse.AggFn.Sum, + field: 'awesomeNumber', + where: `runId:${runId}`, + groupBy: ['testGroup'], + }, + ], + tableVersion: undefined, + teamId, + startTime: now, + endTime: now + ms('10m'), + granularity: '5 minute', + maxNumGroups: 20, + seriesReturnType: clickhouse.SeriesReturnType.Column, + propertyTypeMappingsModel, + }) + ).data; + + const oldData = clickhouse.getLogsChart({ + aggFn: clickhouse.AggFn.Sum, + field: 'awesomeNumber', + q: `runId:${runId}`, + groupBy: 'testGroup', + tableVersion: undefined, + teamId, + startTime: now, + endTime: now + ms('10m'), + granularity: '5 minute', + maxNumGroups: 20, + propertyTypeMappingsModel, + }); + + expect(data.length).toEqual(3); + expect(data).toMatchInlineSnapshot(` +Array [ + Object { + "data": 777, + "group": "group2", + "ts_bucket": 1641340800, + }, + Object { + "data": 77, + "group": "group1", + "ts_bucket": 1641340800, + }, + Object { + "data": 7, + "group": "group1", + "ts_bucket": 1641341100, + }, +] +`); + + expect(data).toMatchObject(oldData); + + jest.clearAllMocks(); + }); + it('clientInsertWithRetries (success)', async () => { jest .spyOn(clickhouse.client, 'insert') @@ -56,7 +1224,8 @@ describe('clickhouse', () => { expect.assertions(2); }); - it('getMetricsChart avoids sending NaN to frontend', async () => { + // TODO: Test this with real data and new chart fn + it.skip('getMetricsChart avoids sending NaN to frontend', async () => { jest .spyOn(clickhouse.client, 'query') .mockResolvedValueOnce({ json: () => Promise.resolve({}) } as any); diff --git a/packages/api/src/clickhouse/index.ts b/packages/api/src/clickhouse/index.ts index f94ef694..ea07df29 100644 --- a/packages/api/src/clickhouse/index.ts +++ b/packages/api/src/clickhouse/index.ts @@ -14,6 +14,7 @@ import ms from 'ms'; import { serializeError } from 'serialize-error'; import SqlString from 'sqlstring'; import { Readable } from 'stream'; +import { z } from 'zod'; import * as config from '@/config'; import { sleep } from '@/utils/common'; @@ -23,6 +24,7 @@ import type { MetricModel, RrwebEventModel, } from '@/utils/logParser'; +import { chartSeriesSchema } from '@/utils/zod'; import { redisClient } from '../utils/redis'; import { @@ -43,6 +45,11 @@ const tracer = opentelemetry.trace.getTracer(__filename); export type SortOrder = 'asc' | 'desc' | null; +export enum SeriesReturnType { + Ratio = 'ratio', + Column = 'column', +} + export enum MetricsDataType { Gauge = 'Gauge', Histogram = 'Histogram', @@ -833,18 +840,19 @@ export const getMetricsChart = async ({ const gaugeMetricSource = SqlString.format( ` - SELECT - timestamp, + SELECT + toStartOfInterval(timestamp, INTERVAL ?) as timestamp, name, - value, + last_value(value) as value, _string_attributes FROM ?? WHERE name = ? AND data_type = ? AND (?) - ORDER BY _timestamp_sort_key ASC + GROUP BY name, _string_attributes, timestamp + ORDER BY timestamp ASC `.trim(), - [tableName, name, dataType, SqlString.raw(whereClause)], + [granularity, tableName, name, dataType, SqlString.raw(whereClause)], ); const query = SqlString.format( @@ -904,6 +912,626 @@ export const getMetricsChart = async ({ return result; }; +export const buildMetricSeriesQuery = async ({ + aggFn, + dataType, + endTime, + granularity, + groupBy, + name, + q, + startTime, + teamId, + sortOrder, +}: { + aggFn: AggFn; + dataType: MetricsDataType; + endTime: number; // unix in ms, + granularity?: Granularity | string; + groupBy?: string; + name: string; + q: string; + startTime: number; // unix in ms + teamId: string; + sortOrder?: 'asc' | 'desc'; +}) => { + const tableName = `default.${TableName.Metric}`; + const propertyTypeMappingsModel = await buildMetricsPropertyTypeMappingsModel( + undefined, // default version + teamId, + ); + + const isRate = isRateAggFn(aggFn); + + const shouldModifyStartTime = isRate; + + // If it's a rate function, then we'll need to look 1 window back to calculate + // the initial rate value. + // We'll filter this extra bucket out later + const modifiedStartTime = shouldModifyStartTime + ? // If granularity is not defined (tables), we'll just look behind 5min + startTime - ms(granularity ?? '5 minute') + : startTime; + + const whereClause = await buildSearchQueryWhereCondition({ + endTime, + propertyTypeMappingsModel, + query: q, + startTime: modifiedStartTime, + }); + const selectClause = [ + granularity != null + ? SqlString.format( + 'toUnixTimestamp(toStartOfInterval(timestamp, INTERVAL ?)) AS ts_bucket', + [granularity], + ) + : "'0' as ts_bucket", + groupBy + ? SqlString.format(`_string_attributes[?] AS group`, [groupBy]) + : "'' AS group", + ]; + + const hasGroupBy = groupBy != '' && groupBy != null; + + if (dataType === MetricsDataType.Gauge || dataType === MetricsDataType.Sum) { + selectClause.push( + aggFn === AggFn.Count + ? 'COUNT(value) as data' + : aggFn === AggFn.Sum + ? `SUM(value) as data` + : aggFn === AggFn.Avg + ? `AVG(value) as data` + : aggFn === AggFn.Max + ? `MAX(value) as data` + : aggFn === AggFn.Min + ? `MIN(value) as data` + : aggFn === AggFn.SumRate + ? `SUM(rate) as data` + : aggFn === AggFn.AvgRate + ? `AVG(rate) as data` + : aggFn === AggFn.MaxRate + ? `MAX(rate) as data` + : aggFn === AggFn.MinRate + ? `MIN(rate) as data` + : `quantile(${ + aggFn === AggFn.P50 || aggFn === AggFn.P50Rate + ? '0.5' + : aggFn === AggFn.P90 || aggFn === AggFn.P90Rate + ? '0.90' + : aggFn === AggFn.P95 || aggFn === AggFn.P95Rate + ? '0.95' + : '0.99' + })(${isRate ? 'rate' : 'value'}) as data`, + ); + } else { + logger.error(`Unsupported data type: ${dataType}`); + } + + const startTimeUnixTs = Math.floor(startTime / 1000); + + // TODO: Can remove the ORDER BY _string_attributes for Gauge metrics + // since they don't get subjected to runningDifference afterwards + const gaugeMetricSource = SqlString.format( + ` + SELECT + ?, + name, + last_value(value) as value, + _string_attributes + FROM ?? + WHERE name = ? + AND data_type = ? + AND (?) + GROUP BY name, _string_attributes, timestamp + ORDER BY _string_attributes, timestamp ASC + `.trim(), + [ + SqlString.raw( + granularity != null + ? `toStartOfInterval(timestamp, INTERVAL ${SqlString.format( + granularity, + )}) as timestamp` + : modifiedStartTime + ? // Manually create the time buckets if we're including the prev time range + `if(timestamp < fromUnixTimestamp(${startTimeUnixTs}), 0, ${startTimeUnixTs}) as timestamp` + : // Otherwise lump everything into one bucket + '0 as timestamp', + ), + tableName, + name, + dataType, + SqlString.raw(whereClause), + ], + ); + + const rateMetricSource = SqlString.format( + ` + SELECT + if( + runningDifference(value) < 0 + OR neighbor(_string_attributes, -1, _string_attributes) != _string_attributes, + nan, + runningDifference(value) + ) AS rate, + timestamp, + _string_attributes, + name + FROM (?) + WHERE isNaN(rate) = 0 + ${shouldModifyStartTime ? 'AND timestamp >= fromUnixTimestamp(?)' : ''} + `.trim(), + [ + SqlString.raw(gaugeMetricSource), + ...(shouldModifyStartTime ? [Math.floor(startTime / 1000)] : []), + ], + ); + + const query = SqlString.format( + ` + WITH metrics AS (?) + SELECT ? + FROM metrics + GROUP BY group, ts_bucket + ORDER BY ts_bucket ASC + ${ + granularity != null + ? `WITH FILL + FROM toUnixTimestamp(toStartOfInterval(toDateTime(?), INTERVAL ?)) + TO toUnixTimestamp(toStartOfInterval(toDateTime(?), INTERVAL ?)) + STEP ?` + : '' + } + `, + [ + SqlString.raw(isRate ? rateMetricSource : gaugeMetricSource), + SqlString.raw(selectClause.join(',')), + ...(granularity != null + ? [ + startTime / 1000, + granularity, + endTime / 1000, + granularity, + ms(granularity) / 1000, + ] + : []), + ], + ); + + return { + query, + hasGroupBy, + sortOrder, + }; +}; + +const buildEventSeriesQuery = async ({ + aggFn, + endTime, + field, + granularity, + groupBy, + propertyTypeMappingsModel, + q, + sortOrder, + startTime, + tableVersion, + teamId, +}: { + aggFn: AggFn; + endTime: number; // unix in ms, + field?: string; + granularity: string | undefined; // can be undefined in the number chart + groupBy: string; + propertyTypeMappingsModel: LogsPropertyTypeMappingsModel; + q: string; + sortOrder?: 'asc' | 'desc'; + startTime: number; // unix in ms + tableVersion: number | undefined; + teamId: string; +}) => { + if (isRateAggFn(aggFn)) { + throw new Error('Rate is not supported in logs chart'); + } + + const tableName = getLogStreamTableName(tableVersion, teamId); + const whereClause = await buildSearchQueryWhereCondition({ + endTime, + propertyTypeMappingsModel, + query: q, + startTime, + }); + + if (field == null && aggFn !== AggFn.Count) { + throw new Error( + 'Field is required for all aggregation functions except Count', + ); + } + + const selectField = + field != null + ? buildSearchColumnName(propertyTypeMappingsModel.get(field), field) + : ''; + + const hasGroupBy = groupBy != '' && groupBy != null; + const isCountFn = aggFn === AggFn.Count; + const groupByField = + hasGroupBy && + buildSearchColumnName(propertyTypeMappingsModel.get(groupBy), groupBy); + + const serializer = new SQLSerializer(propertyTypeMappingsModel); + + const label = SqlString.escape(`${aggFn}(${field})`); + + const selectClause = [ + isCountFn + ? 'toFloat64(count()) as data' + : aggFn === AggFn.Sum + ? `toFloat64(sum(${selectField})) as data` + : aggFn === AggFn.Avg + ? `toFloat64(avg(${selectField})) as data` + : aggFn === AggFn.Max + ? `toFloat64(max(${selectField})) as data` + : aggFn === AggFn.Min + ? `toFloat64(min(${selectField})) as data` + : aggFn === AggFn.CountDistinct + ? `toFloat64(count(distinct ${selectField})) as data` + : `toFloat64(quantile(${ + aggFn === AggFn.P50 + ? '0.5' + : aggFn === AggFn.P90 + ? '0.90' + : aggFn === AggFn.P95 + ? '0.95' + : '0.99' + })(${selectField})) as data`, + granularity != null + ? `toUnixTimestamp(toStartOfInterval(timestamp, INTERVAL ${granularity})) as ts_bucket` + : "'0' as ts_bucket", + groupByField ? `${groupByField} as group` : `'' as group`, // FIXME: should we fallback to use aggFn as group + `${label} as label`, + ].join(','); + + const groupByClause = `ts_bucket ${groupByField ? `, ${groupByField}` : ''}`; + + const query = SqlString.format( + ` + SELECT ? + FROM ?? + WHERE ? AND (?) ? ? + GROUP BY ? + ORDER BY ts_bucket ASC + ${ + granularity != null + ? `WITH FILL + FROM toUnixTimestamp(toStartOfInterval(toDateTime(?), INTERVAL ?)) + TO toUnixTimestamp(toStartOfInterval(toDateTime(?), INTERVAL ?)) + STEP ?` + : '' + }${ + sortOrder === 'asc' || sortOrder === 'desc' ? `, data ${sortOrder}` : '' + } + `, + [ + SqlString.raw(selectClause), + tableName, + buildTeamLogStreamWhereCondition(tableVersion, teamId), + SqlString.raw(whereClause), + SqlString.raw( + !isCountFn && field != null + ? ` AND (${await serializer.isNotNull(field, false)})` + : '', + ), + SqlString.raw( + hasGroupBy + ? ` AND (${await serializer.isNotNull(groupBy, false)})` + : '', + ), + SqlString.raw(groupByClause), + ...(granularity != null + ? [ + startTime / 1000, + granularity, + endTime / 1000, + granularity, + ms(granularity) / 1000, + ] + : []), + ], + ); + + return { + query, + hasGroupBy, + sortOrder, + }; +}; + +export const queryMultiSeriesChart = async ({ + maxNumGroups, + tableVersion, + teamId, + seriesReturnType = SeriesReturnType.Column, + queries, +}: { + maxNumGroups: number; + tableVersion: number | undefined; + teamId: string; + seriesReturnType?: SeriesReturnType; + queries: { query: string; hasGroupBy: boolean; sortOrder?: 'desc' | 'asc' }[]; +}) => { + // For now only supports same-table series with the same groupBy + + const seriesCTEs = queries + .map((q, i) => `series_${i} AS (${q.query})`) + .join(',\n'); + + // Only join on group bys if all queries have group bys + // TODO: This will not work for an array of group by fields + const allQueiesHaveGroupBy = queries.every(q => q.hasGroupBy); + + let seriesIndexWithSorting = -1; + let sortOrder: 'asc' | 'desc' = 'desc'; + for (let i = 0; i < queries.length; i++) { + const q = queries[i]; + if (q.sortOrder === 'asc' || q.sortOrder === 'desc') { + seriesIndexWithSorting = i; + sortOrder = q.sortOrder; + break; + } + } + + let leftJoin = ''; + // Join every series after the first one + for (let i = 1; i < queries.length; i++) { + leftJoin += `LEFT JOIN series_${i} ON series_${i}.ts_bucket=series_0.ts_bucket${ + allQueiesHaveGroupBy ? ` AND series_${i}.group = series_0.group` : '' + }\n`; + } + + const select = + seriesReturnType === 'column' + ? queries + .map((_, i) => { + return `series_${i}.data as "series_${i}.data"`; + }) + .join(',\n') + : 'series_0.data / series_1.data as "series_0.data"'; + + // Return each series data as a separate column + const query = SqlString.format( + `WITH ? + ,raw_groups AS ( + SELECT + ?, + series_0.ts_bucket as ts_bucket, + series_0.group as group + FROM series_0 AS series_0 + ? + ), groups AS ( + SELECT *, ?(?) OVER (PARTITION BY group) as rank_order_by_value + FROM raw_groups + ), final AS ( + SELECT *, DENSE_RANK() OVER (ORDER BY rank_order_by_value ?) as rank + FROM groups + ) + SELECT * + FROM final + WHERE rank <= ? + ORDER BY ts_bucket ASC + ? + `, + [ + SqlString.raw(seriesCTEs), + SqlString.raw(select), + SqlString.raw(leftJoin), + // Setting rank_order_by_value + SqlString.raw(sortOrder === 'asc' ? 'MIN' : 'MAX'), + SqlString.raw( + // If ratio, we judge on series_0 + seriesReturnType === 'ratio' + ? 'series_0.data' + : // If the user specified a sorting order, we use that + seriesIndexWithSorting > -1 + ? `series_${seriesIndexWithSorting}.data` + : // Otherwise we just grab the greatest value + `greatest(${queries.map((_, i) => `series_${i}.data`).join(', ')})`, + ), + // ORDER BY rank_order_by_value .... + SqlString.raw(sortOrder === 'asc' ? 'ASC' : 'DESC'), + maxNumGroups, + // Final row sort ordering + SqlString.raw( + sortOrder === 'asc' || sortOrder === 'desc' + ? `, series_${ + seriesIndexWithSorting > -1 ? seriesIndexWithSorting : 0 + }.data ${sortOrder}` + : '', + ), + ], + ); + + const rows = await client.query({ + query, + format: 'JSON', + clickhouse_settings: { + additional_table_filters: buildLogStreamAdditionalFilters( + tableVersion, + teamId, + ), + }, + }); + + const result = await rows.json< + ResponseJSON<{ + ts_bucket: number; + group: string; + [series_data: `series_${number}.data`]: number; + }> + >(); + return result; +}; + +export const getMultiSeriesChart = async ({ + series, + endTime, + granularity, + maxNumGroups, + propertyTypeMappingsModel, + startTime, + tableVersion, + teamId, + seriesReturnType = SeriesReturnType.Column, +}: { + series: z.infer[]; + endTime: number; // unix in ms, + startTime: number; // unix in ms + granularity: string | undefined; // can be undefined in the number chart + maxNumGroups: number; + propertyTypeMappingsModel?: LogsPropertyTypeMappingsModel; + tableVersion: number | undefined; + teamId: string; + seriesReturnType?: SeriesReturnType; +}) => { + let queries: { + query: string; + hasGroupBy: boolean; + sortOrder?: 'desc' | 'asc'; + }[] = []; + if ( + // Default table is logs + ('table' in series[0] && + (series[0].table === 'logs' || series[0].table == null)) || + !('table' in series[0]) + ) { + if (propertyTypeMappingsModel == null) { + throw new Error('propertyTypeMappingsModel is required for logs chart'); + } + + queries = await Promise.all( + series.map(s => { + if (s.type != 'time' && s.type != 'table') { + throw new Error(`Unsupported series type: ${s.type}`); + } + if (s.table != 'logs' && s.table != null) { + throw new Error(`All series must have the same table`); + } + + return buildEventSeriesQuery({ + aggFn: s.aggFn, + endTime, + field: s.field, + granularity, + groupBy: s.groupBy[0], + propertyTypeMappingsModel, + q: s.where, + sortOrder: s.type === 'table' ? s.sortOrder : undefined, + startTime, + tableVersion, + teamId, + }); + }), + ); + } else if ('table' in series[0] && series[0].table === 'metrics') { + queries = await Promise.all( + series.map(s => { + if (s.type != 'time' && s.type != 'table') { + throw new Error(`Unsupported series type: ${s.type}`); + } + if (s.table != 'metrics') { + throw new Error(`All series must have the same table`); + } + if (s.field == null) { + throw new Error('Metric name is required'); + } + if (s.metricDataType == null) { + throw new Error('Metric data type is required'); + } + + return buildMetricSeriesQuery({ + aggFn: s.aggFn, + endTime, + name: s.field, + granularity, + groupBy: s.groupBy[0], + sortOrder: s.type === 'table' ? s.sortOrder : undefined, + q: s.where, + startTime, + teamId, + dataType: s.metricDataType, + }); + }), + ); + } + + return queryMultiSeriesChart({ + maxNumGroups, + tableVersion, + teamId, + seriesReturnType, + queries, + }); +}; + +export const getMultiSeriesChartLegacyFormat = async ({ + series, + endTime, + granularity, + maxNumGroups, + propertyTypeMappingsModel, + startTime, + tableVersion, + teamId, + seriesReturnType, +}: { + series: z.infer[]; + endTime: number; // unix in ms, + startTime: number; // unix in ms + granularity: string | undefined; // can be undefined in the number chart + maxNumGroups: number; + propertyTypeMappingsModel?: LogsPropertyTypeMappingsModel; + tableVersion: number | undefined; + teamId: string; + seriesReturnType?: SeriesReturnType; +}) => { + const result = await getMultiSeriesChart({ + series, + endTime, + granularity, + maxNumGroups, + propertyTypeMappingsModel, + startTime, + tableVersion, + teamId, + seriesReturnType, + }); + + const flatData = result.data.flatMap(row => { + if (seriesReturnType === 'column') { + return series.map((_, i) => { + return { + ts_bucket: row.ts_bucket, + group: row.group, + data: row[`series_${i}.data`], + }; + }); + } + + // Ratio only has 1 series + return [ + { + ts_bucket: row.ts_bucket, + group: row.group, + data: row['series_0.data'], + }, + ]; + }); + + return { + rows: flatData.length, + data: flatData, + }; +}; + export const getLogsChart = async ({ aggFn, endTime, diff --git a/packages/api/src/fixtures.ts b/packages/api/src/fixtures.ts index 71f93e28..8f8af25c 100644 --- a/packages/api/src/fixtures.ts +++ b/packages/api/src/fixtures.ts @@ -1,6 +1,18 @@ import mongoose from 'mongoose'; import request from 'supertest'; +import * as clickhouse from '@/clickhouse'; +import { + LogsPropertyTypeMappingsModel, + MetricsPropertyTypeMappingsModel, +} from '@/clickhouse/propertyTypeMappingsModel'; +import { + LogPlatform, + LogStreamModel, + LogType, + MetricModel, +} from '@/utils/logParser'; + import * as config from './config'; import { createTeam, getTeam } from './controllers/team'; import { findUserByEmail } from './controllers/user'; @@ -105,3 +117,137 @@ export const getLoggedInAgent = async (server: MockServer) => { user, }; }; +export function buildEvent({ + level, + source = 'test', + timestamp, + platform = LogPlatform.NodeJS, + type = LogType.Log, + end_timestamp = 0, + span_name, + ...properties +}: { + level?: string; + source?: string; + timestamp?: number; // ms timestamp + platform?: LogPlatform; + type?: LogType; + end_timestamp?: number; //ms timestamp + span_name?: string; +} & { + [key: string]: number | string | boolean; +}): LogStreamModel { + const ts = timestamp ?? Date.now(); + + const boolNames: string[] = []; + const boolValues: number[] = []; + const numberNames: string[] = []; + const numberValues: number[] = []; + const stringNames: string[] = []; + const stringValues: string[] = []; + + Object.entries(properties).forEach(([key, value]) => { + if (typeof value === 'boolean') { + boolNames.push(key); + boolValues.push(value ? 1 : 0); + } else if (typeof value === 'number') { + numberNames.push(key); + numberValues.push(value); + } else if (typeof value === 'string') { + stringNames.push(key); + stringValues.push(value); + } + }); + + return { + // TODO: Fix Timestamp Types + // @ts-ignore + timestamp: `${ts}000000`, + // @ts-ignore + observed_timestamp: `${ts}000000`, + _source: source, + _platform: platform, + severity_text: level, + // @ts-ignore + end_timestamp: `${end_timestamp}000000`, + type, + span_name, + 'bool.names': boolNames, + 'bool.values': boolValues, + 'number.names': numberNames, + 'number.values': numberValues, + 'string.names': stringNames, + 'string.values': stringValues, + }; +} + +export function buildMetricSeries({ + tags, + name, + points, + data_type, + is_delta, + is_monotonic, + unit, +}: { + tags: Record; + name: string; + points: { value: number; timestamp: number }[]; + data_type: clickhouse.MetricsDataType; + is_monotonic: boolean; + is_delta: boolean; + unit: string; +}): MetricModel[] { + // @ts-ignore TODO: Fix Timestamp types + return points.map(({ value, timestamp }) => ({ + _string_attributes: tags, + name, + value, + timestamp: `${timestamp}000000`, + data_type, + is_monotonic, + is_delta, + unit, + })); +} + +export function mockLogsPropertyTypeMappingsModel(propertyMap: { + [property: string]: 'bool' | 'number' | 'string'; +}) { + const propertyTypesMappingsModel = new LogsPropertyTypeMappingsModel( + 1, + 'fake team id', + () => Promise.resolve({}), + ); + jest + .spyOn(propertyTypesMappingsModel, 'get') + .mockImplementation((property: string) => { + // eslint-disable-next-line security/detect-object-injection + return propertyMap[property]; + }); + + jest + .spyOn(clickhouse, 'buildLogsPropertyTypeMappingsModel') + .mockImplementation(() => Promise.resolve(propertyTypesMappingsModel)); + + return propertyTypesMappingsModel; +} + +export function mockSpyMetricPropertyTypeMappingsModel(propertyMap: { + [property: string]: 'bool' | 'number' | 'string'; +}) { + const model = new MetricsPropertyTypeMappingsModel(1, 'fake', () => + Promise.resolve({}), + ); + + jest.spyOn(model, 'get').mockImplementation((property: string) => { + // eslint-disable-next-line security/detect-object-injection + return propertyMap[property]; + }); + + jest + .spyOn(clickhouse, 'buildMetricsPropertyTypeMappingsModel') + .mockImplementation(() => Promise.resolve(model)); + + return model; +} diff --git a/packages/api/src/models/dashboard.ts b/packages/api/src/models/dashboard.ts index 9c006ba6..c8e535a6 100644 --- a/packages/api/src/models/dashboard.ts +++ b/packages/api/src/models/dashboard.ts @@ -1,6 +1,8 @@ import mongoose, { Schema } from 'mongoose'; -import { AggFn } from '../clickhouse'; +import { SourceTable } from '@/utils/zod'; + +import { AggFn, SeriesReturnType } from '../clickhouse'; import type { ObjectId } from '.'; // Based on numbro.js format @@ -25,7 +27,7 @@ type Chart = { h: number; series: ( | { - table: string; + table: SourceTable; type: 'time'; aggFn: AggFn; // TODO: Type field: string | undefined; @@ -34,7 +36,7 @@ type Chart = { numberFormat?: NumberFormat; } | { - table: string; + table: SourceTable; type: 'histogram'; field: string | undefined; where: string; @@ -46,7 +48,7 @@ type Chart = { } | { type: 'number'; - table: string; + table: SourceTable; aggFn: AggFn; field: string | undefined; where: string; @@ -54,7 +56,7 @@ type Chart = { } | { type: 'table'; - table: string; + table: SourceTable; aggFn: AggFn; field: string | undefined; where: string; @@ -67,6 +69,7 @@ type Chart = { content: string; } )[]; + seriesReturnType?: SeriesReturnType; }; export interface IDashboard { diff --git a/packages/api/src/routers/api/alerts.ts b/packages/api/src/routers/api/alerts.ts index ed44e334..82b8d3ad 100644 --- a/packages/api/src/routers/api/alerts.ts +++ b/packages/api/src/routers/api/alerts.ts @@ -39,7 +39,7 @@ const zAlert = z .object({ channel: zChannel, interval: z.enum(['1m', '5m', '15m', '30m', '1h', '6h', '12h', '1d']), - threshold: z.number().min(1), + threshold: z.number().min(0), type: z.enum(['presence', 'absence']), source: z.enum(['LOG', 'CHART']).default('LOG'), }) diff --git a/packages/api/src/routers/api/chart.ts b/packages/api/src/routers/api/chart.ts new file mode 100644 index 00000000..ba432470 --- /dev/null +++ b/packages/api/src/routers/api/chart.ts @@ -0,0 +1,105 @@ +import opentelemetry, { SpanStatusCode } from '@opentelemetry/api'; +import express from 'express'; +import { isNumber, parseInt } from 'lodash'; +import { z } from 'zod'; +import { validateRequest } from 'zod-express-middleware'; + +import * as clickhouse from '@/clickhouse'; +import { getTeam } from '@/controllers/team'; +import logger from '@/utils/logger'; +import { chartSeriesSchema } from '@/utils/zod'; + +const router = express.Router(); + +router.post( + '/series', + validateRequest({ + body: z.object({ + series: z.array(chartSeriesSchema), + endTime: z.number(), + granularity: z.nativeEnum(clickhouse.Granularity).optional(), + startTime: z.number(), + seriesReturnType: z.optional(z.nativeEnum(clickhouse.SeriesReturnType)), + }), + }), + async (req, res, next) => { + try { + const teamId = req.user?.team; + const { endTime, granularity, startTime, seriesReturnType, series } = + req.body; + + if (teamId == null) { + return res.sendStatus(403); + } + if (!isNumber(startTime) || !isNumber(endTime)) { + return res.sendStatus(400); + } + + const team = await getTeam(teamId); + if (team == null) { + return res.sendStatus(403); + } + + const propertyTypeMappingsModel = + await clickhouse.buildLogsPropertyTypeMappingsModel( + team.logStreamTableVersion, + teamId.toString(), + startTime, + endTime, + ); + + const propertySet = new Set(); + series.map(s => { + if ('field' in s && s.field != null) { + propertySet.add(s.field); + } + if ('groupBy' in s && s.groupBy.length > 0) { + s.groupBy.map(g => propertySet.add(g)); + } + }); + + // Hack to refresh property cache if needed + const properties = Array.from(propertySet); + + if ( + properties.some(p => { + return !clickhouse.doesLogsPropertyExist( + p, + propertyTypeMappingsModel, + ); + }) + ) { + logger.warn({ + message: `getChart: Property type mappings cache is out of date (${properties.join( + ', ', + )})`, + }); + await propertyTypeMappingsModel.refresh(); + } + + // TODO: expose this to the frontend + const MAX_NUM_GROUPS = 20; + + res.json( + await clickhouse.getMultiSeriesChart({ + series, + endTime: endTime, + granularity, + maxNumGroups: MAX_NUM_GROUPS, + propertyTypeMappingsModel, + startTime: startTime, + tableVersion: team.logStreamTableVersion, + teamId: teamId.toString(), + seriesReturnType, + }), + ); + } catch (e) { + const span = opentelemetry.trace.getActiveSpan(); + span?.recordException(e as Error); + span?.setStatus({ code: SpanStatusCode.ERROR }); + next(e); + } + }, +); + +export default router; diff --git a/packages/api/src/routers/api/index.ts b/packages/api/src/routers/api/index.ts index c1863cbc..af561871 100644 --- a/packages/api/src/routers/api/index.ts +++ b/packages/api/src/routers/api/index.ts @@ -1,4 +1,5 @@ import alertsRouter from './alerts'; +import chartRouter from './chart'; import dashboardRouter from './dashboards'; import logsRouter from './logs'; import logViewsRouter from './logViews'; @@ -20,4 +21,5 @@ export default { sessionsRouter, teamRouter, webhooksRouter, + chartRouter, }; diff --git a/packages/api/src/tasks/__tests__/checkAlerts.test.ts b/packages/api/src/tasks/__tests__/checkAlerts.test.ts index 7e7416f0..cb6d6c05 100644 --- a/packages/api/src/tasks/__tests__/checkAlerts.test.ts +++ b/packages/api/src/tasks/__tests__/checkAlerts.test.ts @@ -1,7 +1,19 @@ +import ms from 'ms'; + +import { + buildEvent, + buildMetricSeries, + clearDBCollections, + closeDB, + getServer, + mockLogsPropertyTypeMappingsModel, + mockSpyMetricPropertyTypeMappingsModel, +} from '@/fixtures'; +import { LogType } from '@/utils/logParser'; + import * as clickhouse from '../../clickhouse'; import { createAlert } from '../../controllers/alerts'; import { createTeam } from '../../controllers/team'; -import { clearDBCollections, closeDB, getServer } from '../../fixtures'; import AlertHistory from '../../models/alertHistory'; import Dashboard from '../../models/dashboard'; import LogView from '../../models/logView'; @@ -254,25 +266,34 @@ describe('checkAlerts', () => { jest .spyOn(slack, 'postMessageToWebhook') .mockResolvedValueOnce(null as any); - jest - .spyOn(clickhouse, 'getLogsChart') - .mockResolvedValueOnce({ - rows: 1, - data: [ - { - data: '11', - group: 'HyperDX', - rank: '1', - rank_order_by_value: '11', - ts_bucket: 1700172600, - }, - ], - } as any) - // no logs found in the next window - .mockResolvedValueOnce({ - rows: 0, - data: [], - } as any); + mockLogsPropertyTypeMappingsModel({ + runId: 'string', + }); + + const runId = Math.random().toString(); // dedup watch mode runs + const teamId = 'test'; + const now = new Date('2023-11-16T22:12:00.000Z'); + // Send events in the last alert window 22:05 - 22:10 + const eventMs = now.getTime() - ms('5m'); + + await clickhouse.bulkInsertTeamLogStream(undefined, teamId, [ + buildEvent({ + span_name: 'HyperDX', + level: 'error', + timestamp: eventMs, + runId, + end_timestamp: eventMs + 100, + type: LogType.Span, + }), + buildEvent({ + span_name: 'HyperDX', + level: 'error', + timestamp: eventMs + 5, + runId, + end_timestamp: eventMs + 7, + type: LogType.Span, + }), + ]); const team = await createTeam({ name: 'My Team' }); const webhook = await new Webhook({ @@ -296,12 +317,21 @@ describe('checkAlerts', () => { { table: 'logs', type: 'time', - aggFn: 'max', + aggFn: 'sum', field: 'duration', - where: 'level:error', + where: `level:error runId:${runId}`, + groupBy: ['span_name'], + }, + { + table: 'logs', + type: 'time', + aggFn: 'min', + field: 'duration', + where: `level:error runId:${runId}`, groupBy: ['span_name'], }, ], + seriesReturnType: 'column', }, { id: 'obil1', @@ -336,9 +366,7 @@ describe('checkAlerts', () => { chartId: '198hki', }); - const now = new Date('2023-11-16T22:12:00.000Z'); - - // shoud fetch 5m of logs + // should fetch 5m of logs await processAlert(now, alert); expect(alert.state).toBe('ALERT'); @@ -359,12 +387,13 @@ describe('checkAlerts', () => { }).sort({ createdAt: 1, }); + expect(alertHistories.length).toBe(2); const [history1, history2] = alertHistories; expect(history1.state).toBe('ALERT'); expect(history1.counts).toBe(1); expect(history1.createdAt).toEqual(new Date('2023-11-16T22:10:00.000Z')); - expect(history1.lastValues.length).toBe(1); + expect(history1.lastValues.length).toBe(2); expect(history1.lastValues.length).toBeGreaterThan(0); expect(history1.lastValues[0].count).toBeGreaterThanOrEqual(1); @@ -372,32 +401,19 @@ describe('checkAlerts', () => { expect(history2.counts).toBe(0); expect(history2.createdAt).toEqual(new Date('2023-11-16T22:15:00.000Z')); - // check if getLogsChart query + webhook were triggered - expect(clickhouse.getLogsChart).toHaveBeenNthCalledWith(1, { - aggFn: 'max', - endTime: 1700172600000, - field: 'duration', - granularity: '5 minute', - groupBy: 'span_name', - maxNumGroups: 20, - propertyTypeMappingsModel: expect.any(Object), - q: 'level:error', - startTime: 1700172300000, - tableVersion: team.logStreamTableVersion, - teamId: team._id.toString(), - }); + // check if webhook was triggered expect(slack.postMessageToWebhook).toHaveBeenNthCalledWith( 1, 'https://hooks.slack.com/services/123', { - text: 'Alert for "Max Duration" in "My Dashboard" - 11 exceeds 10', + text: 'Alert for "Max Duration" in "My Dashboard" - 102 exceeds 10', blocks: [ { text: { text: [ - `**`, + `**`, 'Group: "HyperDX"', - '11 exceeds 10', + '102 exceeds 10', ].join('\n'), type: 'mrkdwn', }, @@ -406,29 +422,111 @@ describe('checkAlerts', () => { ], }, ); + + jest.resetAllMocks(); }); it('CHART alert (metrics table series)', async () => { + const runId = Math.random().toString(); // dedup watch mode runs + jest .spyOn(slack, 'postMessageToWebhook') .mockResolvedValueOnce(null as any); - jest - .spyOn(clickhouse, 'getMetricsChart') - .mockResolvedValueOnce({ - rows: 1, - data: [ - { - data: 11, - group: 'HyperDX', - ts_bucket: 1700172600, - }, + + const now = new Date('2023-11-16T22:12:00.000Z'); + // Need data in 22:00 - 22:05 to calculate a rate for 22:05 - 22:10 + const metricNowTs = new Date('2023-11-16T22:00:00.000Z').getTime(); + + mockSpyMetricPropertyTypeMappingsModel({ + runId: 'string', + host: 'string', + 'cloud.provider': 'string', + }); + + await clickhouse.bulkInsertTeamMetricStream( + buildMetricSeries({ + name: 'redis.memory.rss', + tags: { + host: 'HyperDX', + 'cloud.provider': 'aws', + runId, + series: '1', + }, + data_type: clickhouse.MetricsDataType.Sum, + is_monotonic: true, + is_delta: true, + unit: 'Bytes', + points: [ + { value: 1, timestamp: metricNowTs }, + { value: 8, timestamp: metricNowTs + ms('1m') }, + { value: 8, timestamp: metricNowTs + ms('2m') }, + { value: 9, timestamp: metricNowTs + ms('3m') }, + { value: 15, timestamp: metricNowTs + ms('4m') }, // 15 + { value: 30, timestamp: metricNowTs + ms('5m') }, + { value: 31, timestamp: metricNowTs + ms('6m') }, + { value: 32, timestamp: metricNowTs + ms('7m') }, + { value: 33, timestamp: metricNowTs + ms('8m') }, + { value: 34, timestamp: metricNowTs + ms('9m') }, // 34 + { value: 35, timestamp: metricNowTs + ms('10m') }, + { value: 36, timestamp: metricNowTs + ms('11m') }, ], - } as any) - // no logs found in the next window - .mockResolvedValueOnce({ - rows: 0, - data: [], - } as any); + }), + ); + + await clickhouse.bulkInsertTeamMetricStream( + buildMetricSeries({ + name: 'redis.memory.rss', + tags: { + host: 'HyperDX', + 'cloud.provider': 'aws', + runId, + series: '2', + }, + data_type: clickhouse.MetricsDataType.Sum, + is_monotonic: true, + is_delta: true, + unit: 'Bytes', + points: [ + { value: 1000, timestamp: metricNowTs }, + { value: 8000, timestamp: metricNowTs + ms('1m') }, + { value: 8000, timestamp: metricNowTs + ms('2m') }, + { value: 9000, timestamp: metricNowTs + ms('3m') }, + { value: 15000, timestamp: metricNowTs + ms('4m') }, // 15000 + { value: 30000, timestamp: metricNowTs + ms('5m') }, + { value: 30001, timestamp: metricNowTs + ms('6m') }, + { value: 30002, timestamp: metricNowTs + ms('7m') }, + { value: 30003, timestamp: metricNowTs + ms('8m') }, + { value: 30004, timestamp: metricNowTs + ms('9m') }, // 30004 + { value: 30005, timestamp: metricNowTs + ms('10m') }, + { value: 30006, timestamp: metricNowTs + ms('11m') }, + ], + }), + ); + + await clickhouse.bulkInsertTeamMetricStream( + buildMetricSeries({ + name: 'redis.memory.rss', + tags: { host: 'test2', 'cloud.provider': 'aws', runId, series: '0' }, + data_type: clickhouse.MetricsDataType.Sum, + is_monotonic: true, + is_delta: true, + unit: 'Bytes', + points: [ + { value: 1, timestamp: metricNowTs }, + { value: 8, timestamp: metricNowTs + ms('1m') }, + { value: 8, timestamp: metricNowTs + ms('2m') }, + { value: 9, timestamp: metricNowTs + ms('3m') }, + { value: 15, timestamp: metricNowTs + ms('4m') }, // 15 + { value: 17, timestamp: metricNowTs + ms('5m') }, + { value: 18, timestamp: metricNowTs + ms('6m') }, + { value: 19, timestamp: metricNowTs + ms('7m') }, + { value: 20, timestamp: metricNowTs + ms('8m') }, + { value: 21, timestamp: metricNowTs + ms('9m') }, // 21 + { value: 22, timestamp: metricNowTs + ms('10m') }, + { value: 23, timestamp: metricNowTs + ms('11m') }, + ], + }), + ); const team = await createTeam({ name: 'My Team' }); const webhook = await new Webhook({ @@ -452,12 +550,21 @@ describe('checkAlerts', () => { { table: 'metrics', type: 'time', - aggFn: 'max', - field: 'redis.memory.rss - Gauge', - where: 'cloud.provider:"aws"', + aggFn: 'avg_rate', + field: 'redis.memory.rss - Sum', + where: `cloud.provider:"aws" runId:${runId}`, + groupBy: ['host'], + }, + { + table: 'metrics', + type: 'time', + aggFn: 'min_rate', + field: 'redis.memory.rss - Sum', + where: `cloud.provider:"aws" runId:${runId}`, groupBy: ['host'], }, ], + seriesReturnType: 'ratio', }, { id: 'obil1', @@ -492,9 +599,7 @@ describe('checkAlerts', () => { chartId: '198hki', }); - const now = new Date('2023-11-16T22:12:00.000Z'); - - // shoud fetch 5m of logs + // shoud fetch 5m of metrics await processAlert(now, alert); expect(alert.state).toBe('ALERT'); @@ -527,30 +632,19 @@ describe('checkAlerts', () => { new Date('2023-11-16T22:15:00.000Z'), ); - // check if getLogsChart query + webhook were triggered - expect(clickhouse.getMetricsChart).toHaveBeenNthCalledWith(1, { - aggFn: 'max', - dataType: 'Gauge', - endTime: 1700172600000, - granularity: '5 minute', - groupBy: 'host', - name: 'redis.memory.rss', - q: 'cloud.provider:"aws"', - startTime: 1700172300000, - teamId: team._id.toString(), - }); + // check if webhook was triggered expect(slack.postMessageToWebhook).toHaveBeenNthCalledWith( 1, 'https://hooks.slack.com/services/123', { - text: 'Alert for "Redis Memory" in "My Dashboard" - 11 exceeds 10', + text: 'Alert for "Redis Memory" in "My Dashboard" - 395.3421052631579 exceeds 10', blocks: [ { text: { text: [ - `**`, + `**`, 'Group: "HyperDX"', - '11 exceeds 10', + '395.3421052631579 exceeds 10', ].join('\n'), type: 'mrkdwn', }, diff --git a/packages/api/src/tasks/checkAlerts.ts b/packages/api/src/tasks/checkAlerts.ts index fa41a8eb..adc5fb17 100644 --- a/packages/api/src/tasks/checkAlerts.ts +++ b/packages/api/src/tasks/checkAlerts.ts @@ -155,7 +155,7 @@ const buildLogEventSlackMessage = async ({ endTime: endTime.getTime(), limit: 5, offset: 0, - order: 'desc', // TODO: better to use null + order: 'desc', q: searchQuery, startTime: startTime.getTime(), tableVersion: logView.team.logStreamTableVersion, @@ -336,8 +336,7 @@ export const processAlert = async (now: Date, alert: AlertDocument) => { // Logs Source let checksData: | Awaited> - | Awaited> - | Awaited> + | Awaited> | null = null; let logView: Awaited> | null = null; let targetDashboard: EnhancedDashboard | null = null; @@ -380,17 +379,19 @@ export const processAlert = async (now: Date, alert: AlertDocument) => { ).populate<{ team: ITeam; }>('team'); + if ( dashboard && Array.isArray(dashboard.charts) && dashboard.charts.length === 1 ) { const chart = dashboard.charts[0]; + // Doesn't work for metric alerts yet + const MAX_NUM_GROUPS = 20; // TODO: assuming that the chart has only 1 series for now - const series = chart.series[0]; - if (series.type === 'time' && series.table === 'logs') { + const firstSeries = chart.series[0]; + if (firstSeries.type === 'time' && firstSeries.table === 'logs') { targetDashboard = dashboard; - const MAX_NUM_GROUPS = 20; const startTimeMs = fns.getTime(checkStartTime); const endTimeMs = fns.getTime(checkEndTime); const propertyTypeMappingsModel = @@ -400,51 +401,49 @@ export const processAlert = async (now: Date, alert: AlertDocument) => { startTimeMs, endTimeMs, ); - checksData = await clickhouse.getLogsChart({ - aggFn: series.aggFn, + + checksData = await clickhouse.getMultiSeriesChartLegacyFormat({ + series: chart.series, endTime: endTimeMs, - // @ts-expect-error - field: series.field, granularity: `${windowSizeInMins} minute`, - groupBy: series.groupBy[0], maxNumGroups: MAX_NUM_GROUPS, propertyTypeMappingsModel, - q: series.where, startTime: startTimeMs, tableVersion: dashboard.team.logStreamTableVersion, teamId: dashboard.team._id.toString(), + seriesReturnType: chart.seriesReturnType, }); } else if ( - series.type === 'time' && - series.table === 'metrics' && - series.field + firstSeries.type === 'time' && + firstSeries.table === 'metrics' && + firstSeries.field ) { targetDashboard = dashboard; - let startTimeMs = fns.getTime(checkStartTime); + const startTimeMs = fns.getTime(checkStartTime); const endTimeMs = fns.getTime(checkEndTime); - const [metricName, rawMetricDataType] = series.field.split(' - '); - const metricDataType = z - .nativeEnum(clickhouse.MetricsDataType) - .parse(rawMetricDataType); - if ( - metricDataType === clickhouse.MetricsDataType.Sum && - clickhouse.isRateAggFn(series.aggFn) - ) { - // adjust the time so that we have enough data points to calculate a rate - startTimeMs = fns - .subMinutes(startTimeMs, windowSizeInMins) - .getTime(); - } - checksData = await clickhouse.getMetricsChart({ - aggFn: series.aggFn, - dataType: metricDataType, + checksData = await clickhouse.getMultiSeriesChartLegacyFormat({ + series: chart.series.map(series => { + if ('field' in series && series.field != null) { + const [metricName, rawMetricDataType] = + series.field.split(' - '); + const metricDataType = z + .nativeEnum(clickhouse.MetricsDataType) + .parse(rawMetricDataType); + return { + ...series, + metricDataType, + field: metricName, + }; + } + return series; + }), endTime: endTimeMs, granularity: `${windowSizeInMins} minute`, - groupBy: series.groupBy[0], - name: metricName, - q: series.where, + maxNumGroups: MAX_NUM_GROUPS, startTime: startTimeMs, + tableVersion: dashboard.team.logStreamTableVersion, teamId: dashboard.team._id.toString(), + seriesReturnType: chart.seriesReturnType, }); } } diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts new file mode 100644 index 00000000..c4553884 --- /dev/null +++ b/packages/api/src/utils/zod.ts @@ -0,0 +1,83 @@ +import { z } from 'zod'; + +import { AggFn, MetricsDataType } from '@/clickhouse'; + +export const numberFormatSchema = z.object({ + output: z + .union([ + z.literal('currency'), + z.literal('percent'), + z.literal('byte'), + z.literal('time'), + z.literal('number'), + ]) + .optional(), + mantissa: z.number().optional(), + thousandSeparated: z.boolean().optional(), + average: z.boolean().optional(), + decimalBytes: z.boolean().optional(), + factor: z.number().optional(), + currencySymbol: z.string().optional(), + unit: z.string().optional(), +}); + +export const aggFnSchema = z.nativeEnum(AggFn); + +export const sourceTableSchema = z.union([ + z.literal('logs'), + z.literal('rrweb'), + z.literal('metrics'), +]); + +export type SourceTable = z.infer; + +export const timeChartSeriesSchema = z.object({ + table: z.optional(sourceTableSchema), + type: z.literal('time'), + aggFn: aggFnSchema, + field: z.union([z.string(), z.undefined()]), + where: z.string(), + groupBy: z.array(z.string()), + numberFormat: numberFormatSchema.optional(), + metricDataType: z.optional(z.nativeEnum(MetricsDataType)), +}); + +export const tableChartSeriesSchema = z.object({ + type: z.literal('table'), + table: z.optional(sourceTableSchema), + aggFn: aggFnSchema, + field: z.optional(z.string()), + where: z.string(), + groupBy: z.array(z.string()), + sortOrder: z.optional(z.union([z.literal('desc'), z.literal('asc')])), + numberFormat: numberFormatSchema.optional(), + metricDataType: z.optional(z.nativeEnum(MetricsDataType)), +}); + +export const chartSeriesSchema = z.union([ + timeChartSeriesSchema, + tableChartSeriesSchema, + z.object({ + table: z.optional(sourceTableSchema), + type: z.literal('histogram'), + field: z.union([z.string(), z.undefined()]), + where: z.string(), + }), + z.object({ + type: z.literal('search'), + fields: z.array(z.string()), + where: z.string(), + }), + z.object({ + type: z.literal('number'), + table: z.optional(sourceTableSchema), + aggFn: aggFnSchema, + field: z.union([z.string(), z.undefined()]), + where: z.string(), + numberFormat: numberFormatSchema.optional(), + }), + z.object({ + type: z.literal('markdown'), + content: z.string(), + }), +]); diff --git a/packages/app/src/ChartPage.tsx b/packages/app/src/ChartPage.tsx index cae9f633..26b8b5f8 100644 --- a/packages/app/src/ChartPage.tsx +++ b/packages/app/src/ChartPage.tsx @@ -8,7 +8,7 @@ import { decodeArray, encodeArray } from 'serialize-query-params'; import { StringParam, useQueryParam, withDefault } from 'use-query-params'; import AppNav from './AppNav'; -import { AggFn, ChartSeries, ChartSeriesForm } from './ChartUtils'; +import { ChartSeriesForm } from './ChartUtils'; import DSSelect from './DSSelect'; import HDXLineChart from './HDXLineChart'; import { LogTableWithSidePanel } from './LogTableWithSidePanel'; @@ -16,6 +16,7 @@ import SearchTimeRangePicker, { parseTimeRangeInput, } from './SearchTimeRangePicker'; import { parseTimeQuery, useTimeQuery } from './timeQuery'; +import type { AggFn, ChartSeries, SourceTable } from './types'; import { useQueryParam as useHDXQueryParam } from './useQueryParam'; export const ChartSeriesParam: QueryParamConfig = { @@ -56,21 +57,12 @@ export default function GraphPage() { }, ); - const setTable = (index: number, table: string) => { - setChartSeries( - produce(chartSeries, series => { - if (series?.[index] != null) { - series[index].table = table; - } - }), - ); - }; - const setAggFn = (index: number, fn: AggFn) => { setChartSeries( produce(chartSeries, series => { - if (series?.[index] != null) { - series[index].aggFn = fn; + const s = series?.[index]; + if (s != null && s.type === 'time') { + s.aggFn = fn; } }), ); @@ -78,8 +70,9 @@ export default function GraphPage() { const setField = (index: number, field: string | undefined) => { setChartSeries( produce(chartSeries, series => { - if (series?.[index] != null) { - series[index].field = field; + const s = series?.[index]; + if (s != null && s.type === 'time') { + s.field = field; } }), ); @@ -91,9 +84,10 @@ export default function GraphPage() { ) => { setChartSeries( produce(chartSeries, series => { - if (series?.[index] != null) { - series[index].field = field; - series[index].aggFn = fn; + const s = series?.[index]; + if (s != null && s.type === 'time') { + s.field = field; + s.aggFn = fn; } }), ); @@ -101,8 +95,9 @@ export default function GraphPage() { const setWhere = (index: number, where: string) => { setChartSeries( produce(chartSeries, series => { - if (series?.[index] != null) { - series[index].where = where; + const s = series?.[index]; + if (s != null && s.type === 'time') { + s.where = where; } }), ); @@ -110,8 +105,9 @@ export default function GraphPage() { const setGroupBy = (index: number, groupBy: string | undefined) => { setChartSeries( produce(chartSeries, series => { - if (series?.[index] != null) { - series[index].groupBy = groupBy != null ? [groupBy] : []; + const s = series?.[index]; + if (s != null && s.type === 'time') { + s.groupBy = groupBy != null ? [groupBy] : []; } }), ); @@ -148,7 +144,11 @@ export default function GraphPage() { onSearch(displayedTimeInputValue); const dateRange = parseTimeRangeInput(displayedTimeInputValue); - if (dateRange[0] != null && dateRange[1] != null) { + if ( + dateRange[0] != null && + dateRange[1] != null && + chartSeries[0].type === 'time' + ) { setChartConfig({ // TODO: Support multiple series table: chartSeries[0].table ?? 'logs', @@ -177,6 +177,9 @@ export default function GraphPage() {
e.preventDefault()}>
Create New Chart
{chartSeries.map((series, index) => { + if (series.type !== 'time') { + return null; + } return ( { setChartSeries( produce(chartSeries, series => { - if (series?.[index] != null) { - series[index].table = table; - series[index].aggFn = aggFn; + const s = series?.[index]; + if (s != null && s.type === 'time') { + s.table = table; + s.aggFn = aggFn; } }), ); }} - setTable={table => setTable(index, table)} setWhere={where => setWhere(index, where)} setAggFn={fn => setAggFn(index, fn)} setGroupBy={groupBy => setGroupBy(index, groupBy)} diff --git a/packages/app/src/ChartUtils.tsx b/packages/app/src/ChartUtils.tsx index 1f52986f..5ec93aca 100644 --- a/packages/app/src/ChartUtils.tsx +++ b/packages/app/src/ChartUtils.tsx @@ -9,12 +9,15 @@ import api from './api'; import Checkbox from './Checkbox'; import MetricTagFilterInput from './MetricTagFilterInput'; import SearchInput from './SearchInput'; +import type { AggFn, ChartSeries, SourceTable } from './types'; +import { NumberFormat } from './types'; + export const SORT_ORDER = [ { value: 'asc' as const, label: 'Ascending' }, { value: 'desc' as const, label: 'Descending' }, ]; + export type SortOrder = (typeof SORT_ORDER)[number]['value']; -import type { NumberFormat } from './types'; export const TABLES = [ { value: 'logs' as const, label: 'Logs / Spans' }, @@ -61,14 +64,94 @@ export type Granularity = | '7 day' | '30 day'; -export type ChartSeries = { - table: string; - type: 'time'; - aggFn: AggFn; - field: string | undefined; - where: string; - groupBy: string[]; -}; +const seriesDisplayName = (s: ChartSeries) => + s.type === 'time' || s.type === 'table' + ? `${s.aggFn}${ + s.aggFn !== 'count' + ? `(${ + s.table === 'metrics' + ? s.field?.split(' - ')?.[0] ?? s.field + : s.field + })` + : '()' + }${s.where ? `{${s.where}}` : ''}` + : ''; + +export function seriesColumns({ + series, + seriesReturnType, +}: { + seriesReturnType: 'ratio' | 'column'; + series: ChartSeries[]; +}) { + const seriesMeta = + seriesReturnType === 'ratio' + ? [ + { + dataKey: `series_0.data`, + displayName: `${seriesDisplayName(series[0])}/${seriesDisplayName( + series[1], + )}`, + sortOrder: + 'sortOrder' in series[0] ? series[0].sortOrder : undefined, + }, + ] + : series.map((s, i) => { + return { + dataKey: `series_${i}.data`, + displayName: seriesDisplayName(s), + sortOrder: 'sortOrder' in s ? s.sortOrder : undefined, + }; + }); + + return seriesMeta; +} + +export function seriesToSearchQuery({ + series, + groupByValue, +}: { + series: ChartSeries[]; + groupByValue?: string; +}) { + const queries = series + .map((s, i) => { + if (s.type === 'time' || s.type === 'table' || s.type === 'number') { + const { where, aggFn, field } = s; + return `${where.trim()}${aggFn !== 'count' ? ` ${field}:*` : ''}${ + 'groupBy' in s && s.groupBy != null && s.groupBy.length > 0 + ? ` ${s.groupBy}:${groupByValue}` + : '' + }`.trim(); + } + }) + .filter(q => q != null && q.length > 0); + + const q = + queries.length > 1 + ? queries.map(q => `(${q})`).join(' OR ') + : queries.join(''); + + return q; +} + +export function seriesToUrlSearchQueryParam({ + series, + dateRange, + groupByValue = '*', +}: { + series: ChartSeries[]; + dateRange: [Date, Date]; + groupByValue?: string | undefined; +}) { + const q = seriesToSearchQuery({ series, groupByValue }); + + return new URLSearchParams({ + q, + from: `${dateRange[0].getTime()}`, + to: `${dateRange[1].getTime()}`, + }); +} export function usePropertyOptions(types: ('number' | 'string' | 'bool')[]) { const { data: propertyTypeMappingsResult } = api.usePropertyTypeMappings(); @@ -107,27 +190,38 @@ export function usePropertyOptions(types: ('number' | 'string' | 'bool')[]) { return propertyOptions; } -export function MetricTagSelect({ - value, - setValue, - metricName, -}: { - value: string | undefined | null; - setValue: (value: string | undefined) => void; - metricName: string | undefined; -}) { +function useMetricTagOptions({ metricNames }: { metricNames?: string[] }) { const { data: metricTagsData } = api.useMetricsTags(); const options = useMemo(() => { - const tags = - metricTagsData?.data?.filter(metric => metric.name === metricName)?.[0] - ?.tags ?? []; + let tagNameSet = new Set(); + if (metricNames != null && metricNames.length > 0) { + const firstMetricName = metricNames[0]; // Start the set - const tagNameSet = new Set(); + const tags = + metricTagsData?.data?.filter( + metric => metric.name === firstMetricName, + )?.[0]?.tags ?? []; + tags.forEach(tag => { + Object.keys(tag).forEach(tagName => tagNameSet.add(tagName)); + }); - tags.forEach(tag => { - Object.keys(tag).forEach(tagName => tagNameSet.add(tagName)); - }); + for (let i = 1; i < metricNames.length; i++) { + const tags = + metricTagsData?.data?.filter( + metric => metric.name === metricNames[i], + )?.[0]?.tags ?? []; + const intersection = new Set(); + tags.forEach(tag => { + Object.keys(tag).forEach(tagName => { + if (tagNameSet.has(tagName)) { + intersection.add(tagName); + } + }); + }); + tagNameSet = intersection; + } + } return [ { value: undefined, label: 'None' }, @@ -136,7 +230,21 @@ export function MetricTagSelect({ label: tagName, })), ]; - }, [metricTagsData, metricName]); + }, [metricTagsData, metricNames]); + + return options; +} + +export function MetricTagSelect({ + value, + setValue, + metricNames, +}: { + value: string | undefined | null; + setValue: (value: string | undefined) => void; + metricNames?: string[]; +}) { + const options = useMetricTagOptions({ metricNames }); return ( -
+
-
+
void; setField: (field: string | undefined) => void; setFieldAndAggFn: (field: string | undefined, fn: AggFn) => void; - setTableAndAggFn: (table: string, fn: AggFn) => void; + setTableAndAggFn: (table: SourceTable, fn: AggFn) => void; setGroupBy: (groupBy: string | undefined) => void; setSortOrder?: (sortOrder: SortOrder) => void; - setTable: (table: string) => void; setWhere: (where: string) => void; sortOrder?: string; table: string; @@ -495,7 +581,7 @@ export function ChartSeriesForm({
) : null} {table === 'metrics' ? ( -
+
@@ -629,6 +715,308 @@ export function ChartSeriesForm({ ); } +export function TableSelect({ + table, + setTableAndAggFn, +}: { + setTableAndAggFn: (table: SourceTable, fn: AggFn) => void; + table: string; +}) { + return ( + v.value === aggFn)} + onChange={opt => _setAggFn(opt?.value ?? 'count')} + classNamePrefix="ds-react-select" + /> + ) : ( + v.value === sortOrder)} + onChange={opt => setSortOrder(opt?.value ?? 'desc')} + classNamePrefix="ds-react-select" + /> +
+
+ ) + } + {setNumberFormat && ( +
+ + + Chart Settings + + } + c="dark.2" + mb={8} + /> + +
Number Format
+ + +
+ )} + + ); +} + export function convertDateRangeToGranularityString( dateRange: [Date, Date], maxNumBuckets: number, diff --git a/packages/app/src/DashboardPage.tsx b/packages/app/src/DashboardPage.tsx index 0ccb02dd..bb851c0c 100644 --- a/packages/app/src/DashboardPage.tsx +++ b/packages/app/src/DashboardPage.tsx @@ -27,7 +27,6 @@ import { Badge, Transition } from '@mantine/core'; import api from './api'; import AppNav from './AppNav'; import { convertDateRangeToGranularityString, Granularity } from './ChartUtils'; -import type { Chart } from './EditChartForm'; import { EditHistogramChartForm, EditLineChartForm, @@ -38,17 +37,17 @@ import { } from './EditChartForm'; import GranularityPicker from './GranularityPicker'; import HDXHistogramChart from './HDXHistogramChart'; -import HDXLineChart from './HDXLineChart'; import HDXMarkdownChart from './HDXMarkdownChart'; +import HDXMultiSeriesTableChart from './HDXMultiSeriesTableChart'; +import HDXMultiSeriesLineChart from './HDXMultiSeriesTimeChart'; import HDXNumberChart from './HDXNumberChart'; -import HDXTableChart from './HDXTableChart'; import { LogTableWithSidePanel } from './LogTableWithSidePanel'; import SearchInput from './SearchInput'; import SearchTimeRangePicker from './SearchTimeRangePicker'; import { FloppyIcon, Histogram } from './SVGIcons'; import TabBar from './TabBar'; -import { parseTimeQuery, useNewTimeQuery, useTimeQuery } from './timeQuery'; -import type { Alert } from './types'; +import { parseTimeQuery, useNewTimeQuery } from './timeQuery'; +import type { Alert, Chart } from './types'; import { useConfirm } from './useConfirm'; import { hashCode } from './utils'; import { ZIndexContext } from './zIndex'; @@ -272,11 +271,23 @@ const Tile = forwardRef( className="fs-7 text-muted flex-grow-1 overflow-hidden" onMouseDown={e => e.stopPropagation()} > - {config.type === 'time' && ( - + {chart.series[0].type === 'time' && config.type === 'time' && ( + )} - {config.type === 'table' && ( - + {chart.series[0].type === 'table' && config.type === 'table' && ( + )} {config.type === 'histogram' && ( @@ -322,17 +333,25 @@ const EditChartModal = ({ onClose: () => void; show: boolean; }) => { - const [tab, setTab] = useState< + type Tab = | 'time' | 'search' | 'histogram' | 'markdown' | 'number' | 'table' - | undefined - >(undefined); + | undefined; + + const [tab, setTab] = useState(undefined); const displayedTab = tab ?? chart?.series?.[0]?.type ?? 'time'; + const onTabClick = useCallback( + (newTab: Tab) => { + setTab(newTab); + }, + [setTab], + ); + return ( { - setTab(v); - }} + onClick={onTabClick} /> {displayedTab === 'time' && chart != null && ( { - draft.series[0].type = 'time'; + for (const series of draft.series) { + series.type = 'time'; + } })} alerts={alerts} onSave={onSave} @@ -416,7 +435,9 @@ const EditChartModal = ({ {displayedTab === 'table' && chart != null && ( { - draft.series[0].type = 'table'; + for (const series of draft.series) { + series.type = 'table'; + } })} onSave={onSave} onClose={onClose} @@ -737,6 +758,7 @@ export default function DashboardPage() { groupBy: [], }, ], + seriesReturnType: 'column', }); }; diff --git a/packages/app/src/EditChartForm.tsx b/packages/app/src/EditChartForm.tsx index 80a8d636..22bd50d1 100644 --- a/packages/app/src/EditChartForm.tsx +++ b/packages/app/src/EditChartForm.tsx @@ -1,83 +1,33 @@ -import { useEffect, useMemo, useState } from 'react'; +import { useMemo, useState } from 'react'; +import type { Draft } from 'immer'; import produce from 'immer'; -import { Button, Form, InputGroup, Modal } from 'react-bootstrap'; +import { Button as BSButton, Form, InputGroup } from 'react-bootstrap'; import Select from 'react-select'; -import { Divider, Group, Paper } from '@mantine/core'; +import { Button, Divider, Flex, Group, Paper, Switch } from '@mantine/core'; import { NumberFormatInput } from './components/NumberFormat'; import { intervalToGranularity } from './Alert'; import { AGG_FNS, - AggFn, - ChartSeriesForm, + ChartSeriesFormCompact, convertDateRangeToGranularityString, FieldSelect, + GroupBySelect, + seriesToSearchQuery, + TableSelect, } from './ChartUtils'; import Checkbox from './Checkbox'; import * as config from './config'; import { METRIC_ALERTS_ENABLED } from './config'; import EditChartFormAlerts from './EditChartFormAlerts'; import HDXHistogramChart from './HDXHistogramChart'; -import HDXLineChart from './HDXLineChart'; import HDXMarkdownChart from './HDXMarkdownChart'; +import HDXMultiSeriesTableChart from './HDXMultiSeriesTableChart'; +import HDXMultiSeriesLineChart from './HDXMultiSeriesTimeChart'; import HDXNumberChart from './HDXNumberChart'; -import HDXTableChart from './HDXTableChart'; import { LogTableWithSidePanel } from './LogTableWithSidePanel'; -import type { Alert, NumberFormat } from './types'; -import { hashCode, useDebounce } from './utils'; - -export type Chart = { - id: string; - name: string; - x: number; - y: number; - w: number; - h: number; - series: ( - | { - table: string; - type: 'time'; - aggFn: AggFn; // TODO: Type - field: string | undefined; - where: string; - groupBy: string[]; - numberFormat?: NumberFormat; - } - | { - table: string; - type: 'histogram'; - field: string | undefined; - where: string; - } - | { - type: 'search'; - fields: string[]; - where: string; - } - | { - type: 'number'; - table: string; - aggFn: AggFn; - field: string | undefined; - where: string; - numberFormat?: NumberFormat; - } - | { - type: 'table'; - table: string; - aggFn: AggFn; - field: string | undefined; - where: string; - groupBy: string[]; - sortOrder: 'desc' | 'asc'; - numberFormat?: NumberFormat; - } - | { - type: 'markdown'; - content: string; - } - )[]; -}; +import type { Alert, Chart, TimeChartSeries } from './types'; +import { useDebounce } from './utils'; const DEFAULT_ALERT: Alert = { channel: { @@ -169,16 +119,16 @@ export const EditMarkdownChartForm = ({
- - +
Markdown Preview
@@ -272,16 +222,16 @@ export const EditSearchChartForm = ({
- - +
Search Preview
@@ -465,16 +415,16 @@ export const EditNumberChartForm = ({
- - +
Chart Preview
@@ -535,6 +485,8 @@ export const EditTableChartForm = ({ granularity: convertDateRangeToGranularityString(dateRange, 60), dateRange, numberFormat: editedChart.series[0].numberFormat, + series: editedChart.series, + seriesReturnType: editedChart.seriesReturnType, } : null, [editedChart, dateRange], @@ -573,118 +525,54 @@ export const EditTableChartForm = ({ placeholder="Chart Name" />
- - setEditedChart( - produce(editedChart, draft => { - if (draft.series[0].type === CHART_TYPE) { - draft.series[0].table = table; - } - }), - ) - } - setAggFn={aggFn => - setEditedChart( - produce(editedChart, draft => { - if (draft.series[0].type === CHART_TYPE) { - draft.series[0].aggFn = aggFn; - } - }), - ) - } - setTableAndAggFn={(table, aggFn) => { - setEditedChart( - produce(editedChart, draft => { - if (draft.series[0].type === CHART_TYPE) { - draft.series[0].table = table; - draft.series[0].aggFn = aggFn; - } - }), - ); - }} - setWhere={where => - setEditedChart( - produce(editedChart, draft => { - if (draft.series[0].type === CHART_TYPE) { - draft.series[0].where = where; - } - }), - ) - } - setGroupBy={groupBy => - setEditedChart( - produce(editedChart, draft => { - if (draft.series[0].type === CHART_TYPE) { - if (groupBy != undefined) { - draft.series[0].groupBy[0] = groupBy; - } else { - draft.series[0].groupBy = []; - } - } - }), - ) - } - setField={field => - setEditedChart( - produce(editedChart, draft => { - if (draft.series[0].type === CHART_TYPE) { - draft.series[0].field = field; - } - }), - ) - } - setFieldAndAggFn={(field, aggFn) => { - setEditedChart( - produce(editedChart, draft => { - if (draft.series[0].type === CHART_TYPE) { - draft.series[0].field = field; - draft.series[0].aggFn = aggFn; - } - }), - ); - }} - setSortOrder={sortOrder => - setEditedChart( - produce(editedChart, draft => { - if (draft.series[0].type === CHART_TYPE) { - draft.series[0].sortOrder = sortOrder; - } - }), - ) - } - numberFormat={editedChart.series[0].numberFormat} - setNumberFormat={numberFormat => - setEditedChart( - produce(editedChart, draft => { - if (draft.series[0].type === CHART_TYPE) { - draft.series[0].numberFormat = numberFormat; - } - }), - ) - } +
- - +
Chart Preview
- + { + setEditedChart( + produce(editedChart, draft => { + // We need to clear out all other series sort orders first + for (let i = 0; i < draft.series.length; i++) { + if (i !== seriesIndex) { + const s = draft.series[i]; + if (s.type === CHART_TYPE) { + s.sortOrder = undefined; + } + } + } + + const s = draft.series[seriesIndex]; + if (s.type === CHART_TYPE) { + s.sortOrder = + s.sortOrder == null + ? 'desc' + : s.sortOrder === 'asc' + ? 'desc' + : 'asc'; + } + + return; + }), + ); + }} + />
{editedChart.series[0].table === 'logs' ? ( @@ -822,16 +710,16 @@ export const EditHistogramChartForm = ({
- - +
Chart Preview
@@ -864,6 +752,285 @@ export const EditHistogramChartForm = ({ ); }; +function pushNewSeries(draft: Draft) { + const firstSeries = draft.series[0] as TimeChartSeries; + const { table, type, groupBy, numberFormat } = firstSeries; + draft.series.push({ + table, + type, + aggFn: table === 'logs' ? 'count' : 'avg', + field: '', + where: '', + groupBy, + numberFormat, + }); +} + +export const EditMultiSeriesChartForm = ({ + editedChart, + setEditedChart, + CHART_TYPE, +}: { + editedChart: Chart; + setEditedChart: (chart: Chart) => void; + CHART_TYPE: 'time' | 'table'; +}) => { + if (editedChart.series[0].type !== CHART_TYPE) { + return null; + } + + return ( + <> + {editedChart.series.length > 1 && ( + +
+ + Data Source +
+
+ { + setEditedChart( + produce(editedChart, draft => { + draft.series.forEach((series, i) => { + if (series.type === CHART_TYPE) { + series.table = table; + series.aggFn = aggFn; + } + }); + }), + ); + }} + /> +
+ + )} + {editedChart.series.map((series, i) => { + if (series.type !== CHART_TYPE) { + return null; + } + + return ( +
+ + {editedChart.series.length > 1 && ( + + )} + + } + c="dark.2" + labelPosition="right" + mb={8} + /> + + setEditedChart( + produce(editedChart, draft => { + const draftSeries = draft.series[i]; + if (draftSeries.type === CHART_TYPE) { + draftSeries.aggFn = aggFn; + } + }), + ) + } + setWhere={where => + setEditedChart( + produce(editedChart, draft => { + const draftSeries = draft.series[i]; + if (draftSeries.type === CHART_TYPE) { + draftSeries.where = where; + } + }), + ) + } + setGroupBy={ + editedChart.series.length === 1 + ? groupBy => + setEditedChart( + produce(editedChart, draft => { + const draftSeries = draft.series[i]; + if (draftSeries.type === CHART_TYPE) { + if (groupBy != undefined) { + draftSeries.groupBy[0] = groupBy; + } else { + draftSeries.groupBy = []; + } + } + }), + ) + : undefined + } + setField={field => + setEditedChart( + produce(editedChart, draft => { + const draftSeries = draft.series[i]; + if (draftSeries.type === CHART_TYPE) { + draftSeries.field = field; + } + }), + ) + } + setTableAndAggFn={ + editedChart.series.length === 1 + ? (table, aggFn) => { + setEditedChart( + produce(editedChart, draft => { + const draftSeries = draft.series[i]; + if (draftSeries.type === CHART_TYPE) { + draftSeries.table = table; + draftSeries.aggFn = aggFn; + } + }), + ); + } + : undefined + } + setFieldAndAggFn={(field, aggFn) => { + setEditedChart( + produce(editedChart, draft => { + const draftSeries = draft.series[i]; + if (draftSeries.type === CHART_TYPE) { + draftSeries.field = field; + draftSeries.aggFn = aggFn; + } + }), + ); + }} + /> +
+ ); + })} + + {editedChart.series.length > 1 && ( + +
Group By
+
+ (s as TimeChartSeries).field) + .filter(f => f != null) as string[] + } + setGroupBy={groupBy => { + setEditedChart( + produce(editedChart, draft => { + draft.series.forEach((series, i) => { + if (series.type === CHART_TYPE) { + if (groupBy != undefined) { + series.groupBy[0] = groupBy; + } else { + series.groupBy = []; + } + } + }); + }), + ); + }} + /> +
+ + )} + + + {editedChart.series.length === 1 && ( + + )} + + {editedChart.series.length == 2 && ( + + setEditedChart( + produce(editedChart, draft => { + draft.seriesReturnType = event.currentTarget.checked + ? 'ratio' + : 'column'; + }), + ) + } + /> + )} + + { + setEditedChart( + produce(editedChart, draft => { + draft.series.forEach((series, i) => { + if (series.type === CHART_TYPE) { + series.numberFormat = numberFormat; + } + }); + }), + ); + }} + /> + + + ); +}; + export const EditLineChartForm = ({ isLocalDashboard, chart, @@ -900,6 +1067,8 @@ export const EditLineChartForm = ({ : convertDateRangeToGranularityString(dateRange, 60), dateRange, numberFormat: editedChart.series[0].numberFormat, + series: editedChart.series, + seriesReturnType: editedChart.seriesReturnType, } : null, [editedChart, alertEnabled, editedAlert?.interval, dateRange], @@ -944,95 +1113,12 @@ export const EditLineChartForm = ({ placeholder="Chart Name" />
- - setEditedChart( - produce(editedChart, draft => { - if (draft.series[0].type === CHART_TYPE) { - draft.series[0].table = table; - } - }), - ) - } - setAggFn={aggFn => - setEditedChart( - produce(editedChart, draft => { - if (draft.series[0].type === CHART_TYPE) { - draft.series[0].aggFn = aggFn; - } - }), - ) - } - setWhere={where => - setEditedChart( - produce(editedChart, draft => { - if (draft.series[0].type === CHART_TYPE) { - draft.series[0].where = where; - } - }), - ) - } - setGroupBy={groupBy => - setEditedChart( - produce(editedChart, draft => { - if (draft.series[0].type === CHART_TYPE) { - if (groupBy != undefined) { - draft.series[0].groupBy[0] = groupBy; - } else { - draft.series[0].groupBy = []; - } - } - }), - ) - } - setField={field => - setEditedChart( - produce(editedChart, draft => { - if (draft.series[0].type === CHART_TYPE) { - draft.series[0].field = field; - } - }), - ) - } - setTableAndAggFn={(table, aggFn) => { - setEditedChart( - produce(editedChart, draft => { - if (draft.series[0].type === CHART_TYPE) { - draft.series[0].table = table; - draft.series[0].aggFn = aggFn; - } - }), - ); - }} - setFieldAndAggFn={(field, aggFn) => { - setEditedChart( - produce(editedChart, draft => { - if (draft.series[0].type === CHART_TYPE) { - draft.series[0].field = field; - draft.series[0].aggFn = aggFn; - } - }), - ); - }} - setNumberFormat={numberFormat => - setEditedChart( - produce(editedChart, draft => { - if (draft.series[0].type === CHART_TYPE) { - draft.series[0].numberFormat = numberFormat; - } - }), - ) - } + {isChartAlertsFeatureEnabled && ( - + {isLocalDashboard ? ( Alerts are not available in unsaved dashboards. @@ -1060,22 +1146,22 @@ export const EditLineChartForm = ({ )} -
- - +
Chart Preview
- string; + onSortClick?: (columnNumber: number) => void; +}) => { + //we need a reference to the scrolling element for logic down below + const tableContainerRef = useRef(null); + + const tableWidth = tableContainerRef.current?.clientWidth; + const numColumns = columns.length + 1; + const reactTableColumns: ColumnDef[] = [ + { + accessorKey: 'group', + header: groupColumnName, + size: tableWidth != null ? tableWidth / numColumns : 200, + }, + ...columns.map(({ dataKey, displayName }, i) => ({ + accessorKey: dataKey, + header: displayName, + accessorFn: (row: any) => row[dataKey], + cell: ({ + getValue, + row, + }: { + getValue: Getter; + row: Row; + }) => { + const value = getValue(); + let formattedValue: string | number | null = value ?? null; + if (numberFormat) { + formattedValue = formatNumber(value, numberFormat); + } + if (getRowSearchLink == null) { + return formattedValue; + } + + return ( + + + {formattedValue} + + + ); + }, + size: + i === columns.length - 1 + ? UNDEFINED_WIDTH + : tableWidth != null + ? tableWidth / numColumns + : 200, + enableResizing: i !== columns.length - 1, + })), + ]; + + const table = useReactTable({ + data, + columns: reactTableColumns, + getCoreRowModel: getCoreRowModel(), + enableColumnResizing: true, + columnResizeMode: 'onChange', + }); + + const { rows } = table.getRowModel(); + + const rowVirtualizer = useVirtualizer({ + count: rows.length, + getScrollElement: () => tableContainerRef.current, + estimateSize: useCallback(() => 58, []), + overscan: 10, + paddingEnd: 20, + }); + + const items = rowVirtualizer.getVirtualItems(); + + const [paddingTop, paddingBottom] = + items.length > 0 + ? [ + Math.max(0, items[0].start - rowVirtualizer.options.scrollMargin), + Math.max( + 0, + rowVirtualizer.getTotalSize() - items[items.length - 1].end, + ), + ] + : [0, 0]; + + return ( +
+ + + {table.getHeaderGroups().map(headerGroup => ( + + {headerGroup.headers.map((header, headerIndex) => { + const sortOrder = columns[headerIndex - 1]?.sortOrder; + return ( + + ); + })} + + ))} + + + {paddingTop > 0 && ( + + + )} + {items.map(virtualRow => { + const row = rows[virtualRow.index] as TableRow; + return ( + + {row.getVisibleCells().map(cell => { + return ( + + ); + })} + + ); + })} + {paddingBottom > 0 && ( + + + )} + +
+ {header.isPlaceholder ? null : ( + +
+ {flexRender( + header.column.columnDef.header, + header.getContext(), + )} +
+ + {headerIndex > 0 && onSortClick != null && ( +
onSortClick(headerIndex - 1)} + > + {sortOrder === 'asc' ? ( + + + + ) : sortOrder === 'desc' ? ( + + + + ) : ( + + + + )} +
+ )} + {header.column.getCanResize() && ( +
+ +
+ )} + + + )} +
+
+ {getRowSearchLink == null ? ( +
+ {flexRender( + cell.column.columnDef.cell, + cell.getContext(), + )} +
+ ) : ( + + + {flexRender( + cell.column.columnDef.cell, + cell.getContext(), + )} + + + )} +
+
+
+ ); +}; + +const HDXMultiSeriesTableChart = memo( + ({ + config: { + series, + seriesReturnType = 'column', + // numberFormat, + dateRange, + sortOrder, + numberFormat, + }, + onSettled, + onSortClick, + }: { + config: { + series: ChartSeries[]; + granularity: Granularity; + dateRange: [Date, Date]; + seriesReturnType: 'ratio' | 'column'; + sortOrder: 'asc' | 'desc'; + numberFormat?: NumberFormat; + }; + onSettled?: () => void; + onSortClick?: (seriesIndex: number) => void; + }) => { + const { data, isError, isLoading } = api.useMultiSeriesChart({ + series, + endDate: dateRange[1] ?? new Date(), + startDate: dateRange[0] ?? new Date(), + seriesReturnType, + }); + + const seriesMeta = seriesColumns({ + series, + seriesReturnType, + }); + + const getRowSearchLink = useCallback( + (row: { group: string }) => { + return `/search?${seriesToUrlSearchQueryParam({ + series, + groupByValue: row.group ? `"${row.group}"` : undefined, + dateRange, + })}`; + }, + [series, dateRange], + ); + + return isLoading ? ( +
+ Loading Chart Data... +
+ ) : isError ? ( +
+ Error loading chart, please try again or contact support. +
+ ) : data?.data?.length === 0 ? ( +
+ No data found within time range. +
+ ) : ( +
+ + + ); + }, +); + +export default HDXMultiSeriesTableChart; diff --git a/packages/app/src/HDXMultiSeriesTimeChart.tsx b/packages/app/src/HDXMultiSeriesTimeChart.tsx new file mode 100644 index 00000000..09d0ffcc --- /dev/null +++ b/packages/app/src/HDXMultiSeriesTimeChart.tsx @@ -0,0 +1,508 @@ +import { memo, useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import Link from 'next/link'; +import cx from 'classnames'; +import { add, format } from 'date-fns'; +import pick from 'lodash/pick'; +import { + Bar, + BarChart, + Label, + Legend, + Line, + LineChart, + ReferenceArea, + ReferenceLine, + ResponsiveContainer, + Tooltip, + XAxis, + YAxis, +} from 'recharts'; + +import api from './api'; +import { + convertGranularityToSeconds, + Granularity, + seriesColumns, + seriesToUrlSearchQueryParam, +} from './ChartUtils'; +import type { ChartSeries, NumberFormat } from './types'; +import useUserPreferences, { TimeFormat } from './useUserPreferences'; +import { formatNumber } from './utils'; +import { semanticKeyedColor, TIME_TOKENS, truncateMiddle } from './utils'; + +function ExpandableLegendItem({ value, entry }: any) { + const [expanded, setExpanded] = useState(false); + const { color } = entry; + + return ( + + setExpanded(v => !v)} + title="Click to expand" + > + {expanded ? value : truncateMiddle(`${value}`, 45)} + + + ); +} + +const legendFormatter = (value: string, entry: any) => ( + +); + +const MemoChart = memo(function MemoChart({ + graphResults, + setIsClickActive, + isClickActive, + dateRange, + groupKeys, + lineNames, + alertThreshold, + alertThresholdType, + displayType = 'line', + numberFormat, +}: { + graphResults: any[]; + setIsClickActive: (v: any) => void; + isClickActive: any; + dateRange: [Date, Date]; + groupKeys: string[]; + lineNames: string[]; + alertThreshold?: number; + alertThresholdType?: 'above' | 'below'; + displayType?: 'stacked_bar' | 'line'; + numberFormat?: NumberFormat; +}) { + const ChartComponent = displayType === 'stacked_bar' ? BarChart : LineChart; + + const lines = useMemo(() => { + return groupKeys.map((key, i) => + displayType === 'stacked_bar' ? ( + + ) : ( + + ), + ); + }, [groupKeys, displayType, lineNames]); + + const sizeRef = useRef<[number, number]>([0, 0]); + const timeFormat: TimeFormat = useUserPreferences().timeFormat; + const tsFormat = TIME_TOKENS[timeFormat]; + // Gets the preffered time format from User Preferences, then converts it to a formattable token + + const tickFormatter = useCallback( + (value: number) => + numberFormat + ? formatNumber(value, { + ...numberFormat, + average: true, + mantissa: 0, + unit: undefined, + }) + : new Intl.NumberFormat('en-US', { + notation: 'compact', + compactDisplay: 'short', + }).format(value), + [numberFormat], + ); + + return ( + { + sizeRef.current = [width ?? 1, height ?? 1]; + }} + > + { + if ( + state != null && + state.chartX != null && + state.chartY != null && + state.activeLabel != null + ) { + setIsClickActive({ + x: state.chartX, + y: state.chartY, + activeLabel: state.activeLabel, + xPerc: state.chartX / sizeRef.current[0], + yPerc: state.chartY / sizeRef.current[1], + }); + } else { + // We clicked on the chart but outside of a line + setIsClickActive(undefined); + } + + // TODO: Properly detect clicks outside of the fake tooltip + e.stopPropagation(); + }} + > + format(new Date(tick * 1000), tsFormat)} + minTickGap={50} + tick={{ fontSize: 12, fontFamily: 'IBM Plex Mono, monospace' }} + /> + + {lines} + } + /> + {alertThreshold != null && alertThresholdType === 'below' && ( + + )} + {alertThreshold != null && alertThresholdType === 'above' && ( + + )} + {alertThreshold != null && ( + } + stroke="red" + strokeDasharray="3 3" + /> + )} + + {/** Needs to be at the bottom to prevent re-rendering */} + {isClickActive != null ? ( + + ) : null} + + + ); +}); + +const HDXLineChartTooltip = (props: any) => { + const timeFormat: TimeFormat = useUserPreferences().timeFormat; + const tsFormat = TIME_TOKENS[timeFormat]; + const { active, payload, label, numberFormat } = props; + if (active && payload && payload.length) { + return ( +
+
{format(new Date(label * 1000), tsFormat)}
+ {payload + .sort((a: any, b: any) => b.value - a.value) + .map((p: any) => ( +
+ {p.name ?? p.dataKey}:{' '} + {numberFormat ? formatNumber(p.value, numberFormat) : p.value} +
+ ))} +
+ ); + } + return null; +}; + +const HDXMultiSeriesLineChart = memo( + ({ + config: { series, granularity, dateRange, seriesReturnType = 'column' }, + onSettled, + alertThreshold, + alertThresholdType, + }: { + config: { + series: ChartSeries[]; + granularity: Granularity; + dateRange: [Date, Date]; + seriesReturnType: 'ratio' | 'column'; + }; + onSettled?: () => void; + alertThreshold?: number; + alertThresholdType?: 'above' | 'below'; + }) => { + const { data, isError, isLoading } = api.useMultiSeriesChart({ + series, + granularity, + endDate: dateRange[1] ?? new Date(), + startDate: dateRange[0] ?? new Date(), + seriesReturnType, + }); + + const tsBucketMap = new Map(); + let graphResults: { + ts_bucket: number; + [key: string]: number | undefined; + }[] = []; + + // TODO: FIX THIS COUNTER + let totalGroups = 0; + const groupSet = new Set(); // to count how many unique groups there were + + const lineDataMap: { + [seriesGroup: string]: { + dataKey: string; + displayName: string; + }; + } = {}; + + const seriesMeta = seriesColumns({ + series, + seriesReturnType, + }); + + // Each row of data will contain the ts_bucket, group name + // and a data value per series, we just need to turn them all into keys + if (data != null) { + for (const row of data.data) { + groupSet.add(row.group); + + const tsBucket = tsBucketMap.get(row.ts_bucket) ?? {}; + tsBucketMap.set(row.ts_bucket, { + ...tsBucket, + ts_bucket: row.ts_bucket, + ...seriesMeta.reduce((acc, meta, i) => { + // We set an arbitrary data key that is unique + // per series/group + const dataKey = `series_${i}.data:::${row.group}`; + + const displayName = + series.length === 1 + ? // If there's only one series, just show the group, unless there is no group + row.group + ? `${row.group}` + : meta.displayName + : // Otherwise, show the series and a group if there is any + `${row.group ? `${row.group} • ` : ''}${meta.displayName}`; + + acc[dataKey] = row[meta.dataKey]; + lineDataMap[dataKey] = { + dataKey, + displayName, + }; + return acc; + }, {} as any), + }); + } + graphResults = Array.from(tsBucketMap.values()).sort( + (a, b) => a.ts_bucket - b.ts_bucket, + ); + totalGroups = groupSet.size; + } + + const groupKeys = Object.values(lineDataMap).map(s => s.dataKey); + const lineNames = Object.values(lineDataMap).map(s => s.displayName); + + const [activeClickPayload, setActiveClickPayload] = useState< + | { + x: number; + y: number; + activeLabel: string; + xPerc: number; + yPerc: number; + } + | undefined + >(undefined); + + useEffect(() => { + const onClickHandler = () => { + if (activeClickPayload) { + setActiveClickPayload(undefined); + } + }; + document.addEventListener('click', onClickHandler); + return () => document.removeEventListener('click', onClickHandler); + }, [activeClickPayload]); + + const clickedActiveLabelDate = + activeClickPayload?.activeLabel != null + ? new Date(Number.parseInt(activeClickPayload.activeLabel) * 1000) + : undefined; + + let qparams: URLSearchParams | undefined; + + if (clickedActiveLabelDate != null) { + const to = add(clickedActiveLabelDate, { + seconds: convertGranularityToSeconds(granularity), + }); + qparams = seriesToUrlSearchQueryParam({ + series, + dateRange: [clickedActiveLabelDate, to], + }); + } + + const numberFormat = + series[0].type === 'time' ? series[0]?.numberFormat : undefined; + + const [displayType, setDisplayType] = useState<'stacked_bar' | 'line'>( + 'line', + ); + + return isLoading ? ( +
+ Loading Chart Data... +
+ ) : isError ? ( +
+ Error loading chart, please try again or contact support. +
+ ) : graphResults.length === 0 ? ( +
+ No data found within time range. +
+ ) : ( +
+
+ {activeClickPayload != null && clickedActiveLabelDate != null ? ( +
0.5 + ? (activeClickPayload?.x ?? 0) - 130 + : (activeClickPayload?.x ?? 0) + 4 + }px, ${activeClickPayload?.y ?? 0}px)`, + }} + > + + + View Events + + +
+ ) : null} + {totalGroups > groupKeys.length ? ( +
+ + Only top{' '} + {groupKeys.length} groups shown + +
+ ) : null} +
+ setDisplayType('line')} + > + + + setDisplayType('stacked_bar')} + > + + +
+ +
+
+ ); + }, +); + +export default HDXMultiSeriesLineChart; diff --git a/packages/app/src/HDXNumberChart.tsx b/packages/app/src/HDXNumberChart.tsx index f94e5ee2..d20635db 100644 --- a/packages/app/src/HDXNumberChart.tsx +++ b/packages/app/src/HDXNumberChart.tsx @@ -1,8 +1,7 @@ import { memo } from 'react'; import api from './api'; -import { AggFn } from './ChartUtils'; -import { NumberFormat } from './types'; +import type { AggFn, NumberFormat } from './types'; import { formatNumber } from './utils'; const HDXNumberChart = memo( diff --git a/packages/app/src/HDXTableChart.tsx b/packages/app/src/HDXTableChart.tsx index f1c439ad..1e46aa4f 100644 --- a/packages/app/src/HDXTableChart.tsx +++ b/packages/app/src/HDXTableChart.tsx @@ -11,9 +11,9 @@ import { ColumnDef } from '@tanstack/react-table'; import { useVirtualizer } from '@tanstack/react-virtual'; import api from './api'; -import { AggFn } from './ChartUtils'; import { UNDEFINED_WIDTH } from './tableUtils'; import type { NumberFormat } from './types'; +import { AggFn } from './types'; import { formatNumber } from './utils'; const Table = ({ data, diff --git a/packages/app/src/api.ts b/packages/app/src/api.ts index ddc4aaa8..afdfd643 100644 --- a/packages/app/src/api.ts +++ b/packages/app/src/api.ts @@ -15,6 +15,7 @@ import type { AlertInterval, AlertSource, AlertType, + ChartSeries, LogView, Session, } from './types'; @@ -150,6 +151,65 @@ const api = { ...options, }); }, + useMultiSeriesChart( + { + startDate, + series, + sortOrder, + granularity, + endDate, + seriesReturnType, + }: { + series: ChartSeries[]; + endDate: Date; + granularity?: string; + startDate: Date; + sortOrder?: 'asc' | 'desc'; + seriesReturnType: 'column' | 'ratio'; + }, + options?: UseQueryOptions, + ) { + const enrichedSeries = series.map(s => { + if (s.type != 'search' && s.type != 'markdown' && s.table === 'metrics') { + const [metricName, metricDataType] = (s.field ?? '').split(' - '); + return { + ...s, + field: metricName, + metricDataType, + }; + } + + return s; + }); + const startTime = startDate.getTime(); + const endTime = endDate.getTime(); + return useQuery({ + refetchOnWindowFocus: false, + queryKey: [ + 'chart/series', + enrichedSeries, + endTime, + granularity, + startTime, + sortOrder, + seriesReturnType, + ], + queryFn: () => + server('chart/series', { + method: 'POST', + json: { + series: enrichedSeries, + endTime, + startTime, + granularity, + sortOrder, + seriesReturnType, + }, + }).json(), + retry: 1, + ...options, + }); + }, useLogsChart( { aggFn, diff --git a/packages/app/src/types.ts b/packages/app/src/types.ts index 05306324..cf42b09f 100644 --- a/packages/app/src/types.ts +++ b/packages/app/src/types.ts @@ -159,3 +159,84 @@ export type NumberFormat = { currencySymbol?: string; unit?: string; }; + +export type AggFn = + | 'avg_rate' + | 'avg' + | 'count_distinct' + | 'count' + | 'max_rate' + | 'max' + | 'min_rate' + | 'min' + | 'p50_rate' + | 'p50' + | 'p90_rate' + | 'p90' + | 'p95_rate' + | 'p95' + | 'p99_rate' + | 'p99' + | 'sum_rate' + | 'sum'; + +export type SourceTable = 'logs' | 'rrweb' | 'metrics'; + +export type TimeChartSeries = { + table: SourceTable; + type: 'time'; + aggFn: AggFn; // TODO: Type + field?: string | undefined; + where: string; + groupBy: string[]; + numberFormat?: NumberFormat; +}; + +export type TableChartSeries = { + type: 'table'; + table: SourceTable; + aggFn: AggFn; + field: string | undefined; + where: string; + groupBy: string[]; + sortOrder?: 'desc' | 'asc'; + numberFormat?: NumberFormat; +}; + +export type ChartSeries = + | TimeChartSeries + | TableChartSeries + | { + table: SourceTable; + type: 'histogram'; + field: string | undefined; + where: string; + } + | { + type: 'search'; + fields: string[]; + where: string; + } + | { + type: 'number'; + table: SourceTable; + aggFn: AggFn; + field: string | undefined; + where: string; + numberFormat?: NumberFormat; + } + | { + type: 'markdown'; + content: string; + }; + +export type Chart = { + id: string; + name: string; + x: number; + y: number; + w: number; + h: number; + series: ChartSeries[]; + seriesReturnType: 'ratio' | 'column'; +}; diff --git a/packages/app/styles/app.scss b/packages/app/styles/app.scss index 45cd8a53..2652b2b0 100644 --- a/packages/app/styles/app.scss +++ b/packages/app/styles/app.scss @@ -630,6 +630,17 @@ div.react-datepicker { } .ds-select { + &.w-auto { + .ds-react-select__menu { + width: auto; + } + } + &.text-nowrap { + .ds-react-select__option { + text-wrap: nowrap; + } + } + &.input-bg .ds-react-select__control { background: $input-bg; }