mirror of
https://github.com/hyperdxio/hyperdx
synced 2026-04-21 13:37:15 +00:00
Minor fixes in the external API (#1744)
This commit is contained in:
Himanshu Kapoor
GitHub
parent
fbeaf15202
commit
9ab68432af
4 changed files with 195 additions and 83 deletions
5
.changeset/tasty-rabbits-peel.md
Normal file
5
.changeset/tasty-rabbits-peel.md
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
"@hyperdx/api": patch
|
||||
---
|
||||
|
||||
Minor fixes in the sources external API: 1. avoid inline schemas, 2. use short format timestamps for materializedView.minGranularity
|
||||
|
|
@ -1224,6 +1224,49 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"MetricSourceFrom": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"databaseName"
|
||||
],
|
||||
"properties": {
|
||||
"databaseName": {
|
||||
"type": "string",
|
||||
"description": "ClickHouse database name"
|
||||
},
|
||||
"tableName": {
|
||||
"type": "string",
|
||||
"description": "ClickHouse table name",
|
||||
"nullable": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"MetricTables": {
|
||||
"type": "object",
|
||||
"description": "Mapping of metric data types to table names. At least one must be specified.",
|
||||
"properties": {
|
||||
"gauge": {
|
||||
"type": "string",
|
||||
"description": "Table containing gauge metrics data"
|
||||
},
|
||||
"histogram": {
|
||||
"type": "string",
|
||||
"description": "Table containing histogram metrics data"
|
||||
},
|
||||
"sum": {
|
||||
"type": "string",
|
||||
"description": "Table containing sum metrics data"
|
||||
},
|
||||
"summary": {
|
||||
"type": "string",
|
||||
"description": "Table containing summary metrics data. Note - not yet fully supported by HyperDX"
|
||||
},
|
||||
"exponential histogram": {
|
||||
"type": "string",
|
||||
"description": "Table containing exponential histogram metrics data. Note - not yet fully supported by HyperDX"
|
||||
}
|
||||
}
|
||||
},
|
||||
"HighlightedAttributeExpression": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
|
|
@ -1295,21 +1338,21 @@
|
|||
"type": "string",
|
||||
"description": "The granularity of the timestamp column",
|
||||
"enum": [
|
||||
"1 second",
|
||||
"15 second",
|
||||
"30 second",
|
||||
"1 minute",
|
||||
"5 minute",
|
||||
"15 minute",
|
||||
"30 minute",
|
||||
"1 hour",
|
||||
"2 hour",
|
||||
"6 hour",
|
||||
"12 hour",
|
||||
"1 day",
|
||||
"2 day",
|
||||
"7 day",
|
||||
"30 day"
|
||||
"1s",
|
||||
"15s",
|
||||
"30s",
|
||||
"1m",
|
||||
"5m",
|
||||
"15m",
|
||||
"30m",
|
||||
"1h",
|
||||
"2h",
|
||||
"6h",
|
||||
"12h",
|
||||
"1d",
|
||||
"2d",
|
||||
"7d",
|
||||
"30d"
|
||||
]
|
||||
},
|
||||
"minDate": {
|
||||
|
|
@ -1636,19 +1679,7 @@
|
|||
"type": "string"
|
||||
},
|
||||
"from": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"databaseName"
|
||||
],
|
||||
"properties": {
|
||||
"databaseName": {
|
||||
"type": "string"
|
||||
},
|
||||
"tableName": {
|
||||
"type": "string",
|
||||
"nullable": true
|
||||
}
|
||||
}
|
||||
"$ref": "#/components/schemas/MetricSourceFrom"
|
||||
},
|
||||
"querySettings": {
|
||||
"type": "array",
|
||||
|
|
@ -1658,30 +1689,7 @@
|
|||
"nullable": true
|
||||
},
|
||||
"metricTables": {
|
||||
"type": "object",
|
||||
"description": "Mapping of metric data types to table names. At least one must be specified.",
|
||||
"properties": {
|
||||
"gauge": {
|
||||
"type": "string",
|
||||
"description": "Table containing gauge metrics data"
|
||||
},
|
||||
"histogram": {
|
||||
"type": "string",
|
||||
"description": "Table containing histogram metrics data"
|
||||
},
|
||||
"sum": {
|
||||
"type": "string",
|
||||
"description": "Table containing sum metrics data"
|
||||
},
|
||||
"summary": {
|
||||
"type": "string",
|
||||
"description": "Table containing summary metrics data. Note - not yet fully supported by HyperDX"
|
||||
},
|
||||
"exponential histogram": {
|
||||
"type": "string",
|
||||
"description": "Table containing exponential histogram metrics data. Note - not yet fully supported by HyperDX"
|
||||
}
|
||||
}
|
||||
"$ref": "#/components/schemas/MetricTables"
|
||||
},
|
||||
"timestampValueExpression": {
|
||||
"type": "string",
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import {
|
|||
} from '../../../fixtures';
|
||||
import Connection, { IConnection } from '../../../models/connection';
|
||||
import { Source } from '../../../models/source';
|
||||
import { mapGranularityToExternalFormat } from '../v2/sources';
|
||||
|
||||
describe('External API v2 Sources', () => {
|
||||
const server = getServer();
|
||||
|
|
@ -152,6 +153,19 @@ describe('External API v2 Sources', () => {
|
|||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
databaseName: DEFAULT_DATABASE,
|
||||
tableName: 'traces_mv_15s',
|
||||
dimensionColumns: 'ServiceName',
|
||||
minGranularity: '15 second',
|
||||
timestampColumn: 'Timestamp',
|
||||
aggregatedColumns: [
|
||||
{
|
||||
mvColumn: 'count',
|
||||
aggFn: 'count',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
querySettings: [
|
||||
{
|
||||
|
|
@ -200,7 +214,20 @@ describe('External API v2 Sources', () => {
|
|||
databaseName: DEFAULT_DATABASE,
|
||||
tableName: 'traces_mv',
|
||||
dimensionColumns: 'ServiceName',
|
||||
minGranularity: '1 minute',
|
||||
minGranularity: '1m',
|
||||
timestampColumn: 'Timestamp',
|
||||
aggregatedColumns: [
|
||||
{
|
||||
mvColumn: 'count',
|
||||
aggFn: 'count',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
databaseName: DEFAULT_DATABASE,
|
||||
tableName: 'traces_mv_15s',
|
||||
dimensionColumns: 'ServiceName',
|
||||
minGranularity: '15s',
|
||||
timestampColumn: 'Timestamp',
|
||||
aggregatedColumns: [
|
||||
{
|
||||
|
|
@ -491,3 +518,32 @@ describe('External API v2 Sources', () => {
|
|||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('External API v2 Sources Mapping', () => {
|
||||
describe('mapGranularityToExternalFormat', () => {
|
||||
it.each`
|
||||
input | expected
|
||||
${'1 second'} | ${'1s'}
|
||||
${'1 minute'} | ${'1m'}
|
||||
${'1 hour'} | ${'1h'}
|
||||
${'1 day'} | ${'1d'}
|
||||
`(
|
||||
'maps supported long-form granularity $input to $expected',
|
||||
({ input, expected }) => {
|
||||
expect(mapGranularityToExternalFormat(input)).toBe(expected);
|
||||
},
|
||||
);
|
||||
|
||||
it.each`
|
||||
input | expected
|
||||
${'invalid'} | ${'invalid'}
|
||||
${'1m'} | ${'1m'}
|
||||
${'2 minutes'} | ${'2 minutes'}
|
||||
`(
|
||||
'passes through unsupported or already-short granularity $input',
|
||||
({ input, expected }) => {
|
||||
expect(mapGranularityToExternalFormat(input)).toBe(expected);
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,10 +1,47 @@
|
|||
import { SourceSchema } from '@hyperdx/common-utils/dist/types';
|
||||
import {
|
||||
SourceSchema,
|
||||
type TSourceUnion,
|
||||
} from '@hyperdx/common-utils/dist/types';
|
||||
import express from 'express';
|
||||
|
||||
import { getSources } from '@/controllers/sources';
|
||||
import { SourceDocument } from '@/models/source';
|
||||
import logger from '@/utils/logger';
|
||||
|
||||
export function mapGranularityToExternalFormat(granularity: string): string {
|
||||
const matches = granularity.match(/^(\d+) (second|minute|hour|day)$/);
|
||||
if (matches == null) return granularity;
|
||||
|
||||
const [, amount, unit] = matches;
|
||||
switch (unit) {
|
||||
case 'second':
|
||||
return `${amount}s`;
|
||||
case 'minute':
|
||||
return `${amount}m`;
|
||||
case 'hour':
|
||||
return `${amount}h`;
|
||||
case 'day':
|
||||
return `${amount}d`;
|
||||
default:
|
||||
return granularity;
|
||||
}
|
||||
}
|
||||
|
||||
function mapSourceToExternalSource(source: TSourceUnion): TSourceUnion {
|
||||
if (!('materializedViews' in source)) return source;
|
||||
if (!Array.isArray(source.materializedViews)) return source;
|
||||
|
||||
return {
|
||||
...source,
|
||||
materializedViews: source.materializedViews.map(view => {
|
||||
return {
|
||||
...view,
|
||||
minGranularity: mapGranularityToExternalFormat(view.minGranularity),
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function formatExternalSource(source: SourceDocument) {
|
||||
// Convert to JSON so that any ObjectIds are converted to strings
|
||||
const json = JSON.stringify(source.toJSON({ getters: true }));
|
||||
|
|
@ -12,7 +49,7 @@ function formatExternalSource(source: SourceDocument) {
|
|||
// Parse using the SourceSchema to strip out any fields not defined in the schema
|
||||
const parseResult = SourceSchema.safeParse(JSON.parse(json));
|
||||
if (parseResult.success) {
|
||||
return parseResult.data;
|
||||
return mapSourceToExternalSource(parseResult.data);
|
||||
}
|
||||
|
||||
// If parsing fails, log the error and return undefined
|
||||
|
|
@ -52,6 +89,37 @@ function formatExternalSource(source: SourceDocument) {
|
|||
* tableName:
|
||||
* type: string
|
||||
* description: ClickHouse table name
|
||||
* MetricSourceFrom:
|
||||
* type: object
|
||||
* required:
|
||||
* - databaseName
|
||||
* properties:
|
||||
* databaseName:
|
||||
* type: string
|
||||
* description: ClickHouse database name
|
||||
* tableName:
|
||||
* type: string
|
||||
* description: ClickHouse table name
|
||||
* nullable: true
|
||||
* MetricTables:
|
||||
* type: object
|
||||
* description: Mapping of metric data types to table names. At least one must be specified.
|
||||
* properties:
|
||||
* gauge:
|
||||
* type: string
|
||||
* description: Table containing gauge metrics data
|
||||
* histogram:
|
||||
* type: string
|
||||
* description: Table containing histogram metrics data
|
||||
* sum:
|
||||
* type: string
|
||||
* description: Table containing sum metrics data
|
||||
* summary:
|
||||
* type: string
|
||||
* description: Table containing summary metrics data. Note - not yet fully supported by HyperDX
|
||||
* exponential histogram:
|
||||
* type: string
|
||||
* description: Table containing exponential histogram metrics data. Note - not yet fully supported by HyperDX
|
||||
* HighlightedAttributeExpression:
|
||||
* type: object
|
||||
* required:
|
||||
|
|
@ -106,7 +174,7 @@ function formatExternalSource(source: SourceDocument) {
|
|||
* minGranularity:
|
||||
* type: string
|
||||
* description: The granularity of the timestamp column
|
||||
* enum: [1 second, 15 second, 30 second, 1 minute, 5 minute, 15 minute, 30 minute, 1 hour, 2 hour, 6 hour, 12 hour, 1 day, 2 day, 7 day, 30 day]
|
||||
* enum: [1s, 15s, 30s, 1m, 5m, 15m, 30m, 1h, 2h, 6h, 12h, 1d, 2d, 7d, 30d]
|
||||
* minDate:
|
||||
* type: string
|
||||
* format: date-time
|
||||
|
|
@ -344,39 +412,14 @@ function formatExternalSource(source: SourceDocument) {
|
|||
* connection:
|
||||
* type: string
|
||||
* from:
|
||||
* type: object
|
||||
* required:
|
||||
* - databaseName
|
||||
* properties:
|
||||
* databaseName:
|
||||
* type: string
|
||||
* tableName:
|
||||
* type: string
|
||||
* nullable: true
|
||||
* $ref: '#/components/schemas/MetricSourceFrom'
|
||||
* querySettings:
|
||||
* type: array
|
||||
* items:
|
||||
* $ref: '#/components/schemas/QuerySetting'
|
||||
* nullable: true
|
||||
* metricTables:
|
||||
* type: object
|
||||
* description: Mapping of metric data types to table names. At least one must be specified.
|
||||
* properties:
|
||||
* gauge:
|
||||
* type: string
|
||||
* description: Table containing gauge metrics data
|
||||
* histogram:
|
||||
* type: string
|
||||
* description: Table containing histogram metrics data
|
||||
* sum:
|
||||
* type: string
|
||||
* description: Table containing sum metrics data
|
||||
* summary:
|
||||
* type: string
|
||||
* description: Table containing summary metrics data. Note - not yet fully supported by HyperDX
|
||||
* exponential histogram:
|
||||
* type: string
|
||||
* description: Table containing exponential histogram metrics data. Note - not yet fully supported by HyperDX
|
||||
* $ref: '#/components/schemas/MetricTables'
|
||||
* timestampValueExpression:
|
||||
* type: string
|
||||
* description: DateTime column or expression that is part of your table's primary key.
|
||||
|
|
|
|||
Loading…
Reference in a new issue