Elgato_dark/hyperdx
1
0
Fork 0
mirror of https://github.com/hyperdxio/hyperdx synced 2026-04-21 13:37:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

feat: Add external GET /sources API (#1725)

Browse source 
HDX-3318

# Summary

This PR adds a `GET /sources` endpoint to the external API. This is intended to be a way for users to retrieve the list of available sources, so that they can use a correct source ID when creating or updating dashboards through the external API. Previously, the user had no easy way to view source IDs.

Create/Update/Delete source endpoints may be added in subsequent iterations.

There will be a related PR in control-plane to add this to the OpenAPI.

<img width="2126" height="1345" alt="Screenshot 2026-02-11 at 10 37 54 AM" src="https://github.com/user-attachments/assets/bd5ba25f-75df-495a-a25f-95b3a6a5cae2" />

```
curl --request GET \
  --url http://localhost:8000/api/v2/sources \
  --header 'authorization: Bearer <API Key>'
```
This commit is contained in:
Drew Davis 2026-02-11 11:18:01 -05:00 committed by GitHub
parent a8aa94b0d8
commit 27f478a699
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 1636 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.changeset/chilly-boxes-grab.md Normal file
View file

@ -0,0 +1,5 @@
---
"@hyperdx/api": patch
---
feat: Add external GET /sources API

636
packages/api/openapi.json
View file

@ -1106,6 +1106,597 @@
}
}
}
},
"QuerySetting": {
"type": "object",
"required": [
"setting",
"value"
],
"properties": {
"setting": {
"type": "string",
"description": "ClickHouse setting name"
},
"value": {
"type": "string",
"description": "Setting value"
}
}
},
"SourceFrom": {
"type": "object",
"required": [
"databaseName",
"tableName"
],
"properties": {
"databaseName": {
"type": "string",
"description": "ClickHouse database name"
},
"tableName": {
"type": "string",
"description": "ClickHouse table name"
}
}
},
"HighlightedAttributeExpression": {
"type": "object",
"required": [
"sqlExpression"
],
"properties": {
"sqlExpression": {
"type": "string",
"description": "SQL expression for the attribute"
},
"luceneExpression": {
"type": "string",
"description": "An optional, Lucene version of the sqlExpression expression. If provided, it is used when searching for this attribute value.",
"nullable": true
},
"alias": {
"type": "string",
"description": "Optional alias for the attribute",
"nullable": true
}
}
},
"AggregatedColumn": {
"type": "object",
"required": [
"mvColumn",
"aggFn"
],
"properties": {
"sourceColumn": {
"type": "string",
"description": "Source column name",
"nullable": true
},
"aggFn": {
"type": "string",
"description": "Aggregation function (e.g., count, sum, avg)"
},
"mvColumn": {
"type": "string",
"description": "Materialized view column name"
}
}
},
"MaterializedView": {
"type": "object",
"required": [
"databaseName",
"tableName",
"dimensionColumns",
"minGranularity",
"timestampColumn",
"aggregatedColumns"
],
"properties": {
"databaseName": {
"type": "string",
"description": "Database name for the materialized view"
},
"tableName": {
"type": "string",
"description": "Table name for the materialized view"
},
"dimensionColumns": {
"type": "string",
"description": "Columns which are not pre-aggregated in the materialized view and can be used for filtering and grouping."
},
"minGranularity": {
"type": "string",
"description": "The granularity of the timestamp column",
"enum": [
"1 second",
"15 second",
"30 second",
"1 minute",
"5 minute",
"15 minute",
"30 minute",
"1 hour",
"2 hour",
"6 hour",
"12 hour",
"1 day",
"2 day",
"7 day",
"30 day"
]
},
"minDate": {
"type": "string",
"format": "date-time",
"description": "(Optional) The earliest date and time for which the materialized view contains data. If not provided, then HyperDX will assume that the materialized view contains data for all dates for which the source table contains data.",
"nullable": true
},
"timestampColumn": {
"type": "string",
"description": "Timestamp column name"
},
"aggregatedColumns": {
"type": "array",
"description": "Columns which are pre-aggregated by the materialized view",
"items": {
"$ref": "#/components/schemas/AggregatedColumn"
}
}
}
},
"SourceKind": {
"type": "string",
"enum": [
"log",
"trace",
"session",
"metric"
],
"description": "The type of data source."
},
"LogSource": {
"type": "object",
"required": [
"id",
"name",
"kind",
"connection",
"from",
"defaultTableSelectExpression",
"timestampValueExpression"
],
"properties": {
"id": {
"type": "string"
},
"name": {
"type": "string"
},
"kind": {
"type": "string",
"enum": [
"log"
]
},
"connection": {
"type": "string"
},
"from": {
"$ref": "#/components/schemas/SourceFrom"
},
"querySettings": {
"type": "array",
"items": {
"$ref": "#/components/schemas/QuerySetting"
},
"nullable": true
},
"defaultTableSelectExpression": {
"type": "string",
"description": "Default columns selected in search results (this can be customized per search later)"
},
"timestampValueExpression": {
"type": "string",
"description": "DateTime column or expression that is part of your table's primary key."
},
"serviceNameExpression": {
"type": "string",
"nullable": true
},
"severityTextExpression": {
"type": "string",
"nullable": true
},
"bodyExpression": {
"type": "string",
"nullable": true
},
"eventAttributesExpression": {
"type": "string",
"nullable": true
},
"resourceAttributesExpression": {
"type": "string",
"nullable": true
},
"displayedTimestampValueExpression": {
"type": "string",
"description": "This DateTime column is used to display and order search results.",
"nullable": true
},
"metricSourceId": {
"type": "string",
"description": "HyperDX Source for metrics associated with logs. Optional",
"nullable": true
},
"traceSourceId": {
"type": "string",
"description": "HyperDX Source for traces associated with logs. Optional",
"nullable": true
},
"traceIdExpression": {
"type": "string",
"nullable": true
},
"spanIdExpression": {
"type": "string",
"nullable": true
},
"implicitColumnExpression": {
"type": "string",
"description": "Column used for full text search if no property is specified in a Lucene-based search. Typically the message body of a log.",
"nullable": true
},
"highlightedTraceAttributeExpressions": {
"type": "array",
"description": "Expressions defining trace-level attributes which are displayed in the trace view for the selected trace.",
"items": {
"$ref": "#/components/schemas/HighlightedAttributeExpression"
},
"nullable": true
},
"highlightedRowAttributeExpressions": {
"type": "array",
"description": "Expressions defining row-level attributes which are displayed in the row side panel for the selected row.",
"items": {
"$ref": "#/components/schemas/HighlightedAttributeExpression"
},
"nullable": true
},
"materializedViews": {
"type": "array",
"description": "Configure materialized views for query optimization. These pre-aggregated views can significantly improve query performance on aggregation queries.",
"items": {
"$ref": "#/components/schemas/MaterializedView"
},
"nullable": true
}
}
},
"TraceSource": {
"type": "object",
"required": [
"id",
"name",
"kind",
"connection",
"from",
"timestampValueExpression",
"durationExpression",
"durationPrecision",
"traceIdExpression",
"spanIdExpression",
"parentSpanIdExpression",
"spanNameExpression",
"spanKindExpression"
],
"properties": {
"id": {
"type": "string"
},
"name": {
"type": "string"
},
"kind": {
"type": "string",
"enum": [
"trace"
]
},
"connection": {
"type": "string"
},
"from": {
"$ref": "#/components/schemas/SourceFrom"
},
"querySettings": {
"type": "array",
"items": {
"$ref": "#/components/schemas/QuerySetting"
},
"nullable": true
},
"defaultTableSelectExpression": {
"type": "string",
"description": "Default columns selected in search results (this can be customized per search later)",
"nullable": true
},
"timestampValueExpression": {
"type": "string",
"description": "DateTime column or expression defines the start of the span"
},
"durationExpression": {
"type": "string"
},
"durationPrecision": {
"type": "integer",
"minimum": 0,
"maximum": 9,
"default": 3
},
"traceIdExpression": {
"type": "string"
},
"spanIdExpression": {
"type": "string"
},
"parentSpanIdExpression": {
"type": "string"
},
"spanNameExpression": {
"type": "string"
},
"spanKindExpression": {
"type": "string"
},
"logSourceId": {
"type": "string",
"description": "HyperDX Source for logs associated with traces. Optional",
"nullable": true
},
"sessionSourceId": {
"type": "string",
"description": "HyperDX Source for sessions associated with traces. Optional",
"nullable": true
},
"metricSourceId": {
"type": "string",
"description": "HyperDX Source for metrics associated with traces. Optional",
"nullable": true
},
"statusCodeExpression": {
"type": "string",
"nullable": true
},
"statusMessageExpression": {
"type": "string",
"nullable": true
},
"serviceNameExpression": {
"type": "string",
"nullable": true
},
"resourceAttributesExpression": {
"type": "string",
"nullable": true
},
"eventAttributesExpression": {
"type": "string",
"nullable": true
},
"spanEventsValueExpression": {
"type": "string",
"description": "Expression to extract span events. Used to capture events associated with spans. Expected to be Nested ( Timestamp DateTime64(9), Name LowCardinality(String), Attributes Map(LowCardinality(String), String)",
"nullable": true
},
"implicitColumnExpression": {
"type": "string",
"description": "Column used for full text search if no property is specified in a Lucene-based search. Typically the message body of a log.",
"nullable": true
},
"highlightedTraceAttributeExpressions": {
"type": "array",
"description": "Expressions defining trace-level attributes which are displayed in the trace view for the selected trace.",
"items": {
"$ref": "#/components/schemas/HighlightedAttributeExpression"
},
"nullable": true
},
"highlightedRowAttributeExpressions": {
"type": "array",
"description": "Expressions defining row-level attributes which are displayed in the row side panel for the selected row",
"items": {
"$ref": "#/components/schemas/HighlightedAttributeExpression"
},
"nullable": true
},
"materializedViews": {
"type": "array",
"description": "Configure materialized views for query optimization. These pre-aggregated views can significantly improve query performance on aggregation queries.",
"items": {
"$ref": "#/components/schemas/MaterializedView"
},
"nullable": true
}
}
},
"MetricSource": {
"type": "object",
"required": [
"id",
"name",
"kind",
"connection",
"from",
"metricTables",
"timestampValueExpression",
"resourceAttributesExpression"
],
"properties": {
"id": {
"type": "string"
},
"name": {
"type": "string"
},
"kind": {
"type": "string",
"enum": [
"metric"
]
},
"connection": {
"type": "string"
},
"from": {
"type": "object",
"required": [
"databaseName"
],
"properties": {
"databaseName": {
"type": "string"
},
"tableName": {
"type": "string",
"nullable": true
}
}
},
"querySettings": {
"type": "array",
"items": {
"$ref": "#/components/schemas/QuerySetting"
},
"nullable": true
},
"metricTables": {
"type": "object",
"description": "Mapping of metric data types to table names. At least one must be specified.",
"properties": {
"gauge": {
"type": "string",
"description": "Table containing gauge metrics data"
},
"histogram": {
"type": "string",
"description": "Table containing histogram metrics data"
},
"sum": {
"type": "string",
"description": "Table containing sum metrics data"
},
"summary": {
"type": "string",
"description": "Table containing summary metrics data. Note - not yet fully supported by HyperDX"
},
"exponential histogram": {
"type": "string",
"description": "Table containing exponential histogram metrics data. Note - not yet fully supported by HyperDX"
}
}
},
"timestampValueExpression": {
"type": "string",
"description": "DateTime column or expression that is part of your table's primary key."
},
"resourceAttributesExpression": {
"type": "string",
"description": "Column containing resource attributes for metrics"
},
"logSourceId": {
"type": "string",
"description": "HyperDX Source for logs associated with metrics. Optional",
"nullable": true
}
}
},
"SessionSource": {
"type": "object",
"required": [
"id",
"name",
"kind",
"connection",
"from",
"traceSourceId"
],
"properties": {
"id": {
"type": "string"
},
"name": {
"type": "string"
},
"kind": {
"type": "string",
"enum": [
"session"
]
},
"connection": {
"type": "string"
},
"from": {
"$ref": "#/components/schemas/SourceFrom"
},
"querySettings": {
"type": "array",
"items": {
"$ref": "#/components/schemas/QuerySetting"
},
"nullable": true
},
"timestampValueExpression": {
"type": "string",
"description": "DateTime column or expression that is part of your table's primary key.",
"nullable": true
},
"traceSourceId": {
"type": "string",
"description": "HyperDX Source for traces associated with sessions."
}
}
},
"Source": {
"oneOf": [
{
"$ref": "#/components/schemas/LogSource"
},
{
"$ref": "#/components/schemas/TraceSource"
},
{
"$ref": "#/components/schemas/MetricSource"
},
{
"$ref": "#/components/schemas/SessionSource"
}
],
"discriminator": {
"propertyName": "kind",
"mapping": {
"log": "#/components/schemas/LogSource",
"trace": "#/components/schemas/TraceSource",
"metric": "#/components/schemas/MetricSource",
"session": "#/components/schemas/SessionSource"
}
}
},
"SourcesListResponse": {
"type": "object",
"properties": {
"data": {
"type": "array",
"items": {
"$ref": "#/components/schemas/Source"
}
}
}
}
}
},
@ -2353,6 +2944,51 @@
}
}
}
},
"/api/v2/sources": {
"get": {
"summary": "List Sources",
"description": "Retrieves a list of all sources for the authenticated team",
"operationId": "listSources",
"tags": [
"Sources"
],
"responses": {
"200": {
"description": "Successfully retrieved sources",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/SourcesListResponse"
}
}
}
},
"401": {
"description": "Unauthorized",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/Error"
},
"example": {
"message": "Unauthorized access. API key is missing or invalid."
}
}
}
},
"403": {
"description": "Forbidden",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/Error"
}
}
}
}
}
}
}
}
}

493
packages/api/src/routers/external-api/__tests__/sources.test.ts Normal file
View file

@ -0,0 +1,493 @@
import { MetricsDataType, SourceKind } from '@hyperdx/common-utils/dist/types';
import mongoose from 'mongoose';
import request, { SuperAgentTest } from 'supertest';
import { ITeam } from '@/models/team';
import { IUser } from '@/models/user';
import * as config from '../../../config';
import {
DEFAULT_DATABASE,
DEFAULT_LOGS_TABLE,
getLoggedInAgent,
getServer,
} from '../../../fixtures';
import Connection, { IConnection } from '../../../models/connection';
import { Source } from '../../../models/source';
describe('External API v2 Sources', () => {
const server = getServer();
let agent: SuperAgentTest;
let team: ITeam;
let user: IUser;
let connection: IConnection;
beforeAll(async () => {
await server.start();
});
beforeEach(async () => {
const result = await getLoggedInAgent(server);
agent = result.agent;
team = result.team;
user = result.user;
connection = await Connection.create({
team: team._id,
name: 'Default',
host: config.CLICKHOUSE_HOST,
username: config.CLICKHOUSE_USER,
password: config.CLICKHOUSE_PASSWORD,
});
});
afterEach(async () => {
await server.clearDBs();
});
afterAll(async () => {
await server.stop();
});
// Helper for authenticated requests
const authRequest = (
method: 'get' | 'post' | 'put' | 'delete',
url: string,
) => {
return agent[method](url).set('Authorization', `Bearer ${user?.accessKey}`);
};
describe('GET /api/v2/sources', () => {
const BASE_URL = '/api/v2/sources';
it('should return 401 when user is not authenticated', async () => {
await request(server.getHttpServer()).get(BASE_URL).expect(401);
});
it('should return empty array when no sources exist', async () => {
const response = await authRequest('get', BASE_URL).expect(200);
expect(response.body).toHaveProperty('data');
expect(response.body.data).toEqual([]);
});
it('should return a single log source', async () => {
const logSource = await Source.create({
kind: SourceKind.Log,
team: team._id,
name: 'Test Log Source',
from: {
databaseName: DEFAULT_DATABASE,
tableName: DEFAULT_LOGS_TABLE,
},
timestampValueExpression: 'Timestamp',
defaultTableSelectExpression: '*',
connection: connection._id,
});
const response = await authRequest('get', BASE_URL).expect(200);
expect(response.body.data).toHaveLength(1);
expect(response.body.data[0]).toEqual({
id: logSource._id.toString(),
name: 'Test Log Source',
kind: SourceKind.Log,
connection: connection._id.toString(),
from: {
databaseName: DEFAULT_DATABASE,
tableName: DEFAULT_LOGS_TABLE,
},
timestampValueExpression: 'Timestamp',
defaultTableSelectExpression: '*',
highlightedTraceAttributeExpressions: [],
highlightedRowAttributeExpressions: [],
materializedViews: [],
querySettings: [],
});
});
it('should return a single trace source', async () => {
const traceSource = await Source.create({
kind: SourceKind.Trace,
team: team._id,
name: 'Test Trace Source',
from: {
databaseName: DEFAULT_DATABASE,
tableName: 'otel_traces',
},
timestampValueExpression: 'Timestamp',
durationExpression: 'Duration',
durationPrecision: 3,
traceIdExpression: 'TraceId',
spanIdExpression: 'SpanId',
parentSpanIdExpression: 'ParentSpanId',
spanNameExpression: 'SpanName',
spanKindExpression: 'SpanKind',
connection: connection._id,
highlightedTraceAttributeExpressions: [
{
sqlExpression: "ResourceAttributes['ServiceName']",
alias: 'ServiceName',
luceneExpression: 'ResourceAttributes.ServiceName',
},
],
highlightedRowAttributeExpressions: [
{
sqlExpression: 'TraceId',
alias: 'trace_id',
luceneExpression: 'TraceId',
},
],
materializedViews: [
{
databaseName: DEFAULT_DATABASE,
tableName: 'traces_mv',
dimensionColumns: 'ServiceName',
minGranularity: '1 minute',
timestampColumn: 'Timestamp',
aggregatedColumns: [
{
mvColumn: 'count',
aggFn: 'count',
},
],
},
],
querySettings: [
{
setting: 'max_execution_time',
value: '30',
},
],
});
const response = await authRequest('get', BASE_URL).expect(200);
expect(response.body.data).toHaveLength(1);
expect(response.body.data[0]).toEqual({
id: traceSource._id.toString(),
name: 'Test Trace Source',
from: {
databaseName: DEFAULT_DATABASE,
tableName: 'otel_traces',
},
kind: SourceKind.Trace,
connection: connection._id.toString(),
timestampValueExpression: 'Timestamp',
durationExpression: 'Duration',
durationPrecision: 3,
traceIdExpression: 'TraceId',
spanIdExpression: 'SpanId',
parentSpanIdExpression: 'ParentSpanId',
spanNameExpression: 'SpanName',
spanKindExpression: 'SpanKind',
highlightedTraceAttributeExpressions: [
{
sqlExpression: "ResourceAttributes['ServiceName']",
alias: 'ServiceName',
luceneExpression: 'ResourceAttributes.ServiceName',
},
],
highlightedRowAttributeExpressions: [
{
sqlExpression: 'TraceId',
alias: 'trace_id',
luceneExpression: 'TraceId',
},
],
materializedViews: [
{
databaseName: DEFAULT_DATABASE,
tableName: 'traces_mv',
dimensionColumns: 'ServiceName',
minGranularity: '1 minute',
timestampColumn: 'Timestamp',
aggregatedColumns: [
{
mvColumn: 'count',
aggFn: 'count',
},
],
},
],
querySettings: [
{
setting: 'max_execution_time',
value: '30',
},
],
});
});
it('should return a single metric source', async () => {
const metricSource = await Source.create({
kind: SourceKind.Metric,
team: team._id,
name: 'Test Metric Source',
from: {
databaseName: DEFAULT_DATABASE,
tableName: '',
},
metricTables: {
[MetricsDataType.Gauge.toLowerCase()]: 'otel_metrics_gauge',
[MetricsDataType.Sum.toLowerCase()]: 'otel_metrics_sum',
[MetricsDataType.Histogram.toLowerCase()]: 'otel_metrics_histogram',
},
timestampValueExpression: 'TimeUnix',
resourceAttributesExpression: 'ResourceAttributes',
connection: connection._id,
});
const response = await authRequest('get', BASE_URL).expect(200);
expect(response.body.data).toHaveLength(1);
expect(response.body.data[0]).toEqual({
id: metricSource._id.toString(),
name: 'Test Metric Source',
kind: SourceKind.Metric,
connection: connection._id.toString(),
from: {
databaseName: DEFAULT_DATABASE,
tableName: '',
},
metricTables: {
gauge: 'otel_metrics_gauge',
sum: 'otel_metrics_sum',
histogram: 'otel_metrics_histogram',
},
timestampValueExpression: 'TimeUnix',
resourceAttributesExpression: 'ResourceAttributes',
querySettings: [],
});
});
it('should return a single session source', async () => {
const traceSource = await Source.create({
kind: SourceKind.Trace,
team: team._id,
name: 'Trace Source for Session',
from: {
databaseName: DEFAULT_DATABASE,
tableName: 'otel_traces',
},
timestampValueExpression: 'Timestamp',
durationExpression: 'Duration',
durationPrecision: 3,
traceIdExpression: 'TraceId',
spanIdExpression: 'SpanId',
parentSpanIdExpression: 'ParentSpanId',
spanNameExpression: 'SpanName',
spanKindExpression: 'SpanKind',
connection: connection._id,
});
const sessionSource = await Source.create({
kind: SourceKind.Session,
team: team._id,
name: 'Test Session Source',
from: {
databaseName: DEFAULT_DATABASE,
tableName: 'rrweb_events',
},
traceSourceId: traceSource._id.toString(),
connection: connection._id,
});
const response = await authRequest('get', BASE_URL).expect(200);
expect(response.body.data).toHaveLength(2);
const sessionData = response.body.data.find(
(s: any) => s.kind === SourceKind.Session,
);
expect(sessionData).toEqual({
id: sessionSource._id.toString(),
name: 'Test Session Source',
kind: SourceKind.Session,
connection: connection._id.toString(),
from: {
databaseName: DEFAULT_DATABASE,
tableName: 'rrweb_events',
},
traceSourceId: traceSource._id.toString(),
querySettings: [],
});
});
it('should return multiple sources of different kinds', async () => {
const logSource = await Source.create({
kind: SourceKind.Log,
team: team._id,
name: 'Logs',
from: {
databaseName: DEFAULT_DATABASE,
tableName: DEFAULT_LOGS_TABLE,
},
timestampValueExpression: 'Timestamp',
defaultTableSelectExpression: '*',
connection: connection._id,
});
const traceSource = await Source.create({
kind: SourceKind.Trace,
team: team._id,
name: 'Traces',
from: {
databaseName: DEFAULT_DATABASE,
tableName: 'otel_traces',
},
timestampValueExpression: 'Timestamp',
durationExpression: 'Duration',
durationPrecision: 3,
traceIdExpression: 'TraceId',
spanIdExpression: 'SpanId',
parentSpanIdExpression: 'ParentSpanId',
spanNameExpression: 'SpanName',
spanKindExpression: 'SpanKind',
connection: connection._id,
});
const metricSource = await Source.create({
kind: SourceKind.Metric,
team: team._id,
name: 'Metrics',
from: {
databaseName: DEFAULT_DATABASE,
tableName: '',
},
metricTables: {
[MetricsDataType.Gauge.toLowerCase()]: 'otel_metrics_gauge',
},
timestampValueExpression: 'TimeUnix',
resourceAttributesExpression: 'ResourceAttributes',
connection: connection._id,
});
const response = await authRequest('get', BASE_URL).expect(200);
expect(response.body.data).toHaveLength(3);
const kinds = response.body.data.map((s: any) => s.kind);
expect(kinds).toContain(SourceKind.Log);
expect(kinds).toContain(SourceKind.Trace);
expect(kinds).toContain(SourceKind.Metric);
const ids = response.body.data.map((s: any) => s.id);
expect(ids).toContain(logSource._id.toString());
expect(ids).toContain(traceSource._id.toString());
expect(ids).toContain(metricSource._id.toString());
});
it("should only return sources for the authenticated user's team", async () => {
// Create a source for the current team
const currentTeamSource = await Source.create({
kind: SourceKind.Log,
team: team._id,
name: 'Current Team Source',
from: {
databaseName: DEFAULT_DATABASE,
tableName: DEFAULT_LOGS_TABLE,
},
timestampValueExpression: 'Timestamp',
defaultTableSelectExpression: '*',
connection: connection._id,
});
// Create another team and source
const otherTeamId = new mongoose.Types.ObjectId();
const otherConnection = await Connection.create({
team: otherTeamId,
name: 'Other Team Connection',
host: config.CLICKHOUSE_HOST,
username: config.CLICKHOUSE_USER,
password: config.CLICKHOUSE_PASSWORD,
});
await Source.create({
kind: SourceKind.Log,
team: otherTeamId,
name: 'Other Team Source',
from: {
databaseName: DEFAULT_DATABASE,
tableName: DEFAULT_LOGS_TABLE,
},
timestampValueExpression: 'Timestamp',
defaultTableSelectExpression: '*',
connection: otherConnection._id,
});
const response = await authRequest('get', BASE_URL).expect(200);
// Should only return the current team's source
expect(response.body.data).toHaveLength(1);
expect(response.body.data[0].id).toBe(currentTeamSource._id.toString());
expect(response.body.data[0].name).toBe('Current Team Source');
});
it('should format sources according to SourceSchema', async () => {
await Source.create({
kind: SourceKind.Log,
team: team._id,
name: 'Test Source',
from: {
databaseName: DEFAULT_DATABASE,
tableName: DEFAULT_LOGS_TABLE,
},
timestampValueExpression: 'Timestamp',
defaultTableSelectExpression: '*',
connection: connection._id,
});
const response = await authRequest('get', BASE_URL).expect(200);
expect(response.body.data).toHaveLength(1);
// Verify that MongoDB _id is converted to string id
expect(response.body.data[0]).toHaveProperty('id');
expect(response.body.data[0]).not.toHaveProperty('_id');
expect(typeof response.body.data[0].id).toBe('string');
// Verify connection ObjectId is converted to string
expect(typeof response.body.data[0].connection).toBe('string');
// Verify team field is not included (internal field)
expect(response.body.data[0]).not.toHaveProperty('team');
});
it('should filter out sources that fail schema validation', async () => {
// Create a valid source
const validSource = await Source.create({
kind: SourceKind.Log,
team: team._id,
name: 'Valid Source',
from: {
databaseName: DEFAULT_DATABASE,
tableName: DEFAULT_LOGS_TABLE,
},
timestampValueExpression: 'Timestamp',
defaultTableSelectExpression: '*',
connection: connection._id,
});
// Create an invalid source by bypassing Mongoose validation
// This simulates a source that might exist in the database but doesn't
// match the SourceSchema (e.g., due to schema evolution)
await Source.collection.insertOne({
kind: 'invalid-kind', // Invalid kind
team: team._id,
name: 'Invalid Source',
from: {
databaseName: DEFAULT_DATABASE,
tableName: DEFAULT_LOGS_TABLE,
},
connection: connection._id,
});
const response = await authRequest('get', BASE_URL).expect(200);
// Should only return the valid source
expect(response.body.data).toHaveLength(1);
expect(response.body.data[0].id).toBe(validSource._id.toString());
});
});
});

9
packages/api/src/routers/external-api/v2/index.ts
View file

@ -4,7 +4,7 @@ import { validateUserAccessKey } from '@/middleware/auth';
import alertsRouter from '@/routers/external-api/v2/alerts';
import chartsRouter from '@/routers/external-api/v2/charts';
import dashboardRouter from '@/routers/external-api/v2/dashboards';
import { Api400Error, Api403Error } from '@/utils/errors';
import sourcesRouter from '@/routers/external-api/v2/sources';
import rateLimiter from '@/utils/rateLimiter';
const router = express.Router();
@ -39,4 +39,11 @@ router.use(
dashboardRouter,
);
router.use(
'/sources',
defaultRateLimiter,
validateUserAccessKey,
sourcesRouter,
);
export default router;

494
packages/api/src/routers/external-api/v2/sources.ts Normal file
View file

@ -0,0 +1,494 @@
import { SourceSchema } from '@hyperdx/common-utils/dist/types';
import express from 'express';
import { getSources } from '@/controllers/sources';
import { SourceDocument } from '@/models/source';
import logger from '@/utils/logger';
function formatExternalSource(source: SourceDocument) {
// Convert to JSON so that any ObjectIds are converted to strings
const json = JSON.stringify(source.toJSON({ getters: true }));
// Parse using the SourceSchema to strip out any fields not defined in the schema
const parseResult = SourceSchema.safeParse(JSON.parse(json));
if (parseResult.success) {
return parseResult.data;
}
// If parsing fails, log the error and return undefined
logger.error(
{ source, error: parseResult.error },
'Failed to parse source using SourceSchema:',
);
return undefined;
}
/**
* @openapi
* components:
* schemas:
* QuerySetting:
* type: object
* required:
* - setting
* - value
* properties:
* setting:
* type: string
* description: ClickHouse setting name
* value:
* type: string
* description: Setting value
* SourceFrom:
* type: object
* required:
* - databaseName
* - tableName
* properties:
* databaseName:
* type: string
* description: ClickHouse database name
* tableName:
* type: string
* description: ClickHouse table name
* HighlightedAttributeExpression:
* type: object
* required:
* - sqlExpression
* properties:
* sqlExpression:
* type: string
* description: SQL expression for the attribute
* luceneExpression:
* type: string
* description: An optional, Lucene version of the sqlExpression expression. If provided, it is used when searching for this attribute value.
* nullable: true
* alias:
* type: string
* description: Optional alias for the attribute
* nullable: true
* AggregatedColumn:
* type: object
* required:
* - mvColumn
* - aggFn
* properties:
* sourceColumn:
* type: string
* description: Source column name
* nullable: true
* aggFn:
* type: string
* description: Aggregation function (e.g., count, sum, avg)
* mvColumn:
* type: string
* description: Materialized view column name
* MaterializedView:
* type: object
* required:
* - databaseName
* - tableName
* - dimensionColumns
* - minGranularity
* - timestampColumn
* - aggregatedColumns
* properties:
* databaseName:
* type: string
* description: Database name for the materialized view
* tableName:
* type: string
* description: Table name for the materialized view
* dimensionColumns:
* type: string
* description: Columns which are not pre-aggregated in the materialized view and can be used for filtering and grouping.
* minGranularity:
* type: string
* description: The granularity of the timestamp column
* enum: [1 second, 15 second, 30 second, 1 minute, 5 minute, 15 minute, 30 minute, 1 hour, 2 hour, 6 hour, 12 hour, 1 day, 2 day, 7 day, 30 day]
* minDate:
* type: string
* format: date-time
* description: (Optional) The earliest date and time for which the materialized view contains data. If not provided, then HyperDX will assume that the materialized view contains data for all dates for which the source table contains data.
* nullable: true
* timestampColumn:
* type: string
* description: Timestamp column name
* aggregatedColumns:
* type: array
* description: Columns which are pre-aggregated by the materialized view
* items:
* $ref: '#/components/schemas/AggregatedColumn'
* SourceKind:
* type: string
* enum: [log, trace, session, metric]
* description: The type of data source.
* LogSource:
* type: object
* required:
* - id
* - name
* - kind
* - connection
* - from
* - defaultTableSelectExpression
* - timestampValueExpression
* properties:
* id:
* type: string
* name:
* type: string
* kind:
* type: string
* enum: [log]
* connection:
* type: string
* from:
* $ref: '#/components/schemas/SourceFrom'
* querySettings:
* type: array
* items:
* $ref: '#/components/schemas/QuerySetting'
* nullable: true
* defaultTableSelectExpression:
* type: string
* description: Default columns selected in search results (this can be customized per search later)
* timestampValueExpression:
* type: string
* description: DateTime column or expression that is part of your table's primary key.
* serviceNameExpression:
* type: string
* nullable: true
* severityTextExpression:
* type: string
* nullable: true
* bodyExpression:
* type: string
* nullable: true
* eventAttributesExpression:
* type: string
* nullable: true
* resourceAttributesExpression:
* type: string
* nullable: true
* displayedTimestampValueExpression:
* type: string
* description: This DateTime column is used to display and order search results.
* nullable: true
* metricSourceId:
* type: string
* description: HyperDX Source for metrics associated with logs. Optional
* nullable: true
* traceSourceId:
* type: string
* description: HyperDX Source for traces associated with logs. Optional
* nullable: true
* traceIdExpression:
* type: string
* nullable: true
* spanIdExpression:
* type: string
* nullable: true
* implicitColumnExpression:
* type: string
* description: Column used for full text search if no property is specified in a Lucene-based search. Typically the message body of a log.
* nullable: true
* highlightedTraceAttributeExpressions:
* type: array
* description: Expressions defining trace-level attributes which are displayed in the trace view for the selected trace.
* items:
* $ref: '#/components/schemas/HighlightedAttributeExpression'
* nullable: true
* highlightedRowAttributeExpressions:
* type: array
* description: Expressions defining row-level attributes which are displayed in the row side panel for the selected row.
* items:
* $ref: '#/components/schemas/HighlightedAttributeExpression'
* nullable: true
* materializedViews:
* type: array
* description: Configure materialized views for query optimization. These pre-aggregated views can significantly improve query performance on aggregation queries.
* items:
* $ref: '#/components/schemas/MaterializedView'
* nullable: true
* TraceSource:
* type: object
* required:
* - id
* - name
* - kind
* - connection
* - from
* - timestampValueExpression
* - durationExpression
* - durationPrecision
* - traceIdExpression
* - spanIdExpression
* - parentSpanIdExpression
* - spanNameExpression
* - spanKindExpression
* properties:
* id:
* type: string
* name:
* type: string
* kind:
* type: string
* enum: [trace]
* connection:
* type: string
* from:
* $ref: '#/components/schemas/SourceFrom'
* querySettings:
* type: array
* items:
* $ref: '#/components/schemas/QuerySetting'
* nullable: true
* defaultTableSelectExpression:
* type: string
* description: Default columns selected in search results (this can be customized per search later)
* nullable: true
* timestampValueExpression:
* type: string
* description: DateTime column or expression defines the start of the span
* durationExpression:
* type: string
* durationPrecision:
* type: integer
* minimum: 0
* maximum: 9
* default: 3
* traceIdExpression:
* type: string
* spanIdExpression:
* type: string
* parentSpanIdExpression:
* type: string
* spanNameExpression:
* type: string
* spanKindExpression:
* type: string
* logSourceId:
* type: string
* description: HyperDX Source for logs associated with traces. Optional
* nullable: true
* sessionSourceId:
* type: string
* description: HyperDX Source for sessions associated with traces. Optional
* nullable: true
* metricSourceId:
* type: string
* description: HyperDX Source for metrics associated with traces. Optional
* nullable: true
* statusCodeExpression:
* type: string
* nullable: true
* statusMessageExpression:
* type: string
* nullable: true
* serviceNameExpression:
* type: string
* nullable: true
* resourceAttributesExpression:
* type: string
* nullable: true
* eventAttributesExpression:
* type: string
* nullable: true
* spanEventsValueExpression:
* type: string
* description: Expression to extract span events. Used to capture events associated with spans. Expected to be Nested ( Timestamp DateTime64(9), Name LowCardinality(String), Attributes Map(LowCardinality(String), String)
* nullable: true
* implicitColumnExpression:
* type: string
* description: Column used for full text search if no property is specified in a Lucene-based search. Typically the message body of a log.
* nullable: true
* highlightedTraceAttributeExpressions:
* type: array
* description: Expressions defining trace-level attributes which are displayed in the trace view for the selected trace.
* items:
* $ref: '#/components/schemas/HighlightedAttributeExpression'
* nullable: true
* highlightedRowAttributeExpressions:
* type: array
* description: Expressions defining row-level attributes which are displayed in the row side panel for the selected row
* items:
* $ref: '#/components/schemas/HighlightedAttributeExpression'
* nullable: true
* materializedViews:
* type: array
* description: Configure materialized views for query optimization. These pre-aggregated views can significantly improve query performance on aggregation queries.
* items:
* $ref: '#/components/schemas/MaterializedView'
* nullable: true
* MetricSource:
* type: object
* required:
* - id
* - name
* - kind
* - connection
* - from
* - metricTables
* - timestampValueExpression
* - resourceAttributesExpression
* properties:
* id:
* type: string
* name:
* type: string
* kind:
* type: string
* enum: [metric]
* connection:
* type: string
* from:
* type: object
* required:
* - databaseName
* properties:
* databaseName:
* type: string
* tableName:
* type: string
* nullable: true
* querySettings:
* type: array
* items:
* $ref: '#/components/schemas/QuerySetting'
* nullable: true
* metricTables:
* type: object
* description: Mapping of metric data types to table names. At least one must be specified.
* properties:
* gauge:
* type: string
* description: Table containing gauge metrics data
* histogram:
* type: string
* description: Table containing histogram metrics data
* sum:
* type: string
* description: Table containing sum metrics data
* summary:
* type: string
* description: Table containing summary metrics data. Note - not yet fully supported by HyperDX
* exponential histogram:
* type: string
* description: Table containing exponential histogram metrics data. Note - not yet fully supported by HyperDX
* timestampValueExpression:
* type: string
* description: DateTime column or expression that is part of your table's primary key.
* resourceAttributesExpression:
* type: string
* description: Column containing resource attributes for metrics
* logSourceId:
* type: string
* description: HyperDX Source for logs associated with metrics. Optional
* nullable: true
* SessionSource:
* type: object
* required:
* - id
* - name
* - kind
* - connection
* - from
* - traceSourceId
* properties:
* id:
* type: string
* name:
* type: string
* kind:
* type: string
* enum: [session]
* connection:
* type: string
* from:
* $ref: '#/components/schemas/SourceFrom'
* querySettings:
* type: array
* items:
* $ref: '#/components/schemas/QuerySetting'
* nullable: true
* timestampValueExpression:
* type: string
* description: DateTime column or expression that is part of your table's primary key.
* nullable: true
* traceSourceId:
* type: string
* description: HyperDX Source for traces associated with sessions.
* Source:
* oneOf:
* - $ref: '#/components/schemas/LogSource'
* - $ref: '#/components/schemas/TraceSource'
* - $ref: '#/components/schemas/MetricSource'
* - $ref: '#/components/schemas/SessionSource'
* discriminator:
* propertyName: kind
* mapping:
* log: '#/components/schemas/LogSource'
* trace: '#/components/schemas/TraceSource'
* metric: '#/components/schemas/MetricSource'
* session: '#/components/schemas/SessionSource'
* SourcesListResponse:
* type: object
* properties:
* data:
* type: array
* items:
* $ref: '#/components/schemas/Source'
*/
const router = express.Router();
/**
* @openapi
* /api/v2/sources:
* get:
* summary: List Sources
* description: Retrieves a list of all sources for the authenticated team
* operationId: listSources
* tags: [Sources]
* responses:
* '200':
* description: Successfully retrieved sources
* content:
* application/json:
* schema:
* $ref: '#/components/schemas/SourcesListResponse'
* '401':
* description: Unauthorized
* content:
* application/json:
* schema:
* $ref: '#/components/schemas/Error'
* example:
* message: "Unauthorized access. API key is missing or invalid."
* '403':
* description: Forbidden
* content:
* application/json:
* schema:
* $ref: '#/components/schemas/Error'
*/
router.get('/', async (req, res, next) => {
try {
const teamId = req.user?.team;
if (teamId == null) {
return res.sendStatus(403);
}
const sources: SourceDocument[] = await getSources(teamId.toString());
return res.json({
data: sources.map(formatExternalSource).filter(s => s !== undefined),
});
} catch (e) {
next(e);
}
});
export default router;