diff --git a/.changeset/six-ways-sell.md b/.changeset/six-ways-sell.md new file mode 100644 index 00000000..644b6916 --- /dev/null +++ b/.changeset/six-ways-sell.md @@ -0,0 +1,7 @@ +--- +"@hyperdx/common-utils": patch +"@hyperdx/api": patch +"@hyperdx/app": patch +--- + +fix: change sources to discriminated union diff --git a/packages/api/src/controllers/sources.ts b/packages/api/src/controllers/sources.ts index 8b2b20bd..e2c614e7 100644 --- a/packages/api/src/controllers/sources.ts +++ b/packages/api/src/controllers/sources.ts @@ -1,21 +1,31 @@ -import { SourceKind } from '@hyperdx/common-utils/dist/types'; +import { SourceKind, SourceSchema } from '@hyperdx/common-utils/dist/types'; -import { ISource, Source } from '@/models/source'; +import { + ISourceInput, + LogSource, + MetricSource, + SessionSource, + Source, + TraceSource, +} from '@/models/source'; -/** - * Clean up metricTables property when changing source type away from Metric. - * This prevents metric-specific configuration from persisting when switching - * to Log, Trace, or Session sources. - */ -function cleanSourceData(source: Omit): Omit { - // Only clean metricTables if the source is not a Metric type - if (source.kind !== SourceKind.Metric) { - // explicitly setting to null for mongoose to clear column - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion - source.metricTables = null as any; +// Returns the discriminator model for the given source kind. +// Updates must go through the correct discriminator model so Mongoose +// recognises kind-specific fields (e.g. metricTables on MetricSource). +function getModelForKind(kind: SourceKind) { + switch (kind) { + case SourceKind.Log: + return LogSource; + case SourceKind.Trace: + return TraceSource; + case SourceKind.Session: + return SessionSource; + case SourceKind.Metric: + return MetricSource; + default: + kind satisfies never; + throw new Error(`${kind} is not a valid SourceKind`); } - - return source; } export function getSources(team: string) { @@ -26,19 +36,56 @@ export function getSource(team: string, sourceId: string) { return Source.findOne({ _id: sourceId, team }); } -export function createSource(team: string, source: Omit) { - return Source.create({ ...source, team }); +type DistributiveOmit = T extends T + ? Omit + : never; + +export function createSource( + team: string, + source: DistributiveOmit, +) { + // @ts-expect-error The create method has incompatible type signatures but is actually safe + return getModelForKind(source.kind)?.create({ ...source, team }); } -export function updateSource( +export async function updateSource( team: string, sourceId: string, - source: Omit, + source: DistributiveOmit, ) { - const cleanedSource = cleanSourceData(source); - return Source.findOneAndUpdate({ _id: sourceId, team }, cleanedSource, { - new: true, - }); + const existing = await Source.findOne({ _id: sourceId, team }); + if (!existing) return null; + + // Same kind: simple update through the discriminator model + if (existing.kind === source.kind) { + // @ts-expect-error The findOneAndUpdate method has incompatible type signatures but is actually safe + return getModelForKind(source.kind)?.findOneAndUpdate( + { _id: sourceId, team }, + source, + { new: true }, + ); + } + + // Kind changed: validate through Zod before writing since the raw + // collection bypass skips Mongoose's discriminator validation. + const parseResult = SourceSchema.safeParse(source); + if (!parseResult.success) { + throw new Error( + `Invalid source data: ${parseResult.error.errors.map(e => e.message).join(', ')}`, + ); + } + + // Use replaceOne on the raw collection to swap the entire document + // in place (including the discriminator key). This is a single atomic + // write — the document is never absent from the collection. + const replacement = { + ...parseResult.data, + _id: existing._id, + team: existing.team, + updatedAt: new Date(), + }; + await Source.collection.replaceOne({ _id: existing._id }, replacement); + return getModelForKind(replacement.kind)?.hydrate(replacement); } export function deleteSource(team: string, sourceId: string) { diff --git a/packages/api/src/models/source.ts b/packages/api/src/models/source.ts index dbbeb8e7..1e6fb208 100644 --- a/packages/api/src/models/source.ts +++ b/packages/api/src/models/source.ts @@ -1,17 +1,41 @@ import { + BaseSourceSchema, + LogSourceSchema, MetricsDataType, + MetricSourceSchema, QuerySettings, + SessionSourceSchema, SourceKind, - TSource, + TraceSourceSchema, } from '@hyperdx/common-utils/dist/types'; import mongoose, { Schema } from 'mongoose'; +import z from 'zod'; -type ObjectId = mongoose.Types.ObjectId; +import { objectIdSchema } from '@/utils/zod'; -export interface ISource extends Omit { - team: ObjectId; - connection: ObjectId | string; -} +// ISource is a discriminated union (inherits from TSource) with team added +// and connection widened to ObjectId | string for Mongoose. +// Omit and & distribute over the union, preserving the discriminated structure. +export const ISourceSchema = z.discriminatedUnion('kind', [ + LogSourceSchema.omit({ connection: true }).extend({ + team: objectIdSchema, + connection: objectIdSchema.or(z.string()), + }), + TraceSourceSchema.omit({ connection: true }).extend({ + team: objectIdSchema, + connection: objectIdSchema.or(z.string()), + }), + SessionSourceSchema.omit({ connection: true }).extend({ + team: objectIdSchema, + connection: objectIdSchema.or(z.string()), + }), + MetricSourceSchema.omit({ connection: true }).extend({ + team: objectIdSchema, + connection: objectIdSchema.or(z.string()), + }), +]); +export type ISource = z.infer; +export type ISourceInput = z.input; export type SourceDocument = mongoose.HydratedDocument; @@ -36,91 +60,164 @@ const QuerySetting = new Schema( { _id: false }, ); -export const Source = mongoose.model( - 'Source', - new Schema( - { - kind: { - type: String, - enum: Object.values(SourceKind), - required: true, - }, - team: { - type: mongoose.Schema.Types.ObjectId, - required: true, - ref: 'Team', - }, - from: { - databaseName: String, - tableName: String, - }, - timestampValueExpression: String, - connection: { - type: mongoose.Schema.Types.ObjectId, - required: true, - ref: 'Connection', - }, +// -------------------------- +// Base schema (common fields shared by all source kinds) +// -------------------------- - name: String, - orderByExpression: String, - displayedTimestampValueExpression: String, - implicitColumnExpression: String, - serviceNameExpression: String, - bodyExpression: String, - tableFilterExpression: String, - eventAttributesExpression: String, - resourceAttributesExpression: String, - defaultTableSelectExpression: String, - uniqueRowIdExpression: String, - severityTextExpression: String, - traceIdExpression: String, - spanIdExpression: String, - traceSourceId: String, - sessionSourceId: String, - metricSourceId: String, +type MongooseSourceBase = Omit< + z.infer, + 'connection' +> & { + team: mongoose.Types.ObjectId; + connection: mongoose.Types.ObjectId; +}; - durationExpression: String, - durationPrecision: Number, - parentSpanIdExpression: String, - spanNameExpression: String, - - logSourceId: String, - spanKindExpression: String, - statusCodeExpression: String, - statusMessageExpression: String, - spanEventsValueExpression: String, - highlightedTraceAttributeExpressions: { - type: mongoose.Schema.Types.Array, - }, - highlightedRowAttributeExpressions: { - type: mongoose.Schema.Types.Array, - }, - materializedViews: { - type: mongoose.Schema.Types.Array, - }, - - metricTables: { - type: { - [MetricsDataType.Gauge]: String, - [MetricsDataType.Histogram]: String, - [MetricsDataType.Sum]: String, - [MetricsDataType.Summary]: String, - [MetricsDataType.ExponentialHistogram]: String, - }, - default: undefined, - }, - - querySettings: { - type: [QuerySetting], - validate: { - validator: maxLength(10), - message: '{PATH} exceeds the limit of 10', - }, +const sourceBaseSchema = new Schema( + { + team: { + type: mongoose.Schema.Types.ObjectId, + required: true, + ref: 'Team', + }, + connection: { + type: mongoose.Schema.Types.ObjectId, + required: true, + ref: 'Connection', + }, + name: String, + from: { + databaseName: String, + tableName: String, + }, + timestampValueExpression: String, + querySettings: { + type: [QuerySetting], + validate: { + validator: maxLength(10), + message: '{PATH} exceeds the limit of 10', }, }, - { - toJSON: { virtuals: true }, - timestamps: true, - }, - ), + }, + { + discriminatorKey: 'kind', + toJSON: { virtuals: true }, + timestamps: true, + }, +); + +// Model is typed with the base schema type internally. Consumers use ISource +// (the discriminated union) via the exported type and discriminator models. +const SourceModel = mongoose.model( + 'Source', + sourceBaseSchema, +); +// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion +export const Source = SourceModel as unknown as mongoose.Model; + +// -------------------------- +// Log discriminator +// -------------------------- +type ILogSource = Extract; +export const LogSource = Source.discriminator( + SourceKind.Log, + new Schema({ + defaultTableSelectExpression: String, + serviceNameExpression: String, + severityTextExpression: String, + bodyExpression: String, + eventAttributesExpression: String, + resourceAttributesExpression: String, + displayedTimestampValueExpression: String, + metricSourceId: String, + traceSourceId: String, + traceIdExpression: String, + spanIdExpression: String, + implicitColumnExpression: String, + uniqueRowIdExpression: String, + tableFilterExpression: String, + highlightedTraceAttributeExpressions: { + type: mongoose.Schema.Types.Array, + }, + highlightedRowAttributeExpressions: { + type: mongoose.Schema.Types.Array, + }, + materializedViews: { + type: mongoose.Schema.Types.Array, + }, + orderByExpression: String, + }), +); + +// -------------------------- +// Trace discriminator +// -------------------------- +type ITraceSource = Extract; +export const TraceSource = Source.discriminator( + SourceKind.Trace, + new Schema({ + defaultTableSelectExpression: String, + durationExpression: String, + durationPrecision: Number, + traceIdExpression: String, + spanIdExpression: String, + parentSpanIdExpression: String, + spanNameExpression: String, + spanKindExpression: String, + logSourceId: String, + sessionSourceId: String, + metricSourceId: String, + statusCodeExpression: String, + statusMessageExpression: String, + serviceNameExpression: String, + resourceAttributesExpression: String, + eventAttributesExpression: String, + spanEventsValueExpression: String, + implicitColumnExpression: String, + displayedTimestampValueExpression: String, + highlightedTraceAttributeExpressions: { + type: mongoose.Schema.Types.Array, + }, + highlightedRowAttributeExpressions: { + type: mongoose.Schema.Types.Array, + }, + materializedViews: { + type: mongoose.Schema.Types.Array, + }, + orderByExpression: String, + }), +); + +// -------------------------- +// Session discriminator +// -------------------------- +type ISessionSource = Extract; +export const SessionSource = Source.discriminator( + SourceKind.Session, + new Schema>({ + traceSourceId: String, + resourceAttributesExpression: String, + }), +); + +// -------------------------- +// Metric discriminator +// -------------------------- +type IMetricSource = Extract; +export const MetricSource = Source.discriminator( + SourceKind.Metric, + new Schema>({ + metricTables: { + type: { + [MetricsDataType.Gauge]: String, + [MetricsDataType.Histogram]: String, + [MetricsDataType.Sum]: String, + [MetricsDataType.Summary]: String, + [MetricsDataType.ExponentialHistogram]: String, + }, + default: undefined, + }, + resourceAttributesExpression: String, + serviceNameExpression: String, + logSourceId: String, + }), ); diff --git a/packages/api/src/routers/api/__tests__/dashboard.test.ts b/packages/api/src/routers/api/__tests__/dashboard.test.ts index 22321168..baaacc53 100644 --- a/packages/api/src/routers/api/__tests__/dashboard.test.ts +++ b/packages/api/src/routers/api/__tests__/dashboard.test.ts @@ -3,7 +3,7 @@ import { MetricsDataType, PresetDashboard, SourceKind, - TSourceUnion, + TSource, } from '@hyperdx/common-utils/dist/types'; import { omit } from 'lodash'; import mongoose, { Types } from 'mongoose'; @@ -418,7 +418,7 @@ describe('dashboard router', () => { }); describe('preset dashboards', () => { - const MOCK_SOURCE: Omit, 'id'> = { + const MOCK_SOURCE: Omit, 'id'> = { kind: SourceKind.Log, name: 'Test Source', connection: new Types.ObjectId().toString(), diff --git a/packages/api/src/routers/api/__tests__/sources.test.ts b/packages/api/src/routers/api/__tests__/sources.test.ts index c6151165..95bc6e2c 100644 --- a/packages/api/src/routers/api/__tests__/sources.test.ts +++ b/packages/api/src/routers/api/__tests__/sources.test.ts @@ -1,10 +1,10 @@ -import { SourceKind, TSourceUnion } from '@hyperdx/common-utils/dist/types'; +import { SourceKind, TSource } from '@hyperdx/common-utils/dist/types'; import { Types } from 'mongoose'; import { getLoggedInAgent, getServer } from '@/fixtures'; import { Source } from '@/models/source'; -const MOCK_SOURCE: Omit, 'id'> = { +const MOCK_SOURCE: Omit, 'id'> = { kind: SourceKind.Log, name: 'Test Source', connection: new Types.ObjectId().toString(), @@ -273,7 +273,7 @@ describe('sources router', () => { // Verify the metric source has metricTables const createdSource = await Source.findById(metricSource._id).lean(); - expect(createdSource?.metricTables).toBeDefined(); + expect(createdSource).toHaveProperty('metricTables'); // Update the source to a trace source const traceSource = { @@ -303,9 +303,13 @@ describe('sources router', () => { // Verify the trace source does NOT have metricTables property const updatedSource = await Source.findById(metricSource._id).lean(); - expect(updatedSource?.kind).toBe(SourceKind.Trace); - expect(updatedSource?.metricTables).toBeNull(); - expect(updatedSource?.durationExpression).toBe('Duration'); + if (updatedSource?.kind !== SourceKind.Trace) { + expect(updatedSource?.kind).toBe(SourceKind.Trace); + throw new Error('Source did not update to trace'); + } + expect(updatedSource.kind).toBe(SourceKind.Trace); + expect(updatedSource).not.toHaveProperty('metricTables'); + expect(updatedSource.durationExpression).toBe('Duration'); }); it('PUT /:id - preserves metricTables when source remains Metric, removes when changed to another type', async () => { @@ -352,8 +356,11 @@ describe('sources router', () => { let updatedSource = await Source.findById(metricSource._id).lean(); // Verify the metric source still has metricTables with updated values - expect(updatedSource?.kind).toBe(SourceKind.Metric); - expect(updatedSource?.metricTables).toMatchObject({ + if (updatedSource?.kind !== SourceKind.Metric) { + expect(updatedSource?.kind).toBe(SourceKind.Metric); + throw new Error('Source is not a metric'); + } + expect(updatedSource.metricTables).toMatchObject({ gauge: 'otel_metrics_gauge_v2', sum: 'otel_metrics_sum_v2', }); @@ -378,9 +385,12 @@ describe('sources router', () => { updatedSource = await Source.findById(metricSource._id).lean(); // Verify the source is now a Log and metricTables is removed - expect(updatedSource?.kind).toBe(SourceKind.Log); - expect(updatedSource?.metricTables).toBeNull(); - expect(updatedSource?.severityTextExpression).toBe('SeverityText'); + if (updatedSource?.kind !== SourceKind.Log) { + expect(updatedSource?.kind).toBe(SourceKind.Log); + throw new Error('Source did not update to log'); + } + expect(updatedSource).not.toHaveProperty('metricTables'); + expect(updatedSource.severityTextExpression).toBe('SeverityText'); }); it('DELETE /:id - deletes a source', async () => { @@ -407,4 +417,272 @@ describe('sources router', () => { // This will succeed even if the ID doesn't exist, consistent with the implementation await agent.delete(`/sources/${nonExistentId}`).expect(200); }); + + describe('backward compatibility with legacy flat-model documents', () => { + // These tests insert documents directly into MongoDB (bypassing Mongoose + // validation) to simulate documents created by the old flat Source model, + // which stored ALL fields from all source kinds in a single schema. + + it('reads a legacy Session source without timestampValueExpression', async () => { + const { agent, team } = await getLoggedInAgent(server); + + // Old flat model allowed Session sources without timestampValueExpression + await Source.collection.insertOne({ + kind: SourceKind.Session, + name: 'Legacy Session', + team: team._id, + connection: new Types.ObjectId(), + from: { databaseName: 'default', tableName: 'otel_sessions' }, + traceSourceId: new Types.ObjectId().toString(), + // timestampValueExpression intentionally omitted + }); + + const response = await agent.get('/sources').expect(200); + + expect(response.body).toHaveLength(1); + expect(response.body[0].kind).toBe(SourceKind.Session); + expect(response.body[0].name).toBe('Legacy Session'); + // timestampValueExpression should be absent or undefined in response + }); + + it('reads a legacy Trace source without defaultTableSelectExpression', async () => { + const { agent, team } = await getLoggedInAgent(server); + + // Old flat model allowed Trace sources without defaultTableSelectExpression + await Source.collection.insertOne({ + kind: SourceKind.Trace, + name: 'Legacy Trace', + team: team._id, + connection: new Types.ObjectId(), + from: { databaseName: 'default', tableName: 'otel_traces' }, + timestampValueExpression: 'Timestamp', + durationExpression: 'Duration', + durationPrecision: 9, + traceIdExpression: 'TraceId', + spanIdExpression: 'SpanId', + parentSpanIdExpression: 'ParentSpanId', + spanNameExpression: 'SpanName', + spanKindExpression: 'SpanKind', + // defaultTableSelectExpression intentionally omitted + }); + + const response = await agent.get('/sources').expect(200); + + expect(response.body).toHaveLength(1); + expect(response.body[0].kind).toBe(SourceKind.Trace); + expect(response.body[0].durationExpression).toBe('Duration'); + }); + + it('reads a legacy Trace source with logSourceId: null', async () => { + const { agent, team } = await getLoggedInAgent(server); + + await Source.collection.insertOne({ + kind: SourceKind.Trace, + name: 'Trace with null logSourceId', + team: team._id, + connection: new Types.ObjectId(), + from: { databaseName: 'default', tableName: 'otel_traces' }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: '*', + durationExpression: 'Duration', + durationPrecision: 3, + traceIdExpression: 'TraceId', + spanIdExpression: 'SpanId', + parentSpanIdExpression: 'ParentSpanId', + spanNameExpression: 'SpanName', + spanKindExpression: 'SpanKind', + logSourceId: null, // Old schema allowed .nullable() + }); + + const response = await agent.get('/sources').expect(200); + + expect(response.body).toHaveLength(1); + expect(response.body[0].kind).toBe(SourceKind.Trace); + // logSourceId: null should be readable (Mongoose doesn't reject it) + expect(response.body[0].logSourceId).toBeNull(); + }); + + it('cross-kind fields from legacy flat-model documents are NOT stripped on internal API read', async () => { + const { agent, team } = await getLoggedInAgent(server); + + // Old flat model stored ALL fields regardless of kind. + // NOTE: Mongoose discriminators do NOT strip unknown/cross-kind fields + // from toJSON() output. The discriminator only controls validation on + // write — unknown fields stored in MongoDB are still returned on read. + // This means the internal API response shape may differ from the external + // API (which runs SourceSchema.safeParse() to strip extra fields). + await Source.collection.insertOne({ + kind: SourceKind.Log, + name: 'Flat Model Log', + team: team._id, + connection: new Types.ObjectId(), + from: { databaseName: 'default', tableName: 'otel_logs' }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: 'Body', + bodyExpression: 'Body', + // These fields belong to other kinds but were stored in old flat model + metricTables: { gauge: 'otel_metrics_gauge' }, + durationExpression: 'Duration', + durationPrecision: 9, + sessionSourceId: 'some-session-id', + }); + + const response = await agent.get('/sources').expect(200); + + expect(response.body).toHaveLength(1); + expect(response.body[0].kind).toBe(SourceKind.Log); + expect(response.body[0].bodyExpression).toBe('Body'); + // Cross-kind fields are still present in the internal API response — + // discriminator toJSON does NOT strip them from existing documents. + expect(response.body[0]).toHaveProperty('metricTables'); + expect(response.body[0]).toHaveProperty('durationExpression'); + }); + + it('fails to update a legacy Session source without providing timestampValueExpression', async () => { + const { agent, team } = await getLoggedInAgent(server); + + const result = await Source.collection.insertOne({ + kind: SourceKind.Session, + name: 'Legacy Session', + team: team._id, + connection: new Types.ObjectId(), + from: { databaseName: 'default', tableName: 'otel_sessions' }, + traceSourceId: 'some-trace-source-id', + }); + + // PUT validation (SourceSchema) requires timestampValueExpression + await agent + .put(`/sources/${result.insertedId}`) + .send({ + kind: SourceKind.Session, + id: result.insertedId.toString(), + name: 'Updated Session', + connection: new Types.ObjectId().toString(), + from: { databaseName: 'default', tableName: 'otel_sessions' }, + traceSourceId: 'some-trace-source-id', + // timestampValueExpression intentionally omitted + }) + .expect(400); + }); + + it('successfully updates a legacy Session source when timestampValueExpression is provided', async () => { + const { agent, team } = await getLoggedInAgent(server); + + const connectionId = new Types.ObjectId(); + const result = await Source.collection.insertOne({ + kind: SourceKind.Session, + name: 'Legacy Session', + team: team._id, + connection: connectionId, + from: { databaseName: 'default', tableName: 'otel_sessions' }, + traceSourceId: 'some-trace-source-id', + }); + + await agent + .put(`/sources/${result.insertedId}`) + .send({ + kind: SourceKind.Session, + id: result.insertedId.toString(), + name: 'Updated Session', + connection: connectionId.toString(), + from: { databaseName: 'default', tableName: 'otel_sessions' }, + traceSourceId: 'some-trace-source-id', + timestampValueExpression: 'TimestampTime', + }) + .expect(200); + + const updated = await Source.findById(result.insertedId); + expect(updated?.name).toBe('Updated Session'); + expect(updated?.timestampValueExpression).toBe('TimestampTime'); + }); + + it('cross-kind fields persist in both raw MongoDB and discriminator toJSON', async () => { + const { team } = await getLoggedInAgent(server); + + // Insert a flat-model doc with cross-kind fields + const result = await Source.collection.insertOne({ + kind: SourceKind.Log, + name: 'Flat Log', + team: team._id, + connection: new Types.ObjectId(), + from: { databaseName: 'default', tableName: 'otel_logs' }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: 'Body', + metricTables: { gauge: 'otel_metrics_gauge' }, + }); + + // Raw query shows fields as stored + const rawDoc = await Source.collection.findOne({ + _id: result.insertedId, + }); + expect(rawDoc).toHaveProperty('metricTables'); + + // NOTE: Mongoose discriminator toJSON does NOT strip cross-kind fields. + // Unknown fields stored in MongoDB are still included in toJSON() output. + const hydrated = await Source.findById(result.insertedId); + // @ts-expect-error toJSON has differing type signatures depending on the source, but it's fine at runtime + const json = hydrated?.toJSON({ getters: true }); + expect(json).toHaveProperty('metricTables'); + }); + + it('Source.find() returns correctly typed discriminators for all kinds', async () => { + const { team } = await getLoggedInAgent(server); + const connectionId = new Types.ObjectId(); + + await Source.collection.insertMany([ + { + kind: SourceKind.Log, + name: 'Log', + team: team._id, + connection: connectionId, + from: { databaseName: 'default', tableName: 'otel_logs' }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: 'Body', + }, + { + kind: SourceKind.Trace, + name: 'Trace', + team: team._id, + connection: connectionId, + from: { databaseName: 'default', tableName: 'otel_traces' }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: '*', + durationExpression: 'Duration', + durationPrecision: 3, + traceIdExpression: 'TraceId', + spanIdExpression: 'SpanId', + parentSpanIdExpression: 'ParentSpanId', + spanNameExpression: 'SpanName', + spanKindExpression: 'SpanKind', + }, + { + kind: SourceKind.Session, + name: 'Session', + team: team._id, + connection: connectionId, + from: { databaseName: 'default', tableName: 'otel_sessions' }, + timestampValueExpression: 'TimestampTime', + traceSourceId: 'some-id', + }, + { + kind: SourceKind.Metric, + name: 'Metric', + team: team._id, + connection: connectionId, + from: { databaseName: 'default', tableName: '' }, + timestampValueExpression: 'TimeUnix', + resourceAttributesExpression: 'ResourceAttributes', + metricTables: { gauge: 'otel_metrics_gauge' }, + }, + ]); + + const sources = await Source.find({ team: team._id }).sort({ name: 1 }); + expect(sources).toHaveLength(4); + + expect(sources[0].kind).toBe(SourceKind.Log); + expect(sources[1].kind).toBe(SourceKind.Metric); + expect(sources[2].kind).toBe(SourceKind.Session); + expect(sources[3].kind).toBe(SourceKind.Trace); + }); + }); }); diff --git a/packages/api/src/routers/api/ai.ts b/packages/api/src/routers/api/ai.ts index 5b296ba7..2c748ccf 100644 --- a/packages/api/src/routers/api/ai.ts +++ b/packages/api/src/routers/api/ai.ts @@ -60,16 +60,28 @@ Here are some guidelines: The user is looking to do a query on their data source named: ${source.name} of type ${source.kind}. -The ${source.kind === SourceKind.Log ? 'log level' : 'span status code'} is stored in ${source.severityTextExpression}. -You can identify services via ${source.serviceNameExpression} +${ + source.kind === SourceKind.Log + ? `The log level is stored in ${source.severityTextExpression}.` + : source.kind === SourceKind.Trace + ? `The span status code is stored in ${source.statusCodeExpression}.` + : '' +} +${'serviceNameExpression' in source ? `You can identify services via ${source.serviceNameExpression}` : ''} ${ source.kind === SourceKind.Trace ? `Duration of spans can be queried via ${source.durationExpression} which is expressed in 10^-${source.durationPrecision} seconds of precision. Span names under ${source.spanNameExpression} and span kinds under ${source.spanKindExpression}` - : `The log body can be queried via ${source.bodyExpression}` + : 'bodyExpression' in source + ? `The log body can be queried via ${source.bodyExpression}` + : '' +} +${ + source.kind === SourceKind.Trace || source.kind === SourceKind.Log + ? `Various log/span-specific attributes as a Map can be found under ${source.eventAttributesExpression} while resource attributes that follow the OpenTelemetry semantic convention can be found under ${source.resourceAttributesExpression} +You must use the full field name ex. "column['key']" or "column.key" as it appears.` + : '' } -Various log/span-specific attributes as a Map can be found under ${source.eventAttributesExpression} while resource attributes that follow the OpenTelemetry semantic convention can be found under ${source.resourceAttributesExpression} -You must use the full field name ex. "column['key']" or "column.key" as it appears. The following is a list of properties and example values that exist in the source: ${JSON.stringify(keyValues)} diff --git a/packages/api/src/routers/api/sources.ts b/packages/api/src/routers/api/sources.ts index 840a1d97..7864bfad 100644 --- a/packages/api/src/routers/api/sources.ts +++ b/packages/api/src/routers/api/sources.ts @@ -1,6 +1,6 @@ import { SourceSchema, - sourceSchemaWithout, + SourceSchemaNoId, } from '@hyperdx/common-utils/dist/types'; import express from 'express'; import { z } from 'zod'; @@ -23,14 +23,17 @@ router.get('/', async (req, res, next) => { const sources = await getSources(teamId.toString()); - return res.json(sources.map(s => s.toJSON({ getters: true }))); + return res.json( + sources.map( + // @ts-expect-error source.toJSON has incompatible type signatures but is actually a safe operation + source => source.toJSON({ getters: true }), + ), + ); } catch (e) { next(e); } }); -const SourceSchemaNoId = sourceSchemaWithout({ id: true }); - router.post( '/', validateRequest({ @@ -40,11 +43,10 @@ router.post( try { const { teamId } = getNonNullUserWithTeam(req); - // TODO: HDX-1768 Eliminate type assertion const source = await createSource(teamId.toString(), { ...req.body, - team: teamId, - } as any); + team: teamId.toJSON(), + }); res.json(source); } catch (e) { @@ -65,11 +67,10 @@ router.put( try { const { teamId } = getNonNullUserWithTeam(req); - // TODO: HDX-1768 Eliminate type assertion const source = await updateSource(teamId.toString(), req.params.id, { ...req.body, - team: teamId, - } as any); + team: teamId.toJSON(), + }); if (!source) { res.status(404).send('Source not found'); diff --git a/packages/api/src/routers/external-api/__tests__/sources.test.ts b/packages/api/src/routers/external-api/__tests__/sources.test.ts index 069c5ea9..39ca2703 100644 --- a/packages/api/src/routers/external-api/__tests__/sources.test.ts +++ b/packages/api/src/routers/external-api/__tests__/sources.test.ts @@ -13,7 +13,14 @@ import { getServer, } from '../../../fixtures'; import Connection, { IConnection } from '../../../models/connection'; -import { Source } from '../../../models/source'; +import { + ISource, + LogSource, + MetricSource, + SessionSource, + Source, + TraceSource, +} from '../../../models/source'; import { mapGranularityToExternalFormat } from '../v2/sources'; describe('External API v2 Sources', () => { @@ -73,7 +80,7 @@ describe('External API v2 Sources', () => { }); it('should return a single log source', async () => { - const logSource = await Source.create({ + const logSource = await LogSource.create({ kind: SourceKind.Log, team: team._id, name: 'Test Log Source', @@ -108,7 +115,7 @@ describe('External API v2 Sources', () => { }); it('should return a single trace source', async () => { - const traceSource = await Source.create({ + const traceSource = await TraceSource.create({ kind: SourceKind.Trace, team: team._id, name: 'Test Trace Source', @@ -117,6 +124,7 @@ describe('External API v2 Sources', () => { tableName: 'otel_traces', }, timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: '*', durationExpression: 'Duration', durationPrecision: 3, traceIdExpression: 'TraceId', @@ -188,6 +196,7 @@ describe('External API v2 Sources', () => { kind: SourceKind.Trace, connection: connection._id.toString(), timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: '*', durationExpression: 'Duration', durationPrecision: 3, traceIdExpression: 'TraceId', @@ -247,7 +256,7 @@ describe('External API v2 Sources', () => { }); it('should return a single metric source', async () => { - const metricSource = await Source.create({ + const metricSource = await MetricSource.create({ kind: SourceKind.Metric, team: team._id, name: 'Test Metric Source', @@ -289,7 +298,7 @@ describe('External API v2 Sources', () => { }); it('should return a single session source', async () => { - const traceSource = await Source.create({ + const traceSource = await TraceSource.create({ kind: SourceKind.Trace, team: team._id, name: 'Trace Source for Session', @@ -298,6 +307,7 @@ describe('External API v2 Sources', () => { tableName: 'otel_traces', }, timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: '*', durationExpression: 'Duration', durationPrecision: 3, traceIdExpression: 'TraceId', @@ -308,17 +318,18 @@ describe('External API v2 Sources', () => { connection: connection._id, }); - const sessionSource = await Source.create({ + const sessionSource = await SessionSource.create({ kind: SourceKind.Session, - team: team._id, + team: team._id.toString(), name: 'Test Session Source', from: { databaseName: DEFAULT_DATABASE, tableName: 'rrweb_events', }, + timestampValueExpression: 'Timestamp', traceSourceId: traceSource._id.toString(), - connection: connection._id, - }); + connection: connection._id.toString(), + } satisfies Omit, 'id'>); const response = await authRequest('get', BASE_URL).expect(200); @@ -338,11 +349,12 @@ describe('External API v2 Sources', () => { }, traceSourceId: traceSource._id.toString(), querySettings: [], + timestampValueExpression: 'Timestamp', }); }); it('should return multiple sources of different kinds', async () => { - const logSource = await Source.create({ + const logSource = await LogSource.create({ kind: SourceKind.Log, team: team._id, name: 'Logs', @@ -355,7 +367,7 @@ describe('External API v2 Sources', () => { connection: connection._id, }); - const traceSource = await Source.create({ + const traceSource = await TraceSource.create({ kind: SourceKind.Trace, team: team._id, name: 'Traces', @@ -364,6 +376,7 @@ describe('External API v2 Sources', () => { tableName: 'otel_traces', }, timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: '*', durationExpression: 'Duration', durationPrecision: 3, traceIdExpression: 'TraceId', @@ -374,7 +387,7 @@ describe('External API v2 Sources', () => { connection: connection._id, }); - const metricSource = await Source.create({ + const metricSource = await MetricSource.create({ kind: SourceKind.Metric, team: team._id, name: 'Metrics', @@ -407,7 +420,7 @@ describe('External API v2 Sources', () => { it("should only return sources for the authenticated user's team", async () => { // Create a source for the current team - const currentTeamSource = await Source.create({ + const currentTeamSource = await LogSource.create({ kind: SourceKind.Log, team: team._id, name: 'Current Team Source', @@ -430,7 +443,7 @@ describe('External API v2 Sources', () => { password: config.CLICKHOUSE_PASSWORD, }); - await Source.create({ + await LogSource.create({ kind: SourceKind.Log, team: otherTeamId, name: 'Other Team Source', @@ -452,7 +465,7 @@ describe('External API v2 Sources', () => { }); it('should format sources according to SourceSchema', async () => { - await Source.create({ + await LogSource.create({ kind: SourceKind.Log, team: team._id, name: 'Test Source', @@ -483,7 +496,7 @@ describe('External API v2 Sources', () => { it('should filter out sources that fail schema validation', async () => { // Create a valid source - const validSource = await Source.create({ + const validSource = await LogSource.create({ kind: SourceKind.Log, team: team._id, name: 'Valid Source', @@ -517,6 +530,92 @@ describe('External API v2 Sources', () => { expect(response.body.data[0].id).toBe(validSource._id.toString()); }); }); + + describe('backward compatibility with legacy flat-model documents', () => { + const BASE_URL = '/api/v2/sources'; + + it('returns legacy Session source without timestampValueExpression using default TimestampTime', async () => { + // Legacy Session sources were created before timestampValueExpression was + // required. applyLegacyDefaults() backfills 'TimestampTime' before + // SourceSchema.safeParse(), so these sources still appear in the response. + await Source.collection.insertOne({ + kind: SourceKind.Session, + name: 'Legacy Session', + team: team._id, + connection: connection._id, + from: { databaseName: DEFAULT_DATABASE, tableName: 'otel_sessions' }, + traceSourceId: 'some-trace-source-id', + // timestampValueExpression intentionally omitted + }); + + const response = await authRequest('get', BASE_URL).expect(200); + + expect(response.body.data).toHaveLength(1); + expect(response.body.data[0].kind).toBe(SourceKind.Session); + // Default is applied at read time, not persisted to the database + expect(response.body.data[0].timestampValueExpression).toBe( + 'TimestampTime', + ); + }); + + it('returns Trace source with logSourceId: null (Zod optional accepts null)', async () => { + // Old schema had logSourceId: z.string().optional().nullable() + // New schema removed .nullable() — however, Zod's optional() in + // discriminatedUnion context still accepts null values (they pass + // safeParse). This means logSourceId: null is NOT a breaking change. + await Source.collection.insertOne({ + kind: SourceKind.Trace, + name: 'Trace with null logSourceId', + team: team._id, + connection: connection._id, + from: { databaseName: DEFAULT_DATABASE, tableName: 'otel_traces' }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: '*', + durationExpression: 'Duration', + durationPrecision: 3, + traceIdExpression: 'TraceId', + spanIdExpression: 'SpanId', + parentSpanIdExpression: 'ParentSpanId', + spanNameExpression: 'SpanName', + spanKindExpression: 'SpanKind', + logSourceId: null, + }); + + const response = await authRequest('get', BASE_URL).expect(200); + + // Source IS returned — logSourceId: null passes Zod safeParse + expect(response.body.data).toHaveLength(1); + expect(response.body.data[0].kind).toBe(SourceKind.Trace); + expect(response.body.data[0].logSourceId).toBeNull(); + }); + + it('strips cross-kind fields from legacy flat-model Log source via SourceSchema.safeParse', async () => { + // The external API runs SourceSchema.safeParse() which DOES strip + // unknown/cross-kind fields (unlike Mongoose toJSON which keeps them). + // This is the key difference between internal and external APIs. + await Source.collection.insertOne({ + kind: SourceKind.Log, + name: 'Flat Model Log', + team: team._id, + connection: connection._id, + from: { databaseName: DEFAULT_DATABASE, tableName: DEFAULT_LOGS_TABLE }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: 'Body', + // Cross-kind fields from old flat model + metricTables: { gauge: 'otel_metrics_gauge' }, + durationExpression: 'Duration', + }); + + const response = await authRequest('get', BASE_URL).expect(200); + + expect(response.body.data).toHaveLength(1); + expect(response.body.data[0].kind).toBe(SourceKind.Log); + expect(response.body.data[0].defaultTableSelectExpression).toBe('Body'); + // Cross-kind fields ARE stripped by SourceSchema.safeParse in the external API + expect(response.body.data[0]).not.toHaveProperty('metricTables'); + expect(response.body.data[0]).not.toHaveProperty('durationExpression'); + }); + }); }); describe('External API v2 Sources Mapping', () => { diff --git a/packages/api/src/routers/external-api/v2/charts.ts b/packages/api/src/routers/external-api/v2/charts.ts index 52d7ad92..140c6dc7 100644 --- a/packages/api/src/routers/external-api/v2/charts.ts +++ b/packages/api/src/routers/external-api/v2/charts.ts @@ -4,8 +4,8 @@ import { Granularity } from '@hyperdx/common-utils/dist/core/utils'; import { ChartConfigWithOptDateRange, DisplayType, + SourceKind, } from '@hyperdx/common-utils/dist/types'; -import { SourceKind } from '@hyperdx/common-utils/dist/types'; import opentelemetry, { SpanStatusCode } from '@opentelemetry/api'; import express from 'express'; import _ from 'lodash'; @@ -260,7 +260,7 @@ const buildChartConfigFromRequest = async ( databaseName: source.from.databaseName, tableName: !isMetricSource ? source.from.tableName : '', }, - ...(isMetricSource && { + ...(source.kind === SourceKind.Metric && { metricTables: source.metricTables, }), select: [ diff --git a/packages/api/src/routers/external-api/v2/sources.ts b/packages/api/src/routers/external-api/v2/sources.ts index 6a760916..3f143e64 100644 --- a/packages/api/src/routers/external-api/v2/sources.ts +++ b/packages/api/src/routers/external-api/v2/sources.ts @@ -1,6 +1,7 @@ import { + SourceKind, SourceSchema, - type TSourceUnion, + type TSource, } from '@hyperdx/common-utils/dist/types'; import express from 'express'; @@ -27,7 +28,7 @@ export function mapGranularityToExternalFormat(granularity: string): string { } } -function mapSourceToExternalSource(source: TSourceUnion): TSourceUnion { +function mapSourceToExternalSource(source: TSource): TSource { if (!('materializedViews' in source)) return source; if (!Array.isArray(source.materializedViews)) return source; @@ -42,12 +43,41 @@ function mapSourceToExternalSource(source: TSourceUnion): TSourceUnion { }; } +function applyLegacyDefaults( + parsed: Record, +): Record { + // Legacy Session sources were created before timestampValueExpression was + // required. The old code defaulted it to 'TimestampTime' at query time. + if (parsed.kind === SourceKind.Session && !parsed.timestampValueExpression) { + return { ...parsed, timestampValueExpression: 'TimestampTime' }; + } + return parsed; +} + function formatExternalSource(source: SourceDocument) { // Convert to JSON so that any ObjectIds are converted to strings - const json = JSON.stringify(source.toJSON({ getters: true })); + const json = JSON.stringify( + (() => { + switch (source.kind) { + case SourceKind.Log: + return source.toJSON({ getters: true }); + case SourceKind.Trace: + return source.toJSON({ getters: true }); + case SourceKind.Metric: + return source.toJSON({ getters: true }); + case SourceKind.Session: + return source.toJSON({ getters: true }); + default: + source satisfies never; + return {}; + } + })(), + ); // Parse using the SourceSchema to strip out any fields not defined in the schema - const parseResult = SourceSchema.safeParse(JSON.parse(json)); + const parseResult = SourceSchema.safeParse( + applyLegacyDefaults(JSON.parse(json)), + ); if (parseResult.success) { return mapSourceToExternalSource(parseResult.data); } diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.test.ts index 5fd1843c..b076b912 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.test.ts @@ -1,6 +1,7 @@ import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; import { AlertState, + SourceKind, Tile, WebhookService, } from '@hyperdx/common-utils/dist/types'; @@ -293,16 +294,17 @@ describe('checkAlerts', () => { interval: '1m', }, source: { - id: 'fake-source-id' as any, - kind: 'log' as any, - team: 'team-123' as any, + id: 'fake-source-id', + kind: SourceKind.Log, + team: 'team-123', from: { databaseName: 'default', tableName: 'otel_logs', }, timestampValueExpression: 'Timestamp', - connection: 'connection-123' as any, + connection: 'connection-123', name: 'Logs', + defaultTableSelectExpression: 'Timestamp, Body', }, savedSearch: { _id: 'fake-saved-search-id' as any, diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 779a43f9..d69fbd40 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -23,6 +23,7 @@ import { import { BuilderChartConfigWithOptDateRange, DisplayType, + SourceKind, } from '@hyperdx/common-utils/dist/types'; import * as fns from 'date-fns'; import { chunk, isString } from 'lodash'; @@ -91,7 +92,10 @@ export async function computeAliasWithClauses( metadata: Metadata, ): Promise { const resolvedSelect = - savedSearch.select || source.defaultTableSelectExpression || ''; + savedSearch.select || + ((source.kind === SourceKind.Log || source.kind === SourceKind.Trace) && + source.defaultTableSelectExpression) || + ''; const config: BuilderChartConfigWithOptDateRange = { connection: '', displayType: DisplayType.Search, @@ -99,7 +103,10 @@ export async function computeAliasWithClauses( select: resolvedSelect, where: savedSearch.where, whereLanguage: savedSearch.whereLanguage, - implicitColumnExpression: source.implicitColumnExpression, + implicitColumnExpression: + source.kind === SourceKind.Log || source.kind === SourceKind.Trace + ? source.implicitColumnExpression + : undefined, timestampValueExpression: source.timestampValueExpression, }; const query = await renderChartConfig(config, metadata, source.querySettings); @@ -442,7 +449,10 @@ const getChartConfigFromAlert = ( whereLanguage: savedSearch.whereLanguage, filters: savedSearch.filters?.map(f => ({ ...f })), groupBy: alert.groupBy, - implicitColumnExpression: source.implicitColumnExpression, + implicitColumnExpression: + source.kind === SourceKind.Log || source.kind === SourceKind.Trace + ? source.implicitColumnExpression + : undefined, timestampValueExpression: source.timestampValueExpression, }; } else if (details.taskType === AlertTaskType.TILE) { @@ -457,6 +467,15 @@ const getChartConfigFromAlert = ( tile.config.displayType === DisplayType.StackedBar || tile.config.displayType === DisplayType.Number ) { + // Tile alerts can use Log, Trace, or Metric sources. + // implicitColumnExpression exists on Log and Trace sources; + // metricTables exists on Metric sources. + const implicitColumnExpression = + source.kind === SourceKind.Log || source.kind === SourceKind.Trace + ? source.implicitColumnExpression + : undefined; + const metricTables = + source.kind === SourceKind.Metric ? source.metricTables : undefined; return { connection, dateRange, @@ -466,8 +485,8 @@ const getChartConfigFromAlert = ( from: source.from, granularity: `${windowSizeInMins} minute`, groupBy: tile.config.groupBy, - implicitColumnExpression: source.implicitColumnExpression, - metricTables: source.metricTables, + implicitColumnExpression, + metricTables, select: tile.config.select, timestampValueExpression: source.timestampValueExpression, where: tile.config.where, @@ -671,14 +690,19 @@ export const processAlert = async ( } } - // Optimize chart config with materialized views, if available - const optimizedChartConfig = source?.materializedViews?.length + // Optimize chart config with materialized views, if available. + // materializedViews exists on Log and Trace sources. + const mvSource = + source.kind === SourceKind.Log || source.kind === SourceKind.Trace + ? source + : undefined; + const optimizedChartConfig = mvSource?.materializedViews?.length ? await tryOptimizeConfigWithMaterializedView( chartConfig, metadata, clickhouseClient, undefined, - source, + mvSource, ) : chartConfig; diff --git a/packages/api/src/tasks/checkAlerts/template.ts b/packages/api/src/tasks/checkAlerts/template.ts index 56c4909b..dce38c27 100644 --- a/packages/api/src/tasks/checkAlerts/template.ts +++ b/packages/api/src/tasks/checkAlerts/template.ts @@ -10,6 +10,7 @@ import { AlertChannelType, ChartConfigWithOptDateRange, DisplayType, + SourceKind, WebhookService, zAlertChannelType, } from '@hyperdx/common-utils/dist/types'; @@ -579,6 +580,11 @@ ${targetTemplate}`; if (source == null) { throw new Error(`Source ID is ${alert.source} but source is null`); } + if (source.kind !== SourceKind.Log && source.kind !== SourceKind.Trace) { + throw new Error( + `Expecting SourceKind 'trace' or 'log', got ${source.kind}`, + ); + } // TODO: show group + total count for group-by alerts // fetch sample logs const resolvedSelect = diff --git a/packages/api/src/tasks/usageStats.ts b/packages/api/src/tasks/usageStats.ts index ac017db0..f077c5f5 100644 --- a/packages/api/src/tasks/usageStats.ts +++ b/packages/api/src/tasks/usageStats.ts @@ -1,6 +1,10 @@ import { ResponseJSON } from '@hyperdx/common-utils/dist/clickhouse'; import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; -import { MetricsDataType, SourceKind } from '@hyperdx/common-utils/dist/types'; +import { + MetricsDataType, + SourceKind, + TMetricSource, +} from '@hyperdx/common-utils/dist/types'; import * as HyperDX from '@hyperdx/node-opentelemetry'; import ms from 'ms'; import os from 'os'; @@ -30,8 +34,10 @@ const logger = pino({ function extractTableNames(source: SourceDocument): string[] { const tables: string[] = []; if (source.kind === SourceKind.Metric) { + // Cast to TMetricSource to access metricTables after kind narrowing + const metricSource = source; for (const key of Object.values(MetricsDataType)) { - const metricTable = source.metricTables?.[key]; + const metricTable = metricSource.metricTables?.[key]; if (!metricTable) continue; tables.push(metricTable); } diff --git a/packages/app/src/ChartUtils.tsx b/packages/app/src/ChartUtils.tsx index 39220a93..a6c23572 100644 --- a/packages/app/src/ChartUtils.tsx +++ b/packages/app/src/ChartUtils.tsx @@ -30,6 +30,7 @@ import { MetricsDataType as MetricsDataTypeV2, SourceKind, SQLInterval, + TMetricSource, TSource, } from '@hyperdx/common-utils/dist/types'; import { SegmentedControl } from '@mantine/core'; @@ -630,14 +631,14 @@ function firstGroupColumnIsLogLevel( source: TSource | undefined, groupColumns: ColumnMetaType[], ) { - return ( - source && - groupColumns.length === 1 && - groupColumns[0].name === - (source.kind === SourceKind.Log - ? source.severityTextExpression - : source.statusCodeExpression) - ); + if (!source || groupColumns.length !== 1) return false; + if (source.kind === SourceKind.Log) { + return groupColumns[0].name === source.severityTextExpression; + } + if (source.kind === SourceKind.Trace) { + return groupColumns[0].name === source.statusCodeExpression; + } + return false; } function addResponseToFormattedData({ @@ -905,7 +906,7 @@ export const mapV1AggFnToV2 = (aggFn?: AggFn): AggFnV2 | undefined => { }; export const convertV1GroupByToV2 = ( - metricSource: TSource, + metricSource: TMetricSource, groupBy: string[], ): string => { return groupBy @@ -932,7 +933,7 @@ export const convertV1ChartConfigToV2 = ( }, source: { log?: TSource; - metric?: TSource; + metric?: TMetricSource; trace?: TSource; }, ): BuilderChartConfigWithDateRange => { @@ -1020,12 +1021,18 @@ export function buildEventsSearchUrl({ } const isMetricChart = isMetricChartConfig(config); - if (isMetricChart && source?.logSourceId == null) { - notifications.show({ - color: 'yellow', - message: 'No log source is associated with the selected metric source.', - }); - return null; + if (isMetricChart) { + const logSourceId = + source.kind === SourceKind.Metric || source.kind === SourceKind.Trace + ? source.logSourceId + : undefined; + if (logSourceId == null) { + notifications.show({ + color: 'yellow', + message: 'No log source is associated with the selected metric source.', + }); + return null; + } } let where = config.where; @@ -1089,7 +1096,10 @@ export function buildEventsSearchUrl({ params.where = ''; params.whereLanguage = 'lucene'; params.filters = JSON.stringify([]); - params.source = source?.logSourceId ?? ''; + params.source = + (source.kind === SourceKind.Metric || source.kind === SourceKind.Trace + ? source.logSourceId + : undefined) ?? ''; } // Include the select parameter if provided to preserve custom columns diff --git a/packages/app/src/DBDashboardPage.tsx b/packages/app/src/DBDashboardPage.tsx index cd4339b3..5604f9cc 100644 --- a/packages/app/src/DBDashboardPage.tsx +++ b/packages/app/src/DBDashboardPage.tsx @@ -31,11 +31,13 @@ import { DashboardFilter, DisplayType, Filter, + isLogSource, + isTraceSource, SearchCondition, SearchConditionLanguage, SourceKind, SQLInterval, - TSourceUnion, + TSource, } from '@hyperdx/common-utils/dist/types'; import { ActionIcon, @@ -253,7 +255,10 @@ const Tile = forwardRef( databaseName: source.from?.databaseName || 'default', tableName: tableName || '', }, - implicitColumnExpression: source.implicitColumnExpression, + implicitColumnExpression: + isLogSource(source) || isTraceSource(source) + ? source.implicitColumnExpression + : undefined, filters, metricTables: isMetricSource ? source.metricTables : undefined, }); @@ -528,7 +533,10 @@ const Tile = forwardRef( dateRange, select: queriedConfig.select || - source?.defaultTableSelectExpression || + (source?.kind === SourceKind.Log || + source?.kind === SourceKind.Trace + ? source.defaultTableSelectExpression + : '') || '', groupBy: undefined, granularity: undefined, @@ -1420,7 +1428,7 @@ function DBDashboardPage({ presetConfig }: { presetConfig?: Dashboard }) { convertToDashboardTemplate( dashboard, // TODO: fix this type issue - sources as TSourceUnion[], + sources, connections, ), dashboard?.name, diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index d5a479c7..cfa5079a 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -36,6 +36,8 @@ import { ChartConfigWithDateRange, DisplayType, Filter, + isLogSource, + isTraceSource, SourceKind, TSource, } from '@hyperdx/common-utils/dist/types'; @@ -663,6 +665,7 @@ function useSearchedConfigToChartConfig( ) { const { data: sourceObj, isLoading } = useSource({ id: source, + kinds: [SourceKind.Log, SourceKind.Trace], }); const defaultOrderBy = useDefaultOrderBy(source); @@ -673,11 +676,10 @@ function useSearchedConfigToChartConfig( select: select || defaultSearchConfig?.select || - sourceObj.defaultTableSelectExpression || - '', + sourceObj.defaultTableSelectExpression, from: sourceObj.from, source: sourceObj.id, - ...(sourceObj.tableFilterExpression != null + ...(isLogSource(sourceObj) && sourceObj.tableFilterExpression != null ? { filters: [ { @@ -785,7 +787,10 @@ function optimizeDefaultOrderBy( } export function useDefaultOrderBy(sourceID: string | undefined | null) { - const { data: source } = useSource({ id: sourceID }); + const { data: source } = useSource({ + id: sourceID, + kinds: [SourceKind.Log, SourceKind.Trace], + }); const { data: tableMetadata } = useTableMetadata(tcFromSource(source)); // When source changes, make sure select and orderby fields are set to default @@ -796,7 +801,7 @@ export function useDefaultOrderBy(sourceID: string | undefined | null) { if (trimmedOrderBy) return trimmedOrderBy; return optimizeDefaultOrderBy( source?.timestampValueExpression ?? '', - source?.displayedTimestampValueExpression, + source.displayedTimestampValueExpression, tableMetadata?.sorting_key, ); }, [source, tableMetadata]); @@ -835,6 +840,7 @@ function DBSearchPage() { ); const { data: searchedSource } = useSource({ id: searchedConfig.source, + kinds: [SourceKind.Log, SourceKind.Trace], }); const [analysisMode, setAnalysisMode] = useQueryState( @@ -918,7 +924,11 @@ function DBSearchPage() { } return { select: - _savedSearch?.select ?? searchedSource?.defaultTableSelectExpression, + _savedSearch?.select ?? + (searchedSource?.kind === SourceKind.Log || + searchedSource?.kind === SourceKind.Trace + ? searchedSource.defaultTableSelectExpression + : undefined), where: _savedSearch?.where ?? '', whereLanguage: _savedSearch?.whereLanguage ?? 'lucene', source: _savedSearch?.source, @@ -1807,7 +1817,11 @@ function DBSearchPage() { setAnalysisMode={setAnalysisMode} chartConfig={filtersChartConfig} sourceId={inputSourceObj?.id} - showDelta={!!searchedSource?.durationExpression} + showDelta={ + !!(searchedSource?.kind === SourceKind.Trace + ? searchedSource.durationExpression + : undefined) + } {...searchFilters} /> @@ -1865,18 +1879,20 @@ function DBSearchPage() { )} - {analysisMode === 'delta' && searchedSource != null && ( - - )} + {analysisMode === 'delta' && + searchedSource != null && + isTraceSource(searchedSource) && ( + + )} {analysisMode === 'results' && ( {chartConfig && histogramTimeChartConfig && ( diff --git a/packages/app/src/DBServiceMapPage.tsx b/packages/app/src/DBServiceMapPage.tsx index 56baf0ab..529eb3e7 100644 --- a/packages/app/src/DBServiceMapPage.tsx +++ b/packages/app/src/DBServiceMapPage.tsx @@ -2,7 +2,7 @@ import { useEffect, useState } from 'react'; import dynamic from 'next/dynamic'; import { parseAsInteger, useQueryState } from 'nuqs'; import { useForm, useWatch } from 'react-hook-form'; -import { SourceKind } from '@hyperdx/common-utils/dist/types'; +import { SourceKind, TTraceSource } from '@hyperdx/common-utils/dist/types'; import { Box, Button, @@ -70,12 +70,13 @@ function DBServiceMapPage() { }); const defaultSource = sources?.find( - source => source.kind === SourceKind.Trace, + (source): source is TTraceSource => source.kind === SourceKind.Trace, ); const source = sourceId && sources ? (sources.find( - source => source.id === sourceId && source.kind === SourceKind.Trace, + (source): source is TTraceSource => + source.id === sourceId && source.kind === SourceKind.Trace, ) ?? defaultSource) : defaultSource; diff --git a/packages/app/src/DashboardFiltersModal.tsx b/packages/app/src/DashboardFiltersModal.tsx index f6bd51ac..cfd8ff7a 100644 --- a/packages/app/src/DashboardFiltersModal.tsx +++ b/packages/app/src/DashboardFiltersModal.tsx @@ -118,8 +118,8 @@ const DashboardFilterEditForm = ({ : undefined; const sourceIsMetric = source?.kind === SourceKind.Metric; - const metricTypes = Object.values(MetricsDataType).filter( - type => source?.metricTables?.[type], + const metricTypes = Object.values(MetricsDataType).filter(type => + source?.kind === SourceKind.Metric ? source.metricTables?.[type] : false, ); const [modalContentRef, setModalContentRef] = useState( diff --git a/packages/app/src/KubernetesDashboardPage.tsx b/packages/app/src/KubernetesDashboardPage.tsx index 9a478682..0ecc6ef6 100644 --- a/packages/app/src/KubernetesDashboardPage.tsx +++ b/packages/app/src/KubernetesDashboardPage.tsx @@ -7,13 +7,19 @@ import sub from 'date-fns/sub'; import { useQueryState } from 'nuqs'; import { useForm, useWatch } from 'react-hook-form'; import { convertDateRangeToGranularityString } from '@hyperdx/common-utils/dist/core/utils'; -import { SourceKind, TSource } from '@hyperdx/common-utils/dist/types'; +import { + isLogSource, + isMetricSource, + SourceKind, + TLogSource, + TMetricSource, + TSource, +} from '@hyperdx/common-utils/dist/types'; import { ActionIcon, Alert, Badge, Box, - Button, Card, Flex, Grid, @@ -56,7 +62,7 @@ import { withAppNav } from './layout'; import NamespaceDetailsSidePanel from './NamespaceDetailsSidePanel'; import NodeDetailsSidePanel from './NodeDetailsSidePanel'; import PodDetailsSidePanel from './PodDetailsSidePanel'; -import { useSources } from './source'; +import { useSource, useSources } from './source'; import { parseTimeQuery, useTimeQuery } from './timeQuery'; import { KubePhase } from './types'; import { formatNumber, formatUptime } from './utils'; @@ -143,7 +149,7 @@ export const InfraPodsStatusTable = ({ where, }: { dateRange: [Date, Date]; - metricSource: TSource; + metricSource: TMetricSource; where: string; }) => { const [phaseFilter, setPhaseFilter] = React.useState('running'); @@ -532,7 +538,7 @@ const NodesTable = ({ where, dateRange, }: { - metricSource: TSource; + metricSource: TMetricSource; where: string; dateRange: [Date, Date]; }) => { @@ -736,7 +742,7 @@ const NamespacesTable = ({ where, }: { dateRange: [Date, Date]; - metricSource: TSource; + metricSource: TMetricSource; where: string; }) => { const groupBy = ['k8s.namespace.name']; @@ -965,8 +971,12 @@ export const resolveSourceIds = ( // Find a default metric source that matches the existing log source if (_logSourceId && !_metricSourceId) { - const { connection, metricSourceId: correlatedMetricSourceId } = - findSource(sources, { id: _logSourceId }) ?? {}; + const foundSource = findSource(sources, { id: _logSourceId }); + const connection = foundSource?.connection; + const correlatedMetricSourceId = + foundSource && isLogSource(foundSource) + ? foundSource.metricSourceId + : undefined; const metricSourceId = (correlatedMetricSourceId && findSource(sources, { id: correlatedMetricSourceId })?.id) ?? @@ -977,8 +987,12 @@ export const resolveSourceIds = ( // Find a default log source that matches the existing metric source if (!_logSourceId && _metricSourceId) { - const { connection, logSourceId: correlatedLogSourceId } = - findSource(sources, { id: _metricSourceId }) ?? {}; + const foundSource = findSource(sources, { id: _metricSourceId }); + const connection = foundSource?.connection; + const correlatedLogSourceId = + foundSource && isMetricSource(foundSource) + ? foundSource.logSourceId + : undefined; const logSourceId = (correlatedLogSourceId && findSource(sources, { id: correlatedLogSourceId })?.id) ?? @@ -989,10 +1003,10 @@ export const resolveSourceIds = ( // Find any two correlated log and metric sources const logSourceWithMetricSource = sources.find( - s => + (s): s is TLogSource => s.kind === SourceKind.Log && - s.metricSourceId && - findSource(sources, { id: s.metricSourceId }), + !!s.metricSourceId && + !!findSource(sources, { id: s.metricSourceId }), ); if (logSourceWithMetricSource) { @@ -1036,8 +1050,14 @@ function KubernetesDashboardPage() { [_logSourceId, _metricSourceId, sources], ); - const logSource = sources?.find(s => s.id === logSourceId); - const metricSource = sources?.find(s => s.id === metricSourceId); + const { data: logSource } = useSource({ + id: logSourceId, + kinds: [SourceKind.Log], + }); + const { data: metricSource } = useSource({ + id: metricSourceId, + kinds: [SourceKind.Metric], + }); const { control } = useForm({ values: { @@ -1077,8 +1097,12 @@ function KubernetesDashboardPage() { // Default to the log source's correlated metric source if (watchedLogSourceId && sources) { const logSource = findSource(sources, { id: watchedLogSourceId }); - const correlatedMetricSource = logSource?.metricSourceId - ? findSource(sources, { id: logSource.metricSourceId }) + const logSourceMetricSourceId = + logSource && isLogSource(logSource) + ? logSource.metricSourceId + : undefined; + const correlatedMetricSource = logSourceMetricSourceId + ? findSource(sources, { id: logSourceMetricSourceId }) : undefined; if ( correlatedMetricSource && @@ -1119,8 +1143,12 @@ function KubernetesDashboardPage() { // Default to the metric source's correlated log source if (watchedMetricSourceId && sources) { const metricSource = findSource(sources, { id: watchedMetricSourceId }); - const correlatedLogSource = metricSource?.logSourceId - ? findSource(sources, { id: metricSource.logSourceId }) + const metricSourceLogSourceId = + metricSource && isMetricSource(metricSource) + ? metricSource.logSourceId + : undefined; + const correlatedLogSource = metricSourceLogSourceId + ? findSource(sources, { id: metricSourceLogSourceId }) : undefined; if ( correlatedLogSource && diff --git a/packages/app/src/NamespaceDetailsSidePanel.tsx b/packages/app/src/NamespaceDetailsSidePanel.tsx index ce0967e2..5e0e7719 100644 --- a/packages/app/src/NamespaceDetailsSidePanel.tsx +++ b/packages/app/src/NamespaceDetailsSidePanel.tsx @@ -2,7 +2,7 @@ import * as React from 'react'; import { StringParam, useQueryParam, withDefault } from 'use-query-params'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; import { convertDateRangeToGranularityString } from '@hyperdx/common-utils/dist/core/utils'; -import { TSource } from '@hyperdx/common-utils/dist/types'; +import { TLogSource, TMetricSource } from '@hyperdx/common-utils/dist/types'; import { Badge, Card, @@ -55,7 +55,7 @@ const NamespaceDetails = ({ }: { name: string; dateRange: [Date, Date]; - metricSource?: TSource; + metricSource?: TMetricSource; }) => { const where = `${metricSource?.resourceAttributesExpression}.k8s.namespace.name:"${name}"`; const groupBy = ['k8s.namespace.name']; @@ -138,7 +138,7 @@ function NamespaceLogs({ where, }: { dateRange: [Date, Date]; - logSource: TSource; + logSource: TLogSource; where: string; }) { const [resultType, setResultType] = React.useState<'all' | 'error'>('all'); @@ -226,8 +226,8 @@ export default function NamespaceDetailsSidePanel({ metricSource, logSource, }: { - metricSource: TSource; - logSource: TSource; + metricSource: TMetricSource; + logSource: TLogSource; }) { const [namespaceName, setNamespaceName] = useQueryParam( 'namespaceName', diff --git a/packages/app/src/NodeDetailsSidePanel.tsx b/packages/app/src/NodeDetailsSidePanel.tsx index 4dfc2cab..f4f2c03e 100644 --- a/packages/app/src/NodeDetailsSidePanel.tsx +++ b/packages/app/src/NodeDetailsSidePanel.tsx @@ -2,7 +2,7 @@ import * as React from 'react'; import { StringParam, useQueryParam, withDefault } from 'use-query-params'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; import { convertDateRangeToGranularityString } from '@hyperdx/common-utils/dist/core/utils'; -import { TSource } from '@hyperdx/common-utils/dist/types'; +import { TLogSource, TMetricSource } from '@hyperdx/common-utils/dist/types'; import { Badge, Card, @@ -56,7 +56,7 @@ const NodeDetails = ({ }: { name: string; dateRange: [Date, Date]; - metricSource: TSource; + metricSource: TMetricSource; }) => { const where = `${metricSource.resourceAttributesExpression}.k8s.node.name:"${name}"`; const groupBy = ['k8s.node.name']; @@ -151,7 +151,7 @@ function NodeLogs({ where, }: { dateRange: [Date, Date]; - logSource: TSource; + logSource: TLogSource; where: string; }) { const [resultType, setResultType] = React.useState<'all' | 'error'>('all'); @@ -239,8 +239,8 @@ export default function NodeDetailsSidePanel({ metricSource, logSource, }: { - metricSource: TSource; - logSource: TSource; + metricSource: TMetricSource; + logSource: TLogSource; }) { const [nodeName, setNodeName] = useQueryParam( 'nodeName', diff --git a/packages/app/src/PodDetailsSidePanel.tsx b/packages/app/src/PodDetailsSidePanel.tsx index bc32115e..7e058322 100644 --- a/packages/app/src/PodDetailsSidePanel.tsx +++ b/packages/app/src/PodDetailsSidePanel.tsx @@ -2,7 +2,7 @@ import * as React from 'react'; import { StringParam, useQueryParam, withDefault } from 'use-query-params'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; import { convertDateRangeToGranularityString } from '@hyperdx/common-utils/dist/core/utils'; -import { TSource } from '@hyperdx/common-utils/dist/types'; +import { TLogSource, TMetricSource } from '@hyperdx/common-utils/dist/types'; import { Box, Card, @@ -56,7 +56,7 @@ const PodDetails = ({ podName, }: { dateRange: [Date, Date]; - logSource: TSource; + logSource: TLogSource; podName: string; }) => { const { data: logsData } = useV2LogBatch<{ @@ -134,7 +134,7 @@ function PodLogs({ onRowClick, }: { dateRange: [Date, Date]; - logSource: TSource; + logSource: TLogSource; where: string; rowId: string | null; onRowClick: (rowWhere: RowWhereResult) => void; @@ -220,8 +220,8 @@ export default function PodDetailsSidePanel({ logSource, metricSource, }: { - logSource: TSource; - metricSource: TSource; + logSource: TLogSource; + metricSource: TMetricSource; }) { const [podName, setPodName] = useQueryParam( 'podName', diff --git a/packages/app/src/ServicesDashboardPage.tsx b/packages/app/src/ServicesDashboardPage.tsx index a01a680a..57950060 100644 --- a/packages/app/src/ServicesDashboardPage.tsx +++ b/packages/app/src/ServicesDashboardPage.tsx @@ -1,6 +1,5 @@ import { useCallback, useEffect, useMemo, useState } from 'react'; import dynamic from 'next/dynamic'; -import { pick } from 'lodash'; import { parseAsString, parseAsStringEnum, @@ -11,15 +10,31 @@ import { UseControllerProps, useForm, useWatch } from 'react-hook-form'; import SqlString from 'sqlstring'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; import { convertDateRangeToGranularityString } from '@hyperdx/common-utils/dist/core/utils'; +import type { TSource } from '@hyperdx/common-utils/dist/types'; import { BuilderChartConfigWithDateRange, CteChartConfig, DisplayType, Filter, + isLogSource, + isTraceSource, PresetDashboard, SourceKind, - TSource, + TTraceSource, } from '@hyperdx/common-utils/dist/types'; + +// Extract common chart config fields from a source. +// This avoids union type issues with lodash `pick` on discriminated unions. +function pickSourceConfigFields(source: TSource) { + return { + timestampValueExpression: source.timestampValueExpression, + connection: source.connection, + from: source.from, + ...(isLogSource(source) || isTraceSource(source) + ? { implicitColumnExpression: source.implicitColumnExpression } + : {}), + }; +} import { ActionIcon, Box, @@ -146,7 +161,10 @@ function ServiceSelectControlled({ dateRange: [Date, Date]; onCreate?: () => void; } & UseControllerProps) { - const { data: source } = useSource({ id: sourceId }); + const { data: source } = useSource({ + id: sourceId, + kinds: [SourceKind.Trace], + }); const { expressions } = useServiceDashboardExpressions({ source }); const queriedConfig = { @@ -213,7 +231,7 @@ export function EndpointLatencyChart({ appliedConfig = {}, extraFilters = [], }: { - source: TSource; + source: TTraceSource; dateRange: [Date, Date]; appliedConfig?: AppliedConfig; extraFilters?: Filter[]; @@ -260,12 +278,7 @@ export function EndpointLatencyChart({ ]} config={{ source: source.id, - ...pick(source, [ - 'timestampValueExpression', - 'implicitColumnExpression', - 'connection', - 'from', - ]), + ...pickSourceConfigFields(source), where: appliedConfig.where || '', whereLanguage: (appliedConfig.whereLanguage ?? getStoredLanguage()) || 'sql', @@ -318,12 +331,7 @@ export function EndpointLatencyChart({ toolbarSuffix={[displaySwitcher]} config={{ source: source.id, - ...pick(source, [ - 'timestampValueExpression', - 'implicitColumnExpression', - 'connection', - 'from', - ]), + ...pickSourceConfigFields(source), where: appliedConfig.where || '', whereLanguage: (appliedConfig.whereLanguage ?? getStoredLanguage()) || 'sql', @@ -358,7 +366,10 @@ function HttpTab({ searchedTimeRange: [Date, Date]; appliedConfig: AppliedConfig; }) { - const { data: source } = useSource({ id: appliedConfig.source }); + const { data: source } = useSource({ + id: appliedConfig.source, + kinds: [SourceKind.Trace], + }); const { expressions } = useServiceDashboardExpressions({ source }); const [reqChartType, setReqChartType] = useQueryState( @@ -385,12 +396,7 @@ function HttpTab({ if (reqChartType === 'overall') { return { source: source.id, - ...pick(source, [ - 'timestampValueExpression', - 'implicitColumnExpression', - 'connection', - 'from', - ]), + ...pickSourceConfigFields(source), where: appliedConfig.where || '', whereLanguage: (appliedConfig.whereLanguage ?? getStoredLanguage()) || 'sql', @@ -421,7 +427,10 @@ function HttpTab({ } return { timestampValueExpression: 'series_time_bucket', - implicitColumnExpression: source.implicitColumnExpression, + implicitColumnExpression: + isLogSource(source) || isTraceSource(source) + ? source.implicitColumnExpression + : undefined, connection: source.connection, source: source.id, with: [ @@ -429,7 +438,10 @@ function HttpTab({ name: 'error_series', chartConfig: { timestampValueExpression: source?.timestampValueExpression || '', - implicitColumnExpression: source?.implicitColumnExpression || '', + implicitColumnExpression: + isLogSource(source) || isTraceSource(source) + ? source?.implicitColumnExpression || '' + : '', connection: source?.connection ?? '', from: source?.from ?? { databaseName: '', @@ -595,12 +607,7 @@ function HttpTab({ sourceId={source.id} config={{ source: source.id, - ...pick(source, [ - 'timestampValueExpression', - 'implicitColumnExpression', - 'connection', - 'from', - ]), + ...pickSourceConfigFields(source), where: appliedConfig.where || '', whereLanguage: (appliedConfig.whereLanguage ?? getStoredLanguage()) || 'sql', @@ -639,12 +646,7 @@ function HttpTab({ ]} config={{ source: source.id, - ...pick(source, [ - 'timestampValueExpression', - 'implicitColumnExpression', - 'connection', - 'from', - ]), + ...pickSourceConfigFields(source), where: appliedConfig.where || '', whereLanguage: (appliedConfig.whereLanguage ?? getStoredLanguage()) || 'sql', @@ -724,7 +726,7 @@ function HttpTab({ - {source && ( + {source && isTraceSource(source) && ( ('list'); @@ -891,12 +891,7 @@ function DatabaseTab({ name: 'queries_by_total_time', isSubquery: true, chartConfig: { - ...pick(source, [ - 'timestampValueExpression', - 'implicitColumnExpression', - 'connection', - 'from', - ]), + ...pickSourceConfigFields(source), where: appliedConfig.where || '', whereLanguage: (appliedConfig.whereLanguage ?? getStoredLanguage()) || 'sql', @@ -1014,12 +1009,7 @@ function DatabaseTab({ name: 'queries_by_total_count', isSubquery: true, chartConfig: { - ...pick(source, [ - 'timestampValueExpression', - 'implicitColumnExpression', - 'connection', - 'from', - ]), + ...pickSourceConfigFields(source), where: appliedConfig.where || '', whereLanguage: (appliedConfig.whereLanguage ?? getStoredLanguage()) || 'sql', @@ -1193,12 +1183,7 @@ function DatabaseTab({ ]} config={{ source: source.id, - ...pick(source, [ - 'timestampValueExpression', - 'implicitColumnExpression', - 'connection', - 'from', - ]), + ...pickSourceConfigFields(source), where: appliedConfig.where || '', whereLanguage: (appliedConfig.whereLanguage ?? getStoredLanguage()) || @@ -1279,12 +1264,7 @@ function DatabaseTab({ ]} config={{ source: source.id, - ...pick(source, [ - 'timestampValueExpression', - 'implicitColumnExpression', - 'connection', - 'from', - ]), + ...pickSourceConfigFields(source), where: appliedConfig.where || '', whereLanguage: (appliedConfig.whereLanguage ?? getStoredLanguage()) || @@ -1366,7 +1346,10 @@ function ErrorsTab({ searchedTimeRange: [Date, Date]; appliedConfig: AppliedConfig; }) { - const { data: source } = useSource({ id: appliedConfig.source }); + const { data: source } = useSource({ + id: appliedConfig.source, + kinds: [SourceKind.Trace], + }); const { expressions } = useServiceDashboardExpressions({ source }); return ( @@ -1379,12 +1362,7 @@ function ErrorsTab({ sourceId={source.id} config={{ source: source.id, - ...pick(source, [ - 'timestampValueExpression', - 'implicitColumnExpression', - 'connection', - 'from', - ]), + ...pickSourceConfigFields(source), where: appliedConfig.where || '', whereLanguage: (appliedConfig.whereLanguage ?? getStoredLanguage()) || 'sql', @@ -1403,7 +1381,10 @@ function ErrorsTab({ }, ...getScopedFilters({ appliedConfig, expressions }), ], - groupBy: source.serviceNameExpression || expressions.service, + groupBy: + (isLogSource(source) || isTraceSource(source) + ? source.serviceNameExpression + : undefined) || expressions.service, dateRange: searchedTimeRange, }} /> diff --git a/packages/app/src/SessionSidePanel.tsx b/packages/app/src/SessionSidePanel.tsx index 9e64b68c..7440d300 100644 --- a/packages/app/src/SessionSidePanel.tsx +++ b/packages/app/src/SessionSidePanel.tsx @@ -5,7 +5,8 @@ import { DateRange, SearchCondition, SearchConditionLanguage, - TSource, + TSessionSource, + TTraceSource, } from '@hyperdx/common-utils/dist/types'; import { ActionIcon, Button, Drawer } from '@mantine/core'; import { notifications } from '@mantine/notifications'; @@ -31,8 +32,8 @@ export default function SessionSidePanel({ generateChartUrl, zIndex = 100, }: { - traceSource: TSource; - sessionSource: TSource; + traceSource: TTraceSource; + sessionSource: TSessionSource; sessionId: string; session: Session; dateRange: DateRange['dateRange']; diff --git a/packages/app/src/SessionSubpanel.tsx b/packages/app/src/SessionSubpanel.tsx index adce2519..6ccf93d5 100644 --- a/packages/app/src/SessionSubpanel.tsx +++ b/packages/app/src/SessionSubpanel.tsx @@ -10,7 +10,8 @@ import { DateRange, SearchCondition, SearchConditionLanguage, - TSource, + TSessionSource, + TTraceSource, } from '@hyperdx/common-utils/dist/types'; import { ActionIcon, @@ -55,7 +56,7 @@ function useSessionChartConfigs({ end, tab, }: { - traceSource: TSource; + traceSource: TTraceSource; rumSessionId: string; where: string; whereLanguage?: SearchConditionLanguage; @@ -244,8 +245,8 @@ export default function SessionSubpanel({ whereLanguage = 'lucene', onLanguageChange, }: { - traceSource: TSource; - sessionSource: TSource; + traceSource: TTraceSource; + sessionSource: TSessionSource; session: { serviceName: string }; generateSearchUrl?: (query?: string, timeRange?: [Date, Date]) => string; generateChartUrl?: (config: { diff --git a/packages/app/src/SessionsPage.tsx b/packages/app/src/SessionsPage.tsx index a051cb0b..68d10603 100644 --- a/packages/app/src/SessionsPage.tsx +++ b/packages/app/src/SessionsPage.tsx @@ -249,10 +249,12 @@ export default function SessionsPage() { const sourceId = useWatch({ control, name: 'source' }); const { data: sessionSource, isPending: isSessionSourceLoading } = useSource({ id: sourceId, + kinds: [SourceKind.Session], }); const { data: traceTrace } = useSource({ id: sessionSource?.traceSourceId, + kinds: [SourceKind.Trace], }); // Get all sources and select the first session type source by default @@ -376,7 +378,7 @@ export default function SessionsPage() { const { data: tableData, isLoading: isSessionsLoading } = useSessions({ dateRange: searchedTimeRange, - sessionSource: sessionSource, + sessionSource, traceSource: traceTrace, // TODO: if selectedSession is not null, we should filter by that session id where: appliedConfig.where as SearchCondition, @@ -472,16 +474,6 @@ export default function SessionsPage() { ) : ( <> - {sessionSource && sessionSource.kind !== SourceKind.Session && ( - } - color="gray" - py="xs" - mt="md" - > - Please select a valid session source - - )} {!sessions.length ? ( ) : ( diff --git a/packages/app/src/__tests__/DBSearchPage.test.tsx b/packages/app/src/__tests__/DBSearchPage.test.tsx index 727a21b7..02af61d8 100644 --- a/packages/app/src/__tests__/DBSearchPage.test.tsx +++ b/packages/app/src/__tests__/DBSearchPage.test.tsx @@ -1,3 +1,4 @@ +import { SourceKind } from '@hyperdx/common-utils/dist/types'; import { renderHook } from '@testing-library/react'; import * as sourceModule from '@/source'; @@ -159,6 +160,7 @@ describe('useDefaultOrderBy', () => { for (const testCase of testCases) { it(`${testCase.sortingKey}`, () => { const mockSource = { + kind: SourceKind.Log, timestampValueExpression: testCase.timestampValueExpression || 'Timestamp', displayedTimestampValueExpression: @@ -226,6 +228,7 @@ describe('useDefaultOrderBy', () => { it('should return orderByExpression when set on the source', () => { const mockSource = { + kind: SourceKind.Log, timestampValueExpression: 'Timestamp', orderByExpression: 'Timestamp ASC', }; @@ -253,6 +256,7 @@ describe('useDefaultOrderBy', () => { it('should fall back to optimized order when orderByExpression is empty', () => { const mockSource = { + kind: SourceKind.Log, timestampValueExpression: 'Timestamp', orderByExpression: '', }; @@ -280,6 +284,7 @@ describe('useDefaultOrderBy', () => { it('should fall back to optimized order when orderByExpression is undefined', () => { const mockSource = { + kind: SourceKind.Log, timestampValueExpression: 'Timestamp', }; @@ -306,6 +311,7 @@ describe('useDefaultOrderBy', () => { it('should handle complex Timestamp expressions', () => { const mockSource = { + kind: SourceKind.Log, timestampValueExpression: 'toDateTime(timestamp_ms / 1000)', }; diff --git a/packages/app/src/__tests__/serviceDashboard.test.ts b/packages/app/src/__tests__/serviceDashboard.test.ts index 86eff938..f0892289 100644 --- a/packages/app/src/__tests__/serviceDashboard.test.ts +++ b/packages/app/src/__tests__/serviceDashboard.test.ts @@ -1,5 +1,5 @@ import type { ColumnMeta } from '@hyperdx/common-utils/dist/clickhouse'; -import type { TSource } from '@hyperdx/common-utils/dist/types'; +import type { TTraceSource } from '@hyperdx/common-utils/dist/types'; import { SourceKind } from '@hyperdx/common-utils/dist/types'; import { renderHook } from '@testing-library/react'; @@ -15,7 +15,7 @@ function removeAllWhitespace(str: string) { } describe('Service Dashboard', () => { - const mockSource: TSource = { + const mockSource: TTraceSource = { id: 'test-source', name: 'Test Source', kind: SourceKind.Trace, @@ -25,13 +25,16 @@ describe('Service Dashboard', () => { }, connection: 'test-connection', timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: 'Timestamp', durationExpression: 'Duration', durationPrecision: 9, traceIdExpression: 'TraceId', + spanIdExpression: 'SpanId', + parentSpanIdExpression: 'ParentSpanId', serviceNameExpression: 'ServiceName', spanNameExpression: 'SpanName', spanKindExpression: 'SpanKind', - severityTextExpression: 'StatusCode', + statusCodeExpression: 'StatusCode', }; describe('getExpressions', () => { diff --git a/packages/app/src/__tests__/source.test.ts b/packages/app/src/__tests__/source.test.ts index 65e4da8f..dccf3475 100644 --- a/packages/app/src/__tests__/source.test.ts +++ b/packages/app/src/__tests__/source.test.ts @@ -1,11 +1,11 @@ -import { SourceKind, TSource } from '@hyperdx/common-utils/dist/types'; +import { SourceKind, TTraceSource } from '@hyperdx/common-utils/dist/types'; import { getEventBody } from '../source'; describe('getEventBody', () => { // Added to prevent regression back to HDX-3361 it('returns spanNameExpression for trace kind source when both bodyExpression and spanNameExpression are present', () => { - const source: TSource = { + const source = { kind: SourceKind.Trace, from: { databaseName: 'default', @@ -15,9 +15,14 @@ describe('getEventBody', () => { connection: 'test-connection', name: 'Traces', id: 'test-source-id', - bodyExpression: 'Body', spanNameExpression: 'SpanName', - }; + durationExpression: 'Duration', + durationPrecision: 9, + traceIdExpression: 'TraceId', + spanIdExpression: 'SpanId', + parentSpanIdExpression: 'ParentSpanId', + spanKindExpression: 'SpanKind', + } as TTraceSource; const result = getEventBody(source); diff --git a/packages/app/src/components/AlertPreviewChart.tsx b/packages/app/src/components/AlertPreviewChart.tsx index 76ca07c5..b1baabef 100644 --- a/packages/app/src/components/AlertPreviewChart.tsx +++ b/packages/app/src/components/AlertPreviewChart.tsx @@ -3,10 +3,12 @@ import { aliasMapToWithClauses } from '@hyperdx/common-utils/dist/core/utils'; import { AlertInterval, Filter, + isLogSource, + isTraceSource, SearchCondition, SearchConditionLanguage, + TSource, } from '@hyperdx/common-utils/dist/types'; -import { TSource } from '@hyperdx/common-utils/dist/types'; import { Paper } from '@mantine/core'; import { DBTimeChart } from '@/components/DBTimeChart'; @@ -41,7 +43,9 @@ export const AlertPreviewChart = ({ const resolvedSelect = (select && select.trim().length > 0 ? select - : source.defaultTableSelectExpression) ?? ''; + : isLogSource(source) || isTraceSource(source) + ? source.defaultTableSelectExpression + : undefined) ?? ''; const { data: aliasMap } = useAliasMapFromChartConfig({ select: resolvedSelect, @@ -66,7 +70,10 @@ export const AlertPreviewChart = ({ dateRange: intervalToDateRange(interval), granularity: intervalToGranularity(interval), filters: filters || undefined, - implicitColumnExpression: source.implicitColumnExpression, + implicitColumnExpression: + isLogSource(source) || isTraceSource(source) + ? source.implicitColumnExpression + : undefined, groupBy, with: aliasWith, select: [ diff --git a/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx b/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx index 32e70546..a05f026e 100644 --- a/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx +++ b/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx @@ -6,7 +6,12 @@ import { } from '@hyperdx/common-utils/dist/core/metadata'; import { MACRO_SUGGESTIONS } from '@hyperdx/common-utils/dist/macros'; import { QUERY_PARAMS_BY_DISPLAY_TYPE } from '@hyperdx/common-utils/dist/rawSqlParams'; -import { DisplayType, SourceKind } from '@hyperdx/common-utils/dist/types'; +import { + DisplayType, + isLogSource, + isMetricSource, + isTraceSource, +} from '@hyperdx/common-utils/dist/types'; import { Box, Button, Group, Stack, Text, Tooltip } from '@mantine/core'; import { IconHelpCircle } from '@tabler/icons-react'; @@ -100,11 +105,14 @@ export default function RawSqlChartEditor({ .flatMap(source => { const tables: TableConnection[] = getAllMetricTables(source); - if (source.kind !== SourceKind.Metric) { + if (!isMetricSource(source)) { tables.push(tcFromSource(source)); } - if (source.materializedViews) { + if ( + (isLogSource(source) || isTraceSource(source)) && + source.materializedViews + ) { tables.push( ...source.materializedViews.map(mv => ({ databaseName: mv.databaseName, diff --git a/packages/app/src/components/ChartEditor/__tests__/utils.test.ts b/packages/app/src/components/ChartEditor/__tests__/utils.test.ts index 57079d06..228d6ae1 100644 --- a/packages/app/src/components/ChartEditor/__tests__/utils.test.ts +++ b/packages/app/src/components/ChartEditor/__tests__/utils.test.ts @@ -2,6 +2,7 @@ import type { BuilderChartConfig, BuilderSavedChartConfig, RawSqlSavedChartConfig, + TMetricSource, TSource, } from '@hyperdx/common-utils/dist/types'; import { @@ -38,14 +39,14 @@ const logSource: TSource = { implicitColumnExpression: 'Body', }; -const metricSource: TSource = { +const metricSource: TMetricSource = { id: 'source-metric', name: 'Metric Source', kind: SourceKind.Metric, connection: 'conn-1', from: { databaseName: 'db', tableName: '' }, timestampValueExpression: 'TimeUnix', - metricTables: { gauge: 'gauge_table' } as TSource['metricTables'], + metricTables: { gauge: 'gauge_table' } as TMetricSource['metricTables'], resourceAttributesExpression: 'ResourceAttributes', }; diff --git a/packages/app/src/components/ChartEditor/utils.ts b/packages/app/src/components/ChartEditor/utils.ts index 268054af..e3e5e047 100644 --- a/packages/app/src/components/ChartEditor/utils.ts +++ b/packages/app/src/components/ChartEditor/utils.ts @@ -8,6 +8,9 @@ import { BuilderSavedChartConfig, ChartConfigWithDateRange, DisplayType, + isLogSource, + isMetricSource, + isTraceSource, RawSqlChartConfig, RawSqlSavedChartConfig, SavedChartConfig, @@ -135,11 +138,16 @@ export function convertFormStateToChartConfig( timestampValueExpression: source.timestampValueExpression, dateRange, connection: source.connection, - implicitColumnExpression: source.implicitColumnExpression, - metricTables: source.metricTables, + implicitColumnExpression: + isLogSource(source) || isTraceSource(source) + ? source.implicitColumnExpression + : undefined, + metricTables: isMetricSource(source) ? source.metricTables : undefined, where: form.where ?? '', select: isSelectEmpty - ? source.defaultTableSelectExpression || '' + ? ((isLogSource(source) || isTraceSource(source)) && + source.defaultTableSelectExpression) || + '' : mergedSelect, }; diff --git a/packages/app/src/components/ContextSidePanel.tsx b/packages/app/src/components/ContextSidePanel.tsx index 375b2043..e71ba620 100644 --- a/packages/app/src/components/ContextSidePanel.tsx +++ b/packages/app/src/components/ContextSidePanel.tsx @@ -6,6 +6,8 @@ import { useForm, useWatch } from 'react-hook-form'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; import { BuilderChartConfigWithDateRange, + isLogSource, + isTraceSource, TSource, } from '@hyperdx/common-utils/dist/types'; import { Badge, Flex, Group, SegmentedControl } from '@mantine/core'; @@ -228,7 +230,10 @@ export default function ContextSubpanel({ connection: source.connection, from: source.from, timestampValueExpression: source.timestampValueExpression, - select: source.defaultTableSelectExpression || '', + select: + ((isLogSource(source) || isTraceSource(source)) && + source.defaultTableSelectExpression) || + '', limit: { limit: 200 }, orderBy: `${source.timestampValueExpression} DESC`, where: whereClause, @@ -249,10 +254,7 @@ export default function ContextSubpanel({ originalLanguage, newDateRange, contextBy, - source.connection, - source.defaultTableSelectExpression, - source.from, - source.timestampValueExpression, + source, ]); return ( diff --git a/packages/app/src/components/DBEditTimeChartForm.tsx b/packages/app/src/components/DBEditTimeChartForm.tsx index b2bb84d9..0535efe1 100644 --- a/packages/app/src/components/DBEditTimeChartForm.tsx +++ b/packages/app/src/components/DBEditTimeChartForm.tsx @@ -238,12 +238,15 @@ function ChartSeriesEditorComponent({ }); const groupBy = useWatch({ control, name: 'groupBy' }); + const metricTableSource = + tableSource?.kind === SourceKind.Metric ? tableSource : undefined; + const { data: attributeSuggestions, isLoading: isLoadingAttributes } = useFetchMetricResourceAttrs({ databaseName, metricType, metricName, - tableSource, + tableSource: metricTableSource, isSql: aggConditionLanguage === 'sql', }); @@ -256,7 +259,7 @@ function ChartSeriesEditorComponent({ databaseName, metricType, metricName, - tableSource, + tableSource: metricTableSource, }); const handleAddToWhere = useCallback( @@ -1020,7 +1023,11 @@ export default function EditTimeChartForm({ connection: tableSource.connection, from: tableSource.from, limit: { limit: 200 }, - select: tableSource?.defaultTableSelectExpression || '', + select: + ((tableSource?.kind === SourceKind.Log || + tableSource?.kind === SourceKind.Trace) && + tableSource.defaultTableSelectExpression) || + '', filters: seriesToFilters(queriedConfig.select), filtersLogicalOperator: 'OR' as const, groupBy: undefined, @@ -1364,10 +1371,17 @@ export default function EditTimeChartForm({ control={control} name="select" placeholder={ - tableSource?.defaultTableSelectExpression || + ((tableSource?.kind === SourceKind.Log || + tableSource?.kind === SourceKind.Trace) && + tableSource.defaultTableSelectExpression) || 'SELECT Columns' } - defaultValue={tableSource?.defaultTableSelectExpression} + defaultValue={ + tableSource?.kind === SourceKind.Log || + tableSource?.kind === SourceKind.Trace + ? tableSource.defaultTableSelectExpression + : undefined + } onSubmit={onSubmit} label="SELECT" /> @@ -1649,7 +1663,10 @@ export default function EditTimeChartForm({ typeof queriedConfig.select === 'string' && queriedConfig.select ? queriedConfig.select - : tableSource?.defaultTableSelectExpression || '', + : ((tableSource?.kind === SourceKind.Log || + tableSource?.kind === SourceKind.Trace) && + tableSource.defaultTableSelectExpression) || + '', groupBy: undefined, having: undefined, granularity: undefined, diff --git a/packages/app/src/components/DBInfraPanel.tsx b/packages/app/src/components/DBInfraPanel.tsx index 8df2064e..3f049d90 100644 --- a/packages/app/src/components/DBInfraPanel.tsx +++ b/packages/app/src/components/DBInfraPanel.tsx @@ -4,7 +4,13 @@ import { convertDateRangeToGranularityString, Granularity, } from '@hyperdx/common-utils/dist/core/utils'; -import { TSource } from '@hyperdx/common-utils/dist/types'; +import { + isLogSource, + isTraceSource, + SourceKind, + TMetricSource, + TSource, +} from '@hyperdx/common-utils/dist/types'; import { Box, Card, @@ -35,7 +41,7 @@ const InfraSubpanelGroup = ({ where, }: { fieldPrefix: string; - metricSource: TSource; + metricSource: TMetricSource; timestamp: any; title: string; where: string; @@ -205,7 +211,14 @@ export default ({ rowId: string | undefined | null; source: TSource; }) => { - const { data: metricSource } = useSource({ id: source.metricSourceId }); + const metricSourceId = + isLogSource(source) || isTraceSource(source) + ? source.metricSourceId + : undefined; + const { data: metricSource } = useSource({ + id: metricSourceId, + kinds: [SourceKind.Metric], + }); const podUid = rowData?.__hdx_resource_attributes['k8s.pod.uid']; const nodeName = rowData?.__hdx_resource_attributes['k8s.node.name']; @@ -225,7 +238,7 @@ export default ({ metricSource={metricSource} /> )} - {source && ( + {source && source.kind === SourceKind.Log && ( Pod Timeline diff --git a/packages/app/src/components/DBRowDataPanel.tsx b/packages/app/src/components/DBRowDataPanel.tsx index d456a147..93ed1003 100644 --- a/packages/app/src/components/DBRowDataPanel.tsx +++ b/packages/app/src/components/DBRowDataPanel.tsx @@ -1,7 +1,12 @@ import { useMemo } from 'react'; import { flatten } from 'flat'; import type { ResponseJSON } from '@hyperdx/common-utils/dist/clickhouse'; -import { SourceKind, TSource } from '@hyperdx/common-utils/dist/types'; +import { + isLogSource, + isTraceSource, + SourceKind, + TSource, +} from '@hyperdx/common-utils/dist/types'; import { Box } from '@mantine/core'; import { useQueriedChartConfig } from '@/hooks/useChartConfig'; @@ -35,11 +40,20 @@ export function useRowData({ }) { const eventBodyExpr = getEventBody(source); - const searchedTraceIdExpr = source.traceIdExpression; - const searchedSpanIdExpr = source.spanIdExpression; + const searchedTraceIdExpr = + isLogSource(source) || isTraceSource(source) + ? source.traceIdExpression + : undefined; + const searchedSpanIdExpr = + isLogSource(source) || isTraceSource(source) + ? source.spanIdExpression + : undefined; - const severityTextExpr = - source.severityTextExpression || source.statusCodeExpression; + const severityTextExpr = isLogSource(source) + ? source.severityTextExpression + : isTraceSource(source) + ? source.statusCodeExpression + : undefined; const selectHighlightedRowAttributes = source.kind === SourceKind.Trace || source.kind === SourceKind.Log @@ -91,7 +105,8 @@ export function useRowData({ }, ] : []), - ...(source.serviceNameExpression + ...((isLogSource(source) || isTraceSource(source)) && + source.serviceNameExpression ? [ { valueExpression: source.serviceNameExpression, @@ -107,7 +122,8 @@ export function useRowData({ }, ] : []), - ...(source.eventAttributesExpression + ...((isLogSource(source) || isTraceSource(source)) && + source.eventAttributesExpression ? [ { valueExpression: source.eventAttributesExpression, diff --git a/packages/app/src/components/DBRowOverviewPanel.tsx b/packages/app/src/components/DBRowOverviewPanel.tsx index 11bf6281..3571eb02 100644 --- a/packages/app/src/components/DBRowOverviewPanel.tsx +++ b/packages/app/src/components/DBRowOverviewPanel.tsx @@ -53,7 +53,10 @@ export function RowOverviewPanel({ const jsonColumns = getJSONColumnNames(data?.meta); - const eventAttributesExpr = source.eventAttributesExpression; + const eventAttributesExpr = + source.kind === SourceKind.Log || source.kind === SourceKind.Trace + ? source.eventAttributesExpression + : undefined; const firstRow = useMemo(() => { const firstRow = { ...(data?.data?.[0] ?? {}) }; diff --git a/packages/app/src/components/DBRowSidePanel.tsx b/packages/app/src/components/DBRowSidePanel.tsx index 57b4469e..0f88746c 100644 --- a/packages/app/src/components/DBRowSidePanel.tsx +++ b/packages/app/src/components/DBRowSidePanel.tsx @@ -12,7 +12,15 @@ import { isString } from 'lodash'; import { parseAsStringEnum, useQueryState } from 'nuqs'; import { ErrorBoundary } from 'react-error-boundary'; import { useHotkeys } from 'react-hotkeys-hook'; -import { SourceKind, TSource } from '@hyperdx/common-utils/dist/types'; +import { + isLogSource, + isSessionSource, + isTraceSource, + SourceKind, + TLogSource, + TSource, + TTraceSource, +} from '@hyperdx/common-utils/dist/types'; import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import { Box, Drawer, Flex, Stack } from '@mantine/core'; @@ -66,7 +74,7 @@ export type RowSidePanelContextProps = { dbSqlRowTableConfig?: BuilderChartConfigWithDateRange; isChildModalOpen?: boolean; setChildModalOpen?: (open: boolean) => void; - source?: TSource; + source?: TLogSource | TTraceSource; }; export const RowSidePanelContext = createContext({}); @@ -145,14 +153,21 @@ const DBRowSidePanel = ({ ); const hasOverviewPanel = useMemo(() => { - if ( - source.resourceAttributesExpression || - source.eventAttributesExpression + if (isLogSource(source) || isTraceSource(source)) { + if ( + source.resourceAttributesExpression || + source.eventAttributesExpression + ) { + return true; + } + } else if ( + source.kind === SourceKind.Metric && + source.resourceAttributesExpression ) { return true; } return false; - }, [source.eventAttributesExpression, source.resourceAttributesExpression]); + }, [source]); const defaultTab = source.kind === 'trace' @@ -195,8 +210,9 @@ const DBRowSidePanel = ({ normalizedRow?.['__hdx_severity_text']; const highlightedAttributeValues = useMemo(() => { - const attributeExpressions: TSource['highlightedRowAttributeExpressions'] = - []; + const attributeExpressions: NonNullable< + (TLogSource | TTraceSource)['highlightedRowAttributeExpressions'] + > = []; if ( (source.kind === SourceKind.Trace || source.kind === SourceKind.Log) && source.highlightedRowAttributeExpressions @@ -206,7 +222,10 @@ const DBRowSidePanel = ({ // Add service name expression to all sources, to maintain compatibility with // the behavior prior to the addition of highlightedRowAttributeExpressions - if (source.serviceNameExpression) { + if ( + (isLogSource(source) || isTraceSource(source)) && + source.serviceNameExpression + ) { attributeExpressions.push({ sqlExpression: source.serviceNameExpression, }); @@ -240,15 +259,19 @@ const DBRowSidePanel = ({ const focusDate = timestampDate; const traceId: string | undefined = normalizedRow?.['__hdx_trace_id']; - const childSourceId = - source.kind === 'log' - ? source.traceSourceId - : source.kind === 'trace' - ? source.logSourceId - : undefined; + const childSourceId = isLogSource(source) + ? source.traceSourceId + : isTraceSource(source) + ? source.logSourceId + : undefined; - const traceSourceId = - source.kind === 'trace' ? source.id : source.traceSourceId; + const traceSourceId = isTraceSource(source) + ? source.id + : isLogSource(source) + ? source.traceSourceId + : isSessionSource(source) + ? source.traceSourceId + : undefined; const enableServiceMap = traceId && traceSourceId; diff --git a/packages/app/src/components/DBRowTable.tsx b/packages/app/src/components/DBRowTable.tsx index e98a3c58..36278c36 100644 --- a/packages/app/src/components/DBRowTable.tsx +++ b/packages/app/src/components/DBRowTable.tsx @@ -34,6 +34,7 @@ import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils'; import { BuilderChartConfigWithDateRange, SelectList, + SourceKind, TSource, } from '@hyperdx/common-utils/dist/types'; import { @@ -1669,7 +1670,10 @@ function DBSqlRowTableComponent({ config, samples: 10_000, bodyValueExpression: patternColumn ?? '', - severityTextExpression: source?.severityTextExpression ?? '', + severityTextExpression: + (source?.kind === SourceKind.Log + ? source.severityTextExpression + : undefined) ?? '', totalCount: undefined, enabled: denoiseResults, }); diff --git a/packages/app/src/components/DBSearchPageFilters.tsx b/packages/app/src/components/DBSearchPageFilters.tsx index dc64d103..ce4950f2 100644 --- a/packages/app/src/components/DBSearchPageFilters.tsx +++ b/packages/app/src/components/DBSearchPageFilters.tsx @@ -1120,29 +1120,33 @@ const DBSearchPageFiltersComponent = ({ [filterState], ); + const parentSpanIdExpr = + source?.kind === SourceKind.Trace + ? source.parentSpanIdExpression + : undefined; + const setRootSpansOnly = useCallback( (rootSpansOnly: boolean) => { - if (!source?.parentSpanIdExpression) return; + if (!parentSpanIdExpr) return; if (rootSpansOnly) { if (columns?.some(col => col.name === IS_ROOT_SPAN_COLUMN_NAME)) { setFilterValue(IS_ROOT_SPAN_COLUMN_NAME, true, 'only'); } else { - setFilterValue(source.parentSpanIdExpression, '', 'only'); + setFilterValue(parentSpanIdExpr, '', 'only'); } } else { - clearFilter(source.parentSpanIdExpression); + clearFilter(parentSpanIdExpr); clearFilter(IS_ROOT_SPAN_COLUMN_NAME); } }, - [setFilterValue, clearFilter, source, columns], + [setFilterValue, clearFilter, parentSpanIdExpr, columns], ); const isRootSpansOnly = useMemo(() => { - if (!source?.parentSpanIdExpression || source.kind !== SourceKind.Trace) - return false; + if (!parentSpanIdExpr || source?.kind !== SourceKind.Trace) return false; - const parentSpanIdFilter = filterState?.[source?.parentSpanIdExpression]; + const parentSpanIdFilter = filterState?.[parentSpanIdExpr]; const isRootSpanFilter = filterState?.[IS_ROOT_SPAN_COLUMN_NAME]; return ( (parentSpanIdFilter?.included.size === 1 && @@ -1150,7 +1154,7 @@ const DBSearchPageFiltersComponent = ({ (isRootSpanFilter?.included.size === 1 && isRootSpanFilter?.included.has(true)) ); - }, [filterState, source]); + }, [filterState, source, parentSpanIdExpr]); return ( diff --git a/packages/app/src/components/DBSessionPanel.tsx b/packages/app/src/components/DBSessionPanel.tsx index 18d770c0..b248fd35 100644 --- a/packages/app/src/components/DBSessionPanel.tsx +++ b/packages/app/src/components/DBSessionPanel.tsx @@ -1,5 +1,6 @@ import { useMemo } from 'react'; import Link from 'next/link'; +import { isTraceSource, SourceKind } from '@hyperdx/common-utils/dist/types'; import { Loader } from '@mantine/core'; import useFieldExpressionGenerator from '@/hooks/useFieldExpressionGenerator'; @@ -19,8 +20,10 @@ export const useSessionId = ({ dateRange: [Date, Date]; enabled?: boolean; }) => { - // trace source - const { data: source } = useSource({ id: sourceId }); + const { data: source } = useSource({ + id: sourceId, + kinds: [SourceKind.Trace], + }); const { getFieldExpression } = useFieldExpressionGenerator(source); @@ -99,9 +102,16 @@ export const DBSessionPanel = ({ serviceName: string; setSubDrawerOpen: (open: boolean) => void; }) => { - const { data: traceSource } = useSource({ id: traceSourceId }); + const { data: traceSource } = useSource({ + id: traceSourceId, + kinds: [SourceKind.Trace], + }); const { data: sessionSource, isLoading: isSessionSourceLoading } = useSource({ - id: traceSource?.sessionSourceId, + id: + traceSource && isTraceSource(traceSource) + ? traceSource.sessionSourceId + : undefined, + kinds: [SourceKind.Session], }); if (!traceSource || (!sessionSource && isSessionSourceLoading)) { diff --git a/packages/app/src/components/DBTracePanel.tsx b/packages/app/src/components/DBTracePanel.tsx index 0f2236fc..07bd6c2e 100644 --- a/packages/app/src/components/DBTracePanel.tsx +++ b/packages/app/src/components/DBTracePanel.tsx @@ -2,7 +2,11 @@ import { useEffect, useState } from 'react'; import { useQueryState } from 'nuqs'; import { useForm, useWatch } from 'react-hook-form'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; -import { SourceKind } from '@hyperdx/common-utils/dist/types'; +import { + isLogSource, + isTraceSource, + SourceKind, +} from '@hyperdx/common-utils/dist/types'; import { Button, Center, @@ -112,14 +116,22 @@ export default function DBTracePanel({ setValue: traceIdSetValue, } = useForm<{ traceIdExpression: string }>({ defaultValues: { - traceIdExpression: parentSourceData?.traceIdExpression ?? '', + traceIdExpression: + (parentSourceData && + (isLogSource(parentSourceData) || isTraceSource(parentSourceData)) && + parentSourceData.traceIdExpression) || + '', }, }); useEffect(() => { - if (parentSourceData?.traceIdExpression) { + if ( + parentSourceData && + (isLogSource(parentSourceData) || isTraceSource(parentSourceData)) && + parentSourceData.traceIdExpression + ) { traceIdSetValue('traceIdExpression', parentSourceData.traceIdExpression); } - }, [parentSourceData?.traceIdExpression, traceIdSetValue]); + }, [parentSourceData, traceIdSetValue]); const [showTraceIdInput, setShowTraceIdInput] = useState(false); @@ -137,8 +149,11 @@ export default function DBTracePanel({ - {parentSourceData?.traceIdExpression}:{' '} - {traceId || 'No trace id found for event'} + {parentSourceData && + (isLogSource(parentSourceData) || isTraceSource(parentSourceData)) + ? parentSourceData.traceIdExpression + : ''} + : {traceId || 'No trace id found for event'} {traceId != null && (