fix: Backport Services Dashboard fixes (#1446)

Closes HDX-2950

# Summary

This PR backports a number of fixes on the Services dashboard from the Enterprise Edition repo:

- The Request Error Rate by Endpoint chart now queries just the top 60 endpoints by max per-bucket error rate to avoid OOMs on datasets with many endpoints.
- The top 20 chart is now sorted according to the selected toggle (by time / by error rate)
- The slowest 10% of queries chart has been updated to correctly indicate that it is showing the slowest 5% of queries
- P95 and Median have been unswapped in the 20 Top Most Time Consuming Endpoints query/tooltip
- The top 20 most time consuming queries dashboard no longer errors out when db.statement is a materialized column

This PR also re-structures some queries to match how they've been structured in the enterprise repo, to avoid drift. See [this PR](https://github.com/DeploySentinel/hyperdx-ee/pull/1024) for more details on those changes.

## Demo

https://github.com/user-attachments/assets/014938b3-33da-411b-b754-7c713f7a8ac8
This commit is contained in:
Drew Davis 2025-12-04 15:40:13 -05:00 committed by GitHub
parent c60e646ee2
commit 4b1557d957
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 745 additions and 237 deletions

View file

@ -0,0 +1,5 @@
---
"@hyperdx/app": patch
---
fix: Backport Services Dashboard fixes

View file

@ -522,8 +522,13 @@ export const K8S_NETWORK_NUMBER_FORMAT: NumberFormat = {
output: 'byte',
};
function inferValueColumns(meta: Array<{ name: string; type: string }>) {
return filterColumnMetaByType(meta, [JSDataType.Number]);
function inferValueColumns(
meta: Array<{ name: string; type: string }>,
excluded: Set<string>,
) {
return filterColumnMetaByType(meta, [JSDataType.Number])?.filter(
c => !excluded.has(c.name),
);
}
function inferGroupColumns(meta: Array<{ name: string; type: string }>) {
@ -609,6 +614,7 @@ function addResponseToFormattedData({
source,
currentPeriodDateRange,
isPreviousPeriod,
hiddenSeries = [],
}: {
tsBucketMap: Map<number, Record<string, any>>;
lineDataMap: { [keyName: string]: LineDataWithOptionalColor };
@ -616,6 +622,7 @@ function addResponseToFormattedData({
source?: TSource;
isPreviousPeriod: boolean;
currentPeriodDateRange: [Date, Date];
hiddenSeries?: string[];
}) {
const { meta, data } = response;
if (meta == null) {
@ -629,7 +636,7 @@ function addResponseToFormattedData({
);
}
const valueColumns = inferValueColumns(meta) ?? [];
const valueColumns = inferValueColumns(meta, new Set(hiddenSeries)) ?? [];
const groupColumns = inferGroupColumns(meta) ?? [];
const isSingleValueColumn = valueColumns.length === 1;
const hasGroupColumns = groupColumns.length > 0;
@ -694,6 +701,7 @@ export function formatResponseForTimeChart({
granularity,
generateEmptyBuckets = true,
source,
hiddenSeries = [],
}: {
dateRange: [Date, Date];
granularity?: SQLInterval;
@ -701,6 +709,7 @@ export function formatResponseForTimeChart({
previousPeriodResponse?: ResponseJSON<Record<string, any>>;
generateEmptyBuckets?: boolean;
source?: TSource;
hiddenSeries?: string[];
}) {
const meta = currentPeriodResponse.meta;
@ -709,7 +718,7 @@ export function formatResponseForTimeChart({
}
const timestampColumn = inferTimestampColumn(meta);
const valueColumns = inferValueColumns(meta) ?? [];
const valueColumns = inferValueColumns(meta, new Set(hiddenSeries)) ?? [];
const groupColumns = inferGroupColumns(meta) ?? [];
const isSingleValueColumn = valueColumns.length === 1;
@ -732,6 +741,7 @@ export function formatResponseForTimeChart({
source,
isPreviousPeriod: false,
currentPeriodDateRange: dateRange,
hiddenSeries,
});
if (previousPeriodResponse != null) {
@ -742,6 +752,7 @@ export function formatResponseForTimeChart({
source,
isPreviousPeriod: true,
currentPeriodDateRange: dateRange,
hiddenSeries,
});
}

View file

@ -12,6 +12,7 @@ import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata';
import { DEFAULT_AUTO_GRANULARITY_MAX_BUCKETS } from '@hyperdx/common-utils/dist/core/renderChartConfig';
import {
ChartConfigWithDateRange,
ChartConfigWithOptDateRange,
CteChartConfig,
DisplayType,
Filter,
@ -51,10 +52,12 @@ import { TimePicker } from '@/components/TimePicker';
import WhereLanguageControlled from '@/components/WhereLanguageControlled';
import { useQueriedChartConfig } from '@/hooks/useChartConfig';
import { useDashboardRefresh } from '@/hooks/useDashboardRefresh';
import { useJsonColumns } from '@/hooks/useMetadata';
import { withAppNav } from '@/layout';
import SearchInputV2 from '@/SearchInputV2';
import { getExpressions } from '@/serviceDashboard';
import {
getExpressions,
useServiceDashboardExpressions,
} from '@/serviceDashboard';
import { useSource, useSources } from '@/source';
import { Histogram } from '@/SVGIcons';
import { parseTimeQuery, useNewTimeQuery } from '@/timeQuery';
@ -70,12 +73,17 @@ type AppliedConfig = {
const MAX_NUM_SERIES = HARD_LINES_LIMIT;
function getScopedFilters(
source: TSource,
appliedConfig: AppliedConfig,
function getScopedFilters({
appliedConfig,
expressions,
includeIsSpanKindServer = true,
): Filter[] {
const expressions = getExpressions(source);
includeNonEmptyEndpointFilter = false,
}: {
appliedConfig: AppliedConfig;
expressions: ReturnType<typeof getExpressions>;
includeIsSpanKindServer?: boolean;
includeNonEmptyEndpointFilter?: boolean;
}): Filter[] {
const filters: Filter[] = [];
// Database spans are of kind Client. To be cleaned up in HDX-1219
if (includeIsSpanKindServer) {
@ -90,6 +98,12 @@ function getScopedFilters(
condition: `${expressions.service} IN ('${appliedConfig.service}')`,
});
}
if (includeNonEmptyEndpointFilter) {
filters.push({
type: 'sql',
condition: expressions.isEndpointNonEmpty,
});
}
return filters;
}
@ -103,12 +117,7 @@ function ServiceSelectControlled({
onCreate?: () => void;
} & UseControllerProps<any>) {
const { data: source } = useSource({ id: sourceId });
const { data: jsonColumns = [] } = useJsonColumns({
databaseName: source?.from?.databaseName || '',
tableName: source?.from?.tableName || '',
connectionId: source?.connection || '',
});
const expressions = getExpressions(source, jsonColumns);
const { expressions } = useServiceDashboardExpressions({ source });
const queriedConfig = {
...source,
@ -120,10 +129,10 @@ function ServiceSelectControlled({
select: [
{
alias: 'service',
valueExpression: `distinct(${expressions.service})`,
valueExpression: `distinct(${expressions?.service})`,
},
],
where: `${expressions.service} IS NOT NULL`,
where: `${expressions?.service} IS NOT NULL`,
whereLanguage: 'sql' as const,
limit: { limit: 200 },
};
@ -131,7 +140,7 @@ function ServiceSelectControlled({
const { data, isLoading, isError } = useQueriedChartConfig(queriedConfig, {
placeholderData: (prev: any) => prev,
queryKey: ['service-select', queriedConfig],
enabled: !!source,
enabled: !!source && !!expressions,
});
const values = useMemo(() => {
@ -171,12 +180,7 @@ export function EndpointLatencyChart({
appliedConfig?: AppliedConfig;
extraFilters?: Filter[];
}) {
const { data: jsonColumns = [] } = useJsonColumns({
databaseName: source?.from?.databaseName || '',
tableName: source?.from?.tableName || '',
connectionId: source?.connection || '',
});
const expressions = getExpressions(source, jsonColumns);
const { expressions } = useServiceDashboardExpressions({ source });
const [latencyChartType, setLatencyChartType] = useState<
'line' | 'histogram'
>('line');
@ -210,39 +214,58 @@ export function EndpointLatencyChart({
</Box>
</Group>
{source &&
expressions &&
(latencyChartType === 'line' ? (
<DBTimeChart
showDisplaySwitcher={false}
sourceId={source.id}
hiddenSeries={[
'p95_duration_ns',
'p50_duration_ns',
'avg_duration_ns',
]}
config={{
...source,
where: appliedConfig.where || '',
whereLanguage: appliedConfig.whereLanguage || 'sql',
select: [
// Separate the aggregations from the conversion to ms so that AggregatingMergeTree MVs can be used
{
alias: '95th Percentile',
alias: 'p95_duration_ns',
aggFn: 'quantile',
level: 0.95,
valueExpression: expressions.durationInMillis,
valueExpression: expressions.duration,
aggCondition: '',
},
{
alias: '95th Percentile',
valueExpression: `p95_duration_ns / ${expressions.durationDivisorForMillis}`,
},
{
alias: 'p50_duration_ns',
aggFn: 'quantile',
level: 0.5,
valueExpression: expressions.duration,
aggCondition: '',
},
{
alias: 'Median',
aggFn: 'quantile',
level: 0.5,
valueExpression: expressions.durationInMillis,
valueExpression: `p50_duration_ns / ${expressions.durationDivisorForMillis}`,
},
{
alias: 'avg_duration_ns',
aggFn: 'avg',
valueExpression: expressions.duration,
aggCondition: '',
},
{
alias: 'Avg',
aggFn: 'avg',
valueExpression: expressions.durationInMillis,
aggCondition: '',
valueExpression: `avg_duration_ns / ${expressions.durationDivisorForMillis}`,
},
],
filters: [
...extraFilters,
...getScopedFilters(source, appliedConfig),
...getScopedFilters({ appliedConfig, expressions }),
],
numberFormat: MS_NUMBER_FORMAT,
dateRange,
@ -262,7 +285,7 @@ export function EndpointLatencyChart({
],
filters: [
...extraFilters,
...getScopedFilters(source, appliedConfig),
...getScopedFilters({ appliedConfig, expressions }),
],
dateRange,
}}
@ -280,12 +303,7 @@ function HttpTab({
appliedConfig: AppliedConfig;
}) {
const { data: source } = useSource({ id: appliedConfig.source });
const { data: jsonColumns = [] } = useJsonColumns({
databaseName: source?.from?.databaseName || '',
tableName: source?.from?.tableName || '',
connectionId: source?.connection || '',
});
const expressions = getExpressions(source, jsonColumns);
const { expressions } = useServiceDashboardExpressions({ source });
const [reqChartType, setReqChartType] = useQueryState(
'reqChartType',
@ -305,6 +323,157 @@ function HttpTab({
return window.location.pathname + '?' + searchParams.toString();
}, []);
const requestErrorRateConfig =
useMemo<ChartConfigWithDateRange | null>(() => {
if (!source || !expressions) return null;
if (reqChartType === 'overall') {
return {
...source,
where: appliedConfig.where || '',
whereLanguage: appliedConfig.whereLanguage || 'sql',
displayType: DisplayType.Line,
select: [
{
valueExpression: `countIf(${expressions.isError}) / count()`,
alias: 'error_rate',
},
],
numberFormat: ERROR_RATE_PERCENTAGE_NUMBER_FORMAT,
filters: getScopedFilters({ appliedConfig, expressions }),
dateRange: searchedTimeRange,
} satisfies ChartConfigWithDateRange;
}
return {
timestampValueExpression: 'series_time_bucket',
connection: source.connection,
with: [
{
name: 'error_series',
chartConfig: {
timestampValueExpression: source?.timestampValueExpression || '',
connection: source?.connection ?? '',
from: source?.from ?? {
databaseName: '',
tableName: '',
},
where: appliedConfig.where || '',
whereLanguage: appliedConfig.whereLanguage || 'sql',
select: [
{
valueExpression: '',
aggFn: 'count',
alias: 'error_count',
aggCondition: expressions.isError,
aggConditionLanguage: 'sql',
},
{
valueExpression: '',
aggFn: 'count',
alias: 'total_count',
},
{
valueExpression: `error_count / total_count`,
alias: 'error_rate',
},
{
valueExpression: expressions?.endpoint,
alias: 'endpoint',
},
],
filters: getScopedFilters({ appliedConfig, expressions }),
groupBy: [
{
valueExpression: 'endpoint',
},
],
orderBy: [
{
valueExpression: 'endpoint',
ordering: 'ASC',
},
],
dateRange: searchedTimeRange,
granularity: convertDateRangeToGranularityString(
searchedTimeRange,
DEFAULT_AUTO_GRANULARITY_MAX_BUCKETS,
),
} as ChartConfigWithOptDateRange,
isSubquery: true,
},
// Select the top N series from the search as we don't want to crash the browser.
// Series are selected based on their max error value
{
name: 'selected_error_series',
isSubquery: true,
chartConfig: {
timestampValueExpression: '__hdx_time_bucket',
connection: source.connection,
select: [
{
valueExpression: 'groupArray(error_rate)',
alias: 'error_rate',
},
{ valueExpression: 'endpoint' },
{
valueExpression: 'groupArray(__hdx_time_bucket)',
alias: '__hdx_time_buckets',
},
],
from: { databaseName: '', tableName: 'error_series' },
where: '',
groupBy: 'endpoint',
orderBy: 'max(error_series.error_rate) DESC',
limit: { limit: MAX_NUM_SERIES },
},
},
// CTE that explodes series arrays into rows again for compatibility with DBTimeChart
{
name: 'zipped_error_series',
isSubquery: true,
chartConfig: {
timestampValueExpression: '__hdx_time_bucket',
connection: source.connection,
select: [
{ valueExpression: 'endpoint' },
{
valueExpression:
'arrayJoin(arrayZip(error_rate, __hdx_time_buckets))',
alias: 'zipped',
},
],
from: {
databaseName: '',
tableName: 'selected_error_series',
},
where: '',
},
},
],
select: [
{
valueExpression: 'tupleElement(zipped, 1)',
alias: 'Error Rate %',
},
{
valueExpression: 'endpoint',
},
{
valueExpression: 'tupleElement(zipped, 2)',
alias: 'series_time_bucket',
},
],
from: {
databaseName: '',
tableName: 'zipped_error_series',
},
where: '',
dateRange: searchedTimeRange,
displayType: DisplayType.Line,
numberFormat: ERROR_RATE_PERCENTAGE_NUMBER_FORMAT,
groupBy: 'zipped, endpoint',
} satisfies ChartConfigWithDateRange;
}, [source, searchedTimeRange, appliedConfig, expressions, reqChartType]);
return (
<Grid mt="md" grow={false} w="100%" maw="100%" overflow="hidden">
<Grid.Col span={6}>
@ -321,32 +490,12 @@ function HttpTab({
]}
/>
</Group>
{source && (
{source && requestErrorRateConfig && (
<DBTimeChart
sourceId={source.id}
config={{
...source,
where: appliedConfig.where || '',
whereLanguage: appliedConfig.whereLanguage || 'sql',
displayType:
reqChartType === 'overall'
? DisplayType.Line
: DisplayType.StackedBar,
select: [
{
valueExpression: `countIf(${expressions.isError}) / count()`,
alias: 'Error Rate %',
},
],
numberFormat: ERROR_RATE_PERCENTAGE_NUMBER_FORMAT,
filters: getScopedFilters(source, appliedConfig),
groupBy:
reqChartType === 'overall'
? undefined
: source.spanNameExpression || expressions.spanName,
dateRange: searchedTimeRange,
}}
config={requestErrorRateConfig}
showDisplaySwitcher={false}
disableQueryChunking
/>
)}
</ChartBox>
@ -356,7 +505,7 @@ function HttpTab({
<Group justify="space-between" align="center" mb="sm">
<Text size="sm">Request Throughput</Text>
</Group>
{source && (
{source && expressions && (
<DBTimeChart
sourceId={source.id}
config={{
@ -377,7 +526,7 @@ function HttpTab({
},
],
numberFormat: { ...INTEGER_NUMBER_FORMAT, unit: 'requests' },
filters: getScopedFilters(source, appliedConfig),
filters: getScopedFilters({ appliedConfig, expressions }),
dateRange: searchedTimeRange,
}}
/>
@ -390,58 +539,89 @@ function HttpTab({
<Text size="sm">20 Top Most Time Consuming Endpoints</Text>
</Group>
{source && (
{source && expressions && (
<DBListBarChart
groupColumn="Endpoint"
valueColumn="Total (ms)"
getRowSearchLink={getRowSearchLink}
hiddenSeries={[
'duration_ns',
'total_requests',
'duration_p95_ns',
'duration_p50_ns',
'error_requests',
]}
config={{
...source,
where: appliedConfig.where || '',
whereLanguage: appliedConfig.whereLanguage || 'sql',
select: [
// Separate the aggregations from the conversion to ms and rate so that AggregatingMergeTree MVs can be used
{
alias: 'Endpoint',
valueExpression:
source.spanNameExpression || expressions.spanName,
valueExpression: expressions.endpoint,
},
{
alias: 'duration_ns',
aggFn: 'sum',
valueExpression: expressions.duration,
aggCondition: '',
},
{
alias: 'Total (ms)',
aggFn: 'sum',
valueExpression: expressions.durationInMillis,
valueExpression: `duration_ns / ${expressions.durationDivisorForMillis}`,
aggCondition: '',
},
{
alias: 'total_requests',
aggFn: 'count',
valueExpression: '',
},
{
alias: 'Req/Min',
valueExpression: `
count() /
total_requests /
age('mi', toDateTime(${startTime / 1000}), toDateTime(${endTime / 1000}))`,
},
{
alias: 'P95 (ms)',
alias: 'duration_p95_ns',
aggFn: 'quantile',
valueExpression: expressions.durationInMillis,
level: 0.95,
valueExpression: expressions.duration,
aggCondition: '',
},
{
alias: 'P95 (ms)',
valueExpression: `duration_p95_ns / ${expressions.durationDivisorForMillis}`,
},
{
alias: 'duration_p50_ns',
aggFn: 'quantile',
level: 0.5,
valueExpression: expressions.duration,
aggCondition: '',
},
{
alias: 'Median (ms)',
aggFn: 'quantile',
valueExpression: expressions.durationInMillis,
aggCondition: '',
level: 0.95,
valueExpression: `duration_p50_ns / ${expressions.durationDivisorForMillis}`,
},
{
alias: 'error_requests',
aggFn: 'count',
valueExpression: '',
aggCondition: expressions.isError,
aggConditionLanguage: 'sql',
},
{
alias: 'Errors/Min',
valueExpression: `countIf(${expressions.isError}) /
valueExpression: `error_requests /
age('mi', toDateTime(${startTime / 1000}), toDateTime(${endTime / 1000}))`,
},
],
selectGroupBy: false,
groupBy: source.spanNameExpression || expressions.spanName,
groupBy: expressions.endpoint,
orderBy: '"Total (ms)" DESC',
filters: getScopedFilters(source, appliedConfig),
filters: getScopedFilters({ appliedConfig, expressions }),
dateRange: searchedTimeRange,
numberFormat: MS_NUMBER_FORMAT,
limit: { limit: 20 },
@ -482,45 +662,85 @@ function HttpTab({
]}
/>
</Group>
{source && (
{source && expressions && (
<DBTableChart
getRowSearchLink={getRowSearchLink}
hiddenColumns={[
'total_count',
'p95_duration_ns',
'p50_duration_ns',
'duration_sum_ns',
'error_count',
]}
config={{
...source,
where: appliedConfig.where || '',
whereLanguage: appliedConfig.whereLanguage || 'sql',
select: [
// Separate the aggregations from the conversion to ms and rate so that AggregatingMergeTree MVs can be used
{
alias: 'Endpoint',
valueExpression:
source.spanNameExpression || expressions.spanName,
valueExpression: expressions.endpoint,
},
{
alias: 'total_count',
valueExpression: '',
aggFn: 'count',
},
{
alias: 'Req/Min',
valueExpression: `round(count() /
valueExpression: `round(total_count /
age('mi', toDateTime(${startTime / 1000}), toDateTime(${endTime / 1000})), 1)`,
},
{
alias: 'p95_duration_ns',
valueExpression: expressions.duration,
aggFn: 'quantile',
level: 0.95,
},
{
alias: 'P95 (ms)',
valueExpression: `round(quantile(0.95)(${expressions.durationInMillis}), 2)`,
valueExpression: `round(p95_duration_ns / ${expressions.durationDivisorForMillis}, 2)`,
},
{
alias: 'p50_duration_ns',
valueExpression: expressions.duration,
aggFn: 'quantile',
level: 0.5,
},
{
alias: 'Median (ms)',
valueExpression: `round(quantile(0.5)(${expressions.durationInMillis}), 2)`,
valueExpression: `round(p50_duration_ns / ${expressions.durationDivisorForMillis}, 2)`,
},
{
alias: 'duration_sum_ns',
valueExpression: expressions.duration,
aggFn: 'sum',
},
{
alias: 'Total (ms)',
valueExpression: `round(sum(${expressions.durationInMillis}), 2)`,
valueExpression: `round(duration_sum_ns / ${expressions.durationDivisorForMillis}, 2)`,
},
{
alias: 'error_count',
valueExpression: '',
aggCondition: expressions.isError,
aggConditionLanguage: 'sql',
aggFn: 'count',
},
{
alias: 'Errors/Min',
valueExpression: `round(countIf(${expressions.isError}) /
valueExpression: `round(error_count /
age('mi', toDateTime(${startTime / 1000}), toDateTime(${endTime / 1000})), 1)`,
},
],
filters: getScopedFilters(source, appliedConfig),
filters: getScopedFilters({
appliedConfig,
expressions,
includeNonEmptyEndpointFilter: true,
}),
selectGroupBy: false,
groupBy: source.spanNameExpression || expressions.spanName,
groupBy: expressions.endpoint,
dateRange: searchedTimeRange,
orderBy:
topEndpointsChartType === 'time'
@ -545,12 +765,7 @@ function DatabaseTab({
appliedConfig: AppliedConfig;
}) {
const { data: source } = useSource({ id: appliedConfig.source });
const { data: jsonColumns = [] } = useJsonColumns({
databaseName: source?.from?.databaseName || '',
tableName: source?.from?.tableName || '',
connectionId: source?.connection || '',
});
const expressions = getExpressions(source, jsonColumns);
const { expressions } = useServiceDashboardExpressions({ source });
const [chartType, setChartType] = useState<'table' | 'list'>('list');
@ -562,7 +777,7 @@ function DatabaseTab({
const totalTimePerQueryConfig =
useMemo<ChartConfigWithDateRange | null>(() => {
if (!source) return null;
if (!source || !expressions) return null;
return {
with: [
@ -591,7 +806,11 @@ function DatabaseTab({
],
groupBy: 'Statement',
filters: [
...getScopedFilters(source, appliedConfig, false),
...getScopedFilters({
expressions,
appliedConfig,
includeIsSpanKindServer: false,
}),
{ type: 'sql', condition: expressions.isDbSpan },
],
// Date range and granularity add an `__hdx_time_bucket` column to select and group by
@ -671,18 +890,11 @@ function DatabaseTab({
timestampValueExpression: 'series_time_bucket',
connection: source.connection,
} satisfies ChartConfigWithDateRange;
}, [
appliedConfig,
expressions.dbStatement,
expressions.durationInMillis,
expressions.isDbSpan,
searchedTimeRange,
source,
]);
}, [appliedConfig, expressions, searchedTimeRange, source]);
const totalThroughputPerQueryConfig =
useMemo<ChartConfigWithDateRange | null>(() => {
if (!source) return null;
if (!source || !expressions) return null;
return {
with: [
@ -711,7 +923,11 @@ function DatabaseTab({
],
groupBy: 'Statement',
filters: [
...getScopedFilters(source, appliedConfig, false),
...getScopedFilters({
expressions,
appliedConfig,
includeIsSpanKindServer: false,
}),
{ type: 'sql', condition: expressions.isDbSpan },
],
// Date range and granularity add an `__hdx_time_bucket` column to select and group by
@ -794,13 +1010,7 @@ function DatabaseTab({
timestampValueExpression: 'series_time_bucket',
connection: source.connection,
} satisfies ChartConfigWithDateRange;
}, [
appliedConfig,
expressions.dbStatement,
expressions.isDbSpan,
searchedTimeRange,
source,
]);
}, [appliedConfig, expressions, searchedTimeRange, source]);
return (
<Grid mt="md" grow={false} w="100%" maw="100%" overflow="hidden">
@ -861,6 +1071,7 @@ function DatabaseTab({
</Box>
</Group>
{source &&
expressions &&
(chartType === 'list' ? (
<DBListBarChart
groupColumn="Statement"
@ -872,7 +1083,7 @@ function DatabaseTab({
where: appliedConfig.where || '',
whereLanguage: appliedConfig.whereLanguage || 'sql',
dateRange: searchedTimeRange,
groupBy: expressions.dbStatement,
groupBy: 'Statement',
selectGroupBy: false,
orderBy: '"Total" DESC',
select: [
@ -897,17 +1108,10 @@ function DatabaseTab({
aggFn: 'quantile',
valueExpression: expressions.durationInMillis,
aggCondition: '',
level: 0.5,
},
{
alias: 'Median (ms)',
aggFn: 'quantile',
valueExpression: expressions.durationInMillis,
aggCondition: '',
level: 0.95,
},
{
alias: 'Median',
alias: 'Median (ms)',
aggFn: 'quantile',
valueExpression: expressions.durationInMillis,
aggCondition: '',
@ -915,7 +1119,11 @@ function DatabaseTab({
},
],
filters: [
...getScopedFilters(source, appliedConfig, false),
...getScopedFilters({
appliedConfig,
expressions,
includeIsSpanKindServer: false,
}),
{ type: 'sql', condition: expressions.isDbSpan },
],
limit: { limit: 20 },
@ -929,7 +1137,7 @@ function DatabaseTab({
where: appliedConfig.where || '',
whereLanguage: appliedConfig.whereLanguage || 'sql',
dateRange: searchedTimeRange,
groupBy: expressions.dbStatement,
groupBy: 'Statement',
orderBy: '"Total" DESC',
selectGroupBy: false,
select: [
@ -954,17 +1162,10 @@ function DatabaseTab({
aggFn: 'quantile',
valueExpression: expressions.durationInMillis,
aggCondition: '',
level: 0.5,
},
{
alias: 'Median (ms)',
aggFn: 'quantile',
valueExpression: expressions.durationInMillis,
aggCondition: '',
level: 0.95,
},
{
alias: 'Median',
alias: 'Median (ms)',
aggFn: 'quantile',
valueExpression: expressions.durationInMillis,
aggCondition: '',
@ -972,7 +1173,11 @@ function DatabaseTab({
},
],
filters: [
...getScopedFilters(source, appliedConfig, false),
...getScopedFilters({
appliedConfig,
expressions,
includeIsSpanKindServer: false,
}),
{ type: 'sql', condition: expressions.isDbSpan },
],
limit: { limit: 20 },
@ -994,12 +1199,7 @@ function ErrorsTab({
appliedConfig: AppliedConfig;
}) {
const { data: source } = useSource({ id: appliedConfig.source });
const { data: jsonColumns = [] } = useJsonColumns({
databaseName: source?.from?.databaseName || '',
tableName: source?.from?.tableName || '',
connectionId: source?.connection || '',
});
const expressions = getExpressions(source, jsonColumns);
const { expressions } = useServiceDashboardExpressions({ source });
return (
<Grid mt="md" grow={false} w="100%" maw="100%" overflow="hidden">
@ -1008,7 +1208,7 @@ function ErrorsTab({
<Group justify="space-between" align="center" mb="sm">
<Text size="sm">Error Events per Service</Text>
</Group>
{source && (
{source && expressions && (
<DBTimeChart
sourceId={source.id}
config={{
@ -1021,13 +1221,13 @@ function ErrorsTab({
valueExpression: `count()`,
},
],
numberFormat: ERROR_RATE_PERCENTAGE_NUMBER_FORMAT,
numberFormat: INTEGER_NUMBER_FORMAT,
filters: [
{
type: 'sql',
condition: expressions.isError,
},
...getScopedFilters(source, appliedConfig),
...getScopedFilters({ appliedConfig, expressions }),
],
groupBy: source.serviceNameExpression || expressions.service,
dateRange: searchedTimeRange,

View file

@ -1,9 +1,13 @@
import type { ColumnMeta } from '@hyperdx/common-utils/dist/clickhouse';
import type { TSource } from '@hyperdx/common-utils/dist/types';
import { SourceKind } from '@hyperdx/common-utils/dist/types';
import { renderHook } from '@testing-library/react';
import * as metadataModule from '../hooks/useMetadata';
import {
getExpressions,
makeCoalescedFieldsAccessQuery,
useServiceDashboardExpressions,
} from '../serviceDashboard';
function removeAllWhitespace(str: string) {
@ -32,7 +36,7 @@ describe('Service Dashboard', () => {
describe('getExpressions', () => {
it('should use map syntax for non-JSON columns by default', () => {
const expressions = getExpressions(mockSource, []);
const expressions = getExpressions(mockSource, [], []);
expect(expressions.k8sResourceName).toBe(
"SpanAttributes['k8s.resource.name']",
@ -48,7 +52,7 @@ describe('Service Dashboard', () => {
});
it('should use backtick syntax when SpanAttributes is a JSON column', () => {
const expressions = getExpressions(mockSource, ['SpanAttributes']);
const expressions = getExpressions(mockSource, [], ['SpanAttributes']);
expect(expressions.k8sResourceName).toBe(
'SpanAttributes.`k8s.resource.name`',
@ -65,7 +69,7 @@ describe('Service Dashboard', () => {
});
it('should work with empty jsonColumns array', () => {
const expressions = getExpressions(mockSource);
const expressions = getExpressions(mockSource, [], []);
// Should default to map syntax
expect(expressions.k8sResourceName).toBe(
@ -138,4 +142,208 @@ describe('Service Dashboard', () => {
);
});
});
describe('useServiceDashboardExpressions', () => {
const mockColumns: ColumnMeta[] = [
{ name: 'Duration', type: 'UInt64' },
{ name: 'TraceId', type: 'String' },
{ name: 'ServiceName', type: 'String' },
{ name: 'SpanName', type: 'String' },
{ name: 'SpanKind', type: 'String' },
{ name: 'StatusCode', type: 'String' },
{ name: 'SpanAttributes', type: 'Map(String, String)' },
] as ColumnMeta[];
const mockJsonColumns: string[] = [];
beforeEach(() => {
jest.clearAllMocks();
});
it('should return loading state when source is undefined', () => {
jest.spyOn(metadataModule, 'useJsonColumns').mockReturnValue({
data: undefined,
isLoading: false,
} as any);
jest.spyOn(metadataModule, 'useColumns').mockReturnValue({
data: undefined,
isLoading: false,
} as any);
const { result } = renderHook(() =>
useServiceDashboardExpressions({ source: undefined }),
);
expect(result.current.isLoading).toBe(true);
expect(result.current.expressions).toBeUndefined();
});
it('should return loading state when columns are loading', () => {
jest.spyOn(metadataModule, 'useJsonColumns').mockReturnValue({
data: undefined,
isLoading: true,
} as any);
jest.spyOn(metadataModule, 'useColumns').mockReturnValue({
data: undefined,
isLoading: true,
} as any);
const { result } = renderHook(() =>
useServiceDashboardExpressions({ source: mockSource }),
);
expect(result.current.isLoading).toBe(true);
expect(result.current.expressions).toBeUndefined();
});
it('should return loading state when jsonColumns are loading', () => {
jest.spyOn(metadataModule, 'useJsonColumns').mockReturnValue({
data: undefined,
isLoading: true,
} as any);
jest.spyOn(metadataModule, 'useColumns').mockReturnValue({
data: mockColumns,
isLoading: false,
} as any);
const { result } = renderHook(() =>
useServiceDashboardExpressions({ source: mockSource }),
);
expect(result.current.isLoading).toBe(true);
expect(result.current.expressions).toBeUndefined();
});
it('should return expressions when data is loaded with non-JSON columns', () => {
jest.spyOn(metadataModule, 'useJsonColumns').mockReturnValue({
data: mockJsonColumns,
isLoading: false,
} as any);
jest.spyOn(metadataModule, 'useColumns').mockReturnValue({
data: mockColumns,
isLoading: false,
} as any);
const { result } = renderHook(() =>
useServiceDashboardExpressions({ source: mockSource }),
);
expect(result.current.isLoading).toBe(false);
expect(result.current.expressions).toBeDefined();
expect(result.current.expressions?.duration).toBe('Duration');
expect(result.current.expressions?.service).toBe('ServiceName');
expect(result.current.expressions?.spanName).toBe('SpanName');
expect(result.current.expressions?.traceId).toBe('TraceId');
expect(result.current.expressions?.k8sResourceName).toBe(
"SpanAttributes['k8s.resource.name']",
);
});
it('should return expressions when data is loaded with JSON columns', () => {
jest.spyOn(metadataModule, 'useJsonColumns').mockReturnValue({
data: ['SpanAttributes'],
isLoading: false,
} as any);
jest.spyOn(metadataModule, 'useColumns').mockReturnValue({
data: mockColumns,
isLoading: false,
} as any);
const { result } = renderHook(() =>
useServiceDashboardExpressions({ source: mockSource }),
);
expect(result.current.isLoading).toBe(false);
expect(result.current.expressions).toBeDefined();
expect(result.current.expressions?.k8sResourceName).toBe(
'SpanAttributes.`k8s.resource.name`',
);
});
it('should use materialized endpoint column when available', () => {
const columnsWithEndpoint: ColumnMeta[] = [
...mockColumns,
{ name: 'endpoint', type: 'String' },
] as ColumnMeta[];
jest.spyOn(metadataModule, 'useJsonColumns').mockReturnValue({
data: mockJsonColumns,
isLoading: false,
} as any);
jest.spyOn(metadataModule, 'useColumns').mockReturnValue({
data: columnsWithEndpoint,
isLoading: false,
} as any);
const { result } = renderHook(() =>
useServiceDashboardExpressions({ source: mockSource }),
);
expect(result.current.isLoading).toBe(false);
expect(result.current.expressions?.endpoint).toBe('endpoint');
});
it('should fallback to spanName when materialized endpoint column is not available', () => {
jest.spyOn(metadataModule, 'useJsonColumns').mockReturnValue({
data: mockJsonColumns,
isLoading: false,
} as any);
jest.spyOn(metadataModule, 'useColumns').mockReturnValue({
data: mockColumns,
isLoading: false,
} as any);
const { result } = renderHook(() =>
useServiceDashboardExpressions({ source: mockSource }),
);
expect(result.current.isLoading).toBe(false);
expect(result.current.expressions?.endpoint).toBe('SpanName');
});
it('should include filter expressions', () => {
jest.spyOn(metadataModule, 'useJsonColumns').mockReturnValue({
data: mockJsonColumns,
isLoading: false,
} as any);
jest.spyOn(metadataModule, 'useColumns').mockReturnValue({
data: mockColumns,
isLoading: false,
} as any);
const { result } = renderHook(() =>
useServiceDashboardExpressions({ source: mockSource }),
);
expect(result.current.isLoading).toBe(false);
expect(result.current.expressions?.isError).toBe(
"lower(StatusCode) = 'error'",
);
expect(result.current.expressions?.isSpanKindServer).toContain(
'SpanKind IN',
);
expect(result.current.expressions?.isEndpointNonEmpty).toContain(
'NOT empty(',
);
});
it('should include auxiliary expressions', () => {
jest.spyOn(metadataModule, 'useJsonColumns').mockReturnValue({
data: mockJsonColumns,
isLoading: false,
} as any);
jest.spyOn(metadataModule, 'useColumns').mockReturnValue({
data: mockColumns,
isLoading: false,
} as any);
const { result } = renderHook(() =>
useServiceDashboardExpressions({ source: mockSource }),
);
expect(result.current.isLoading).toBe(false);
expect(result.current.expressions?.durationInMillis).toBe('Duration/1e6');
expect(result.current.expressions?.durationDivisorForMillis).toBe('1e6');
});
});
});

View file

@ -180,6 +180,7 @@ export default function DBListBarChart({
enabled,
valueColumn,
groupColumn,
hiddenSeries = [],
}: {
config: ChartConfigWithDateRange;
onSettled?: () => void;
@ -189,6 +190,7 @@ export default function DBListBarChart({
enabled?: boolean;
valueColumn: string;
groupColumn: string;
hiddenSeries?: string[];
}) {
const queriedConfig = omit(config, ['granularity']);
const { data, isLoading, isError, error } = useQueriedChartConfig(
@ -206,12 +208,14 @@ export default function DBListBarChart({
return [];
}
return Object.keys(rows?.[0]).map(key => ({
dataKey: key,
displayName: key,
numberFormat: config.numberFormat,
}));
}, [config.numberFormat, data]);
return Object.keys(rows?.[0])
.filter(key => !hiddenSeries.includes(key))
.map(key => ({
dataKey: key,
displayName: key,
numberFormat: config.numberFormat,
}));
}, [config.numberFormat, data, hiddenSeries]);
return isLoading && !data ? (
<div className="d-flex h-100 w-100 align-items-center justify-content-center text-muted">

View file

@ -19,11 +19,13 @@ export default function DBTableChart({
getRowSearchLink,
enabled = true,
queryKeyPrefix,
hiddenColumns = [],
}: {
config: ChartConfigWithOptTimestamp;
getRowSearchLink?: (row: any) => string | null;
queryKeyPrefix?: string;
enabled?: boolean;
hiddenColumns?: string[];
}) {
const [sort, setSort] = useState<SortingState>([]);
@ -32,7 +34,14 @@ export default function DBTableChart({
if (!_config.limit) {
_config.limit = { limit: 200 };
}
if (_config.groupBy && typeof _config.groupBy === 'string') {
// Set a default orderBy if groupBy is set but orderBy is not,
// so that the set of rows within the limit is stable.
if (
_config.groupBy &&
typeof _config.groupBy === 'string' &&
!_config.orderBy
) {
_config.orderBy = _config.groupBy;
}
@ -80,14 +89,24 @@ export default function DBTableChart({
groupByKeys = queriedConfig.groupBy.split(',').map(v => v.trim());
}
return Object.keys(rows?.[0]).map(key => ({
// If it's an alias, wrap in quotes to support a variety of formats (ex "Time (ms)", "Req/s", etc)
id: aliasMap.includes(key) ? `"${key}"` : key,
dataKey: key,
displayName: key,
numberFormat: groupByKeys.includes(key) ? undefined : config.numberFormat,
}));
}, [config.numberFormat, aliasMap, queriedConfig.groupBy, data]);
return Object.keys(rows?.[0])
.filter(key => !hiddenColumns.includes(key))
.map(key => ({
// If it's an alias, wrap in quotes to support a variety of formats (ex "Time (ms)", "Req/s", etc)
id: aliasMap.includes(key) ? `"${key}"` : key,
dataKey: key,
displayName: key,
numberFormat: groupByKeys.includes(key)
? undefined
: config.numberFormat,
}));
}, [
config.numberFormat,
aliasMap,
queriedConfig.groupBy,
data,
hiddenColumns,
]);
return isLoading && !data ? (
<div className="d-flex h-100 w-100 align-items-center justify-content-center text-muted">

View file

@ -195,19 +195,7 @@ function ActiveTimeTooltip({
);
}
function DBTimeChartComponent({
config,
disableQueryChunking,
enabled = true,
logReferenceTimestamp,
onTimeRangeSelect,
queryKeyPrefix,
referenceLines,
setDisplayType,
showDisplaySwitcher = true,
showLegend = true,
sourceId,
}: {
type DBTimeChartComponentProps = {
config: ChartConfigWithDateRange;
disableQueryChunking?: boolean;
enabled?: boolean;
@ -220,7 +208,24 @@ function DBTimeChartComponent({
showDisplaySwitcher?: boolean;
showLegend?: boolean;
sourceId?: string;
}) {
/** Names of series that should not be shown in the chart */
hiddenSeries?: string[];
};
function DBTimeChartComponent({
config,
disableQueryChunking,
enabled = true,
logReferenceTimestamp,
onTimeRangeSelect,
queryKeyPrefix,
referenceLines,
setDisplayType,
showDisplaySwitcher = true,
showLegend = true,
sourceId,
hiddenSeries,
}: DBTimeChartComponentProps) {
const [isErrorExpanded, errorExpansion] = useDisclosure(false);
const {
displayType: displayTypeProp,
@ -312,6 +317,7 @@ function DBTimeChartComponent({
granularity,
generateEmptyBuckets: fillNulls !== false,
source,
hiddenSeries,
});
} catch (e) {
console.error(e);
@ -326,6 +332,7 @@ function DBTimeChartComponent({
source,
config.compareToPreviousPeriod,
previousPeriodData,
hiddenSeries,
]);
// To enable backward compatibility, allow non-controlled usage of displayType

View file

@ -8,8 +8,7 @@ import { ChartBox } from '@/components/ChartBox';
import { DBTimeChart } from '@/components/DBTimeChart';
import { DrawerBody, DrawerHeader } from '@/components/DrawerUtils';
import SlowestEventsTile from '@/components/ServiceDashboardSlowestEventsTile';
import { useJsonColumns } from '@/hooks/useMetadata';
import { getExpressions } from '@/serviceDashboard';
import { useServiceDashboardExpressions } from '@/serviceDashboard';
import { useSource } from '@/source';
import { useZIndex, ZIndexContext } from '@/zIndex';
@ -25,12 +24,7 @@ export default function ServiceDashboardDbQuerySidePanel({
searchedTimeRange: [Date, Date];
}) {
const { data: source } = useSource({ id: sourceId });
const { data: jsonColumns = [] } = useJsonColumns({
databaseName: source?.from?.databaseName || '',
tableName: source?.from?.tableName || '',
connectionId: source?.connection || '',
});
const expressions = getExpressions(source, jsonColumns);
const { expressions } = useServiceDashboardExpressions({ source });
const [dbQuery, setDbQuery] = useQueryState('dbquery', parseAsString);
const onClose = useCallback(() => {
@ -44,13 +38,13 @@ export default function ServiceDashboardDbQuerySidePanel({
const filters: Filter[] = [
{
type: 'sql',
condition: `${expressions.dbStatement} IN ('${dbQuery}')`,
condition: `${expressions?.dbStatement} IN ('${dbQuery}')`,
},
];
if (service) {
filters.push({
type: 'sql',
condition: `${expressions.service} IN ('${service}')`,
condition: `${expressions?.service} IN ('${service}')`,
});
}
return filters;
@ -97,7 +91,7 @@ export default function ServiceDashboardDbQuerySidePanel({
<Group justify="space-between" align="center" mb="sm">
<Text size="sm">Total Query Time</Text>
</Group>
{source && (
{source && expressions && (
<DBTimeChart
sourceId={sourceId}
config={{
@ -155,7 +149,7 @@ export default function ServiceDashboardDbQuerySidePanel({
<Grid.Col span={12}>
{source && (
<SlowestEventsTile
title="Slowest 10% of Queries"
title="Slowest 5% of Queries"
source={source}
dateRange={searchedTimeRange}
extraFilters={dbQueryFilters}

View file

@ -6,8 +6,8 @@ import { ChartBox } from '@/components/ChartBox';
import DBListBarChart from '@/components/DBListBarChart';
import { useJsonColumns } from '@/hooks/useMetadata';
import {
getExpressions,
makeCoalescedFieldsAccessQuery,
useServiceDashboardExpressions,
} from '@/serviceDashboard';
const MAX_NUM_GROUPS = 200;
@ -28,9 +28,9 @@ export default function ServiceDashboardEndpointPerformanceChart({
tableName: source?.from?.tableName || '',
connectionId: source?.connection || '',
});
const expressions = getExpressions(source, jsonColumns);
const { expressions } = useServiceDashboardExpressions({ source });
if (!source) {
if (!source || !expressions) {
return null;
}

View file

@ -12,8 +12,7 @@ import { DBTimeChart } from '@/components/DBTimeChart';
import { DrawerBody, DrawerHeader } from '@/components/DrawerUtils';
import ServiceDashboardEndpointPerformanceChart from '@/components/ServiceDashboardEndpointPerformanceChart';
import SlowestEventsTile from '@/components/ServiceDashboardSlowestEventsTile';
import { useJsonColumns } from '@/hooks/useMetadata';
import { getExpressions } from '@/serviceDashboard';
import { useServiceDashboardExpressions } from '@/serviceDashboard';
import { EndpointLatencyChart } from '@/ServicesDashboardPage';
import { useSource } from '@/source';
import { useZIndex, ZIndexContext } from '@/zIndex';
@ -30,12 +29,7 @@ export default function ServiceDashboardEndpointSidePanel({
searchedTimeRange: [Date, Date];
}) {
const { data: source } = useSource({ id: sourceId });
const { data: jsonColumns = [] } = useJsonColumns({
databaseName: source?.from?.databaseName || '',
tableName: source?.from?.tableName || '',
connectionId: source?.connection || '',
});
const expressions = getExpressions(source, jsonColumns);
const { expressions } = useServiceDashboardExpressions({ source });
const [endpoint, setEndpoint] = useQueryState('endpoint', parseAsString);
const onClose = useCallback(() => {
@ -46,6 +40,8 @@ export default function ServiceDashboardEndpointSidePanel({
const drawerZIndex = contextZIndex + 10;
const endpointFilters = useMemo(() => {
if (!expressions) return [];
const filters: Filter[] = [
{
type: 'sql',
@ -102,16 +98,30 @@ export default function ServiceDashboardEndpointSidePanel({
<Group justify="space-between" align="center" mb="sm">
<Text size="sm">Request Error Rate</Text>
</Group>
{source && (
{source && expressions && (
<DBTimeChart
sourceId={source.id}
hiddenSeries={['total_count', 'error_count']}
config={{
...source,
where: '',
whereLanguage: 'sql',
select: [
// Separate the aggregations from the conversion to rate so that AggregatingMergeTree MVs can be used
{
valueExpression: `countIf(${expressions.isError}) / count()`,
valueExpression: '',
aggFn: 'count',
alias: 'error_count',
aggCondition: expressions.isError,
aggConditionLanguage: 'sql',
},
{
valueExpression: '',
aggFn: 'count',
alias: 'total_count',
},
{
valueExpression: `error_count / total_count`,
alias: 'Error Rate %',
},
],
@ -129,7 +139,7 @@ export default function ServiceDashboardEndpointSidePanel({
<Group justify="space-between" align="center" mb="sm">
<Text size="sm">Request Throughput</Text>
</Group>
{source && (
{source && expressions && (
<DBTimeChart
sourceId={source.id}
config={{
@ -172,12 +182,15 @@ export default function ServiceDashboardEndpointSidePanel({
/>
</Grid.Col>
<Grid.Col span={12}>
<SlowestEventsTile
title="Slowest 10% of Transactions"
source={source}
dateRange={searchedTimeRange}
extraFilters={endpointFilters}
/>
{/* Ensure expressions exists to ensure that endpointFilters has set */}
{expressions && (
<SlowestEventsTile
title="Slowest 5% of Transactions"
source={source}
dateRange={searchedTimeRange}
extraFilters={endpointFilters}
/>
)}
</Grid.Col>
</Grid>
</DrawerBody>

View file

@ -4,8 +4,7 @@ import { Box, Code, Group, Text } from '@mantine/core';
import { ChartBox } from '@/components/ChartBox';
import { useQueriedChartConfig } from '@/hooks/useChartConfig';
import { useJsonColumns } from '@/hooks/useMetadata';
import { getExpressions } from '@/serviceDashboard';
import { useServiceDashboardExpressions } from '@/serviceDashboard';
import { SQLPreview } from './ChartSQLPreview';
import DBSqlRowTableWithSideBar from './DBSqlRowTableWithSidebar';
@ -27,12 +26,7 @@ export default function SlowestEventsTile({
enabled?: boolean;
extraFilters?: Filter[];
}) {
const { data: jsonColumns = [] } = useJsonColumns({
databaseName: source?.from?.databaseName || '',
tableName: source?.from?.tableName || '',
connectionId: source?.connection || '',
});
const expressions = getExpressions(source, jsonColumns);
const { expressions } = useServiceDashboardExpressions({ source });
const { data, isLoading, isError, error } = useQueriedChartConfig(
{
@ -40,12 +34,16 @@ export default function SlowestEventsTile({
where: '',
whereLanguage: 'sql',
select: [
// Separate the aggregations from the conversion to ms so that AggregatingMergeTree MVs can be used
{
alias: 'p95_duration_ns',
aggFn: 'quantile',
valueExpression: expressions?.duration ?? 'Duration',
level: 0.95,
},
{
alias: 'p95',
aggFn: 'quantile',
aggCondition: '',
valueExpression: expressions.durationInMillis,
level: 0.95,
valueExpression: `p95_duration_ns / ${expressions?.durationDivisorForMillis}`,
},
],
dateRange,
@ -54,7 +52,7 @@ export default function SlowestEventsTile({
{
placeholderData: (prev: any) => prev,
queryKey: [queryKeyPrefix, source],
enabled,
enabled: enabled && !!expressions,
},
);
@ -103,7 +101,8 @@ export default function SlowestEventsTile({
No data found within time range.
</div>
) : (
source && (
source &&
expressions && (
<>
<DBSqlRowTableWithSideBar
isNestedPanel

View file

@ -1,5 +1,10 @@
import { useMemo } from 'react';
import { ColumnMeta } from '@hyperdx/common-utils/dist/clickhouse';
import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata';
import { TSource } from '@hyperdx/common-utils/dist/types';
import { useColumns, useJsonColumns } from './hooks/useMetadata';
const COALESCE_FIELDS_LIMIT = 100;
// Helper function to format field access based on column type
@ -131,21 +136,31 @@ function getDefaults({
};
}
export function getExpressions(source?: TSource, jsonColumns: string[] = []) {
const ENDPOINT_MATERIALIZED_COLUMN_NAME = 'endpoint';
export function getExpressions(
source: TSource,
columns: ColumnMeta[],
jsonColumns: string[],
) {
const spanAttributeField =
source?.eventAttributesExpression || 'SpanAttributes';
const isAttributeFieldJSON = jsonColumns.includes(spanAttributeField);
const defaults = getDefaults({ spanAttributeField, isAttributeFieldJSON });
const hasMaterializedEndpointColumn = !!columns.find(
col => col.name === ENDPOINT_MATERIALIZED_COLUMN_NAME,
);
const fieldExpressions = {
// General
duration: source?.durationExpression || defaults.duration,
durationPrecision: source?.durationPrecision || defaults.durationPrecision,
traceId: source?.traceIdExpression || defaults.traceId,
service: source?.serviceNameExpression || defaults.service,
spanName: source?.spanNameExpression || defaults.spanName,
spanKind: source?.spanKindExpression || defaults.spanKind,
severityText: source?.severityTextExpression || defaults.severityText,
duration: source.durationExpression || defaults.duration,
durationPrecision: source.durationPrecision || defaults.durationPrecision,
traceId: source.traceIdExpression || defaults.traceId,
service: source.serviceNameExpression || defaults.service,
spanName: source.spanNameExpression || defaults.spanName,
spanKind: source.spanKindExpression || defaults.spanKind,
severityText: source.severityTextExpression || defaults.severityText,
// HTTP
httpHost: defaults.httpHost,
@ -159,19 +174,52 @@ export function getExpressions(source?: TSource, jsonColumns: string[] = []) {
dbStatement: defaults.dbStatement,
};
const auxExpressions = {
endpoint: hasMaterializedEndpointColumn
? ENDPOINT_MATERIALIZED_COLUMN_NAME
: fieldExpressions.spanName,
/** An expression for reading the Span duration in milliseconds. Using this will prevent the use of aggregating materialized views which aggregate `Duration` instead of `Duration/1e6` */
durationInMillis: `${fieldExpressions.duration}/1e${fieldExpressions.durationPrecision - 3}`,
/** The divisor used to convert the Span duration to milliseconds */
durationDivisorForMillis: `1e${fieldExpressions.durationPrecision - 3}`,
};
const filterExpressions = {
isEndpointNonEmpty: `NOT empty(${auxExpressions.endpoint})`,
isError: `lower(${fieldExpressions.severityText}) = 'error'`,
isSpanKindServer: `${fieldExpressions.spanKind} IN ('Server', 'SPAN_KIND_SERVER')`,
isDbSpan: `${fieldExpressions.dbStatement} <> ''`,
};
const auxExpressions = {
durationInMillis: `${fieldExpressions.duration}/1e${fieldExpressions.durationPrecision - 3}`, // precision is per second
};
return {
...fieldExpressions,
...filterExpressions,
...auxExpressions,
};
}
export function useServiceDashboardExpressions({
source,
}: {
source: TSource | undefined;
}) {
const tableConnection = useMemo(() => tcFromSource(source), [source]);
const { data: jsonColumns, isLoading: isJsonColumnsLoading } =
useJsonColumns(tableConnection);
const { data: columns = [], isLoading: isColumnsLoading } =
useColumns(tableConnection);
const isLoading = !source || isJsonColumnsLoading || isColumnsLoading;
const expressions = useMemo(() => {
if (isLoading || !jsonColumns || !columns) return undefined;
return getExpressions(source, columns, jsonColumns);
}, [source, columns, jsonColumns, isLoading]);
return {
expressions,
isLoading,
};
}