diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 32da3ac9..a8f7fc53 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -123,6 +123,9 @@ services: PORT: ${HYPERDX_API_PORT} REDIS_URL: redis://redis:6379 USAGE_STATS_ENABLED: ${USAGE_STATS_ENABLED:-false} + # Event deltas create large SQL queries so we need to increase + # the max header size for proxied SQL requests + NODE_OPTIONS: '--max-http-header-size=131072' volumes: - ./packages/api/src:/app/src networks: @@ -147,6 +150,9 @@ services: OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel-collector:4318' OTEL_SERVICE_NAME: 'hdx-oss-dev-app' PORT: ${HYPERDX_APP_PORT} + # Event deltas create large SQL queries so we need to increase + # the max header size for proxied SQL requests + NODE_OPTIONS: '--max-http-header-size=131072' volumes: - ./packages/app/mdx.d.ts:/app/mdx.d.ts - ./packages/app/next-env.d.ts:/app/next-env.d.ts diff --git a/packages/app/src/components/DBDeltaChart.tsx b/packages/app/src/components/DBDeltaChart.tsx index 8e076100..1a2d6d0b 100644 --- a/packages/app/src/components/DBDeltaChart.tsx +++ b/packages/app/src/components/DBDeltaChart.tsx @@ -258,9 +258,9 @@ export default function DBDeltaChart({ config: ChartConfigWithOptDateRange; outlierSqlCondition: string; }) { - const { data: outlierData } = useQueriedChartConfig({ + const { data: outlierPartIds } = useQueriedChartConfig({ ...config, - select: '*', + select: '_part, _part_offset', filters: [ ...(config.filters ?? []), { @@ -268,21 +268,38 @@ export default function DBDeltaChart({ condition: `${outlierSqlCondition}`, }, ], - // TODO: Introduce sampling granules for large results - // filters: [ - // ...(config.filters ?? []), - // { - // type: 'sql', - // condition: 'indexHint()', - // }, - // ], orderBy: [{ ordering: 'DESC', valueExpression: 'rand()' }], limit: { limit: 1000 }, }); - const { data: inlierData } = useQueriedChartConfig({ + const { data: outlierData } = useQueriedChartConfig( + { + ...config, + select: '*', + filters: [ + ...(config.filters ?? []), + { + type: 'sql', + condition: `${outlierSqlCondition}`, + }, + { + type: 'sql', + condition: `indexHint((_part, _part_offset) IN (${outlierPartIds?.data + ?.map((r: any) => `('${r._part}', ${r._part_offset})`) + ?.join(',')}))`, + }, + ], + orderBy: [{ ordering: 'DESC', valueExpression: 'rand()' }], + limit: { limit: 1000 }, + }, + { + enabled: (outlierPartIds?.data?.length ?? 0) > 0, + }, + ); + + const { data: inlierPartIds } = useQueriedChartConfig({ ...config, - select: '*', + select: '_part, _part_offset', filters: [ ...(config.filters ?? []), { @@ -290,18 +307,35 @@ export default function DBDeltaChart({ condition: `NOT (${outlierSqlCondition})`, }, ], - // TODO: Introduce sampling granules for large results - // filters: [ - // ...(config.filters ?? []), - // { - // type: 'sql', - // condition: 'indexHint()', - // }, - // ], orderBy: [{ ordering: 'DESC', valueExpression: 'rand()' }], limit: { limit: 1000 }, }); + const { data: inlierData } = useQueriedChartConfig( + { + ...config, + select: '*', + filters: [ + ...(config.filters ?? []), + { + type: 'sql', + condition: `NOT (${outlierSqlCondition})`, + }, + { + type: 'sql', + condition: `indexHint((_part, _part_offset) IN (${inlierPartIds?.data + ?.map((r: any) => `('${r._part}', ${r._part_offset})`) + ?.join(',')}))`, + }, + ], + orderBy: [{ ordering: 'DESC', valueExpression: 'rand()' }], + limit: { limit: 1000 }, + }, + { + enabled: (inlierPartIds?.data?.length ?? 0) > 0, + }, + ); + // TODO: Is loading state const { sortedProperties, outlierValueOccurences, inlierValueOccurences } = useMemo(() => { diff --git a/packages/app/src/hooks/useChartConfig.tsx b/packages/app/src/hooks/useChartConfig.tsx index e4330ec2..be985203 100644 --- a/packages/app/src/hooks/useChartConfig.tsx +++ b/packages/app/src/hooks/useChartConfig.tsx @@ -16,7 +16,7 @@ import { export function useQueriedChartConfig( config: ChartConfigWithOptDateRange, - options?: UseQueryOptions>, + options?: Partial>>, ) { return useQuery, ClickHouseQueryError | Error>({ queryKey: [config],