Elgato_dark/hyperdx
1
0
Fork 0
mirror of https://github.com/hyperdxio/hyperdx synced 2026-04-21 13:37:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

Split shuffling and fetch steps to speed up event deltas (#530)

Browse source
This commit is contained in:
Mike Shi 2024-12-16 16:35:19 -08:00 committed by GitHub
parent 9993fb2097
commit 48a6145776
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 61 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
docker-compose.dev.yml
View file

@ -123,6 +123,9 @@ services:
PORT: ${HYPERDX_API_PORT}
REDIS_URL: redis://redis:6379
USAGE_STATS_ENABLED: ${USAGE_STATS_ENABLED:-false}
# Event deltas create large SQL queries so we need to increase
# the max header size for proxied SQL requests
NODE_OPTIONS: '--max-http-header-size=131072'
volumes:
- ./packages/api/src:/app/src
networks:
@ -147,6 +150,9 @@ services:
OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel-collector:4318'
OTEL_SERVICE_NAME: 'hdx-oss-dev-app'
PORT: ${HYPERDX_APP_PORT}
# Event deltas create large SQL queries so we need to increase
# the max header size for proxied SQL requests
NODE_OPTIONS: '--max-http-header-size=131072'
volumes:
- ./packages/app/mdx.d.ts:/app/mdx.d.ts
- ./packages/app/next-env.d.ts:/app/next-env.d.ts

74
packages/app/src/components/DBDeltaChart.tsx
View file

@ -258,9 +258,9 @@ export default function DBDeltaChart({
config: ChartConfigWithOptDateRange;
outlierSqlCondition: string;
}) {
const { data: outlierData } = useQueriedChartConfig({
const { data: outlierPartIds } = useQueriedChartConfig({
...config,
select: '*',
select: '_part, _part_offset',
filters: [
...(config.filters ?? []),
{
@ -268,21 +268,38 @@ export default function DBDeltaChart({
condition: `${outlierSqlCondition}`,
},
],
// TODO: Introduce sampling granules for large results
// filters: [
// ...(config.filters ?? []),
// {
// type: 'sql',
// condition: 'indexHint()',
// },
// ],
orderBy: [{ ordering: 'DESC', valueExpression: 'rand()' }],
limit: { limit: 1000 },
});
const { data: inlierData } = useQueriedChartConfig({
const { data: outlierData } = useQueriedChartConfig(
{
...config,
select: '*',
filters: [
...(config.filters ?? []),
{
type: 'sql',
condition: `${outlierSqlCondition}`,
},
{
type: 'sql',
condition: `indexHint((_part, _part_offset) IN (${outlierPartIds?.data
?.map((r: any) => `('${r._part}', ${r._part_offset})`)
?.join(',')}))`,
},
],
orderBy: [{ ordering: 'DESC', valueExpression: 'rand()' }],
limit: { limit: 1000 },
},
{
enabled: (outlierPartIds?.data?.length ?? 0) > 0,
},
);
const { data: inlierPartIds } = useQueriedChartConfig({
...config,
select: '*',
select: '_part, _part_offset',
filters: [
...(config.filters ?? []),
{
@ -290,18 +307,35 @@ export default function DBDeltaChart({
condition: `NOT (${outlierSqlCondition})`,
},
],
// TODO: Introduce sampling granules for large results
// filters: [
// ...(config.filters ?? []),
// {
// type: 'sql',
// condition: 'indexHint()',
// },
// ],
orderBy: [{ ordering: 'DESC', valueExpression: 'rand()' }],
limit: { limit: 1000 },
});
const { data: inlierData } = useQueriedChartConfig(
{
...config,
select: '*',
filters: [
...(config.filters ?? []),
{
type: 'sql',
condition: `NOT (${outlierSqlCondition})`,
},
{
type: 'sql',
condition: `indexHint((_part, _part_offset) IN (${inlierPartIds?.data
?.map((r: any) => `('${r._part}', ${r._part_offset})`)
?.join(',')}))`,
},
],
orderBy: [{ ordering: 'DESC', valueExpression: 'rand()' }],
limit: { limit: 1000 },
},
{
enabled: (inlierPartIds?.data?.length ?? 0) > 0,
},
);
// TODO: Is loading state
const { sortedProperties, outlierValueOccurences, inlierValueOccurences } =
useMemo(() => {

2
packages/app/src/hooks/useChartConfig.tsx
View file

@ -16,7 +16,7 @@ import {
export function useQueriedChartConfig(
config: ChartConfigWithOptDateRange,
options?: UseQueryOptions<ResponseJSON<any>>,
options?: Partial<UseQueryOptions<ResponseJSON<any>>>,
) {
return useQuery<ResponseJSON<any>, ClickHouseQueryError | Error>({
queryKey: [config],