feat: new team setting for number of filters to fetch (#2020)

.changeset/modern-dryers-fail.md

@@ -0,0 +1,7 @@
+---
+"@hyperdx/common-utils": minor
+"@hyperdx/api": minor
+"@hyperdx/app": minor
+---
+
+feat: new team setting for number of filters to fetch

packages/api/src/models/team.ts

@@ -41,6 +41,7 @@ export default mongoose.model<ITeam>(
       queryTimeout: Number,
       fieldMetadataDisabled: Boolean,
       parallelizeWhenPossible: Boolean,
+      filterKeysFetchLimit: Number,
     },
     {
       timestamps: true,

packages/app/src/components/TeamSettings/TeamQueryConfigSection.tsx

@@ -19,7 +19,11 @@ import { IconHelpCircle, IconPencil } from '@tabler/icons-react';
 
 import api from '@/api';
 import SelectControlled from '@/components/SelectControlled';
-import { DEFAULT_QUERY_TIMEOUT, DEFAULT_SEARCH_ROW_LIMIT } from '@/defaults';
+import {
+  DEFAULT_FILTER_KEYS_FETCH_LIMIT,
+  DEFAULT_QUERY_TIMEOUT,
+  DEFAULT_SEARCH_ROW_LIMIT,
+} from '@/defaults';
 import { useBrandDisplayName } from '@/theme/ThemeProvider';
 
 type ClickhouseSettingType = 'number' | 'boolean';
@@ -262,6 +266,17 @@ export default function TeamQueryConfigSection() {
             min={0}
             displayValue={displayValueWithUnit('rows')}
           />
+          <ClickhouseSettingForm
+            settingKey="filterKeysFetchLimit"
+            label="Filter Keys Fetch Limit"
+            tooltip="The number of filter keys to fetch when clicking 'More filters' on the search page"
+            type="number"
+            defaultValue={DEFAULT_FILTER_KEYS_FETCH_LIMIT}
+            placeholder={`default = ${DEFAULT_FILTER_KEYS_FETCH_LIMIT}`}
+            min={1}
+            max={1000}
+            displayValue={displayValueWithUnit('keys')}
+          />
           <ClickhouseSettingForm
             settingKey="fieldMetadataDisabled"
             label="Field Metadata Queries"

packages/app/src/defaults.ts

@@ -3,6 +3,7 @@ import type { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist
 // Limit defaults
 export const DEFAULT_SEARCH_ROW_LIMIT = 200;
 export const DEFAULT_QUERY_TIMEOUT = 60; // max_execution_time, seconds
+export const DEFAULT_FILTER_KEYS_FETCH_LIMIT = 20;
 
 export function searchChartConfigDefaults(
   team: any | undefined | null,

packages/app/src/hooks/useMetadata.tsx

@@ -21,6 +21,7 @@ import {
 import api from '@/api';
 import { IS_LOCAL_MODE } from '@/config';
 import { LOCAL_STORE_CONNECTIONS_KEY } from '@/connection';
+import { DEFAULT_FILTER_KEYS_FETCH_LIMIT } from '@/defaults';
 import { getMetadata } from '@/metadata';
 import { useSource, useSources } from '@/source';
 import { toArray } from '@/utils';
@@ -225,14 +226,19 @@ export function useMultipleGetKeyValues(
   const chartConfigsArr = toArray(chartConfigs);
 
   const { enabled = true } = options || {};
+  const { data: me, isLoading: isLoadingMe } = api.useMe();
   const { data: sources, isLoading: isLoadingSources } = useSources();
 
+  const maxKeys =
+    me?.team?.filterKeysFetchLimit ?? DEFAULT_FILTER_KEYS_FETCH_LIMIT;
+
   const query = useQuery<{ key: string; value: string[] }[]>({
     queryKey: [
       'useMetadata.useGetKeyValues',
       ...chartConfigsArr.map(cc => ({ ...cc })),
       ...keys,
       disableRowLimit,
+      maxKeys,
     ],
     queryFn: async ({ signal }) => {
       return (
@@ -243,7 +249,7 @@ export function useMultipleGetKeyValues(
               : undefined;
             return metadata.getKeyValuesWithMVs({
               chartConfig,
-              keys: keys.slice(0, 20), // Limit to 20 keys for now, otherwise request fails (max header size)
+              keys: keys.slice(0, maxKeys),
               limit,
               disableRowLimit,
               source,
@@ -256,7 +262,7 @@ export function useMultipleGetKeyValues(
     staleTime: 1000 * 60 * 5, // Cache every 5 min
     placeholderData: keepPreviousData,
     ...options,
-    enabled: !!enabled && !!keys.length && !isLoadingSources,
+    enabled: !!enabled && !!keys.length && !isLoadingSources && !isLoadingMe,
   });
 
   return {

packages/common-utils/src/types.ts

@@ -786,6 +786,7 @@ export const TeamClickHouseSettingsSchema = z.object({
   queryTimeout: z.number().optional(),
   metadataMaxRowsToRead: z.number().optional(),
   parallelizeWhenPossible: z.boolean().optional(),
+  filterKeysFetchLimit: z.number().optional(),
 });
 export type TeamClickHouseSettings = z.infer<
   typeof TeamClickHouseSettingsSchema