feat: add load more to filters and add limit back (#912)

.changeset/light-sloths-hope.md

@@ -0,0 +1,6 @@
+---
+"@hyperdx/common-utils": patch
+"@hyperdx/app": patch
+---
+
+feat: add load more to features and improve querying

packages/app/src/components/DBSearchPageFilters.tsx

@@ -1,5 +1,4 @@
-import { useEffect, useMemo, useState } from 'react';
-import { DEFAULT_MAX_ROWS_TO_READ } from '@hyperdx/common-utils/dist/metadata';
+import { useCallback, useEffect, useMemo, useState } from 'react';
 import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
 import {
   Box,
@@ -22,6 +21,7 @@ import { IconSearch } from '@tabler/icons-react';
 import { useExplainQuery } from '@/hooks/useExplainQuery';
 import { useAllFields, useGetKeyValues } from '@/hooks/useMetadata';
 import useResizable from '@/hooks/useResizable';
+import { getMetadata } from '@/metadata';
 import { FilterStateHook, usePinnedFilters } from '@/searchFilters';
 import { mergePath } from '@/utils';
 
@@ -42,13 +42,19 @@ export const TextButton = ({
   onClick,
   label,
   ms,
+  display,
 }: {
   onClick?: VoidFunction;
   label: React.ReactNode;
   ms?: MantineStyleProps['ms'];
+  display?: MantineStyleProps['display'];
 }) => {
   return (
-    <UnstyledButton onClick={onClick} className={classes.textButton}>
+    <UnstyledButton
+      display={display}
+      onClick={onClick}
+      className={classes.textButton}
+    >
       <Text size="xxs" c="gray.6" lh={1} ms={ms}>
         {label}
       </Text>
@@ -136,6 +142,9 @@ export type FilterGroupProps = {
   onExcludeClick: (value: string) => void;
   onPinClick: (value: string) => void;
   isPinned: (value: string) => boolean;
+  onLoadMore: (key: string) => void;
+  loadMoreLoading: boolean;
+  hasLoadedMore: boolean;
 };
 
 const MAX_FILTER_GROUP_ITEMS = 10;
@@ -151,6 +160,9 @@ export const FilterGroup = ({
   onExcludeClick,
   isPinned,
   onPinClick,
+  onLoadMore,
+  loadMoreLoading,
+  hasLoadedMore,
 }: FilterGroupProps) => {
   const [search, setSearch] = useState('');
   const [isExpanded, setExpanded] = useState(false);
@@ -201,8 +213,8 @@ export const FilterGroup = ({
       if (aExcluded && !bExcluded) return -1;
       if (!aExcluded && bExcluded) return 1;
 
-      // Finally sort alphabetically
-      return a.value.localeCompare(b.value);
+      // Finally sort alphabetically/numerically
+      return a.value.localeCompare(b.value, undefined, { numeric: true });
     };
 
     // If expanded or small list, sort everything
@@ -312,6 +324,28 @@ export const FilterGroup = ({
             />
           </div>
         )}
+        {onLoadMore && (!showExpandButton || isExpanded) && (
+          <div className="d-flex m-1">
+            {loadMoreLoading ? (
+              <Group m={6} gap="xs">
+                <Loader size={12} color="gray.6" />
+                <Text c="dimmed" size="xs">
+                  Loading more...
+                </Text>
+              </Group>
+            ) : (
+              <TextButton
+                display={hasLoadedMore ? 'none' : undefined}
+                label={
+                  <>
+                    <span className="bi-chevron-down" /> Load more
+                  </>
+                }
+                onClick={() => onLoadMore(name)}
+              />
+            )}
+          </div>
+        )}
       </Stack>
     </Stack>
   );
@@ -382,7 +416,10 @@ export const DBSearchPageFilters = ({
           Object.keys(filterState).includes(field.path), // keep selected fields
       )
       .map(({ path }) => path)
-      .filter(path => !['Body', 'Timestamp'].includes(path));
+      .filter(
+        path =>
+          !['body', 'timestamp', '_hdx_body'].includes(path.toLowerCase()),
+      );
 
     return strings;
   }, [data, filterState, showMoreFields]);
@@ -395,14 +432,13 @@ export const DBSearchPageFilters = ({
   useEffect(() => {
     if (!isLive) {
       setDateRange(chartConfig.dateRange);
-      setDisableRowLimit(false);
+      setExtraFacets({});
     }
   }, [chartConfig.dateRange, isLive]);
 
   const showRefreshButton = isLive && dateRange !== chartConfig.dateRange;
 
-  const [disableRowLimit, setDisableRowLimit] = useState(false);
-  const keyLimit = 100;
+  const keyLimit = 20;
   const {
     data: facets,
     isLoading: isFacetsLoading,
@@ -411,18 +447,45 @@ export const DBSearchPageFilters = ({
     chartConfigs: { ...chartConfig, dateRange },
     limit: keyLimit,
     keys: datum,
-    disableRowLimit,
   });
-  useEffect(() => {
-    if (
-      numRows > DEFAULT_MAX_ROWS_TO_READ &&
-      facets &&
-      facets.length < keyLimit
-    ) {
-      setDisableRowLimit(true);
-    }
-  }, [numRows, keyLimit, facets]);
 
+  const [extraFacets, setExtraFacets] = useState<Record<string, string[]>>({});
+  const [loadMoreLoadingKeys, setLoadMoreLoadingKeys] = useState<Set<string>>(
+    new Set(),
+  );
+  const loadMoreFilterValuesForKey = useCallback(
+    async (key: string) => {
+      setLoadMoreLoadingKeys(prev => new Set(prev).add(key));
+      try {
+        const metadata = getMetadata();
+        const newKeyVals = await metadata.getKeyValues({
+          chartConfig: {
+            ...chartConfig,
+            dateRange,
+          },
+          keys: [key],
+          limit: 200,
+          disableRowLimit: true,
+        });
+        const newValues = newKeyVals[0].value;
+        if (newValues.length > 0) {
+          setExtraFacets(prev => ({
+            ...prev,
+            [key]: [...(prev[key] || []), ...newValues],
+          }));
+        }
+      } catch (error) {
+        console.error('failed to fetch more keys', error);
+      } finally {
+        setLoadMoreLoadingKeys(prev => {
+          const newSet = new Set(prev);
+          newSet.delete(key);
+          return newSet;
+        });
+      }
+    },
+    [chartConfig, setExtraFacets, dateRange],
+  );
   const shownFacets = useMemo(() => {
     const _facets: { key: string; value: string[] }[] = [];
     for (const facet of facets ?? []) {
@@ -431,11 +494,25 @@ export const DBSearchPageFilters = ({
       const hasSelectedValues =
         filter && (filter.included.size > 0 || filter.excluded.size > 0);
       if (facet.value?.length > 0 || hasSelectedValues) {
-        _facets.push(facet);
+        const extraValues = extraFacets[facet.key];
+        if (extraValues && extraValues.length > 0) {
+          const allValues = facet.value.slice();
+          for (const extraValue of extraValues) {
+            if (!allValues.includes(extraValue)) {
+              allValues.push(extraValue);
+            }
+          }
+          _facets.push({
+            key: facet.key,
+            value: allValues,
+          });
+        } else {
+          _facets.push(facet);
+        }
       }
     }
     return _facets;
-  }, [facets, filterState]);
+  }, [facets, filterState, extraFacets]);
 
   const showClearAllButton = useMemo(
     () =>
@@ -571,6 +648,9 @@ export const DBSearchPageFilters = ({
               }}
               onPinClick={value => toggleFilterPin(facet.key, value)}
               isPinned={value => isFilterPinned(facet.key, value)}
+              onLoadMore={loadMoreFilterValuesForKey}
+              loadMoreLoading={loadMoreLoadingKeys.has(facet.key)}
+              hasLoadedMore={Boolean(extraFacets[facet.key])}
             />
           ))}
 

packages/app/src/components/__tests__/DBSearchPageFilters.test.tsx

@@ -18,6 +18,9 @@ describe('FilterGroup', () => {
     onExcludeClick: jest.fn(),
     onPinClick: jest.fn(),
     isPinned: jest.fn(),
+    onLoadMore: jest.fn(),
+    loadMoreLoading: false,
+    hasLoadedMore: false,
   };
 
   it('should sort options alphabetically by default', () => {

packages/app/src/hooks/useExplainQuery.tsx

@@ -6,9 +6,13 @@ import { getClickhouseClient } from '@/clickhouse';
 import { getMetadata } from '@/metadata';
 
 export function useExplainQuery(
-  config: ChartConfigWithDateRange,
+  _config: ChartConfigWithDateRange,
   options?: Omit<UseQueryOptions<any>, 'queryKey' | 'queryFn'>,
 ) {
+  const config = {
+    ..._config,
+    with: undefined,
+  };
   const clickhouseClient = getClickhouseClient();
   const { data, isLoading, error } = useQuery({
     queryKey: ['explain', config],

packages/common-utils/src/__tests__/metadata.test.ts

@@ -264,7 +264,7 @@ describe('Metadata', () => {
       expect(mockClickhouseClient.query).toHaveBeenCalledWith(
         expect.objectContaining({
           clickhouse_settings: {
-            max_rows_to_read: 1e6,
+            max_rows_to_read: String(3e6),
             read_overflow_mode: 'break',
           },
         }),
@@ -296,7 +296,7 @@ describe('Metadata', () => {
       expect(mockClickhouseClient.query).toHaveBeenCalledWith(
         expect.objectContaining({
           clickhouse_settings: {
-            max_rows_to_read: 1e6,
+            max_rows_to_read: String(3e6),
             read_overflow_mode: 'break',
           },
         }),

packages/common-utils/src/clickhouse.ts

@@ -353,6 +353,16 @@ const standardModeFetch: typeof fetch = (input, init) => {
   return fetch(input, init);
 };
 
+interface QueryInputs<Format extends DataFormat> {
+  query: string;
+  format?: Format;
+  abort_signal?: AbortSignal;
+  query_params?: Record<string, any>;
+  clickhouse_settings?: ClickHouseSettings;
+  connectionId?: string;
+  queryId?: string;
+}
+
 export type ClickhouseClientOptions = {
   host: string;
   username?: string;
@@ -363,31 +373,57 @@ export class ClickhouseClient {
   private readonly host: string;
   private readonly username?: string;
   private readonly password?: string;
+  /*
+   * Some clickhouse db's (the demo instance for example) make the
+   * max_rows_to_read setting readonly and the query will fail if you try to
+   * query with max_rows_to_read specified
+   */
+  private maxRowReadOnly: boolean;
 
   constructor({ host, username, password }: ClickhouseClientOptions) {
     this.host = host;
     this.username = username;
     this.password = password;
+    this.maxRowReadOnly = false;
+  }
+
+  async query<Format extends DataFormat>(
+    props: QueryInputs<Format>,
+  ): Promise<BaseResultSet<ReadableStream, Format>> {
+    let attempts = 0;
+    // retry query if fails
+    while (attempts < 2) {
+      try {
+        const res = await this.__query(props);
+        return res;
+      } catch (error: any) {
+        if (
+          !this.maxRowReadOnly &&
+          error.type === 'READONLY' &&
+          error.message.includes('max_rows_to_read')
+        ) {
+          // Indicate that the CH instance does not accept the max_rows_to_read setting
+          this.maxRowReadOnly = true;
+        } else {
+          throw error;
+        }
+      }
+      attempts++;
+    }
+    // should never get here
+    throw new Error('ClickHouseClient query impossible codepath');
   }
 
   // https://github.com/ClickHouse/clickhouse-js/blob/1ebdd39203730bb99fad4c88eac35d9a5e96b34a/packages/client-web/src/connection/web_connection.ts#L151
-  async query<Format extends DataFormat>({
+  private async __query<Format extends DataFormat>({
     query,
     format = 'JSON' as Format,
     query_params = {},
     abort_signal,
-    clickhouse_settings,
+    clickhouse_settings: external_clickhouse_settings,
     connectionId,
     queryId,
-  }: {
-    query: string;
-    format?: Format;
-    abort_signal?: AbortSignal;
-    query_params?: Record<string, any>;
-    clickhouse_settings?: Record<string, any>;
-    connectionId?: string;
-    queryId?: string;
-  }): Promise<BaseResultSet<ReadableStream, Format>> {
+  }: QueryInputs<Format>): Promise<BaseResultSet<ReadableStream, Format>> {
     let debugSql = '';
     try {
       debugSql = parameterizedQueryToSql({ sql: query, params: query_params });
@@ -403,14 +439,22 @@ export class ClickhouseClient {
     // eslint-disable-next-line no-console
     console.log('--------------------------------------------------------');
 
+    let clickhouse_settings = structuredClone(
+      external_clickhouse_settings || {},
+    );
+    if (clickhouse_settings?.max_rows_to_read && this.maxRowReadOnly) {
+      delete clickhouse_settings['max_rows_to_read'];
+    }
+
     if (isBrowser) {
       // TODO: check if we can use the client-web directly
       const { createClient } = await import('@clickhouse/client-web');
 
-      const clickhouse_settings: ClickHouseSettings = {
+      clickhouse_settings = {
         date_time_output_format: 'iso',
         wait_end_of_query: 0,
         cancel_http_readonly_queries_on_client_close: 1,
+        ...clickhouse_settings,
       };
       const http_headers = {
         ...(connectionId && connectionId !== 'local'
@@ -455,11 +499,6 @@ export class ClickhouseClient {
         url: this.host,
         username: this.username,
         password: this.password,
-        clickhouse_settings: {
-          date_time_output_format: 'iso',
-          wait_end_of_query: 0,
-          cancel_http_readonly_queries_on_client_close: 1,
-        },
       });
 
       // TODO: Custom error handling
@@ -468,7 +507,12 @@ export class ClickhouseClient {
         query_params,
         format,
         abort_signal,
-        clickhouse_settings,
+        clickhouse_settings: {
+          date_time_output_format: 'iso',
+          wait_end_of_query: 0,
+          cancel_http_readonly_queries_on_client_close: 1,
+          ...clickhouse_settings,
+        },
         query_id: queryId,
       }) as unknown as Promise<BaseResultSet<ReadableStream, Format>>;
     } else {

packages/common-utils/src/metadata.ts

@@ -11,7 +11,9 @@ import {
 import { renderChartConfig } from '@/renderChartConfig';
 import type { ChartConfig, ChartConfigWithDateRange, TSource } from '@/types';
 
-export const DEFAULT_MAX_ROWS_TO_READ = 1e6;
+// If filters initially are taking too long to load, decrease this number.
+// Between 1e6 - 5e6 is a good range.
+export const DEFAULT_MAX_ROWS_TO_READ = 3e6;
 
 export class MetadataCache {
   private cache = new Map<string, any>();
@@ -268,7 +270,7 @@ export class Metadata {
           query_params: sql.params,
           connectionId,
           clickhouse_settings: {
-            max_rows_to_read: DEFAULT_MAX_ROWS_TO_READ,
+            max_rows_to_read: String(DEFAULT_MAX_ROWS_TO_READ),
             read_overflow_mode: 'break',
           },
         })
@@ -341,7 +343,7 @@ export class Metadata {
             query_params: sql.params,
             connectionId,
             clickhouse_settings: {
-              max_rows_to_read: DEFAULT_MAX_ROWS_TO_READ,
+              max_rows_to_read: String(DEFAULT_MAX_ROWS_TO_READ),
               read_overflow_mode: 'break',
             },
           })
@@ -458,7 +460,7 @@ export class Metadata {
             connectionId: chartConfig.connection,
             clickhouse_settings: !disableRowLimit
               ? {
-                  max_rows_to_read: DEFAULT_MAX_ROWS_TO_READ,
+                  max_rows_to_read: String(DEFAULT_MAX_ROWS_TO_READ),
                   read_overflow_mode: 'break',
                 }
               : undefined,

packages/common-utils/src/types.ts

@@ -22,6 +22,8 @@ export enum DisplayType {
   Markdown = 'markdown',
 }
 
+export type KeyValue<Key = string, Value = string> = { key: Key; value: Value };
+
 export const MetricTableSchema = z.object(
   Object.values(MetricsDataType).reduce(
     (acc, key) => ({