feat: Add ClickHouse JSON Type Support (#969)

- Upgrades ClickHouse to 25.6, fixes breaking config change, needed for latest JSON type
- Upgrades OTel Collector to 0.129.1, fixes breaking config change, needed for latest JSON support in exporter
- Upgrades OTel OpAMP Supervisor to 0.128.0
- Fixes features to support JSON type columns in OTel in HyperDX (filtering, searching, graphing, opening rows, etc.)

Requires users to set `BETA_CH_OTEL_JSON_SCHEMA_ENABLED=true` in `ch-server` and `OTEL_AGENT_FEATURE_GATE_ARG='--feature-gates=clickhouse.json'` in `otel-collector` to enable JSON schema. Users must start a new ClickHouse DB or migrate their own table manually to enable as it is not schema compatible and migration is not automatic.

Closes HDX-1849, HDX-1969, HDX-1849, HDX-1966, HDX-1964

Co-authored-by: Tom Alexander <3245235+teeohhem@users.noreply.github.com>

.changeset/heavy-pets-walk.md

@@ -0,0 +1,7 @@
+---
+"@hyperdx/common-utils": patch
+"@hyperdx/api": patch
+"@hyperdx/app": patch
+---
+
+feat: Add ClickHouse JSON Type Support

docker-compose.ci.yml

@@ -22,7 +22,7 @@ services:
     depends_on:
       - ch-server
   ch-server:
-    image: clickhouse/clickhouse-server:24-alpine
+    image: clickhouse/clickhouse-server:25.6-alpine
     environment:
       # default settings
       CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1

docker-compose.dev.yml

@@ -30,6 +30,11 @@ services:
       HYPERDX_API_KEY: ${HYPERDX_API_KEY}
       HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
       OPAMP_SERVER_URL: 'http://host.docker.internal:${HYPERDX_OPAMP_PORT}'
+      # Uncomment to enable stdout logging for the OTel collector
+      # OTEL_SUPERVISOR_PASSTHROUGH_LOGS: 'true'
+      # Uncomment to enable JSON schema in ClickHouse
+      # Be sure to also set BETA_CH_OTEL_JSON_SCHEMA_ENABLED to 'true' in ch-server
+      # OTEL_AGENT_FEATURE_GATE_ARG: '--feature-gates=clickhouse.json'
     volumes:
       - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml
       - ./docker/otel-collector/supervisor_docker.yaml:/etc/otel/supervisor.yaml
@@ -46,7 +51,7 @@ services:
       ch-server:
         condition: service_healthy
   ch-server:
-    image: clickhouse/clickhouse-server:24-alpine
+    image: clickhouse/clickhouse-server:25.6-alpine
     ports:
       - 8123:8123 # http api
       - 9000:9000 # native
@@ -54,6 +59,9 @@ services:
       # default settings
       CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
       HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: ${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE}
+      # Set to 'true' to allow for proper OTel JSON Schema creation
+      # Be sure to also set the OTEL_AGENT_FEATURE_GATE_ARG env in otel-collector
+      # BETA_CH_OTEL_JSON_SCHEMA_ENABLED: 'true'
     volumes:
       - ./docker/clickhouse/local/config.xml:/etc/clickhouse-server/config.xml
       - ./docker/clickhouse/local/users.xml:/etc/clickhouse-server/users.xml

docker-compose.yml

@@ -77,7 +77,7 @@ services:
       - ch-server
       - db
   ch-server:
-    image: clickhouse/clickhouse-server:24-alpine
+    image: clickhouse/clickhouse-server:25.6-alpine
     # WARNING: Exposing the database port will make it accessible from outside the container,
     # potentially allowing unauthorized access. If you uncomment the ports below,
     # ensure to secure your database (e.g., with strong authentication, proper network rules, and firewalls).

docker/clickhouse/local/config.xml

@@ -3,8 +3,8 @@
     <logger>
         <level>debug</level>
         <console>true</console>
-        <log remove="remove"/>
-        <errorlog remove="remove"/>
+        <log remove="remove" />
+        <errorlog remove="remove" />
     </logger>
 
     <listen_host>0.0.0.0</listen_host>
@@ -23,7 +23,12 @@
     <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
     <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
 
-    <users_config>users.xml</users_config>
+    <user_directories>
+        <users_xml>
+            <path>users.xml</path>
+        </users_xml>
+    </user_directories>
+    <!-- <users_config>users.xml</users_config> -->
     <default_profile>default</default_profile>
     <default_database>default</default_database>
     <timezone>UTC</timezone>
@@ -46,7 +51,8 @@
         <flush_interval_milliseconds>7500</flush_interval_milliseconds>
     </query_log>
 
-    <!-- Metric log contains rows with current values of ProfileEvents, CurrentMetrics collected with "collect_interval_milliseconds" interval. -->
+    <!-- Metric log contains rows with current values of ProfileEvents, CurrentMetrics collected
+    with "collect_interval_milliseconds" interval. -->
     <metric_log>
         <database>system</database>
         <table>metric_log</table>
@@ -113,7 +119,8 @@
     </processors_profile_log>
 
     <!-- Uncomment if use part log.
-         Part log contains information about all actions with parts in MergeTree tables (creation, deletion, merges, downloads).-->
+         Part log contains information about all actions with parts in MergeTree tables (creation, deletion,
+    merges, downloads).-->
     <part_log>
         <database>system</database>
         <table>part_log</table>

docker/clickhouse/local/init-db.sh

@@ -1,6 +1,11 @@
 #!/bin/bash
 set -e
 
+# We don't have a JSON schema yet, so let's let the collector create the tables
+if [ "$BETA_CH_OTEL_JSON_SCHEMA_ENABLED" = "true" ]; then
+  exit 0
+fi
+
 DATABASE=${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE:-default}
 
 clickhouse client -n <<EOFSQL

docker/otel-collector/Dockerfile

@@ -1,6 +1,6 @@
 ## base #############################################################################################
-FROM otel/opentelemetry-collector-contrib:0.126.0 AS col
-FROM otel/opentelemetry-collector-opampsupervisor:0.126.0 AS supervisor
+FROM otel/opentelemetry-collector-contrib:0.129.1 AS col
+FROM otel/opentelemetry-collector-opampsupervisor:0.128.0 AS supervisor
 
 # From: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/aa5c3aa4c7ec174361fcaf908de8eaca72263078/cmd/opampsupervisor/Dockerfile#L18
 FROM alpine:latest@sha256:a8560b36e8b8210634f77d9f7f9efd7ffa463e380b75e2e74aff4511df3ef88c AS prep

docker/otel-collector/config.yaml

@@ -78,7 +78,8 @@ processors:
             SEVERITY_NUMBER_TRACE
           # Infer: else
           - set(log.severity_text, "info") where log.severity_number == 0
-          - set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_number == 0
+          - set(log.severity_number, SEVERITY_NUMBER_INFO) where
+            log.severity_number == 0
       - context: log
         error_mode: ignore
         statements:
@@ -142,7 +143,12 @@ extensions:
 service:
   telemetry:
     metrics:
-      address: ':8888'
+      readers:
+        - pull:
+            exporter:
+              prometheus:
+                host: '0.0.0.0'
+                port: 8888
     logs:
       level: ${HYPERDX_LOG_LEVEL}
   extensions: [health_check]

docker/otel-collector/supervisor_docker.yaml

@@ -20,6 +20,8 @@ agent:
   executable: /otelcontribcol
   config_files:
     - /etc/otelcol-contrib/config.yaml
+  args:
+    - ${env:OTEL_AGENT_FEATURE_GATE_ARG}
   passthrough_logs: ${env:OTEL_SUPERVISOR_PASSTHROUGH_LOGS}
 
 storage:

packages/app/jest.config.js

@@ -8,7 +8,7 @@ module.exports = {
   },
   moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],
   testRegex: '(/__tests__/.*|(\\.|/)(test|spec))\\.(jsx?|tsx?)$',
-  transformIgnorePatterns: ['/node_modules/(?!(ky|ky-universal))'],
+  transformIgnorePatterns: ['/node_modules/(?!(ky|ky-universal|flat))'],
   moduleNameMapper: {
     '\\.(css|less|scss|sass)$': 'identity-obj-proxy',
     '^@/(.*)$': '<rootDir>/src/$1',

packages/app/package.json

@@ -53,6 +53,7 @@
     "crypto-randomuuid": "^1.0.0",
     "date-fns": "^2.28.0",
     "date-fns-tz": "^2.0.0",
+    "flat": "^6.0.1",
     "fuse.js": "^6.6.2",
     "http-proxy-middleware": "^3.0.5",
     "immer": "^9.0.21",

packages/app/src/components/DBRowDataPanel.tsx

@@ -1,4 +1,5 @@
 import { useMemo } from 'react';
+import { ResponseJSON } from '@clickhouse/client';
 import { SourceKind, TSource } from '@hyperdx/common-utils/dist/types';
 import { Box } from '@mantine/core';
 
@@ -113,6 +114,10 @@ export function useRowData({
   );
 }
 
+export function getJSONColumnNames(meta: ResponseJSON['meta'] | undefined) {
+  return meta?.filter(m => m.type === 'JSON').map(m => m.name) ?? [];
+}
+
 export function RowDataPanel({
   source,
   rowId,
@@ -130,10 +135,12 @@ export function RowDataPanel({
     return firstRow;
   }, [data]);
 
+  const jsonColumns = getJSONColumnNames(data?.meta);
+
   return (
     <div className="flex-grow-1 bg-body overflow-auto">
       <Box mx="md" my="sm">
-        <DBRowJsonViewer data={firstRow} />
+        <DBRowJsonViewer data={firstRow} jsonColumns={jsonColumns} />
       </Box>
     </div>
   );

packages/app/src/components/DBRowJsonViewer.test.tsx

@@ -1,3 +1,4 @@
+import React from 'react';
 import { fireEvent, screen, within } from '@testing-library/react';
 
 import { DBRowJsonViewer } from './DBRowJsonViewer';

packages/app/src/components/DBRowJsonViewer.tsx

@@ -122,7 +122,13 @@ function HyperJsonMenu() {
   );
 }
 
-export function DBRowJsonViewer({ data }: { data: any }) {
+export function DBRowJsonViewer({
+  data,
+  jsonColumns = [],
+}: {
+  data: any;
+  jsonColumns?: string[];
+}) {
   const {
     onPropertyAddClick,
     generateSearchUrl,
@@ -152,7 +158,19 @@ export function DBRowJsonViewer({ data }: { data: any }) {
   const getLineActions = useCallback<GetLineActions>(
     ({ keyPath, value }) => {
       const actions: LineAction[] = [];
-      const fieldPath = mergePath(keyPath);
+      let fieldPath = mergePath(keyPath);
+      const isJsonColumn =
+        keyPath.length > 0 && jsonColumns?.includes(keyPath[0]);
+
+      if (isJsonColumn) {
+        fieldPath = keyPath.join('.');
+        if (keyPath.length > 1) {
+          fieldPath = `${keyPath[0]}.${keyPath
+            .slice(1)
+            .map(k => `\`${k}\``)
+            .join('.')}`;
+        }
+      }
 
       // Add to Filters action (strings only)
       // FIXME: TOTAL HACK To disallow adding timestamp to filters
@@ -173,7 +191,10 @@ export function DBRowJsonViewer({ data }: { data: any }) {
           ),
           title: 'Add to Filters',
           onClick: () => {
-            onPropertyAddClick(fieldPath, value);
+            onPropertyAddClick(
+              isJsonColumn ? `toString(${fieldPath})` : fieldPath,
+              value,
+            );
             notifications.show({
               color: 'green',
               message: `Added "${fieldPath} = ${value}" to filters`,
@@ -305,6 +326,7 @@ export function DBRowJsonViewer({ data }: { data: any }) {
       onPropertyAddClick,
       rowData,
       toggleColumn,
+      jsonColumns,
     ],
   );
 

packages/app/src/components/DBRowOverviewPanel.tsx

@@ -1,4 +1,5 @@
 import { useCallback, useContext, useMemo } from 'react';
+import { flatten } from 'flat';
 import isString from 'lodash/isString';
 import pickBy from 'lodash/pickBy';
 import { SourceKind, TSource } from '@hyperdx/common-utils/dist/types';
@@ -6,7 +7,7 @@ import { Accordion, Box, Divider, Flex, Text } from '@mantine/core';
 
 import { getEventBody } from '@/source';
 
-import { useRowData } from './DBRowDataPanel';
+import { getJSONColumnNames, useRowData } from './DBRowDataPanel';
 import { DBRowJsonViewer } from './DBRowJsonViewer';
 import { RowSidePanelContext } from './DBRowSidePanel';
 import DBRowSidePanelHeader from './DBRowSidePanelHeader';
@@ -29,6 +30,8 @@ export function RowOverviewPanel({
   const { onPropertyAddClick, generateSearchUrl } =
     useContext(RowSidePanelContext);
 
+  const jsonColumns = getJSONColumnNames(data?.meta);
+
   const eventAttributesExpr = source.eventAttributesExpression;
 
   const firstRow = useMemo(() => {
@@ -65,8 +68,18 @@ export function RowOverviewPanel({
     return false;
   });
 
-  const resourceAttributes = firstRow?.__hdx_resource_attributes ?? EMPTY_OBJ;
-  const eventAttributes = firstRow?.__hdx_event_attributes ?? EMPTY_OBJ;
+  // memo
+  const resourceAttributes = useMemo(() => {
+    return flatten<string, Record<string, string>>(
+      firstRow?.__hdx_resource_attributes ?? EMPTY_OBJ,
+    );
+  }, [firstRow?.__hdx_resource_attributes]);
+
+  const _eventAttributes = firstRow?.__hdx_event_attributes ?? EMPTY_OBJ;
+  const flattenedEventAttributes = useMemo(() => {
+    return flatten<string, Record<string, string>>(_eventAttributes);
+  }, [_eventAttributes]);
+
   const dataAttributes =
     eventAttributesExpr &&
     firstRow?.[eventAttributesExpr] &&
@@ -188,7 +201,7 @@ export function RowOverviewPanel({
             <Accordion.Panel>
               <Box px="md">
                 <NetworkPropertySubpanel
-                  eventAttributes={eventAttributes}
+                  eventAttributes={flattenedEventAttributes}
                   onPropertyAddClick={onPropertyAddClick}
                   generateSearchUrl={_generateSearchUrl}
                 />
@@ -242,7 +255,10 @@ export function RowOverviewPanel({
             </Accordion.Control>
             <Accordion.Panel>
               <Box px="md">
-                <DBRowJsonViewer data={topLevelAttributes} />
+                <DBRowJsonViewer
+                  data={topLevelAttributes}
+                  jsonColumns={jsonColumns}
+                />
               </Box>
             </Accordion.Panel>
           </Accordion.Item>
@@ -256,7 +272,10 @@ export function RowOverviewPanel({
           </Accordion.Control>
           <Accordion.Panel>
             <Box px="md">
-              <DBRowJsonViewer data={filteredEventAttributes} />
+              <DBRowJsonViewer
+                data={filteredEventAttributes}
+                jsonColumns={jsonColumns}
+              />
             </Box>
           </Accordion.Panel>
         </Accordion.Item>

packages/app/src/components/DBSearchPageFilters.tsx

@@ -511,6 +511,13 @@ const DBSearchPageFiltersComponent = ({
         }
       }
     }
+    // get remaining filterState that are not in _facets
+    const remainingFilterState = Object.keys(filterState).filter(
+      key => !_facets.some(facet => facet.key === key),
+    );
+    for (const key of remainingFilterState) {
+      _facets.push({ key, value: Array.from(filterState[key].included) });
+    }
     return _facets;
   }, [facets, filterState, extraFacets]);
 

packages/app/src/hooks/useRowWhere.tsx

@@ -82,7 +82,7 @@ export default function useRowWhere({
 
               // Currently we can't distinguish null or 'null'
               if (value === 'null') {
-                return SqlString.format(`isNull(??)`, SqlString.raw(column));
+                return SqlString.format(`isNull(??)`, [column]);
               }
               if (value.length > 1000 || column.length > 1000) {
                 throw new Error('Search value/object key too large.');

smoke-tests/otel-collector/docker-compose.yaml

@@ -19,7 +19,7 @@ services:
       retries: 5
       start_period: 10s
   otel-collector:
-    image: otel/opentelemetry-collector-contrib:0.126.0
+    image: otel/opentelemetry-collector-contrib:0.129.1
     volumes:
       - ../../docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml
       - ./receiver-config.yaml:/etc/otelcol-contrib/receiver-config.yaml

yarn.lock

@@ -4589,6 +4589,7 @@ __metadata:
     date-fns: "npm:^2.28.0"
     date-fns-tz: "npm:^2.0.0"
     eslint-config-next: "npm:^14.2.29"
+    flat: "npm:^6.0.1"
     fuse.js: "npm:^6.6.2"
     http-proxy-middleware: "npm:^3.0.5"
     identity-obj-proxy: "npm:^3.0.0"
@@ -16837,6 +16838,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"flat@npm:^6.0.1":
+  version: 6.0.1
+  resolution: "flat@npm:6.0.1"
+  bin:
+    flat: cli.js
+  checksum: 10c0/9dc0dbe6e2acc012512a53130d9ba1c82c1a596cdca91b23d11716348361c4a68928409bb4433c4493a17595c3efd0cab9f09e23dd3f9962a58af225c3efc23a
+  languageName: node
+  linkType: hard
+
 "flatted@npm:^3.1.0":
   version: 3.2.7
   resolution: "flatted@npm:3.2.7"