[HDX-3994] Deprecate clickhouse.json feature gate in favor of per-exporter json config (#2119)

## Summary

Deprecate the upstream-deprecated `--feature-gates=clickhouse.json` CLI flag in favor of the per-exporter `json: true` config option, as recommended by the OpenTelemetry ClickHouse exporter v0.149.0.

This introduces a new env var `HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE` that controls JSON mode at the exporter config level. The old `OTEL_AGENT_FEATURE_GATE_ARG` env var remains backward-compatible — when it contains `clickhouse.json`, the entrypoint strips that gate, maps it to the new env var, and prints a deprecation warning. Other feature gates are preserved and passed through to the collector.

**Key changes:**
- **`docker/otel-collector/entrypoint.sh`** — Detects `clickhouse.json` in `OTEL_AGENT_FEATURE_GATE_ARG`, strips it, sets `HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE=true`, and prints a deprecation warning. Remaining feature gates are still passed through to the collector in both standalone and supervisor modes.
- **`docker/otel-collector/config.standalone.yaml`** — Added `json: ${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE:-false}` to both ClickHouse exporter configs
- **`packages/api/src/opamp/controllers/opampController.ts`** — Added `json` field to the `CollectorConfig` type and both ClickHouse exporter configs for OpAMP-managed collectors
- **`docker/otel-collector/supervisor_docker.yaml.tmpl`** — Feature gate pass-through preserved for non-`clickhouse.json` gates (entrypoint strips the deprecated gate before supervisor template renders)
- **`smoke-tests/otel-collector/`** — Added a JSON-enabled otel-collector service and smoke tests verifying:
  - `ResourceAttributes` and `LogAttributes` columns in `otel_logs` are `JSON` type (not `Map`)
  - Log data with various attribute types (string, int, boolean) is inserted and queryable via JSON path access

### How to test locally or on Vercel

1. Run `yarn dev` to start the dev stack
2. Verify the `otel-collector-json` container starts without errors (the `clickhouse.json` feature gate is stripped, not passed to the collector)
3. Check container logs for the deprecation warning when `OTEL_AGENT_FEATURE_GATE_ARG` contains `clickhouse.json`
4. Verify the non-JSON `otel-collector` service continues to work normally (json defaults to false)
5. Run smoke tests: `cd smoke-tests/otel-collector && bats json-exporter.bats`

### References

- Linear Issue: https://linear.app/hyperdx/issue/HDX-3994
- Upstream deprecation: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/clickhouseexporter#experimental-json-support

.changeset/deprecate-clickhouse-json-feature-gate.md

@@ -0,0 +1,11 @@
+---
+'@hyperdx/otel-collector': patch
+---
+
+refactor: Deprecate clickhouse.json feature gate in favor of per-exporter json config
+
+Replace the upstream-deprecated `--feature-gates=clickhouse.json` CLI flag with
+the per-exporter `json: true` config option controlled by
+`HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE`. The old
+`OTEL_AGENT_FEATURE_GATE_ARG` is still supported for backward compatibility but
+prints a deprecation warning when `clickhouse.json` is detected.

docker-compose.dev.yml

@@ -80,9 +80,9 @@ services:
       CUSTOM_OTELCOL_CONFIG_FILE: '/etc/otelcol-contrib/custom.config.yaml'
       # Uncomment to enable stdout logging for the OTel collector
       OTEL_SUPERVISOR_LOGS: 'true'
-      # Uncomment to enable JSON schema in ClickHouse
+      # Enable JSON schema in the ClickHouse exporter (per-exporter config)
       # Be sure to also set BETA_CH_OTEL_JSON_SCHEMA_ENABLED to 'true' in ch-server
-      OTEL_AGENT_FEATURE_GATE_ARG: '--feature-gates=clickhouse.json'
+      HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE: 'true'
     volumes:
       - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml
       - ./docker/otel-collector/supervisor_docker.yaml.tmpl:/etc/otel/supervisor.yaml.tmpl
@@ -109,7 +109,7 @@ services:
       CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
       HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: ${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE}
       # Set to 'true' to allow for proper OTel JSON Schema creation
-      # Be sure to also set the OTEL_AGENT_FEATURE_GATE_ARG env in otel-collector
+      # Be sure to also set HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE in otel-collector
       # BETA_CH_OTEL_JSON_SCHEMA_ENABLED: 'true'
     volumes:
       - ./docker/clickhouse/local/config.xml:/etc/clickhouse-server/config.xml

docker/otel-collector/config.standalone.yaml

@@ -33,6 +33,7 @@ exporters:
     logs_table_name: hyperdx_sessions
     timeout: 5s
     create_schema: ${env:HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA:-false}
+    json: ${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE:-false}
     retry_on_failure:
       enabled: true
       initial_interval: 5s
@@ -46,6 +47,7 @@ exporters:
     ttl: 720h
     timeout: 5s
     create_schema: ${env:HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA:-false}
+    json: ${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE:-false}
     retry_on_failure:
       enabled: true
       initial_interval: 5s

docker/otel-collector/entrypoint.sh

@@ -1,8 +1,26 @@
 #!/bin/sh
 set -e
 
-# Fall back to legacy schema when the ClickHouse JSON feature gate is enabled
+# DEPRECATED: The clickhouse.json feature gate has been removed upstream.
+# When OTEL_AGENT_FEATURE_GATE_ARG contains clickhouse.json, strip it and
+# map it to HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE instead. Other feature gates
+# are preserved and passed through to the collector.
 if echo "$OTEL_AGENT_FEATURE_GATE_ARG" | grep -q "clickhouse.json"; then
+  echo "WARNING: '--feature-gates=clickhouse.json' is deprecated and no longer supported by the collector."
+  echo "WARNING: Use HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE=true instead. This flag will be removed in a future release."
+  export HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE=true
+
+  # Strip clickhouse.json from the feature gates, keeping any other gates
+  REMAINING_GATES=$(echo "$OTEL_AGENT_FEATURE_GATE_ARG" | sed 's/--feature-gates=//' | tr ',' '\n' | grep -v 'clickhouse.json' | tr '\n' ',' | sed 's/,$//')
+  if [ -n "$REMAINING_GATES" ]; then
+    export OTEL_AGENT_FEATURE_GATE_ARG="--feature-gates=$REMAINING_GATES"
+  else
+    unset OTEL_AGENT_FEATURE_GATE_ARG
+  fi
+fi
+
+# Fall back to legacy schema when ClickHouse JSON exporter mode is enabled
+if [ "$HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE" = "true" ]; then
   export HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA=true
 fi
 
@@ -39,7 +57,7 @@ if [ -z "$OPAMP_SERVER_URL" ]; then
     COLLECTOR_ARGS="$COLLECTOR_ARGS --config $CUSTOM_OTELCOL_CONFIG_FILE"
   fi
 
-  # Pass feature gates to the collector in standalone mode
+  # Pass remaining feature gates to the collector in standalone mode
   if [ -n "$OTEL_AGENT_FEATURE_GATE_ARG" ]; then
     COLLECTOR_ARGS="$COLLECTOR_ARGS $OTEL_AGENT_FEATURE_GATE_ARG"
   fi

docker/otel-collector/supervisor_docker.yaml.tmpl

@@ -23,8 +23,8 @@ agent:
 {{- if getenv "CUSTOM_OTELCOL_CONFIG_FILE" }}
     - {{ getenv "CUSTOM_OTELCOL_CONFIG_FILE" }}
 {{- end }}
-  args:
 {{- if getenv "OTEL_AGENT_FEATURE_GATE_ARG" }}
+  args:
     - {{ getenv "OTEL_AGENT_FEATURE_GATE_ARG" }}
 {{- end }}
 

packages/api/src/opamp/controllers/opampController.ts

@@ -82,6 +82,7 @@ type CollectorConfig = {
       logs_table_name: string;
       timeout: string;
       create_schema: string;
+      json: string;
       retry_on_failure: {
         enabled: boolean;
         initial_interval: string;
@@ -97,6 +98,7 @@ type CollectorConfig = {
       ttl: string;
       timeout: string;
       create_schema: string;
+      json: string;
       retry_on_failure: {
         enabled: boolean;
         initial_interval: string;
@@ -205,6 +207,7 @@ export const buildOtelCollectorConfig = (
         timeout: '5s',
         create_schema:
           '${env:HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA:-false}',
+        json: '${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE:-false}',
         retry_on_failure: {
           enabled: true,
           initial_interval: '5s',
@@ -221,6 +224,7 @@ export const buildOtelCollectorConfig = (
         timeout: '5s',
         create_schema:
           '${env:HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA:-false}',
+        json: '${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE:-false}',
         retry_on_failure: {
           enabled: true,
           initial_interval: '5s',

smoke-tests/otel-collector/data/json-exporter/basic-insert/assert_query.sql

@@ -0,0 +1,11 @@
+SELECT
+    Body,
+    toString(LogAttributes.`user.id`),
+    toString(LogAttributes.`request.method`),
+    toString(LogAttributes.`http.status_code`),
+    toString(LogAttributes.error)
+FROM otel_json.otel_logs
+WHERE toString(ResourceAttributes.`suite-id`) = 'json-exporter'
+  AND toString(ResourceAttributes.`test-id`) = 'basic-insert'
+ORDER BY Timestamp
+FORMAT CSV

smoke-tests/otel-collector/data/json-exporter/basic-insert/expected.snap

@@ -0,0 +1,3 @@
+"JSON exporter log with string attributes","user-123","GET","",""
+"JSON exporter log with integer attribute","","","200",""
+"JSON exporter log with boolean attribute","","","","false"

smoke-tests/otel-collector/data/json-exporter/basic-insert/input.json

@@ -0,0 +1,83 @@
+{
+  "resourceLogs": [
+    {
+      "resource": {
+        "attributes": [
+          {
+            "key": "suite-id",
+            "value": {
+              "stringValue": "json-exporter"
+            }
+          },
+          {
+            "key": "test-id",
+            "value": {
+              "stringValue": "basic-insert"
+            }
+          },
+          {
+            "key": "service.name",
+            "value": {
+              "stringValue": "test-service"
+            }
+          }
+        ]
+      },
+      "scopeLogs": [
+        {
+          "scope": {},
+          "logRecords": [
+            {
+              "timeUnixNano": "1901999590000000000",
+              "body": {
+                "stringValue": "JSON exporter log with string attributes"
+              },
+              "attributes": [
+                {
+                  "key": "user.id",
+                  "value": {
+                    "stringValue": "user-123"
+                  }
+                },
+                {
+                  "key": "request.method",
+                  "value": {
+                    "stringValue": "GET"
+                  }
+                }
+              ]
+            },
+            {
+              "timeUnixNano": "1901999590000000001",
+              "body": {
+                "stringValue": "JSON exporter log with integer attribute"
+              },
+              "attributes": [
+                {
+                  "key": "http.status_code",
+                  "value": {
+                    "intValue": "200"
+                  }
+                }
+              ]
+            },
+            {
+              "timeUnixNano": "1901999590000000002",
+              "body": {
+                "stringValue": "JSON exporter log with boolean attribute"
+              },
+              "attributes": [
+                {
+                  "key": "error",
+                  "value": {
+                    "boolValue": false
+                  }
+                }
+              ]
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

smoke-tests/otel-collector/data/json-exporter/column-types/assert_query.sql

@@ -0,0 +1,7 @@
+SELECT name, type
+FROM system.columns
+WHERE database = 'otel_json'
+  AND table = 'otel_logs'
+  AND name IN ('ResourceAttributes', 'LogAttributes')
+ORDER BY name
+FORMAT CSV

smoke-tests/otel-collector/data/json-exporter/column-types/expected.snap

@@ -0,0 +1,2 @@
+"LogAttributes","JSON"
+"ResourceAttributes","JSON"

smoke-tests/otel-collector/docker-compose.yaml

@@ -43,6 +43,31 @@ services:
       ch-server:
         condition: service_healthy
 
+  otel-collector-json:
+    build:
+      context: ../..
+      dockerfile: docker/otel-collector/Dockerfile
+      target: dev
+      args:
+        OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION:-0.149.0}
+        OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION:-1.55.0}
+    environment:
+      - CLICKHOUSE_ENDPOINT=tcp://ch-server:9000?dial_timeout=10s
+      - CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT=ch-server:9363
+      - CLICKHOUSE_USER=default
+      - CLICKHOUSE_PASSWORD=
+      - HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=otel_json
+      - HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE=true
+      - HYPERDX_LOG_LEVEL=info
+      # OPAMP_SERVER_URL is intentionally not set to run in standalone mode
+    ports:
+      - 14318:4318 # OTLP http receiver
+    networks:
+      - internal
+    depends_on:
+      ch-server:
+        condition: service_healthy
+
 networks:
   internal:
     name: 'smoke-test-internal-network'

smoke-tests/otel-collector/json-exporter.bats

@@ -0,0 +1,14 @@
+#!/usr/bin/env bats
+
+load 'test_helpers/utilities.bash'
+load 'test_helpers/assertions.bash'
+
+@test "HDX-3994: JSON exporter creates otel_logs table with JSON column types for ResourceAttributes and LogAttributes" {
+    assert_test_data "data/json-exporter/column-types"
+}
+
+@test "HDX-3994: JSON exporter inserts log data with attributes accessible via JSON path" {
+    emit_otel_data "http://localhost:14318" "data/json-exporter/basic-insert"
+    sleep 2
+    assert_test_data "data/json-exporter/basic-insert"
+}

smoke-tests/otel-collector/setup_suite.bash

@@ -7,6 +7,7 @@ setup_suite() {
     validate_env
     docker compose up --build --detach
     wait_for_ready "otel-collector"
+    wait_for_ready "otel-collector-json"
 }
 
 teardown_suite() {