From cb841457f2bb17f1c0e48b5827edb22fda37fa50 Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Wed, 15 Apr 2026 08:49:45 -0700 Subject: [PATCH] [HDX-3994] Deprecate clickhouse.json feature gate in favor of per-exporter json config (#2119) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Deprecate the upstream-deprecated `--feature-gates=clickhouse.json` CLI flag in favor of the per-exporter `json: true` config option, as recommended by the OpenTelemetry ClickHouse exporter v0.149.0. This introduces a new env var `HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE` that controls JSON mode at the exporter config level. The old `OTEL_AGENT_FEATURE_GATE_ARG` env var remains backward-compatible — when it contains `clickhouse.json`, the entrypoint strips that gate, maps it to the new env var, and prints a deprecation warning. Other feature gates are preserved and passed through to the collector. **Key changes:** - **`docker/otel-collector/entrypoint.sh`** — Detects `clickhouse.json` in `OTEL_AGENT_FEATURE_GATE_ARG`, strips it, sets `HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE=true`, and prints a deprecation warning. Remaining feature gates are still passed through to the collector in both standalone and supervisor modes. - **`docker/otel-collector/config.standalone.yaml`** — Added `json: ${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE:-false}` to both ClickHouse exporter configs - **`packages/api/src/opamp/controllers/opampController.ts`** — Added `json` field to the `CollectorConfig` type and both ClickHouse exporter configs for OpAMP-managed collectors - **`docker/otel-collector/supervisor_docker.yaml.tmpl`** — Feature gate pass-through preserved for non-`clickhouse.json` gates (entrypoint strips the deprecated gate before supervisor template renders) - **`smoke-tests/otel-collector/`** — Added a JSON-enabled otel-collector service and smoke tests verifying: - `ResourceAttributes` and `LogAttributes` columns in `otel_logs` are `JSON` type (not `Map`) - Log data with various attribute types (string, int, boolean) is inserted and queryable via JSON path access ### How to test locally or on Vercel 1. Run `yarn dev` to start the dev stack 2. Verify the `otel-collector-json` container starts without errors (the `clickhouse.json` feature gate is stripped, not passed to the collector) 3. Check container logs for the deprecation warning when `OTEL_AGENT_FEATURE_GATE_ARG` contains `clickhouse.json` 4. Verify the non-JSON `otel-collector` service continues to work normally (json defaults to false) 5. Run smoke tests: `cd smoke-tests/otel-collector && bats json-exporter.bats` ### References - Linear Issue: https://linear.app/hyperdx/issue/HDX-3994 - Upstream deprecation: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/clickhouseexporter#experimental-json-support --- .../deprecate-clickhouse-json-feature-gate.md | 11 +++ docker-compose.dev.yml | 6 +- docker/otel-collector/config.standalone.yaml | 2 + docker/otel-collector/entrypoint.sh | 22 ++++- .../supervisor_docker.yaml.tmpl | 2 +- .../src/opamp/controllers/opampController.ts | 4 + .../basic-insert/assert_query.sql | 11 +++ .../json-exporter/basic-insert/expected.snap | 3 + .../json-exporter/basic-insert/input.json | 83 +++++++++++++++++++ .../column-types/assert_query.sql | 7 ++ .../json-exporter/column-types/expected.snap | 2 + .../otel-collector/docker-compose.yaml | 25 ++++++ smoke-tests/otel-collector/json-exporter.bats | 14 ++++ smoke-tests/otel-collector/setup_suite.bash | 1 + 14 files changed, 187 insertions(+), 6 deletions(-) create mode 100644 .changeset/deprecate-clickhouse-json-feature-gate.md create mode 100644 smoke-tests/otel-collector/data/json-exporter/basic-insert/assert_query.sql create mode 100644 smoke-tests/otel-collector/data/json-exporter/basic-insert/expected.snap create mode 100644 smoke-tests/otel-collector/data/json-exporter/basic-insert/input.json create mode 100644 smoke-tests/otel-collector/data/json-exporter/column-types/assert_query.sql create mode 100644 smoke-tests/otel-collector/data/json-exporter/column-types/expected.snap create mode 100644 smoke-tests/otel-collector/json-exporter.bats diff --git a/.changeset/deprecate-clickhouse-json-feature-gate.md b/.changeset/deprecate-clickhouse-json-feature-gate.md new file mode 100644 index 00000000..2b7aef6f --- /dev/null +++ b/.changeset/deprecate-clickhouse-json-feature-gate.md @@ -0,0 +1,11 @@ +--- +'@hyperdx/otel-collector': patch +--- + +refactor: Deprecate clickhouse.json feature gate in favor of per-exporter json config + +Replace the upstream-deprecated `--feature-gates=clickhouse.json` CLI flag with +the per-exporter `json: true` config option controlled by +`HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE`. The old +`OTEL_AGENT_FEATURE_GATE_ARG` is still supported for backward compatibility but +prints a deprecation warning when `clickhouse.json` is detected. diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 6dfc36b4..dd2e5747 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -80,9 +80,9 @@ services: CUSTOM_OTELCOL_CONFIG_FILE: '/etc/otelcol-contrib/custom.config.yaml' # Uncomment to enable stdout logging for the OTel collector OTEL_SUPERVISOR_LOGS: 'true' - # Uncomment to enable JSON schema in ClickHouse + # Enable JSON schema in the ClickHouse exporter (per-exporter config) # Be sure to also set BETA_CH_OTEL_JSON_SCHEMA_ENABLED to 'true' in ch-server - OTEL_AGENT_FEATURE_GATE_ARG: '--feature-gates=clickhouse.json' + HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE: 'true' volumes: - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml - ./docker/otel-collector/supervisor_docker.yaml.tmpl:/etc/otel/supervisor.yaml.tmpl @@ -109,7 +109,7 @@ services: CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: ${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE} # Set to 'true' to allow for proper OTel JSON Schema creation - # Be sure to also set the OTEL_AGENT_FEATURE_GATE_ARG env in otel-collector + # Be sure to also set HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE in otel-collector # BETA_CH_OTEL_JSON_SCHEMA_ENABLED: 'true' volumes: - ./docker/clickhouse/local/config.xml:/etc/clickhouse-server/config.xml diff --git a/docker/otel-collector/config.standalone.yaml b/docker/otel-collector/config.standalone.yaml index f43189f2..46e3958b 100644 --- a/docker/otel-collector/config.standalone.yaml +++ b/docker/otel-collector/config.standalone.yaml @@ -33,6 +33,7 @@ exporters: logs_table_name: hyperdx_sessions timeout: 5s create_schema: ${env:HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA:-false} + json: ${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE:-false} retry_on_failure: enabled: true initial_interval: 5s @@ -46,6 +47,7 @@ exporters: ttl: 720h timeout: 5s create_schema: ${env:HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA:-false} + json: ${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE:-false} retry_on_failure: enabled: true initial_interval: 5s diff --git a/docker/otel-collector/entrypoint.sh b/docker/otel-collector/entrypoint.sh index 3aae225c..272473e4 100644 --- a/docker/otel-collector/entrypoint.sh +++ b/docker/otel-collector/entrypoint.sh @@ -1,8 +1,26 @@ #!/bin/sh set -e -# Fall back to legacy schema when the ClickHouse JSON feature gate is enabled +# DEPRECATED: The clickhouse.json feature gate has been removed upstream. +# When OTEL_AGENT_FEATURE_GATE_ARG contains clickhouse.json, strip it and +# map it to HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE instead. Other feature gates +# are preserved and passed through to the collector. if echo "$OTEL_AGENT_FEATURE_GATE_ARG" | grep -q "clickhouse.json"; then + echo "WARNING: '--feature-gates=clickhouse.json' is deprecated and no longer supported by the collector." + echo "WARNING: Use HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE=true instead. This flag will be removed in a future release." + export HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE=true + + # Strip clickhouse.json from the feature gates, keeping any other gates + REMAINING_GATES=$(echo "$OTEL_AGENT_FEATURE_GATE_ARG" | sed 's/--feature-gates=//' | tr ',' '\n' | grep -v 'clickhouse.json' | tr '\n' ',' | sed 's/,$//') + if [ -n "$REMAINING_GATES" ]; then + export OTEL_AGENT_FEATURE_GATE_ARG="--feature-gates=$REMAINING_GATES" + else + unset OTEL_AGENT_FEATURE_GATE_ARG + fi +fi + +# Fall back to legacy schema when ClickHouse JSON exporter mode is enabled +if [ "$HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE" = "true" ]; then export HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA=true fi @@ -39,7 +57,7 @@ if [ -z "$OPAMP_SERVER_URL" ]; then COLLECTOR_ARGS="$COLLECTOR_ARGS --config $CUSTOM_OTELCOL_CONFIG_FILE" fi - # Pass feature gates to the collector in standalone mode + # Pass remaining feature gates to the collector in standalone mode if [ -n "$OTEL_AGENT_FEATURE_GATE_ARG" ]; then COLLECTOR_ARGS="$COLLECTOR_ARGS $OTEL_AGENT_FEATURE_GATE_ARG" fi diff --git a/docker/otel-collector/supervisor_docker.yaml.tmpl b/docker/otel-collector/supervisor_docker.yaml.tmpl index 3ee29c02..f91e3cc5 100644 --- a/docker/otel-collector/supervisor_docker.yaml.tmpl +++ b/docker/otel-collector/supervisor_docker.yaml.tmpl @@ -23,8 +23,8 @@ agent: {{- if getenv "CUSTOM_OTELCOL_CONFIG_FILE" }} - {{ getenv "CUSTOM_OTELCOL_CONFIG_FILE" }} {{- end }} - args: {{- if getenv "OTEL_AGENT_FEATURE_GATE_ARG" }} + args: - {{ getenv "OTEL_AGENT_FEATURE_GATE_ARG" }} {{- end }} diff --git a/packages/api/src/opamp/controllers/opampController.ts b/packages/api/src/opamp/controllers/opampController.ts index 5f70ec76..1b1ca941 100644 --- a/packages/api/src/opamp/controllers/opampController.ts +++ b/packages/api/src/opamp/controllers/opampController.ts @@ -82,6 +82,7 @@ type CollectorConfig = { logs_table_name: string; timeout: string; create_schema: string; + json: string; retry_on_failure: { enabled: boolean; initial_interval: string; @@ -97,6 +98,7 @@ type CollectorConfig = { ttl: string; timeout: string; create_schema: string; + json: string; retry_on_failure: { enabled: boolean; initial_interval: string; @@ -205,6 +207,7 @@ export const buildOtelCollectorConfig = ( timeout: '5s', create_schema: '${env:HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA:-false}', + json: '${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE:-false}', retry_on_failure: { enabled: true, initial_interval: '5s', @@ -221,6 +224,7 @@ export const buildOtelCollectorConfig = ( timeout: '5s', create_schema: '${env:HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA:-false}', + json: '${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE:-false}', retry_on_failure: { enabled: true, initial_interval: '5s', diff --git a/smoke-tests/otel-collector/data/json-exporter/basic-insert/assert_query.sql b/smoke-tests/otel-collector/data/json-exporter/basic-insert/assert_query.sql new file mode 100644 index 00000000..9a1af1a4 --- /dev/null +++ b/smoke-tests/otel-collector/data/json-exporter/basic-insert/assert_query.sql @@ -0,0 +1,11 @@ +SELECT + Body, + toString(LogAttributes.`user.id`), + toString(LogAttributes.`request.method`), + toString(LogAttributes.`http.status_code`), + toString(LogAttributes.error) +FROM otel_json.otel_logs +WHERE toString(ResourceAttributes.`suite-id`) = 'json-exporter' + AND toString(ResourceAttributes.`test-id`) = 'basic-insert' +ORDER BY Timestamp +FORMAT CSV diff --git a/smoke-tests/otel-collector/data/json-exporter/basic-insert/expected.snap b/smoke-tests/otel-collector/data/json-exporter/basic-insert/expected.snap new file mode 100644 index 00000000..e067132f --- /dev/null +++ b/smoke-tests/otel-collector/data/json-exporter/basic-insert/expected.snap @@ -0,0 +1,3 @@ +"JSON exporter log with string attributes","user-123","GET","","" +"JSON exporter log with integer attribute","","","200","" +"JSON exporter log with boolean attribute","","","","false" \ No newline at end of file diff --git a/smoke-tests/otel-collector/data/json-exporter/basic-insert/input.json b/smoke-tests/otel-collector/data/json-exporter/basic-insert/input.json new file mode 100644 index 00000000..24ad1fa4 --- /dev/null +++ b/smoke-tests/otel-collector/data/json-exporter/basic-insert/input.json @@ -0,0 +1,83 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { + "key": "suite-id", + "value": { + "stringValue": "json-exporter" + } + }, + { + "key": "test-id", + "value": { + "stringValue": "basic-insert" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "test-service" + } + } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1901999590000000000", + "body": { + "stringValue": "JSON exporter log with string attributes" + }, + "attributes": [ + { + "key": "user.id", + "value": { + "stringValue": "user-123" + } + }, + { + "key": "request.method", + "value": { + "stringValue": "GET" + } + } + ] + }, + { + "timeUnixNano": "1901999590000000001", + "body": { + "stringValue": "JSON exporter log with integer attribute" + }, + "attributes": [ + { + "key": "http.status_code", + "value": { + "intValue": "200" + } + } + ] + }, + { + "timeUnixNano": "1901999590000000002", + "body": { + "stringValue": "JSON exporter log with boolean attribute" + }, + "attributes": [ + { + "key": "error", + "value": { + "boolValue": false + } + } + ] + } + ] + } + ] + } + ] +} diff --git a/smoke-tests/otel-collector/data/json-exporter/column-types/assert_query.sql b/smoke-tests/otel-collector/data/json-exporter/column-types/assert_query.sql new file mode 100644 index 00000000..9c64a48b --- /dev/null +++ b/smoke-tests/otel-collector/data/json-exporter/column-types/assert_query.sql @@ -0,0 +1,7 @@ +SELECT name, type +FROM system.columns +WHERE database = 'otel_json' + AND table = 'otel_logs' + AND name IN ('ResourceAttributes', 'LogAttributes') +ORDER BY name +FORMAT CSV diff --git a/smoke-tests/otel-collector/data/json-exporter/column-types/expected.snap b/smoke-tests/otel-collector/data/json-exporter/column-types/expected.snap new file mode 100644 index 00000000..2afffb46 --- /dev/null +++ b/smoke-tests/otel-collector/data/json-exporter/column-types/expected.snap @@ -0,0 +1,2 @@ +"LogAttributes","JSON" +"ResourceAttributes","JSON" \ No newline at end of file diff --git a/smoke-tests/otel-collector/docker-compose.yaml b/smoke-tests/otel-collector/docker-compose.yaml index 070bdb25..5df6e6ed 100644 --- a/smoke-tests/otel-collector/docker-compose.yaml +++ b/smoke-tests/otel-collector/docker-compose.yaml @@ -43,6 +43,31 @@ services: ch-server: condition: service_healthy + otel-collector-json: + build: + context: ../.. + dockerfile: docker/otel-collector/Dockerfile + target: dev + args: + OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION:-0.149.0} + OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION:-1.55.0} + environment: + - CLICKHOUSE_ENDPOINT=tcp://ch-server:9000?dial_timeout=10s + - CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT=ch-server:9363 + - CLICKHOUSE_USER=default + - CLICKHOUSE_PASSWORD= + - HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=otel_json + - HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE=true + - HYPERDX_LOG_LEVEL=info + # OPAMP_SERVER_URL is intentionally not set to run in standalone mode + ports: + - 14318:4318 # OTLP http receiver + networks: + - internal + depends_on: + ch-server: + condition: service_healthy + networks: internal: name: 'smoke-test-internal-network' diff --git a/smoke-tests/otel-collector/json-exporter.bats b/smoke-tests/otel-collector/json-exporter.bats new file mode 100644 index 00000000..0b8fe070 --- /dev/null +++ b/smoke-tests/otel-collector/json-exporter.bats @@ -0,0 +1,14 @@ +#!/usr/bin/env bats + +load 'test_helpers/utilities.bash' +load 'test_helpers/assertions.bash' + +@test "HDX-3994: JSON exporter creates otel_logs table with JSON column types for ResourceAttributes and LogAttributes" { + assert_test_data "data/json-exporter/column-types" +} + +@test "HDX-3994: JSON exporter inserts log data with attributes accessible via JSON path" { + emit_otel_data "http://localhost:14318" "data/json-exporter/basic-insert" + sleep 2 + assert_test_data "data/json-exporter/basic-insert" +} diff --git a/smoke-tests/otel-collector/setup_suite.bash b/smoke-tests/otel-collector/setup_suite.bash index 783e1966..ae386964 100644 --- a/smoke-tests/otel-collector/setup_suite.bash +++ b/smoke-tests/otel-collector/setup_suite.bash @@ -7,6 +7,7 @@ setup_suite() { validate_env docker compose up --build --detach wait_for_ready "otel-collector" + wait_for_ready "otel-collector-json" } teardown_suite() {