mirror of
https://github.com/hyperdxio/hyperdx
synced 2026-04-21 13:37:15 +00:00
cb841457f2
## Summary
Deprecate the upstream-deprecated `--feature-gates=clickhouse.json` CLI flag in favor of the per-exporter `json: true` config option, as recommended by the OpenTelemetry ClickHouse exporter v0.149.0.
This introduces a new env var `HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE` that controls JSON mode at the exporter config level. The old `OTEL_AGENT_FEATURE_GATE_ARG` env var remains backward-compatible — when it contains `clickhouse.json`, the entrypoint strips that gate, maps it to the new env var, and prints a deprecation warning. Other feature gates are preserved and passed through to the collector.
**Key changes:**
- **`docker/otel-collector/entrypoint.sh`** — Detects `clickhouse.json` in `OTEL_AGENT_FEATURE_GATE_ARG`, strips it, sets `HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE=true`, and prints a deprecation warning. Remaining feature gates are still passed through to the collector in both standalone and supervisor modes.
- **`docker/otel-collector/config.standalone.yaml`** — Added `json: ${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE:-false}` to both ClickHouse exporter configs
- **`packages/api/src/opamp/controllers/opampController.ts`** — Added `json` field to the `CollectorConfig` type and both ClickHouse exporter configs for OpAMP-managed collectors
- **`docker/otel-collector/supervisor_docker.yaml.tmpl`** — Feature gate pass-through preserved for non-`clickhouse.json` gates (entrypoint strips the deprecated gate before supervisor template renders)
- **`smoke-tests/otel-collector/`** — Added a JSON-enabled otel-collector service and smoke tests verifying:
- `ResourceAttributes` and `LogAttributes` columns in `otel_logs` are `JSON` type (not `Map`)
- Log data with various attribute types (string, int, boolean) is inserted and queryable via JSON path access
### How to test locally or on Vercel
1. Run `yarn dev` to start the dev stack
2. Verify the `otel-collector-json` container starts without errors (the `clickhouse.json` feature gate is stripped, not passed to the collector)
3. Check container logs for the deprecation warning when `OTEL_AGENT_FEATURE_GATE_ARG` contains `clickhouse.json`
4. Verify the non-JSON `otel-collector` service continues to work normally (json defaults to false)
5. Run smoke tests: `cd smoke-tests/otel-collector && bats json-exporter.bats`
### References
- Linear Issue: https://linear.app/hyperdx/issue/HDX-3994
- Upstream deprecation: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/clickhouseexporter#experimental-json-support
77 lines
2.3 KiB
YAML
77 lines
2.3 KiB
YAML
# This configuration is derived from packages/api/src/opamp/controllers/opampController.ts
|
|
# When updating this file, ensure it stays in sync with buildOtelCollectorConfig()
|
|
|
|
receivers:
|
|
otlp/hyperdx:
|
|
protocols:
|
|
grpc:
|
|
include_metadata: true
|
|
endpoint: "0.0.0.0:4317"
|
|
http:
|
|
cors:
|
|
allowed_origins: ["*"]
|
|
allowed_headers: ["*"]
|
|
include_metadata: true
|
|
endpoint: "0.0.0.0:4318"
|
|
|
|
connectors:
|
|
routing/logs:
|
|
default_pipelines: [logs/out-default]
|
|
error_mode: ignore
|
|
table:
|
|
- context: log
|
|
statement: route() where IsMatch(attributes["rr-web.event"], ".*")
|
|
pipelines: [logs/out-rrweb]
|
|
|
|
exporters:
|
|
clickhouse/rrweb:
|
|
database: ${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE}
|
|
endpoint: ${env:CLICKHOUSE_ENDPOINT}
|
|
password: ${env:CLICKHOUSE_PASSWORD}
|
|
username: ${env:CLICKHOUSE_USER}
|
|
ttl: 720h
|
|
logs_table_name: hyperdx_sessions
|
|
timeout: 5s
|
|
create_schema: ${env:HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA:-false}
|
|
json: ${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE:-false}
|
|
retry_on_failure:
|
|
enabled: true
|
|
initial_interval: 5s
|
|
max_interval: 30s
|
|
max_elapsed_time: 300s
|
|
clickhouse:
|
|
database: ${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE}
|
|
endpoint: ${env:CLICKHOUSE_ENDPOINT}
|
|
password: ${env:CLICKHOUSE_PASSWORD}
|
|
username: ${env:CLICKHOUSE_USER}
|
|
ttl: 720h
|
|
timeout: 5s
|
|
create_schema: ${env:HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA:-false}
|
|
json: ${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE:-false}
|
|
retry_on_failure:
|
|
enabled: true
|
|
initial_interval: 5s
|
|
max_interval: 30s
|
|
max_elapsed_time: 300s
|
|
|
|
service:
|
|
pipelines:
|
|
traces:
|
|
receivers: [otlp/hyperdx]
|
|
processors: [memory_limiter, batch]
|
|
exporters: [clickhouse]
|
|
metrics:
|
|
receivers: [otlp/hyperdx]
|
|
processors: [memory_limiter, batch]
|
|
exporters: [clickhouse]
|
|
logs/in:
|
|
receivers: [otlp/hyperdx]
|
|
exporters: [routing/logs]
|
|
logs/out-default:
|
|
receivers: [routing/logs]
|
|
processors: [memory_limiter, transform, batch]
|
|
exporters: [clickhouse]
|
|
logs/out-rrweb:
|
|
receivers: [routing/logs]
|
|
processors: [memory_limiter, batch]
|
|
exporters: [clickhouse/rrweb]
|