feat: Enforce ClickStack schemas by default (#1682)

- Introduce a new flag `HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA` (default to false) to otel collector - Custom ClickStack schemas should be enforced by default - ClickHouse tables migration logs should be stored in `clickstack_db_version_xxx` tables - The collector will run the migration at startup and retry if it fails to connect to the database (using exponential backoff). - Fully backward compatible Ref: HDX-3301
2026-04-21 13:37:15 +00:00 · 2026-02-02 17:39:20 +01:00 · 2026-02-02 17:39:20 +01:00 · 6f4c8efba0
commit 6f4c8efba0
parent 5da3f3a21e
16 changed files with 358 additions and 63 deletions
--- a/.changeset/modern-ducks-grab.md
+++ b/.changeset/modern-ducks-grab.md
@ -0,0 +1,5 @@
+---
+"@hyperdx/otel-collector": minor
+---
+
+feat: Enforce ClickStack schemas by default
--- a/docker-compose.ci.yml
+++ b/docker-compose.ci.yml
@ -29,7 +29,6 @@ services:
    volumes:
      - ./docker/clickhouse/local/config.xml:/etc/clickhouse-server/config.xml
      - ./docker/clickhouse/local/users.xml:/etc/clickhouse-server/users.xml
-      - ./docker/clickhouse/local/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh
    restart: on-failure
    ports:
      - 8123:8123 # http api
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@ -60,6 +60,7 @@ services:
      CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s'
      CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT: 'ch-server:9363'
      HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: 'otel_json'
+      HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA: 'true'
      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
      HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
      OPAMP_SERVER_URL: 'http://host.docker.internal:${HYPERDX_OPAMP_PORT}'
@ -95,7 +96,6 @@ services:
    volumes:
      - ./docker/clickhouse/local/config.xml:/etc/clickhouse-server/config.xml
      - ./docker/clickhouse/local/users.xml:/etc/clickhouse-server/users.xml
-      - ./docker/clickhouse/local/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh
      - .volumes/ch_data_dev:/var/lib/clickhouse
      - .volumes/ch_logs_dev:/var/log/clickhouse-server
    restart: on-failure
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -32,6 +32,8 @@ services:
      HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: ${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE}
      HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
      OPAMP_SERVER_URL: 'http://app:${HYPERDX_OPAMP_PORT}'
+      # TODO: use new schema
+      HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA: 'true'
    ports:
      - '13133:13133' # health_check extension
      - '24225:24225' # fluentd receiver
--- a/docker/clickhouse/local/init-db.sh
+++ b/docker/clickhouse/local/init-db.sh
@ -1,25 +0,0 @@
-#!/bin/bash
-set -e
-
-# We don't have a JSON schema yet, so let's let the collector create the tables
-if [ "$BETA_CH_OTEL_JSON_SCHEMA_ENABLED" = "true" ]; then
-  exit 0
-fi
-
-DATABASE=${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE:-default}
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-SCHEMAS_DIR="${SCRIPT_DIR}/schemas"
-
-# Create database
-clickhouse client -n <<EOFSQL
-CREATE DATABASE IF NOT EXISTS ${DATABASE};
-EOFSQL
-
-# Execute schema files with variable substitution
-export DATABASE
-for schema_file in "${SCHEMAS_DIR}"/*.sql; do
-  if [ -f "$schema_file" ]; then
-    echo "Applying schema: $(basename "$schema_file")"
-    envsubst < "$schema_file" | clickhouse client -n
-  fi
-done
--- a/docker/clickhouse/local/schemas/hyperdx_sessions.sql
+++ b/docker/clickhouse/local/schemas/hyperdx_sessions.sql
@ -1,36 +0,0 @@
-CREATE TABLE IF NOT EXISTS ${DATABASE}.hyperdx_sessions
-(
-    `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
-    `TimestampTime` DateTime DEFAULT toDateTime(Timestamp),
-    `TraceId` String CODEC(ZSTD(1)),
-    `SpanId` String CODEC(ZSTD(1)),
-    `TraceFlags` UInt8,
-    `SeverityText` LowCardinality(String) CODEC(ZSTD(1)),
-    `SeverityNumber` UInt8,
-    `ServiceName` LowCardinality(String) CODEC(ZSTD(1)),
-    `Body` String CODEC(ZSTD(1)),
-    `ResourceSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)),
-    `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
-    `ScopeSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)),
-    `ScopeName` String CODEC(ZSTD(1)),
-    `ScopeVersion` LowCardinality(String) CODEC(ZSTD(1)),
-    `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
-    `LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
-    `__hdx_materialized_rum.sessionId` String MATERIALIZED ResourceAttributes['rum.sessionId'] CODEC(ZSTD(1)),
-    `__hdx_materialized_type` LowCardinality(String) MATERIALIZED toString(simpleJSONExtractInt(Body, 'type')) CODEC(ZSTD(1)),
-    INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
-    INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE bloom_filter(0.001) GRANULARITY 1,
-    INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-    INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-    INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-    INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-    INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-    INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-    INDEX idx_body Body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8
-)
-ENGINE = MergeTree
-PARTITION BY toDate(TimestampTime)
-PRIMARY KEY (ServiceName, TimestampTime)
-ORDER BY (ServiceName, TimestampTime, Timestamp)
-TTL TimestampTime + toIntervalDay(30)
-SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
--- a/docker/otel-collector/Dockerfile
+++ b/docker/otel-collector/Dockerfile
@ -2,6 +2,7 @@
 FROM otel/opentelemetry-collector-contrib:0.129.1 AS col
 FROM otel/opentelemetry-collector-opampsupervisor:0.128.0 AS supervisor
 FROM hairyhenderson/gomplate:v4.3.3-alpine AS gomplate
+FROM kukymbr/goose-docker@sha256:0cd025636df126e7f66472861ca4db3683bc649be46cd1f6ef1a316209058e23 AS goose

 # From: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/aa5c3aa4c7ec174361fcaf908de8eaca72263078/cmd/opampsupervisor/Dockerfile#L18
 FROM alpine:latest@sha256:a8560b36e8b8210634f77d9f7f9efd7ffa463e380b75e2e74aff4511df3ef88c AS base
@ -19,6 +20,9 @@ RUN apk add --no-cache ca-certificates && \
 # Copy gomplate binary from the gomplate image
 COPY --from=gomplate /bin/gomplate /usr/local/bin/gomplate

+# Copy goose binary from the goose image
+COPY --from=goose /bin/goose /usr/local/bin/goose
+
 USER ${USER_UID}:${USER_GID}

 COPY --from=supervisor --chmod=755 /usr/local/bin/opampsupervisor /opampsupervisor
@ -40,6 +44,7 @@ LABEL org.opencontainers.image.vendor="HyperDX" \
 COPY --chown=10001:10001 ./config.yaml /etc/otelcol-contrib/config.yaml
 COPY --chown=10001:10001 ./config.standalone.yaml /etc/otelcol-contrib/standalone-config.yaml
 COPY --chown=10001:10001 ./supervisor_docker.yaml.tmpl /etc/otel/supervisor.yaml.tmpl
+COPY --chown=10001:10001 ./schema /etc/otel/schema

 EXPOSE 4317 4318 13133

@ -57,6 +62,7 @@ LABEL org.opencontainers.image.vendor="HyperDX" \
 COPY --chown=10001:10001 ./config.yaml /etc/otelcol-contrib/config.yaml
 COPY --chown=10001:10001 ./config.standalone.yaml /etc/otelcol-contrib/standalone-config.yaml
 COPY --chown=10001:10001 ./supervisor_docker.yaml.tmpl /etc/otel/supervisor.yaml.tmpl
+COPY --chown=10001:10001 ./schema /etc/otel/schema

 EXPOSE 4317 4318 13133

--- a/docker/otel-collector/config.standalone.yaml
+++ b/docker/otel-collector/config.standalone.yaml
@ -32,6 +32,7 @@ exporters:
    ttl: 720h
    logs_table_name: hyperdx_sessions
    timeout: 5s
+    create_schema: ${env:HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA:-false}
    retry_on_failure:
      enabled: true
      initial_interval: 5s
@ -44,6 +45,7 @@ exporters:
    username: ${env:CLICKHOUSE_USER}
    ttl: 720h
    timeout: 5s
+    create_schema: ${env:HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA:-false}
    retry_on_failure:
      enabled: true
      initial_interval: 5s
--- a/docker/otel-collector/entrypoint.sh
+++ b/docker/otel-collector/entrypoint.sh
@ -1,6 +1,102 @@
 #!/bin/sh
 set -e

+# Run ClickHouse schema migrations if not using legacy schema creation
+if [ "$HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA" != "true" ]; then
+  echo "========================================"
+  echo "Running ClickHouse schema migrations..."
+  echo "========================================"
+
+  # Set connection defaults
+  DB_NAME="${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE:-default}"
+  DB_USER="${CLICKHOUSE_USER:-default}"
+  DB_PASSWORD="${CLICKHOUSE_PASSWORD:-}"
+  echo "Target database: $DB_NAME"
+
+  # Build goose connection string from environment variables
+  # CLICKHOUSE_ENDPOINT format: tcp://host:port, http://host:port, or https://host:port
+  # Note: database is not specified here since SQL files use ${DATABASE} prefix explicitly
+  case "$CLICKHOUSE_ENDPOINT" in
+    *\?*) GOOSE_DBSTRING="${CLICKHOUSE_ENDPOINT}&username=${DB_USER}&password=${DB_PASSWORD}" ;;
+    *)    GOOSE_DBSTRING="${CLICKHOUSE_ENDPOINT}?username=${DB_USER}&password=${DB_PASSWORD}" ;;
+  esac
+
+  # Create temporary directory for processed SQL files
+  TEMP_SCHEMA_DIR="/tmp/schema"
+  mkdir -p "$TEMP_SCHEMA_DIR"
+
+  # Copy and process SQL files, replacing ${DATABASE} macro with actual database name
+  echo "Preparing SQL files with database: $DB_NAME"
+  cp -r /etc/otel/schema/* "$TEMP_SCHEMA_DIR/"
+  find "$TEMP_SCHEMA_DIR" -name "*.sql" -exec sed -i "s/\${DATABASE}/${DB_NAME}/g" {} \;
+
+  # Track migration status
+  MIGRATION_ERRORS=0
+
+  # Run migrations for each telemetry type
+  for schema_dir in "$TEMP_SCHEMA_DIR"/*/; do
+    if [ -d "$schema_dir" ]; then
+      telemetry_type=$(basename "$schema_dir")
+      echo "----------------------------------------"
+      echo "Migrating $telemetry_type schemas..."
+      echo "Directory: $schema_dir"
+
+      # List SQL files to be executed
+      for sql_file in "$schema_dir"/*.sql; do
+        if [ -f "$sql_file" ]; then
+          echo "  - $(basename "$sql_file")"
+        fi
+      done
+
+      # Run goose migration with exponential backoff retry for connection issues
+      MAX_RETRIES=5
+      RETRY_COUNT=0
+      RETRY_DELAY=1
+      MIGRATION_SUCCESS=false
+
+      # For _init schema, use 'default' database for version table since target DB doesn't exist yet
+      if [ "$telemetry_type" = "_init" ]; then
+        GOOSE_TABLE="default.clickstack_db_version_${telemetry_type}"
+      else
+        GOOSE_TABLE="${DB_NAME}.clickstack_db_version_${telemetry_type}"
+      fi
+
+      while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
+        if goose -table "$GOOSE_TABLE" -dir "$schema_dir" clickhouse "$GOOSE_DBSTRING" up; then
+          echo "SUCCESS: $telemetry_type migrations completed"
+          MIGRATION_SUCCESS=true
+          break
+        else
+          RETRY_COUNT=$((RETRY_COUNT + 1))
+          if [ $RETRY_COUNT -lt $MAX_RETRIES ]; then
+            echo "RETRY: $telemetry_type migration failed, retrying in ${RETRY_DELAY}s... (attempt $RETRY_COUNT/$MAX_RETRIES)"
+            sleep $RETRY_DELAY
+            RETRY_DELAY=$((RETRY_DELAY * 2))
+          fi
+        fi
+      done
+
+      if [ "$MIGRATION_SUCCESS" = false ]; then
+        echo "ERROR: $telemetry_type migrations failed after $MAX_RETRIES attempts"
+        MIGRATION_ERRORS=$((MIGRATION_ERRORS + 1))
+      fi
+    fi
+  done
+
+  # Cleanup temporary directory
+  rm -rf "$TEMP_SCHEMA_DIR"
+
+  echo "========================================"
+  if [ $MIGRATION_ERRORS -gt 0 ]; then
+    echo "Schema migrations failed with $MIGRATION_ERRORS error(s)"
+    echo "========================================"
+    exit 1
+  else
+    echo "Schema migrations completed successfully"
+    echo "========================================"
+  fi
+fi
+
 # Check if OPAMP_SERVER_URL is defined to determine mode
 if [ -z "$OPAMP_SERVER_URL" ]; then
  # Standalone mode - run collector directly without supervisor
--- a/docker/otel-collector/schema/README.md
+++ b/docker/otel-collector/schema/README.md
@ -0,0 +1,3 @@
+# Tables and schemas used by ClickStack
+
+All schemas in this directory are referenced from [Tables and schemas used by ClickStack](https://clickhouse.com/docs/use-cases/observability/clickstack/ingesting-data/schemas).
--- a/docker/otel-collector/schema/_init/00001_create_database.sql
+++ b/docker/otel-collector/schema/_init/00001_create_database.sql
@ -0,0 +1,3 @@
+-- +goose Up
+-- +goose NO TRANSACTION
+CREATE DATABASE IF NOT EXISTS ${DATABASE};
--- a/docker/otel-collector/schema/logs/00001_otel_logs.sql
+++ b/docker/otel-collector/schema/logs/00001_otel_logs.sql
@ -1,3 +1,4 @@
+-- +goose Up
 CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs
 (
  `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
@ -39,3 +40,4 @@ PRIMARY KEY (ServiceName, TimestampTime)
 ORDER BY (ServiceName, TimestampTime, Timestamp)
 TTL TimestampTime + toIntervalDay(30)
 SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
+
--- a/docker/otel-collector/schema/metrics/00001_otel_metrics.sql
+++ b/docker/otel-collector/schema/metrics/00001_otel_metrics.sql
@ -0,0 +1,195 @@
+-- +goose Up
+CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_gauge
+(
+    `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    `ResourceSchemaUrl` String CODEC(ZSTD(1)),
+    `ScopeName` String CODEC(ZSTD(1)),
+    `ScopeVersion` String CODEC(ZSTD(1)),
+    `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    `ScopeDroppedAttrCount` UInt32 CODEC(ZSTD(1)),
+    `ScopeSchemaUrl` String CODEC(ZSTD(1)),
+    `ServiceName` LowCardinality(String) CODEC(ZSTD(1)),
+    `MetricName` String CODEC(ZSTD(1)),
+    `MetricDescription` String CODEC(ZSTD(1)),
+    `MetricUnit` String CODEC(ZSTD(1)),
+    `Attributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    `StartTimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
+    `TimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
+    `Value` Float64 CODEC(ZSTD(1)),
+    `Flags` UInt32 CODEC(ZSTD(1)),
+    `Exemplars.FilteredAttributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)),
+    `Exemplars.TimeUnix` Array(DateTime64(9)) CODEC(ZSTD(1)),
+    `Exemplars.Value` Array(Float64) CODEC(ZSTD(1)),
+    `Exemplars.SpanId` Array(String) CODEC(ZSTD(1)),
+    `Exemplars.TraceId` Array(String) CODEC(ZSTD(1)),
+    INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1
+)
+ENGINE = MergeTree
+PARTITION BY toDate(TimeUnix)
+ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix));
+
+-- +goose StatementBegin
+CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_sum
+(
+    `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    `ResourceSchemaUrl` String CODEC(ZSTD(1)),
+    `ScopeName` String CODEC(ZSTD(1)),
+    `ScopeVersion` String CODEC(ZSTD(1)),
+    `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    `ScopeDroppedAttrCount` UInt32 CODEC(ZSTD(1)),
+    `ScopeSchemaUrl` String CODEC(ZSTD(1)),
+    `ServiceName` LowCardinality(String) CODEC(ZSTD(1)),
+    `MetricName` String CODEC(ZSTD(1)),
+    `MetricDescription` String CODEC(ZSTD(1)),
+    `MetricUnit` String CODEC(ZSTD(1)),
+    `Attributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    `StartTimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
+    `TimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
+    `Value` Float64 CODEC(ZSTD(1)),
+    `Flags` UInt32 CODEC(ZSTD(1)),
+    `Exemplars.FilteredAttributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)),
+    `Exemplars.TimeUnix` Array(DateTime64(9)) CODEC(ZSTD(1)),
+    `Exemplars.Value` Array(Float64) CODEC(ZSTD(1)),
+    `Exemplars.SpanId` Array(String) CODEC(ZSTD(1)),
+    `Exemplars.TraceId` Array(String) CODEC(ZSTD(1)),
+    `AggregationTemporality` Int32 CODEC(ZSTD(1)),
+    `IsMonotonic` Bool CODEC(ZSTD(1)),
+    INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1
+)
+ENGINE = MergeTree
+PARTITION BY toDate(TimeUnix)
+ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix));
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_histogram
+(
+    `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    `ResourceSchemaUrl` String CODEC(ZSTD(1)),
+    `ScopeName` String CODEC(ZSTD(1)),
+    `ScopeVersion` String CODEC(ZSTD(1)),
+    `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    `ScopeDroppedAttrCount` UInt32 CODEC(ZSTD(1)),
+    `ScopeSchemaUrl` String CODEC(ZSTD(1)),
+    `ServiceName` LowCardinality(String) CODEC(ZSTD(1)),
+    `MetricName` String CODEC(ZSTD(1)),
+    `MetricDescription` String CODEC(ZSTD(1)),
+    `MetricUnit` String CODEC(ZSTD(1)),
+    `Attributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    `StartTimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
+    `TimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
+    `Count` UInt64 CODEC(Delta(8), ZSTD(1)),
+    `Sum` Float64 CODEC(ZSTD(1)),
+    `BucketCounts` Array(UInt64) CODEC(ZSTD(1)),
+    `ExplicitBounds` Array(Float64) CODEC(ZSTD(1)),
+    `Exemplars.FilteredAttributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)),
+    `Exemplars.TimeUnix` Array(DateTime64(9)) CODEC(ZSTD(1)),
+    `Exemplars.Value` Array(Float64) CODEC(ZSTD(1)),
+    `Exemplars.SpanId` Array(String) CODEC(ZSTD(1)),
+    `Exemplars.TraceId` Array(String) CODEC(ZSTD(1)),
+    `Flags` UInt32 CODEC(ZSTD(1)),
+    `Min` Float64 CODEC(ZSTD(1)),
+    `Max` Float64 CODEC(ZSTD(1)),
+    `AggregationTemporality` Int32 CODEC(ZSTD(1)),
+    INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1
+)
+ENGINE = MergeTree
+PARTITION BY toDate(TimeUnix)
+ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix));
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_exponential_histogram
+(
+    `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    `ResourceSchemaUrl` String CODEC(ZSTD(1)),
+    `ScopeName` String CODEC(ZSTD(1)),
+    `ScopeVersion` String CODEC(ZSTD(1)),
+    `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    `ScopeDroppedAttrCount` UInt32 CODEC(ZSTD(1)),
+    `ScopeSchemaUrl` String CODEC(ZSTD(1)),
+    `ServiceName` LowCardinality(String) CODEC(ZSTD(1)),
+    `MetricName` String CODEC(ZSTD(1)),
+    `MetricDescription` String CODEC(ZSTD(1)),
+    `MetricUnit` String CODEC(ZSTD(1)),
+    `Attributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    `StartTimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
+    `TimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
+    `Count` UInt64 CODEC(Delta(8), ZSTD(1)),
+    `Sum` Float64 CODEC(ZSTD(1)),
+    `Scale` Int32 CODEC(ZSTD(1)),
+    `ZeroCount` UInt64 CODEC(ZSTD(1)),
+    `PositiveOffset` Int32 CODEC(ZSTD(1)),
+    `PositiveBucketCounts` Array(UInt64) CODEC(ZSTD(1)),
+    `NegativeOffset` Int32 CODEC(ZSTD(1)),
+    `NegativeBucketCounts` Array(UInt64) CODEC(ZSTD(1)),
+    `Exemplars.FilteredAttributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)),
+    `Exemplars.TimeUnix` Array(DateTime64(9)) CODEC(ZSTD(1)),
+    `Exemplars.Value` Array(Float64) CODEC(ZSTD(1)),
+    `Exemplars.SpanId` Array(String) CODEC(ZSTD(1)),
+    `Exemplars.TraceId` Array(String) CODEC(ZSTD(1)),
+    `Flags` UInt32 CODEC(ZSTD(1)),
+    `Min` Float64 CODEC(ZSTD(1)),
+    `Max` Float64 CODEC(ZSTD(1)),
+    `AggregationTemporality` Int32 CODEC(ZSTD(1)),
+    INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1
+)
+ENGINE = MergeTree
+PARTITION BY toDate(TimeUnix)
+ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix));
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_summary
+(
+    `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    `ResourceSchemaUrl` String CODEC(ZSTD(1)),
+    `ScopeName` String CODEC(ZSTD(1)),
+    `ScopeVersion` String CODEC(ZSTD(1)),
+    `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    `ScopeDroppedAttrCount` UInt32 CODEC(ZSTD(1)),
+    `ScopeSchemaUrl` String CODEC(ZSTD(1)),
+    `ServiceName` LowCardinality(String) CODEC(ZSTD(1)),
+    `MetricName` String CODEC(ZSTD(1)),
+    `MetricDescription` String CODEC(ZSTD(1)),
+    `MetricUnit` String CODEC(ZSTD(1)),
+    `Attributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    `StartTimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
+    `TimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
+    `Count` UInt64 CODEC(Delta(8), ZSTD(1)),
+    `Sum` Float64 CODEC(ZSTD(1)),
+    `ValueAtQuantiles.Quantile` Array(Float64) CODEC(ZSTD(1)),
+    `ValueAtQuantiles.Value` Array(Float64) CODEC(ZSTD(1)),
+    `Flags` UInt32 CODEC(ZSTD(1)),
+    INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1
+)
+ENGINE = MergeTree
+PARTITION BY toDate(TimeUnix)
+ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix));
+-- +goose StatementEnd
+
--- a/docker/otel-collector/schema/sessions/00001_hyperdx_sessions.sql
+++ b/docker/otel-collector/schema/sessions/00001_hyperdx_sessions.sql
@ -0,0 +1,35 @@
+-- +goose Up
+CREATE TABLE IF NOT EXISTS ${DATABASE}.hyperdx_sessions
+(
+  `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
+  `TimestampTime` DateTime DEFAULT toDateTime(Timestamp),
+  `TraceId` String CODEC(ZSTD(1)),
+  `SpanId` String CODEC(ZSTD(1)),
+  `TraceFlags` UInt8,
+  `SeverityText` LowCardinality(String) CODEC(ZSTD(1)),
+  `SeverityNumber` UInt8,
+  `ServiceName` LowCardinality(String) CODEC(ZSTD(1)),
+  `Body` String CODEC(ZSTD(1)),
+  `ResourceSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)),
+  `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+  `ScopeSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)),
+  `ScopeName` String CODEC(ZSTD(1)),
+  `ScopeVersion` LowCardinality(String) CODEC(ZSTD(1)),
+  `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+  `LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+  INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
+  INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+  INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+  INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+  INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+  INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+  INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+  INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8
+)
+ENGINE = MergeTree
+PARTITION BY toDate(TimestampTime)
+PRIMARY KEY (ServiceName, TimestampTime)
+ORDER BY (ServiceName, TimestampTime, Timestamp)
+TTL TimestampTime + toIntervalDay(30)
+SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
+
--- a/docker/otel-collector/schema/traces/00001_otel_traces.sql
+++ b/docker/otel-collector/schema/traces/00001_otel_traces.sql
@ -1,3 +1,4 @@
+-- +goose Up
 CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces
 (
    `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
@ -37,3 +38,4 @@ PARTITION BY toDate(Timestamp)
 ORDER BY (ServiceName, SpanName, toDateTime(Timestamp))
 TTL toDate(Timestamp) + toIntervalDay(30)
 SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
+
--- a/packages/api/src/opamp/controllers/opampController.ts
+++ b/packages/api/src/opamp/controllers/opampController.ts
@ -81,6 +81,7 @@ type CollectorConfig = {
      ttl: string;
      logs_table_name: string;
      timeout: string;
+      create_schema: string;
      retry_on_failure: {
        enabled: boolean;
        initial_interval: string;
@ -95,6 +96,7 @@ type CollectorConfig = {
      password: string;
      ttl: string;
      timeout: string;
+      create_schema: string;
      retry_on_failure: {
        enabled: boolean;
        initial_interval: string;
@ -199,6 +201,8 @@ export const buildOtelCollectorConfig = (teams: ITeam[]): CollectorConfig => {
        ttl: '720h',
        logs_table_name: 'hyperdx_sessions',
        timeout: '5s',
+        create_schema:
+          '${env:HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA:-false}',
        retry_on_failure: {
          enabled: true,
          initial_interval: '5s',
@ -213,6 +217,8 @@ export const buildOtelCollectorConfig = (teams: ITeam[]): CollectorConfig => {
        password: '${env:CLICKHOUSE_PASSWORD}',
        ttl: '720h',
        timeout: '5s',
+        create_schema:
+          '${env:HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA:-false}',
        retry_on_failure: {
          enabled: true,
          initial_interval: '5s',