diff --git a/.changeset/otel-collector-add-core-extensions.md b/.changeset/otel-collector-add-core-extensions.md new file mode 100644 index 00000000..5bb6904f --- /dev/null +++ b/.changeset/otel-collector-add-core-extensions.md @@ -0,0 +1,13 @@ +--- +'@hyperdx/otel-collector': minor +--- + +feat: Add missing core extensions, commonly-used contrib processors/receivers, and filestorage extension + +Add the two missing core extensions (memorylimiterextension, zpagesextension), +12 commonly-used contrib processors (attributes, filter, resource, k8sattributes, +tailsampling, probabilisticsampler, span, groupbyattrs, redaction, logdedup, +metricstransform, cumulativetodelta), 4 commonly-used contrib receivers +(filelog, dockerstats, k8scluster, kubeletstats), and the filestorage extension +(used for persistent sending queue in the clickhouse exporter) to +builder-config.yaml. diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7d1a5ef6..c7c84862 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -47,7 +47,7 @@ jobs: - name: Run unit tests run: make ci-unit integration: - timeout-minutes: 8 + timeout-minutes: 16 runs-on: ubuntu-24.04 steps: - name: Checkout @@ -93,7 +93,7 @@ jobs: working-directory: ./packages/otel-collector run: go test ./... otel-smoke-test: - timeout-minutes: 8 + timeout-minutes: 16 runs-on: ubuntu-24.04 steps: - name: Checkout diff --git a/docker/hyperdx/Dockerfile b/docker/hyperdx/Dockerfile index 7e8a53d8..d7ffaea7 100644 --- a/docker/hyperdx/Dockerfile +++ b/docker/hyperdx/Dockerfile @@ -29,7 +29,9 @@ ARG OTEL_COLLECTOR_CORE_VERSION COPY --from=ocb-bin /usr/local/bin/ocb /usr/local/bin/ocb WORKDIR /build COPY packages/otel-collector/builder-config.yaml . -RUN sed -i "s/__OTEL_COLLECTOR_VERSION__/${OTEL_COLLECTOR_VERSION}/g; s/__OTEL_COLLECTOR_CORE_VERSION__/${OTEL_COLLECTOR_CORE_VERSION}/g" builder-config.yaml && \ +RUN --mount=type=cache,target=/go/pkg/mod,id=ocb-gomod \ + --mount=type=cache,target=/root/.cache/go-build,id=ocb-gobuild \ + sed -i "s/__OTEL_COLLECTOR_VERSION__/${OTEL_COLLECTOR_VERSION}/g; s/__OTEL_COLLECTOR_CORE_VERSION__/${OTEL_COLLECTOR_CORE_VERSION}/g" builder-config.yaml && \ CGO_ENABLED=0 ocb --config=builder-config.yaml # Build the Go migration tool with full TLS support for ClickHouse diff --git a/docker/otel-collector/Dockerfile b/docker/otel-collector/Dockerfile index 15bfc95d..1d69a089 100644 --- a/docker/otel-collector/Dockerfile +++ b/docker/otel-collector/Dockerfile @@ -19,7 +19,9 @@ ARG OTEL_COLLECTOR_CORE_VERSION COPY --from=ocb-bin /usr/local/bin/ocb /usr/local/bin/ocb WORKDIR /build COPY packages/otel-collector/builder-config.yaml . -RUN sed -i "s/__OTEL_COLLECTOR_VERSION__/${OTEL_COLLECTOR_VERSION}/g; s/__OTEL_COLLECTOR_CORE_VERSION__/${OTEL_COLLECTOR_CORE_VERSION}/g" builder-config.yaml && \ +RUN --mount=type=cache,target=/go/pkg/mod,id=ocb-gomod \ + --mount=type=cache,target=/root/.cache/go-build,id=ocb-gobuild \ + sed -i "s/__OTEL_COLLECTOR_VERSION__/${OTEL_COLLECTOR_VERSION}/g; s/__OTEL_COLLECTOR_CORE_VERSION__/${OTEL_COLLECTOR_CORE_VERSION}/g" builder-config.yaml && \ CGO_ENABLED=0 ocb --config=builder-config.yaml # Build the Go migration tool with full TLS support for ClickHouse diff --git a/packages/otel-collector/README.md b/packages/otel-collector/README.md index e788d7fd..1b6e7aa3 100644 --- a/packages/otel-collector/README.md +++ b/packages/otel-collector/README.md @@ -3,8 +3,8 @@ Custom-built OpenTelemetry Collector for HyperDX, compiled via [OCB (OpenTelemetry Collector Builder)](https://github.com/open-telemetry/opentelemetry-collector/tree/main/cmd/builder). This replaces the pre-built `otel/opentelemetry-collector-contrib` image with a -binary that includes only the components HyperDX needs, plus any custom -receivers/processors added in this package. +binary that includes the components HyperDX needs plus commonly-used core and +contrib components, and any custom receivers/processors added in this package. ## Architecture @@ -50,26 +50,44 @@ builds. ## Included components -All components referenced in config files and the OpAMP controller: +Components used by HyperDX internally are marked with their config references. +Components marked "user configs" are included so users can reference them in +custom OTel configurations without rebuilding the collector. ### Receivers -| Component | Module | Used in | -| --------------- | ------- | ------------------------------------------------- | -| `nop` | core | OpAMP controller | -| `otlp` | core | standalone configs, OpAMP controller, smoke tests | -| `fluentforward` | contrib | standalone configs, OpAMP controller, smoke tests | -| `hostmetrics` | contrib | custom.config.yaml | -| `prometheus` | contrib | OpAMP controller, smoke tests | +| Component | Module | Used in | +| ---------------- | ------- | ------------------------------------------------- | +| `nop` | core | OpAMP controller | +| `otlp` | core | standalone configs, OpAMP controller, smoke tests | +| `dockerstats` | contrib | user configs | +| `filelog` | contrib | user configs | +| `fluentforward` | contrib | standalone configs, OpAMP controller, smoke tests | +| `hostmetrics` | contrib | custom.config.yaml | +| `k8scluster` | contrib | user configs | +| `kubeletstats` | contrib | user configs | +| `prometheus` | contrib | OpAMP controller, smoke tests | ### Processors -| Component | Module | Used in | -| ------------------- | ------- | ------------------------------------------------- | -| `batch` | core | config.yaml, standalone configs, OpAMP controller | -| `memory_limiter` | core | config.yaml, standalone configs, OpAMP controller | -| `resourcedetection` | contrib | config.yaml | -| `transform` | contrib | config.yaml, standalone configs, OpAMP controller | +| Component | Module | Used in | +| ---------------------- | ------- | ------------------------------------------------- | +| `batch` | core | config.yaml, standalone configs, OpAMP controller | +| `memory_limiter` | core | config.yaml, standalone configs, OpAMP controller | +| `attributes` | contrib | user configs | +| `cumulativetodelta` | contrib | user configs | +| `filter` | contrib | user configs | +| `groupbyattrs` | contrib | user configs | +| `k8sattributes` | contrib | user configs | +| `logdedup` | contrib | user configs | +| `metricstransform` | contrib | user configs | +| `probabilisticsampler` | contrib | user configs | +| `redaction` | contrib | user configs | +| `resourcedetection` | contrib | config.yaml | +| `resource` | contrib | user configs | +| `span` | contrib | user configs | +| `tailsampling` | contrib | user configs | +| `transform` | contrib | config.yaml, standalone configs, OpAMP controller | ### Exporters @@ -92,7 +110,10 @@ All components referenced in config files and the OpAMP controller: | Component | Module | Used in | | ----------------- | ------- | ---------------------------------------- | +| `memorylimiter` | core | user configs | +| `zpages` | core | user configs | | `bearertokenauth` | contrib | standalone-auth config, OpAMP controller | +| `file_storage` | contrib | OpAMP controller (sending queue storage) | | `health_check` | contrib | config.yaml, standalone-auth config | | `opamp` | contrib | used by OpAMP supervisor | | `pprof` | contrib | included for debugging/profiling | diff --git a/packages/otel-collector/builder-config.yaml b/packages/otel-collector/builder-config.yaml index 1b2acdbf..ff3fbe5f 100644 --- a/packages/otel-collector/builder-config.yaml +++ b/packages/otel-collector/builder-config.yaml @@ -27,12 +27,24 @@ receivers: go.opentelemetry.io/collector/receiver/otlpreceiver v__OTEL_COLLECTOR_VERSION__ # Contrib + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/dockerstatsreceiver + v__OTEL_COLLECTOR_VERSION__ + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/filelogreceiver + v__OTEL_COLLECTOR_VERSION__ - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/fluentforwardreceiver v__OTEL_COLLECTOR_VERSION__ - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver v__OTEL_COLLECTOR_VERSION__ + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/k8sclusterreceiver + v__OTEL_COLLECTOR_VERSION__ + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kubeletstatsreceiver + v__OTEL_COLLECTOR_VERSION__ - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v__OTEL_COLLECTOR_VERSION__ @@ -46,9 +58,45 @@ processors: go.opentelemetry.io/collector/processor/memorylimiterprocessor v__OTEL_COLLECTOR_VERSION__ # Contrib + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor + v__OTEL_COLLECTOR_VERSION__ + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/processor/cumulativetodeltaprocessor + v__OTEL_COLLECTOR_VERSION__ + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor + v__OTEL_COLLECTOR_VERSION__ + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/processor/groupbyattrsprocessor + v__OTEL_COLLECTOR_VERSION__ + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor + v__OTEL_COLLECTOR_VERSION__ + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/processor/logdedupprocessor + v__OTEL_COLLECTOR_VERSION__ + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricstransformprocessor + v__OTEL_COLLECTOR_VERSION__ + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor + v__OTEL_COLLECTOR_VERSION__ + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/processor/redactionprocessor + v__OTEL_COLLECTOR_VERSION__ - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v__OTEL_COLLECTOR_VERSION__ + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor + v__OTEL_COLLECTOR_VERSION__ + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/processor/spanprocessor + v__OTEL_COLLECTOR_VERSION__ + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor + v__OTEL_COLLECTOR_VERSION__ - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor v__OTEL_COLLECTOR_VERSION__ @@ -83,10 +131,20 @@ connectors: v__OTEL_COLLECTOR_VERSION__ extensions: + # Core + - gomod: + go.opentelemetry.io/collector/extension/memorylimiterextension + v__OTEL_COLLECTOR_VERSION__ + - gomod: + go.opentelemetry.io/collector/extension/zpagesextension + v__OTEL_COLLECTOR_VERSION__ # Contrib - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/bearertokenauthextension v__OTEL_COLLECTOR_VERSION__ + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage/filestorage + v__OTEL_COLLECTOR_VERSION__ - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension v__OTEL_COLLECTOR_VERSION__