mirror of
https://github.com/hyperdxio/hyperdx
synced 2026-04-21 13:37:15 +00:00
[HDX-3929] Migrate OTel Collector build to use OCB (OpenTelemetry Collector Builder) (#2109)
This commit is contained in:
parent
28b81dd8a9
commit
28f374ef12
10 changed files with 449 additions and 10 deletions
8
.changeset/bump-otel-collector-to-0149.md
Normal file
8
.changeset/bump-otel-collector-to-0149.md
Normal file
|
|
@ -0,0 +1,8 @@
|
||||||
|
---
|
||||||
|
'@hyperdx/otel-collector': minor
|
||||||
|
---
|
||||||
|
|
||||||
|
feat: Bump OTel Collector from 0.147.0 to 0.149.0
|
||||||
|
|
||||||
|
Upgrade the OpenTelemetry Collector and all its components from v0.147.0 to
|
||||||
|
v0.149.0 (core providers from v1.53.0 to v1.55.0).
|
||||||
12
.changeset/migrate-otel-collector-to-ocb.md
Normal file
12
.changeset/migrate-otel-collector-to-ocb.md
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
---
|
||||||
|
'@hyperdx/otel-collector': minor
|
||||||
|
---
|
||||||
|
|
||||||
|
feat: Migrate OTel Collector build to use OCB (OpenTelemetry Collector Builder)
|
||||||
|
|
||||||
|
Replace the pre-built otel/opentelemetry-collector-contrib image with a custom
|
||||||
|
binary built via OCB. This enables adding custom receiver/processor components
|
||||||
|
in the future while including only the components HyperDX needs. The collector
|
||||||
|
version is now centralized in `.env` via `OTEL_COLLECTOR_VERSION` and
|
||||||
|
`OTEL_COLLECTOR_CORE_VERSION`, with `builder-config.yaml` using templatized
|
||||||
|
placeholders substituted at Docker build time.
|
||||||
6
.env
6
.env
|
|
@ -38,5 +38,11 @@ HDX_DEV_OTEL_HTTP_PORT=4318
|
||||||
HDX_DEV_OTEL_METRICS_PORT=8888
|
HDX_DEV_OTEL_METRICS_PORT=8888
|
||||||
HDX_DEV_OTEL_JSON_HTTP_PORT=14318
|
HDX_DEV_OTEL_JSON_HTTP_PORT=14318
|
||||||
|
|
||||||
|
# Otel Collector version (used as Docker build arg for image tags and component versions)
|
||||||
|
# When bumping, look up the core version from the upstream manifest:
|
||||||
|
# https://github.com/open-telemetry/opentelemetry-collector-releases/blob/main/distributions/otelcol-contrib/manifest.yaml
|
||||||
|
OTEL_COLLECTOR_VERSION=0.149.0
|
||||||
|
OTEL_COLLECTOR_CORE_VERSION=1.55.0
|
||||||
|
|
||||||
# Otel/Clickhouse config
|
# Otel/Clickhouse config
|
||||||
HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=default
|
HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=default
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,9 @@ services:
|
||||||
context: .
|
context: .
|
||||||
dockerfile: docker/otel-collector/Dockerfile
|
dockerfile: docker/otel-collector/Dockerfile
|
||||||
target: dev
|
target: dev
|
||||||
|
args:
|
||||||
|
OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION}
|
||||||
|
OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION}
|
||||||
environment:
|
environment:
|
||||||
CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s'
|
CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s'
|
||||||
HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: ${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE}
|
HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: ${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE}
|
||||||
|
|
|
||||||
|
|
@ -26,6 +26,9 @@ services:
|
||||||
context: .
|
context: .
|
||||||
dockerfile: docker/otel-collector/Dockerfile
|
dockerfile: docker/otel-collector/Dockerfile
|
||||||
target: dev
|
target: dev
|
||||||
|
args:
|
||||||
|
OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION}
|
||||||
|
OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION}
|
||||||
environment:
|
environment:
|
||||||
CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s'
|
CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s'
|
||||||
CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT: 'ch-server:9363'
|
CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT: 'ch-server:9363'
|
||||||
|
|
@ -63,6 +66,9 @@ services:
|
||||||
context: .
|
context: .
|
||||||
dockerfile: docker/otel-collector/Dockerfile
|
dockerfile: docker/otel-collector/Dockerfile
|
||||||
target: dev
|
target: dev
|
||||||
|
args:
|
||||||
|
OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION}
|
||||||
|
OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION}
|
||||||
environment:
|
environment:
|
||||||
CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s'
|
CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s'
|
||||||
CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT: 'ch-server:9363'
|
CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT: 'ch-server:9363'
|
||||||
|
|
|
||||||
|
|
@ -9,15 +9,29 @@
|
||||||
|
|
||||||
ARG NODE_VERSION=22.22
|
ARG NODE_VERSION=22.22
|
||||||
ARG CLICKHOUSE_VERSION=26.1
|
ARG CLICKHOUSE_VERSION=26.1
|
||||||
ARG OTEL_COLLECTOR_VERSION=0.147.0
|
ARG OTEL_COLLECTOR_VERSION=0.149.0
|
||||||
ARG OTEL_COLLECTOR_OPAMPSUPERVISOR_VERSION=0.147.0
|
ARG OTEL_COLLECTOR_CORE_VERSION=1.55.0
|
||||||
|
|
||||||
# base #############################################################################################
|
# base #############################################################################################
|
||||||
# == Otel Collector Image ==
|
# == Otel Collector Image ==
|
||||||
FROM otel/opentelemetry-collector-contrib:${OTEL_COLLECTOR_VERSION} AS otel_collector_base
|
FROM otel/opentelemetry-collector-opampsupervisor:${OTEL_COLLECTOR_VERSION} AS otel_collector_opampsupervisor_base
|
||||||
FROM otel/opentelemetry-collector-opampsupervisor:${OTEL_COLLECTOR_OPAMPSUPERVISOR_VERSION} AS otel_collector_opampsupervisor_base
|
|
||||||
FROM kukymbr/goose-docker@sha256:0cd025636df126e7f66472861ca4db3683bc649be46cd1f6ef1a316209058e23 AS goose
|
FROM kukymbr/goose-docker@sha256:0cd025636df126e7f66472861ca4db3683bc649be46cd1f6ef1a316209058e23 AS goose
|
||||||
|
|
||||||
|
# Build the custom HyperDX collector binary using OCB (OpenTelemetry Collector Builder).
|
||||||
|
# This replaces the pre-built otel/opentelemetry-collector-contrib image so we can
|
||||||
|
# include custom receiver/processor components alongside the standard contrib ones.
|
||||||
|
# Note: The official OCB image may ship an older Go than the contrib modules require,
|
||||||
|
# so we copy the ocb binary into a golang base with a newer toolchain.
|
||||||
|
FROM otel/opentelemetry-collector-builder:${OTEL_COLLECTOR_VERSION} AS ocb-bin
|
||||||
|
FROM golang:1.26-alpine AS ocb-builder
|
||||||
|
ARG OTEL_COLLECTOR_VERSION
|
||||||
|
ARG OTEL_COLLECTOR_CORE_VERSION
|
||||||
|
COPY --from=ocb-bin /usr/local/bin/ocb /usr/local/bin/ocb
|
||||||
|
WORKDIR /build
|
||||||
|
COPY packages/otel-collector/builder-config.yaml .
|
||||||
|
RUN sed -i "s/__OTEL_COLLECTOR_VERSION__/${OTEL_COLLECTOR_VERSION}/g; s/__OTEL_COLLECTOR_CORE_VERSION__/${OTEL_COLLECTOR_CORE_VERSION}/g" builder-config.yaml && \
|
||||||
|
CGO_ENABLED=0 ocb --config=builder-config.yaml
|
||||||
|
|
||||||
# Build the Go migration tool with full TLS support for ClickHouse
|
# Build the Go migration tool with full TLS support for ClickHouse
|
||||||
FROM golang:1.26-alpine AS migrate-builder
|
FROM golang:1.26-alpine AS migrate-builder
|
||||||
WORKDIR /build
|
WORKDIR /build
|
||||||
|
|
@ -132,7 +146,7 @@ ARG USER_GID=10001
|
||||||
ENV CODE_VERSION=$CODE_VERSION
|
ENV CODE_VERSION=$CODE_VERSION
|
||||||
ENV OTEL_RESOURCE_ATTRIBUTES="service.version=$CODE_VERSION"
|
ENV OTEL_RESOURCE_ATTRIBUTES="service.version=$CODE_VERSION"
|
||||||
# Copy from otel collector bases
|
# Copy from otel collector bases
|
||||||
COPY --from=otel_collector_base --chmod=755 /otelcol-contrib /otelcontribcol
|
COPY --from=ocb-builder --chmod=755 /build/output/otelcol-hyperdx /otelcontribcol
|
||||||
COPY --from=otel_collector_opampsupervisor_base --chmod=755 /usr/local/bin/opampsupervisor /usr/local/bin/opampsupervisor
|
COPY --from=otel_collector_opampsupervisor_base --chmod=755 /usr/local/bin/opampsupervisor /usr/local/bin/opampsupervisor
|
||||||
|
|
||||||
# Copy Node.js runtime from node base
|
# Copy Node.js runtime from node base
|
||||||
|
|
|
||||||
|
|
@ -1,12 +1,29 @@
|
||||||
## base #############################################################################################
|
## base #############################################################################################
|
||||||
FROM otel/opentelemetry-collector-contrib:0.147.0 AS col
|
ARG OTEL_COLLECTOR_VERSION=0.149.0
|
||||||
FROM otel/opentelemetry-collector-opampsupervisor:0.147.0 AS supervisor
|
ARG OTEL_COLLECTOR_CORE_VERSION=1.55.0
|
||||||
|
|
||||||
|
FROM otel/opentelemetry-collector-opampsupervisor:${OTEL_COLLECTOR_VERSION} AS supervisor
|
||||||
FROM hairyhenderson/gomplate:v4.3.3-alpine AS gomplate
|
FROM hairyhenderson/gomplate:v4.3.3-alpine AS gomplate
|
||||||
FROM kukymbr/goose-docker@sha256:0cd025636df126e7f66472861ca4db3683bc649be46cd1f6ef1a316209058e23 AS goose
|
FROM kukymbr/goose-docker@sha256:0cd025636df126e7f66472861ca4db3683bc649be46cd1f6ef1a316209058e23 AS goose
|
||||||
|
|
||||||
# Build the Go migration tool with full TLS support for ClickHouse
|
# Build the custom HyperDX collector binary using OCB (OpenTelemetry Collector Builder).
|
||||||
|
# This replaces the pre-built otel/opentelemetry-collector-contrib image so we can
|
||||||
|
# include custom receiver/processor components alongside the standard contrib ones.
|
||||||
# Note: Build context must be repo root (use: docker build -f docker/otel-collector/Dockerfile .)
|
# Note: Build context must be repo root (use: docker build -f docker/otel-collector/Dockerfile .)
|
||||||
FROM golang:1.25-alpine AS migrate-builder
|
# Note: The official OCB image may ship an older Go than the contrib modules require,
|
||||||
|
# so we copy the ocb binary into a golang base with a newer toolchain.
|
||||||
|
FROM otel/opentelemetry-collector-builder:${OTEL_COLLECTOR_VERSION} AS ocb-bin
|
||||||
|
FROM golang:1.26-alpine AS ocb-builder
|
||||||
|
ARG OTEL_COLLECTOR_VERSION
|
||||||
|
ARG OTEL_COLLECTOR_CORE_VERSION
|
||||||
|
COPY --from=ocb-bin /usr/local/bin/ocb /usr/local/bin/ocb
|
||||||
|
WORKDIR /build
|
||||||
|
COPY packages/otel-collector/builder-config.yaml .
|
||||||
|
RUN sed -i "s/__OTEL_COLLECTOR_VERSION__/${OTEL_COLLECTOR_VERSION}/g; s/__OTEL_COLLECTOR_CORE_VERSION__/${OTEL_COLLECTOR_CORE_VERSION}/g" builder-config.yaml && \
|
||||||
|
CGO_ENABLED=0 ocb --config=builder-config.yaml
|
||||||
|
|
||||||
|
# Build the Go migration tool with full TLS support for ClickHouse
|
||||||
|
FROM golang:1.26-alpine AS migrate-builder
|
||||||
WORKDIR /build
|
WORKDIR /build
|
||||||
COPY packages/otel-collector/go.mod packages/otel-collector/go.sum ./
|
COPY packages/otel-collector/go.mod packages/otel-collector/go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
|
|
@ -38,7 +55,7 @@ COPY --from=migrate-builder /migrate /usr/local/bin/migrate
|
||||||
USER ${USER_UID}:${USER_GID}
|
USER ${USER_UID}:${USER_GID}
|
||||||
|
|
||||||
COPY --from=supervisor --chmod=755 /usr/local/bin/opampsupervisor /opampsupervisor
|
COPY --from=supervisor --chmod=755 /usr/local/bin/opampsupervisor /opampsupervisor
|
||||||
COPY --from=col --chmod=755 /otelcol-contrib /otelcontribcol
|
COPY --from=ocb-builder --chmod=755 /build/output/otelcol-hyperdx /otelcontribcol
|
||||||
|
|
||||||
# Copy entrypoint and log tail wrapper scripts
|
# Copy entrypoint and log tail wrapper scripts
|
||||||
COPY --chmod=755 docker/otel-collector/entrypoint.sh /entrypoint.sh
|
COPY --chmod=755 docker/otel-collector/entrypoint.sh /entrypoint.sh
|
||||||
|
|
|
||||||
255
packages/otel-collector/README.md
Normal file
255
packages/otel-collector/README.md
Normal file
|
|
@ -0,0 +1,255 @@
|
||||||
|
# HyperDX OTel Collector
|
||||||
|
|
||||||
|
Custom-built OpenTelemetry Collector for HyperDX, compiled via
|
||||||
|
[OCB (OpenTelemetry Collector Builder)](https://github.com/open-telemetry/opentelemetry-collector/tree/main/cmd/builder).
|
||||||
|
This replaces the pre-built `otel/opentelemetry-collector-contrib` image with a
|
||||||
|
binary that includes only the components HyperDX needs, plus any custom
|
||||||
|
receivers/processors added in this package.
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
### Build process
|
||||||
|
|
||||||
|
The collector binary is built during `docker build` via a multi-stage
|
||||||
|
Dockerfile:
|
||||||
|
|
||||||
|
1. The `ocb` binary is copied from the official
|
||||||
|
`otel/opentelemetry-collector-builder:<version>` image
|
||||||
|
2. It's placed into a `golang:<version>-alpine` base (the official OCB image may
|
||||||
|
ship an older Go than the contrib modules require)
|
||||||
|
3. Version placeholders in `builder-config.yaml` are substituted via `sed`
|
||||||
|
4. `ocb` compiles the custom binary from the resolved manifest
|
||||||
|
|
||||||
|
### Key files
|
||||||
|
|
||||||
|
| File | Purpose |
|
||||||
|
| --------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
|
| `builder-config.yaml` | OCB manifest — declares which components to include. Uses `__OTEL_COLLECTOR_VERSION__` and `__OTEL_COLLECTOR_CORE_VERSION__` placeholders substituted at build time. |
|
||||||
|
| `cmd/migrate/main.go` | Go-based ClickHouse migration tool (goose) with full TLS support |
|
||||||
|
| `go.mod` / `go.sum` | Go module for the migration tool |
|
||||||
|
|
||||||
|
### Dockerfiles that build this collector
|
||||||
|
|
||||||
|
| Dockerfile | Description |
|
||||||
|
| ---------------------------------- | -------------------------------------------------------------------- |
|
||||||
|
| `docker/otel-collector/Dockerfile` | Standalone collector image (dev + prod targets) |
|
||||||
|
| `docker/hyperdx/Dockerfile` | All-in-one image (includes collector, ClickHouse, MongoDB, API, App) |
|
||||||
|
|
||||||
|
### Version configuration
|
||||||
|
|
||||||
|
The collector version is controlled by two variables:
|
||||||
|
|
||||||
|
- **`OTEL_COLLECTOR_VERSION`** — The contrib/core component version (e.g.
|
||||||
|
`0.149.0`)
|
||||||
|
- **`OTEL_COLLECTOR_CORE_VERSION`** — The core confmap provider version (e.g.
|
||||||
|
`1.55.0`)
|
||||||
|
|
||||||
|
These are defined in the root `.env` file and passed as Docker build args. Both
|
||||||
|
Dockerfiles also have matching `ARG` defaults as fallbacks for standalone
|
||||||
|
builds.
|
||||||
|
|
||||||
|
## Included components
|
||||||
|
|
||||||
|
All components referenced in config files and the OpAMP controller:
|
||||||
|
|
||||||
|
### Receivers
|
||||||
|
|
||||||
|
| Component | Module | Used in |
|
||||||
|
| --------------- | ------- | ------------------------------------------------- |
|
||||||
|
| `nop` | core | OpAMP controller |
|
||||||
|
| `otlp` | core | standalone configs, OpAMP controller, smoke tests |
|
||||||
|
| `fluentforward` | contrib | standalone configs, OpAMP controller, smoke tests |
|
||||||
|
| `hostmetrics` | contrib | custom.config.yaml |
|
||||||
|
| `prometheus` | contrib | OpAMP controller, smoke tests |
|
||||||
|
|
||||||
|
### Processors
|
||||||
|
|
||||||
|
| Component | Module | Used in |
|
||||||
|
| ------------------- | ------- | ------------------------------------------------- |
|
||||||
|
| `batch` | core | config.yaml, standalone configs, OpAMP controller |
|
||||||
|
| `memory_limiter` | core | config.yaml, standalone configs, OpAMP controller |
|
||||||
|
| `resourcedetection` | contrib | config.yaml |
|
||||||
|
| `transform` | contrib | config.yaml, standalone configs, OpAMP controller |
|
||||||
|
|
||||||
|
### Exporters
|
||||||
|
|
||||||
|
| Component | Module | Used in |
|
||||||
|
| ------------ | ------- | ------------------------------------ |
|
||||||
|
| `clickhouse` | contrib | standalone configs, OpAMP controller |
|
||||||
|
| `debug` | core | OpAMP controller |
|
||||||
|
| `nop` | core | OpAMP controller |
|
||||||
|
| `otlp` | core | included for utility |
|
||||||
|
| `otlphttp` | core | custom.config.yaml |
|
||||||
|
|
||||||
|
### Connectors
|
||||||
|
|
||||||
|
| Component | Module | Used in |
|
||||||
|
| --------- | ------- | ------------------------------------ |
|
||||||
|
| `forward` | core | included for utility |
|
||||||
|
| `routing` | contrib | standalone configs, OpAMP controller |
|
||||||
|
|
||||||
|
### Extensions
|
||||||
|
|
||||||
|
| Component | Module | Used in |
|
||||||
|
| ----------------- | ------- | ---------------------------------------- |
|
||||||
|
| `bearertokenauth` | contrib | standalone-auth config, OpAMP controller |
|
||||||
|
| `health_check` | contrib | config.yaml, standalone-auth config |
|
||||||
|
| `opamp` | contrib | used by OpAMP supervisor |
|
||||||
|
| `pprof` | contrib | included for debugging/profiling |
|
||||||
|
|
||||||
|
### Confmap Providers
|
||||||
|
|
||||||
|
| Provider | Module |
|
||||||
|
| -------- | ------ |
|
||||||
|
| `env` | core |
|
||||||
|
| `file` | core |
|
||||||
|
| `http` | core |
|
||||||
|
| `https` | core |
|
||||||
|
| `yaml` | core |
|
||||||
|
|
||||||
|
## Upgrading the OTel Collector version
|
||||||
|
|
||||||
|
### Step 1: Look up the core version
|
||||||
|
|
||||||
|
Check the upstream contrib manifest to find the core provider version that
|
||||||
|
corresponds to the new contrib version:
|
||||||
|
|
||||||
|
https://github.com/open-telemetry/opentelemetry-collector-releases/blob/main/distributions/otelcol-contrib/manifest.yaml
|
||||||
|
|
||||||
|
Look at the `providers:` section — the core version follows a different scheme
|
||||||
|
(e.g. contrib `0.149.0` corresponds to core `1.55.0`).
|
||||||
|
|
||||||
|
### Step 2: Update these files
|
||||||
|
|
||||||
|
**`.env`** (root) — primary source of truth:
|
||||||
|
|
||||||
|
```
|
||||||
|
OTEL_COLLECTOR_VERSION=<new_version>
|
||||||
|
OTEL_COLLECTOR_CORE_VERSION=<new_core_version>
|
||||||
|
```
|
||||||
|
|
||||||
|
**`docker/otel-collector/Dockerfile`** — ARG defaults:
|
||||||
|
|
||||||
|
```dockerfile
|
||||||
|
ARG OTEL_COLLECTOR_VERSION=<new_version>
|
||||||
|
ARG OTEL_COLLECTOR_CORE_VERSION=<new_core_version>
|
||||||
|
```
|
||||||
|
|
||||||
|
**`docker/hyperdx/Dockerfile`** — ARG defaults:
|
||||||
|
|
||||||
|
```dockerfile
|
||||||
|
ARG OTEL_COLLECTOR_VERSION=<new_version>
|
||||||
|
ARG OTEL_COLLECTOR_CORE_VERSION=<new_core_version>
|
||||||
|
```
|
||||||
|
|
||||||
|
**`smoke-tests/otel-collector/docker-compose.yaml`** — fallback defaults:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
args:
|
||||||
|
OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION:-<new_version>}
|
||||||
|
OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION:-<new_core_version>}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Files you do NOT need to change
|
||||||
|
|
||||||
|
- `packages/otel-collector/builder-config.yaml` — uses placeholders, substituted
|
||||||
|
at build time
|
||||||
|
- `docker-compose.dev.yml` — reads from `.env` automatically
|
||||||
|
- `docker-compose.ci.yml` — reads from `.env` automatically
|
||||||
|
|
||||||
|
### Step 3: Verify
|
||||||
|
|
||||||
|
Build the image and check the version:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker build -f docker/otel-collector/Dockerfile --target dev -t hdx-otel-test .
|
||||||
|
docker run --rm --entrypoint /otelcontribcol hdx-otel-test --version
|
||||||
|
docker rmi hdx-otel-test
|
||||||
|
```
|
||||||
|
|
||||||
|
## Adding a custom component
|
||||||
|
|
||||||
|
There are two approaches for including custom components, depending on whether
|
||||||
|
the component source lives in this monorepo or is published as a standalone Go
|
||||||
|
module.
|
||||||
|
|
||||||
|
### Approach A: Local source (monorepo development)
|
||||||
|
|
||||||
|
Best for active development — iterate on the component alongside the rest of the
|
||||||
|
codebase without needing to publish a Go module tag on every change.
|
||||||
|
|
||||||
|
1. Create the Go module under this package (e.g.
|
||||||
|
`packages/otel-collector/receiver/myreceiver/`) with its own `go.mod`.
|
||||||
|
|
||||||
|
2. Add an entry to `builder-config.yaml` under the appropriate section with a
|
||||||
|
`path:` directive pointing to the local module:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
receivers:
|
||||||
|
- gomod:
|
||||||
|
github.com/hyperdxio/hyperdx/packages/otel-collector/receiver/myreceiver
|
||||||
|
v0.0.0
|
||||||
|
path: ./receiver/myreceiver
|
||||||
|
```
|
||||||
|
|
||||||
|
3. **Update the Dockerfiles** to copy the component source into the OCB build
|
||||||
|
stage. Currently, only `builder-config.yaml` is copied. In both
|
||||||
|
`docker/otel-collector/Dockerfile` and `docker/hyperdx/Dockerfile`, expand
|
||||||
|
the `COPY` line in the `ocb-builder` stage:
|
||||||
|
|
||||||
|
```dockerfile
|
||||||
|
# Before (only copies the manifest):
|
||||||
|
COPY packages/otel-collector/builder-config.yaml .
|
||||||
|
|
||||||
|
# After (also copies custom component source):
|
||||||
|
COPY packages/otel-collector/builder-config.yaml .
|
||||||
|
COPY packages/otel-collector/receiver/ ./receiver/
|
||||||
|
```
|
||||||
|
|
||||||
|
Or, to copy everything at once (simpler, but any change to files in
|
||||||
|
`packages/otel-collector/` will invalidate the Docker layer cache for the OCB
|
||||||
|
build stage):
|
||||||
|
|
||||||
|
```dockerfile
|
||||||
|
COPY packages/otel-collector/ .
|
||||||
|
```
|
||||||
|
|
||||||
|
4. Add the component to the relevant OTel config files and/or the OpAMP
|
||||||
|
controller (`packages/api/src/opamp/controllers/opampController.ts`).
|
||||||
|
|
||||||
|
### Approach B: Published Go module (remote reference)
|
||||||
|
|
||||||
|
Best for stable, versioned components — especially useful for components shared
|
||||||
|
across repos or distributed to external users. No Dockerfile `COPY` changes
|
||||||
|
needed.
|
||||||
|
|
||||||
|
1. Publish the component as a Go module with a tagged version (e.g.
|
||||||
|
`github.com/hyperdxio/my-otel-receiver v0.1.0`). For modules in a monorepo
|
||||||
|
subdirectory, the Git tag must include the module path prefix (e.g.
|
||||||
|
`packages/otel-collector/receiver/myreceiver/v0.1.0`).
|
||||||
|
|
||||||
|
2. Add an entry to `builder-config.yaml` **without** a `path:` directive — OCB
|
||||||
|
will fetch it via the Go module proxy, just like contrib components:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
receivers:
|
||||||
|
- gomod: github.com/hyperdxio/my-otel-receiver v0.1.0
|
||||||
|
```
|
||||||
|
|
||||||
|
No Dockerfile changes needed — OCB downloads it during `go mod download`.
|
||||||
|
|
||||||
|
3. Add the component to the relevant OTel config files and/or the OpAMP
|
||||||
|
controller.
|
||||||
|
|
||||||
|
### Which approach to use?
|
||||||
|
|
||||||
|
| | Local (`path:`) | Remote (`gomod:`) |
|
||||||
|
| --------------------------- | -------------------- | --------------------------------- |
|
||||||
|
| Dockerfile COPY needed | Yes | No |
|
||||||
|
| Works with unpublished code | Yes | No — must be tagged and published |
|
||||||
|
| Dev iteration speed | Fast — rebuild image | Must push, tag, wait for proxy |
|
||||||
|
| Monorepo friendly | Yes | Requires proper Go module tagging |
|
||||||
|
| Best for | Active development | Stable/shared components |
|
||||||
|
|
||||||
|
For most HyperDX development, **Approach A (local source)** is recommended. Use
|
||||||
|
Approach B when the component is mature and independently versioned.
|
||||||
115
packages/otel-collector/builder-config.yaml
Normal file
115
packages/otel-collector/builder-config.yaml
Normal file
|
|
@ -0,0 +1,115 @@
|
||||||
|
# OpenTelemetry Collector Builder (OCB) manifest for HyperDX.
|
||||||
|
# This replaces the pre-built otel/opentelemetry-collector-contrib image with a
|
||||||
|
# custom-built binary that includes only the components HyperDX needs, plus any
|
||||||
|
# custom receivers/processors we add in the future.
|
||||||
|
#
|
||||||
|
# Version placeholders are replaced at Docker build time via sed:
|
||||||
|
# __OTEL_COLLECTOR_VERSION__ -> contrib/core component version (e.g. 0.149.0)
|
||||||
|
# __OTEL_COLLECTOR_CORE_VERSION__ -> core confmap provider version (e.g. 1.55.0)
|
||||||
|
# Both values are defined in the root .env file.
|
||||||
|
#
|
||||||
|
# The upstream contrib manifest for reference:
|
||||||
|
# https://github.com/open-telemetry/opentelemetry-collector-releases/blob/main/distributions/otelcol-contrib/manifest.yaml
|
||||||
|
|
||||||
|
dist:
|
||||||
|
name: otelcol-hyperdx
|
||||||
|
description: HyperDX OpenTelemetry Collector
|
||||||
|
version: __OTEL_COLLECTOR_VERSION__
|
||||||
|
output_path: /build/output
|
||||||
|
build_tags: 'grpcnotrace'
|
||||||
|
|
||||||
|
receivers:
|
||||||
|
# Core
|
||||||
|
- gomod:
|
||||||
|
go.opentelemetry.io/collector/receiver/nopreceiver
|
||||||
|
v__OTEL_COLLECTOR_VERSION__
|
||||||
|
- gomod:
|
||||||
|
go.opentelemetry.io/collector/receiver/otlpreceiver
|
||||||
|
v__OTEL_COLLECTOR_VERSION__
|
||||||
|
# Contrib
|
||||||
|
- gomod:
|
||||||
|
github.com/open-telemetry/opentelemetry-collector-contrib/receiver/fluentforwardreceiver
|
||||||
|
v__OTEL_COLLECTOR_VERSION__
|
||||||
|
- gomod:
|
||||||
|
github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver
|
||||||
|
v__OTEL_COLLECTOR_VERSION__
|
||||||
|
- gomod:
|
||||||
|
github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver
|
||||||
|
v__OTEL_COLLECTOR_VERSION__
|
||||||
|
|
||||||
|
processors:
|
||||||
|
# Core
|
||||||
|
- gomod:
|
||||||
|
go.opentelemetry.io/collector/processor/batchprocessor
|
||||||
|
v__OTEL_COLLECTOR_VERSION__
|
||||||
|
- gomod:
|
||||||
|
go.opentelemetry.io/collector/processor/memorylimiterprocessor
|
||||||
|
v__OTEL_COLLECTOR_VERSION__
|
||||||
|
# Contrib
|
||||||
|
- gomod:
|
||||||
|
github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor
|
||||||
|
v__OTEL_COLLECTOR_VERSION__
|
||||||
|
- gomod:
|
||||||
|
github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor
|
||||||
|
v__OTEL_COLLECTOR_VERSION__
|
||||||
|
|
||||||
|
exporters:
|
||||||
|
# Core
|
||||||
|
- gomod:
|
||||||
|
go.opentelemetry.io/collector/exporter/debugexporter
|
||||||
|
v__OTEL_COLLECTOR_VERSION__
|
||||||
|
- gomod:
|
||||||
|
go.opentelemetry.io/collector/exporter/nopexporter
|
||||||
|
v__OTEL_COLLECTOR_VERSION__
|
||||||
|
- gomod:
|
||||||
|
go.opentelemetry.io/collector/exporter/otlpexporter
|
||||||
|
v__OTEL_COLLECTOR_VERSION__
|
||||||
|
- gomod:
|
||||||
|
go.opentelemetry.io/collector/exporter/otlphttpexporter
|
||||||
|
v__OTEL_COLLECTOR_VERSION__
|
||||||
|
# Contrib
|
||||||
|
- gomod:
|
||||||
|
github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter
|
||||||
|
v__OTEL_COLLECTOR_VERSION__
|
||||||
|
|
||||||
|
connectors:
|
||||||
|
# Core
|
||||||
|
- gomod:
|
||||||
|
go.opentelemetry.io/collector/connector/forwardconnector
|
||||||
|
v__OTEL_COLLECTOR_VERSION__
|
||||||
|
# Contrib
|
||||||
|
- gomod:
|
||||||
|
github.com/open-telemetry/opentelemetry-collector-contrib/connector/routingconnector
|
||||||
|
v__OTEL_COLLECTOR_VERSION__
|
||||||
|
|
||||||
|
extensions:
|
||||||
|
# Contrib
|
||||||
|
- gomod:
|
||||||
|
github.com/open-telemetry/opentelemetry-collector-contrib/extension/bearertokenauthextension
|
||||||
|
v__OTEL_COLLECTOR_VERSION__
|
||||||
|
- gomod:
|
||||||
|
github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension
|
||||||
|
v__OTEL_COLLECTOR_VERSION__
|
||||||
|
- gomod:
|
||||||
|
github.com/open-telemetry/opentelemetry-collector-contrib/extension/opampextension
|
||||||
|
v__OTEL_COLLECTOR_VERSION__
|
||||||
|
- gomod:
|
||||||
|
github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension
|
||||||
|
v__OTEL_COLLECTOR_VERSION__
|
||||||
|
|
||||||
|
providers:
|
||||||
|
- gomod:
|
||||||
|
go.opentelemetry.io/collector/confmap/provider/envprovider
|
||||||
|
v__OTEL_COLLECTOR_CORE_VERSION__
|
||||||
|
- gomod:
|
||||||
|
go.opentelemetry.io/collector/confmap/provider/fileprovider
|
||||||
|
v__OTEL_COLLECTOR_CORE_VERSION__
|
||||||
|
- gomod:
|
||||||
|
go.opentelemetry.io/collector/confmap/provider/httpprovider
|
||||||
|
v__OTEL_COLLECTOR_CORE_VERSION__
|
||||||
|
- gomod:
|
||||||
|
go.opentelemetry.io/collector/confmap/provider/httpsprovider
|
||||||
|
v__OTEL_COLLECTOR_CORE_VERSION__
|
||||||
|
- gomod:
|
||||||
|
go.opentelemetry.io/collector/confmap/provider/yamlprovider
|
||||||
|
v__OTEL_COLLECTOR_CORE_VERSION__
|
||||||
|
|
@ -23,6 +23,9 @@ services:
|
||||||
context: ../..
|
context: ../..
|
||||||
dockerfile: docker/otel-collector/Dockerfile
|
dockerfile: docker/otel-collector/Dockerfile
|
||||||
target: dev
|
target: dev
|
||||||
|
args:
|
||||||
|
OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION:-0.149.0}
|
||||||
|
OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION:-1.55.0}
|
||||||
environment:
|
environment:
|
||||||
- CLICKHOUSE_ENDPOINT=tcp://ch-server:9000?dial_timeout=10s
|
- CLICKHOUSE_ENDPOINT=tcp://ch-server:9000?dial_timeout=10s
|
||||||
- CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT=ch-server:9363
|
- CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT=ch-server:9363
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue