mirror of
https://github.com/hyperdxio/hyperdx
synced 2026-04-21 13:37:15 +00:00
0a4fb15df2
## Summary Update `packages/otel-collector/builder-config.yaml` to include commonly-used components from the upstream [opentelemetry-collector](https://github.com/open-telemetry/opentelemetry-collector) core and [opentelemetry-collector-contrib](https://github.com/open-telemetry/opentelemetry-collector-contrib) distributions. This gives users more flexibility in their custom OTel configs without pulling in the entire contrib distribution (which causes very long compile times). Also adds Go module and build cache mounts to the OCB Docker build stage for faster rebuilds, and bumps CI timeouts for integration and smoke test jobs to account for the larger binary. ### Core extensions added (2) - `memorylimiterextension` — memory-based limiting at the extension level - `zpagesextension` — zPages debugging endpoints ### Contrib receivers added (4) - `dockerstatsreceiver` — container metrics from Docker - `filelogreceiver` — tail log files - `k8sclusterreceiver` — Kubernetes cluster-level metrics - `kubeletstatsreceiver` — node/pod/container metrics from kubelet ### Contrib processors added (12) - `attributesprocessor` — insert/update/delete/hash attributes - `cumulativetodeltaprocessor` — convert cumulative metrics to delta - `filterprocessor` — drop unwanted telemetry - `groupbyattrsprocessor` — reassign resource attributes - `k8sattributesprocessor` — enrich telemetry with k8s metadata - `logdedupprocessor` — deduplicate repeated log entries - `metricstransformprocessor` — rename/aggregate/transform metrics - `probabilisticsamplerprocessor` — percentage-based sampling - `redactionprocessor` — mask/remove sensitive data - `resourceprocessor` — modify resource attributes - `spanprocessor` — rename spans, extract attributes - `tailsamplingprocessor` — sample traces based on policies ### Contrib extensions added (1) - `filestorage` — persistent file-based storage (used by clickhouse exporter sending queue in EE OpAMP controller) ### Other changes - **Docker cache mounts**: Added `--mount=type=cache` for Go module and build caches in the OCB builder stage of both `docker/otel-collector/Dockerfile` and `docker/hyperdx/Dockerfile` - **CI timeouts**: Bumped `integration` and `otel-smoke-test` jobs from 8 to 16 minutes in `.github/workflows/main.yml` All existing HyperDX-specific components are preserved unchanged. ### How to test locally or on Vercel 1. Build the OTel Collector Docker image — verify OCB resolves all listed modules 2. Provide a custom OTel config that uses one of the newly-added components and verify it loads 3. Verify existing HyperDX OTel pipeline still functions ### References - Linear Issue: https://linear.app/clickhouse/issue/HDX-4029 - Upstream core builder-config: https://github.com/open-telemetry/opentelemetry-collector/blob/main/cmd/otelcorecol/builder-config.yaml - Upstream contrib builder-config: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/cmd/otelcontribcol/builder-config.yaml
108 lines
5.5 KiB
Docker
108 lines
5.5 KiB
Docker
## base #############################################################################################
|
|
ARG OTEL_COLLECTOR_VERSION=0.149.0
|
|
ARG OTEL_COLLECTOR_CORE_VERSION=1.55.0
|
|
|
|
FROM otel/opentelemetry-collector-opampsupervisor:${OTEL_COLLECTOR_VERSION} AS supervisor
|
|
FROM hairyhenderson/gomplate:v4.3.3-alpine AS gomplate
|
|
FROM kukymbr/goose-docker@sha256:0cd025636df126e7f66472861ca4db3683bc649be46cd1f6ef1a316209058e23 AS goose
|
|
|
|
# Build the custom HyperDX collector binary using OCB (OpenTelemetry Collector Builder).
|
|
# This replaces the pre-built otel/opentelemetry-collector-contrib image so we can
|
|
# include custom receiver/processor components alongside the standard contrib ones.
|
|
# Note: Build context must be repo root (use: docker build -f docker/otel-collector/Dockerfile .)
|
|
# Note: The official OCB image may ship an older Go than the contrib modules require,
|
|
# so we copy the ocb binary into a golang base with a newer toolchain.
|
|
FROM otel/opentelemetry-collector-builder:${OTEL_COLLECTOR_VERSION} AS ocb-bin
|
|
FROM golang:1.26-alpine AS ocb-builder
|
|
ARG OTEL_COLLECTOR_VERSION
|
|
ARG OTEL_COLLECTOR_CORE_VERSION
|
|
COPY --from=ocb-bin /usr/local/bin/ocb /usr/local/bin/ocb
|
|
WORKDIR /build
|
|
COPY packages/otel-collector/builder-config.yaml .
|
|
RUN --mount=type=cache,target=/go/pkg/mod,id=ocb-gomod \
|
|
--mount=type=cache,target=/root/.cache/go-build,id=ocb-gobuild \
|
|
sed -i "s/__OTEL_COLLECTOR_VERSION__/${OTEL_COLLECTOR_VERSION}/g; s/__OTEL_COLLECTOR_CORE_VERSION__/${OTEL_COLLECTOR_CORE_VERSION}/g" builder-config.yaml && \
|
|
CGO_ENABLED=0 ocb --config=builder-config.yaml
|
|
|
|
# Build the Go migration tool with full TLS support for ClickHouse
|
|
FROM golang:1.26-alpine AS migrate-builder
|
|
WORKDIR /build
|
|
COPY packages/otel-collector/go.mod packages/otel-collector/go.sum ./
|
|
RUN go mod download
|
|
COPY packages/otel-collector/ ./
|
|
RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /migrate ./cmd/migrate
|
|
|
|
# From: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/aa5c3aa4c7ec174361fcaf908de8eaca72263078/cmd/opampsupervisor/Dockerfile#L18
|
|
FROM alpine:3.23@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659 AS base
|
|
|
|
ARG USER_UID=10001
|
|
ARG USER_GID=10001
|
|
|
|
# Install certs, create user/group, and make the writable data dir
|
|
RUN apk add --no-cache ca-certificates && \
|
|
addgroup -S -g ${USER_GID} otel && \
|
|
adduser -S -u ${USER_UID} -G otel otel && \
|
|
install -d -m 0777 -o ${USER_UID} -g ${USER_GID} /etc/otel/supervisor-data && \
|
|
install -d -m 0755 -o ${USER_UID} -g ${USER_GID} /etc/otelcol-contrib
|
|
|
|
# Copy gomplate binary from the gomplate image
|
|
COPY --from=gomplate /bin/gomplate /usr/local/bin/gomplate
|
|
|
|
# Copy goose binary for shell-based migrations (default)
|
|
COPY --from=goose /bin/goose /usr/local/bin/goose
|
|
|
|
# Copy migrate binary (Go-based goose migration tool with full TLS support)
|
|
COPY --from=migrate-builder /migrate /usr/local/bin/migrate
|
|
|
|
USER ${USER_UID}:${USER_GID}
|
|
|
|
COPY --from=supervisor --chmod=755 /usr/local/bin/opampsupervisor /opampsupervisor
|
|
COPY --from=ocb-builder --chmod=755 /build/output/otelcol-hyperdx /otelcontribcol
|
|
|
|
# Copy entrypoint and log tail wrapper scripts
|
|
COPY --chmod=755 docker/otel-collector/entrypoint.sh /entrypoint.sh
|
|
COPY --chmod=755 docker/otel-collector/log-tailer.sh /log-tailer.sh
|
|
|
|
## dev ##############################################################################################
|
|
FROM base AS dev
|
|
|
|
LABEL org.opencontainers.image.vendor="HyperDX" \
|
|
org.opencontainers.image.title="HyperDX OTel Collector (Dev)" \
|
|
org.opencontainers.image.description="OpenTelemetry Collector for HyperDX development" \
|
|
org.opencontainers.image.source="https://github.com/hyperdxio/hyperdx" \
|
|
org.opencontainers.image.licenses="MIT"
|
|
|
|
COPY --chown=10001:10001 docker/otel-collector/config.yaml /etc/otelcol-contrib/config.yaml
|
|
COPY --chown=10001:10001 docker/otel-collector/config.standalone.yaml /etc/otelcol-contrib/standalone-config.yaml
|
|
COPY --chown=10001:10001 docker/otel-collector/config.standalone.auth.yaml /etc/otelcol-contrib/standalone-auth-config.yaml
|
|
COPY --chown=10001:10001 docker/otel-collector/supervisor_docker.yaml.tmpl /etc/otel/supervisor.yaml.tmpl
|
|
COPY --chown=10001:10001 docker/otel-collector/schema /etc/otel/schema
|
|
|
|
EXPOSE 4317 4318 13133
|
|
|
|
HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \
|
|
CMD wget -q --spider http://localhost:13133/ || exit 1
|
|
|
|
ENTRYPOINT ["/entrypoint.sh", "/opampsupervisor"]
|
|
|
|
## prod #############################################################################################
|
|
FROM base AS prod
|
|
|
|
LABEL org.opencontainers.image.vendor="HyperDX" \
|
|
org.opencontainers.image.title="HyperDX OTel Collector" \
|
|
org.opencontainers.image.description="OpenTelemetry Collector for HyperDX observability platform" \
|
|
org.opencontainers.image.source="https://github.com/hyperdxio/hyperdx" \
|
|
org.opencontainers.image.licenses="MIT"
|
|
|
|
COPY --chown=10001:10001 docker/otel-collector/config.yaml /etc/otelcol-contrib/config.yaml
|
|
COPY --chown=10001:10001 docker/otel-collector/config.standalone.yaml /etc/otelcol-contrib/standalone-config.yaml
|
|
COPY --chown=10001:10001 docker/otel-collector/config.standalone.auth.yaml /etc/otelcol-contrib/standalone-auth-config.yaml
|
|
COPY --chown=10001:10001 docker/otel-collector/supervisor_docker.yaml.tmpl /etc/otel/supervisor.yaml.tmpl
|
|
COPY --chown=10001:10001 docker/otel-collector/schema /etc/otel/schema
|
|
|
|
EXPOSE 4317 4318 13133
|
|
|
|
HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \
|
|
CMD wget -q --spider http://localhost:13133/ || exit 1
|
|
|
|
ENTRYPOINT ["/entrypoint.sh", "/opampsupervisor"]
|