From f84b6debb7284b92d3a68519f068048b5f20fc04 Mon Sep 17 00:00:00 2001 From: undercover-cactus Date: Fri, 28 Nov 2025 14:38:05 +0100 Subject: [PATCH] feat: statically build binary (#292) Co-authored-by: Gonza Montiel --- .../build-prod-binary/action.yml | 21 +++++++++ .../workflows/actions/setup-env/action.yml | 24 ++++++---- .../workflows/task-build-static-operator.yml | 46 +++++++++++++++++++ .github/workflows/task-publish-binary.yml | 7 +++ docker/datahaven-production.Dockerfile | 7 ++- operator/.cargo/config.toml | 2 +- operator/Cargo.lock | 12 +++++ operator/Cargo.toml | 4 ++ operator/node/Cargo.toml | 5 ++ operator/scripts/verify-licenses.sh | 1 + 10 files changed, 118 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/task-build-static-operator.yml diff --git a/.github/workflow-templates/build-prod-binary/action.yml b/.github/workflow-templates/build-prod-binary/action.yml index 0e491fdf..55fba473 100644 --- a/.github/workflow-templates/build-prod-binary/action.yml +++ b/.github/workflow-templates/build-prod-binary/action.yml @@ -6,6 +6,9 @@ inputs: target: description: The CPU target for the binary required: true + build: + description: If we should statically build it or not + required: false runs: using: "composite" @@ -24,6 +27,7 @@ runs: --tag prod --no-cache \ --build-arg="COMMIT=${{ github.event.inputs.sha }}" \ --build-arg="RUSTFLAGS=-C target-cpu=${{ inputs.target }}" \ + --build-arg="BUILD=${{ inputs.build }}" \ --file ./docker/datahaven-production.Dockerfile \ . @@ -46,12 +50,29 @@ runs: docker rmi prod - name: Save DataHaven node binary + if: inputs.build != 'static' shell: bash run: | mkdir -p build cp datahaven-node build/datahaven-node-${{ inputs.target }} + + - name: Save DataHaven node binary + if: inputs.build == 'static' + shell: bash + run: | + mkdir -p build + cp datahaven-node build/datahaven-node-${{ inputs.target }}-static + - name: Upload binary + if: inputs.build != 'static' uses: actions/upload-artifact@v4 with: name: datahaven-binaries-${{inputs.target}} path: build/datahaven-node-${{inputs.target }} + + - name: Upload binary + if: inputs.build == 'static' + uses: actions/upload-artifact@v4 + with: + name: datahaven-binaries-${{inputs.target}}-static + path: build/datahaven-node-${{inputs.target }}-static diff --git a/.github/workflows/actions/setup-env/action.yml b/.github/workflows/actions/setup-env/action.yml index 5407aeef..a4b5b98c 100644 --- a/.github/workflows/actions/setup-env/action.yml +++ b/.github/workflows/actions/setup-env/action.yml @@ -12,6 +12,11 @@ inputs: required: false default: "true" + skip-libpq: + description: "Indicate to skip postgres lib install or not" + required: false + default: "false" + runs: using: "composite" steps: @@ -68,9 +73,16 @@ runs: shell: bash run: sudo apt-get update && sudo apt-get install -y libpq-dev libclang-dev - # Auto-install missing dependencies when install-deps is false (for self-hosted runners) + # Install protoc only when install-deps is true + - name: Install Protoc + if: inputs.install-deps == 'true' + uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ github.token }} + + # Auto-install missing dependencies when install-deps is false (for self-hosted runners) can be skipped when statically build the node - name: Setup system dependencies (self-hosted) - if: inputs.install-deps == 'false' + if: inputs.install-deps == 'false' && inputs.skip-libpq == 'false' shell: bash run: | echo "Checking and installing system dependencies locally if needed..." @@ -172,13 +184,7 @@ runs: echo "All required system dependencies ready!" - # Install protoc only when install-deps is true - - name: Install Protoc - if: inputs.install-deps == 'true' - uses: arduino/setup-protoc@v3 - with: - repo-token: ${{ github.token }} - + # Auto-install protoc when install-deps is false (for self-hosted runners) - name: Setup Protoc (self-hosted) if: inputs.install-deps == 'false' diff --git a/.github/workflows/task-build-static-operator.yml b/.github/workflows/task-build-static-operator.yml new file mode 100644 index 00000000..add5b029 --- /dev/null +++ b/.github/workflows/task-build-static-operator.yml @@ -0,0 +1,46 @@ +name: DataHaven Operator Static Build + +on: + pull_request: + paths: + # Only check the build if we are adding a dependency to save runner time + - 'operator/Cargo.toml' + - 'operator/Cargo.lock' + +jobs: + build-node: + name: Build operator binary + runs-on: + group: DH-runners + env: + RUSTC_WRAPPER: "sccache" + CARGO_INCREMENTAL: "0" + CARGO_TERM_COLOR: always + SCCACHE_GHA_ENABLED: "true" + defaults: + run: + working-directory: ./operator + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 1 + + - uses: ./.github/workflows/actions/setup-env + with: + cache-key: BUILD-RELEASE + install-deps: false + skip-libpq: true + + - name: Set build flags + run: echo "RUSTFLAGS=${{ env.RUSTFLAGS }} -C linker=clang -C link-arg=-fuse-ld=mold" >> $GITHUB_ENV + + - name: Build node binary + run: | + cargo build --release --locked --features fast-runtime,static + + - name: Test binary + run: | + ldd ./target/release/datahaven-node + ./target/release/datahaven-node --version + diff --git a/.github/workflows/task-publish-binary.yml b/.github/workflows/task-publish-binary.yml index 9e19bd89..1a04ab34 100644 --- a/.github/workflows/task-publish-binary.yml +++ b/.github/workflows/task-publish-binary.yml @@ -37,6 +37,12 @@ jobs: strategy: matrix: cpu: ["x86-64", "skylake", "znver3"] + build: ["default", "static"] + exclude: + - cpu: skylake + build: static + - cpu: znver3 + build: static steps: - name: Checkout uses: actions/checkout@v5 @@ -46,6 +52,7 @@ jobs: uses: ./.github/workflow-templates/build-prod-binary with: target: ${{ matrix.cpu }} + build: ${{ matrix.build }} ####### Prepare and publish the release draft ####### diff --git a/docker/datahaven-production.Dockerfile b/docker/datahaven-production.Dockerfile index 9c8f6226..c7257b71 100644 --- a/docker/datahaven-production.Dockerfile +++ b/docker/datahaven-production.Dockerfile @@ -7,6 +7,7 @@ FROM docker.io/library/ubuntu:22.04 AS builder # Branch or tag to build DataHaven from ARG COMMIT="main" ARG RUSTFLAGS="" +ARG BUILD="" ENV RUSTFLAGS=$RUSTFLAGS ENV DEBIAN_FRONTEND=noninteractive ENV PROTOC_VER=21.12 @@ -37,7 +38,11 @@ WORKDIR /datahaven RUN rustc --print target-cpus RUN echo "*** Building DataHaven ***" -RUN cargo build --profile=production --all +RUN if [ "$BUILD" = "static" ]; then \ + cargo build --profile=production --all --features static + else \ + cargo build --profile=production --all + fi FROM debian:stable-slim LABEL maintainer="steve@moonsonglabs.com" diff --git a/operator/.cargo/config.toml b/operator/.cargo/config.toml index f42e1463..b7a46ab6 100644 --- a/operator/.cargo/config.toml +++ b/operator/.cargo/config.toml @@ -3,4 +3,4 @@ lto = "thin" opt-level = 2 [registries.crates-io] -protocol = "sparse" \ No newline at end of file +protocol = "sparse" diff --git a/operator/Cargo.lock b/operator/Cargo.lock index 5e98f5e9..da644ca0 100644 --- a/operator/Cargo.lock +++ b/operator/Cargo.lock @@ -3100,6 +3100,7 @@ dependencies = [ "pallet-transaction-payment-rpc", "pallet-transaction-payment-rpc-runtime-api", "parity-scale-codec", + "pq-sys", "sc-basic-authorship", "sc-cli", "sc-client-api", @@ -11921,6 +11922,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "pq-src" +version = "0.3.10+libpq-18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56ef39ce621f4993d6084fdcd4cbf1e01c84bdba53109cfad095d2cf441b85b9" +dependencies = [ + "cc", + "openssl-sys", +] + [[package]] name = "pq-sys" version = "0.7.4" @@ -11929,6 +11940,7 @@ checksum = "089d5dc8f44104b719912ad4478fd558b59a431ce19ef9101f637be8c656b90a" dependencies = [ "libc", "pkg-config", + "pq-src", "vcpkg", ] diff --git a/operator/Cargo.toml b/operator/Cargo.toml index 67586bc7..d19fab19 100644 --- a/operator/Cargo.toml +++ b/operator/Cargo.toml @@ -304,6 +304,10 @@ shp-types = { git = "https://github.com/Moonsong-Labs/storage-hub.git", tag = "v ## Precompiles pallet-evm-precompile-file-system = { git = "https://github.com/Moonsong-Labs/storage-hub.git", tag = "v0.1.4", default-features = false } +# Static linking +#### Needed to build static binaries #### +pq-sys = { version = "0.7.4" } + # The list of dependencies below (which can be both direct and indirect dependencies) are crates # that are suspected to be CPU-intensive, and that are unlikely to require debugging (as some of # their debug info might be missing) or to require to be frequently recompiled. We compile these diff --git a/operator/node/Cargo.toml b/operator/node/Cargo.toml index 831f71c5..22e0f645 100644 --- a/operator/node/Cargo.toml +++ b/operator/node/Cargo.toml @@ -132,6 +132,10 @@ serde = { workspace = true, default-features = true } cumulus-client-service = { workspace = true } toml = { workspace = true } +# Static linking +#### Needed to build static binaries #### +pq-sys = { workspace = true, optional = true } + [build-dependencies] substrate-build-script-utils = { workspace = true, default-features = true } @@ -144,6 +148,7 @@ std = [ "datahaven-testnet-runtime/std", "shp-opaque/std" ] +static = ["pq-sys", "pq-sys/bundled"] # Dependencies that are only required if runtime benchmarking should be build. runtime-benchmarks = [ diff --git a/operator/scripts/verify-licenses.sh b/operator/scripts/verify-licenses.sh index 6362029c..11ed3ae5 100755 --- a/operator/scripts/verify-licenses.sh +++ b/operator/scripts/verify-licenses.sh @@ -51,6 +51,7 @@ AUTHORS=( ) NAMES=( "ring" # v0.16.20 has null license metadata but contains Apache-2.0 AND ISC LICENSE file + "pq-src" # License is the same as postgres "shp-tx-implicits-runtime-api" ) licenses_filter=$(printf ' .license != "%s" and' "${LICENSES[@]}")