diff --git a/.github/workflow-templates/build-prod-binary/action.yml b/.github/workflow-templates/build-prod-binary/action.yml new file mode 100644 index 00000000..1134f80f --- /dev/null +++ b/.github/workflow-templates/build-prod-binary/action.yml @@ -0,0 +1,57 @@ +name: Build Production Binary +description: | + Builds production a DataHaven binary for a given CPU target + +inputs: + target: + description: The CPU target for the binary + required: true + +runs: + using: "composite" + steps: + - name: Download sources from artifacts + uses: actions/download-artifact@v4 + with: + name: datahaven-sources + path: . + - name: Build production DataHaven + shell: bash + run: | + # Build DataHaven + # (we don't use volumes because of ownership/permissions issues) + docker build \ + --tag prod --no-cache \ + --build-arg="COMMIT=${{ github.event.inputs.sha }}" \ + --build-arg="RUSTFLAGS=-C target-cpu=${{ inputs.target }}" \ + --file ./docker/datahaven-production.Dockerfile + . # Use current directory as build context + + # Copy DataHaven binary + docker rm -f dummy 2> /dev/null | true + docker create -ti --name dummy prod bash + docker cp dummy:/datahaven/datahaven-node datahaven-node + docker rm -f dummy + + GLIBC_VERSION="$(objdump -T datahaven-node | grep "GLIBC_" | sed 's/.*GLIBC_\([.0-9]*\).*/\1/g' | sort -Vu | tail -1)" + + if [[ $GLIBC_VERSION == "2.34" ]]; then + echo "✅ Using expected GLIBC version: ${GLIBC_VERSION}"; + else + echo "❌ Unexpected GLIBC version: ${GLIBC_VERSION}"; + exit 1; + fi + + # Cleanup + docker rmi prod + + - name: Save DataHaven node binary + shell: bash + run: | + mkdir -p build + cp datahaven-node build/datahaven-node-${{ inputs.target }} + - name: Upload binary + uses: actions/upload-artifact@v4 + with: + name: datahavenbinaries-${{inputs.target}} + path: build/datahaven-node-${{inputs.target }} diff --git a/.github/workflow-templates/publish-docker/action.yml b/.github/workflow-templates/publish-docker/action.yml new file mode 100644 index 00000000..88283faa --- /dev/null +++ b/.github/workflow-templates/publish-docker/action.yml @@ -0,0 +1,69 @@ +name: Publish docker image +description: | + Publish docker image tags to dockerhub + +inputs: + dockerhub_username: + description: "Dockerhub username" + required: true + dockerhub_password: + description: "Dockerhub password" + required: true + image_tags: + description: "Image tags" + required: true + image_title: + description: "Image title" + required: true + image_description: + description: "Image description" + required: true + image_url: + description: "Image url" + required: true + image_source: + description: "Image source" + required: true + image_created: + description: "Image creation timestamp" + required: true + image_revision: + description: "Image revision" + required: true + image_licenses: + description: "Image licenses" + required: true + +runs: + using: "composite" + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.8.0 + with: + version: latest + driver-opts: | + image=moby/buildkit:master + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ inputs.dockerhub_username }} + password: ${{ inputs.dockerhub_password }} + - name: Build and push moonbeam + id: docker_build + uses: docker/build-push-action@v6 + with: + context: . + file: ./docker/moonbeam.Dockerfile + platforms: linux/amd64 + push: true + tags: ${{ inputs.image_tags }} + labels: | + org.opencontainers.image.title=${{ inputs.image_title }} + org.opencontainers.image.description=${{ inputs.image_title }} + org.opencontainers.image.url=${{ inputs.image_url }} + org.opencontainers.image.source=${{ inputs.image_source }} + org.opencontainers.image.created=${{ inputs.image_created }} + org.opencontainers.image.revision=${{ inputs.image_revision }} + org.opencontainers.image.licenses=${{ inputs.image_licenses }} diff --git a/.github/workflows/task-publish-binary b/.github/workflows/task-publish-binary new file mode 100644 index 00000000..649fb14a --- /dev/null +++ b/.github/workflows/task-publish-binary @@ -0,0 +1,135 @@ +name: Publish Binary Draft + +# The code (like generate-release-body) will be taken from the tag version, not master +on: + workflow_dispatch: + inputs: + from: + description: tag (ex. v0.1.0) to retrieve commit diff from + required: true + to: + description: tag (ex. v0.2.0) to generate release note and binaries from + required: true + +jobs: + build-binary: + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + matrix: + cpu: ["x86-64", "skylake", "znver3"] + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + ref: ${{ github.event.inputs.to }} + fetch-depth: 0 + - name: Upload sources as artifact + uses: actions/upload-artifact@v4 + with: + name: datahaven-sources + path: . + retention-days: 1 + - name: Cargo build + uses: ./.github/workflow-templates/build-prod-binary + with: + target: ${{ matrix.cpu }} + + ####### Prepare and publish the release draft ####### + + publish-draft-release: + runs-on: ubuntu-latest + permissions: + contents: write + needs: ["build-binary"] + outputs: + release_url: ${{ steps.create-release.outputs.html_url }} + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + ref: ${{ github.event.inputs.to }} + fetch-depth: 0 + - uses: actions/download-artifact@v5 + with: + pattern: datahaven-binaries-* + merge-multiple: true + path: build + - name: Use Node.js + uses: actions/setup-node@v5 + with: + node-version-file: "tools/.nvmrc" + - name: Prepare DataHaven binary for release body generation + working-directory: build + run: | + mv datahaven-node-x86-64 datahaven-node + - name: Generate release body + id: generate-release-body + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + working-directory: tools + run: | + yarn + yarn -s run ts-node github/generate-release-body.ts --owner "${{ github.repository_owner }}" --repo "$(basename ${{ github.repository }})" --from "${{ github.event.inputs.from }}" --to "${{ github.event.inputs.to }}" --srtool-report-folder '../build/' > ../body.md + - name: Prepare binary assets + working-directory: build + run: | + # Prepare binaries for upload + cp datahaven-node ../datahaven-node + cp datahaven-node-skylake ../datahaven-node-skylake + cp datahaven-node-znver3 ../datahaven-node-znver3 + - name: Create draft release with binaries + id: create-release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ github.event.inputs.to }} + name: DataHaven ${{ github.event.inputs.to }} + body_path: body.md + draft: true + files: | + datahaven-node + datahaven-node-skylake + datahaven-node-znver3 + + ####### Publish Release Candidate Docker Image ####### + + docker-release-candidate: + runs-on: ubuntu-latest + needs: ["build-binary"] + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + ref: ${{ github.event.inputs.to }} + - uses: actions/download-artifact@v5 + with: + pattern: datahaven-binaries-* + merge-multiple: true + path: build + - name: Rename binary for Docker + working-directory: build + run: | + mv datahaven-node-x86-64 datahaven-node + - name: Prepare Docker metadata + id: prep + run: | + DOCKER_IMAGE=datahavenxyz/datahaven + VERSION="${{ github.event.inputs.to }}" + TAG="${VERSION}-rc" + + echo "tags=${DOCKER_IMAGE}:${TAG}" >> $GITHUB_OUTPUT + echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT + - name: Cargo build + uses: ./.github/workflow-templates/publish-docker + with: + dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }} + dockerhub_password: ${{ secrets.DOCKERHUB_TOKEN }} + image_tags: ${{ steps.prep.outputs.tags }} + image_title: ${{ github.event.repository.name }} + image_description: ${{ github.event.repository.description }} + image_url: ${{ github.event.repository.html_url }} + image_source: ${{ github.event.repository.clone_url }} + image_created: ${{ steps.prep.outputs.created }} + image_revision: ${{ github.sha }} + image_licenses: ${{ github.event.repository.license.spdx_id }} diff --git a/docker/datahaven-production.Dockerfile b/docker/datahaven-production.Dockerfile new file mode 100644 index 00000000..95bf3a56 --- /dev/null +++ b/docker/datahaven-production.Dockerfile @@ -0,0 +1,67 @@ +# Production Node for DataHaven +# +# Requires to run from repository root and to copy the binary in the build folder (part of the release workflow) + +FROM docker.io/library/ubuntu:22.04 AS builder + +# Branch or tag to build DataHaven from +ARG COMMIT="main" +ARG RUSTFLAGS="" +ENV RUSTFLAGS=$RUSTFLAGS +ENV DEBIAN_FRONTEND=noninteractive +ENV PROTOC_VER=21.12 + +WORKDIR / + +RUN echo "*** Installing Basic dependencies ***" +RUN apt-get update && apt-get install -y ca-certificates && update-ca-certificates +RUN apt install --assume-yes git clang curl libpq-dev libssl-dev llvm libudev-dev make protobuf-compiler pkg-config +RUN echo "Installing protoc v${PROTOC_VER}..." \ +RUN curl -Lo protoc.zip "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VER}/protoc-${PROTOC_VER}-linux-x86_64.zip" \ + && unzip -q protoc.zip -d /usr/local/ \ + && rm protoc.zip \ + +RUN set -e + +RUN echo "*** Installing Rust environment ***" +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:$PATH" +RUN rustup default stable +# rustup version are pinned in the rust-toolchain file + +COPY ./operator /datahaven/datahaven +WORKDIR /datahaven/datahaven + +# Print target cpu +RUN rustc --print target-cpus + +RUN echo "*** Building DataHaven ***" +RUN cargo build --profile=production --all + +FROM debian:stable-slim +LABEL maintainer="steve@moonsonglabs.com" +LABEL description="Production Binary for DataHaven Nodes" + +RUN useradd -m -u 1000 -U -s /bin/sh -d /datahaven datahaven && \ + mkdir -p /datahaven/.local/share && \ + mkdir /data && \ + chown -R datahaven:datahaven /data && \ + ln -s /data /datahaven/.local/share/datahaven && \ + rm -rf /usr/sbin + +USER datahaven + +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=builder --chown=datahaven /datahaven/target/production/datahaven-node /datahaven/datahaven-node + +RUN chmod uog+x /datahaven/datahaven-node + +# 30333 for parachain p2p +# 30334 for relaychain p2p +# 9944 for Websocket & RPC call +# 9615 for Prometheus (metrics) +EXPOSE 30333 30334 9944 9615 + +VOLUME ["/data"] + +ENTRYPOINT ["/datahaven/datahaven-node"] diff --git a/docker/datahaven.Dockerfile b/docker/datahaven.Dockerfile new file mode 100644 index 00000000..2a8ab432 --- /dev/null +++ b/docker/datahaven.Dockerfile @@ -0,0 +1,35 @@ +# DataHaven Binary +# +# Requires to run from repository root and to copy the binary in the build folder (part of the release workflow) + +FROM debian:stable AS builder + +RUN apt-get update && apt-get install -y ca-certificates && update-ca-certificates + +FROM debian:stable-slim +LABEL maintainer="steve@moonsonglabs.com" +LABEL description="DataHaven Binary" + +RUN useradd -m -u 1000 -U -s /bin/sh -d /datahaven datahaven && \ + mkdir -p /datahaven/.local/share && \ + mkdir /data && \ + chown -R datahaven:datahaven /data && \ + ln -s /data /datahaven/.local/share/datahaven && \ + rm -rf /usr/sbin + +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt + +USER datahaven + +COPY --chown=datahaven build/* /datahaven +RUN chmod uog+x /datahaven/datahaven* + +# 30333 for parachain p2p +# 30334 for relaychain p2p +# 9944 for Websocket & RPC call +# 9615 for Prometheus (metrics) +EXPOSE 30333 30334 9944 9615 + +VOLUME ["/data"] + +ENTRYPOINT ["/datahaven/datahaven-node"] diff --git a/operator/Cargo.toml b/operator/Cargo.toml index 78aa3629..da62e197 100644 --- a/operator/Cargo.toml +++ b/operator/Cargo.toml @@ -283,3 +283,86 @@ shp-types = { git = "https://github.com/Moonsong-Labs/storage-hub.git", tag = "v cumulus-client-service = { git = "https://github.com/paritytech/polkadot-sdk.git", tag = "polkadot-stable2412-6", default-features = false } ## Precompiles pallet-evm-precompile-file-system = { git = "https://github.com/Moonsong-Labs/storage-hub.git", tag = "v0.0.2-alpha", default-features = false } + +# The list of dependencies below (which can be both direct and indirect dependencies) are crates +# that are suspected to be CPU-intensive, and that are unlikely to require debugging (as some of +# their debug info might be missing) or to require to be frequently recompiled. We compile these +# dependencies with `opt-level=3` even in "dev" mode in order to make "dev" mode more usable. +# The majority of these crates are cryptographic libraries. +# +# Note that this does **not** affect crates that depend on DataHaven. In other words, if you add +# a dependency on DataHaven, you have to copy-paste this list in your own `Cargo.toml` (assuming +# that you want the same list). This list is only relevant when running `cargo build` from within +# the DataHaven workspace. +# +# If you see an error mentioning "profile package spec ... did not match any packages", it +# probably concerns this list. +# +# This list is ordered alphabetically. +[profile.dev.package] +blake2 = { opt-level = 3 } +blake2b_simd = { opt-level = 3 } +chacha20poly1305 = { opt-level = 3 } +cranelift-codegen = { opt-level = 3 } +cranelift-wasm = { opt-level = 3 } +crc32fast = { opt-level = 3 } +crossbeam-deque = { opt-level = 3 } +crypto-mac = { opt-level = 3 } +curve25519-dalek = { opt-level = 3 } +ed25519-zebra = { opt-level = 3 } +futures-channel = { opt-level = 3 } +hash-db = { opt-level = 3 } +hashbrown = { opt-level = 3 } +hmac = { opt-level = 3 } +httparse = { opt-level = 3 } +integer-sqrt = { opt-level = 3 } +k256 = { opt-level = 3 } +keccak = { opt-level = 3 } +libm = { opt-level = 3 } +librocksdb-sys = { opt-level = 3 } +libsecp256k1 = { opt-level = 3 } +libz-sys = { opt-level = 3 } +mio = { opt-level = 3 } +nalgebra = { opt-level = 3 } +num-bigint = { opt-level = 3 } +parking_lot = { opt-level = 3 } +parking_lot_core = { opt-level = 3 } +percent-encoding = { opt-level = 3 } +primitive-types = { opt-level = 3 } +ring = { opt-level = 3 } +rustls = { opt-level = 3 } +secp256k1 = { opt-level = 3 } +sha2 = { opt-level = 3 } +sha3 = { opt-level = 3 } +smallvec = { opt-level = 3 } +snow = { opt-level = 3 } +twox-hash = { opt-level = 3 } +uint = { opt-level = 3 } +wasmi = { opt-level = 3 } +x25519-dalek = { opt-level = 3 } +yamux = { opt-level = 3 } +zeroize = { opt-level = 3 } + +# make sure dev builds with backtrace do +# not slow us down +[profile.dev.package.backtrace] +inherits = "release" + +[profile.production] +inherits = "release" +debug-assertions = false # Disable debug-assert! for production builds +codegen-units = 1 +incremental = false +lto = true + +[profile.release] +debug-assertions = true # Enable debug-assert! for non-production profiles +opt-level = 3 +# Moonbeam runtime requires unwinding. +panic = "unwind" + +[profile.testnet] +debug = 1 # debug symbols are useful for profilers +debug-assertions = true # Enable debug-assert! for non-production profiles +inherits = "release" +overflow-checks = true diff --git a/tools/.nvmrc b/tools/.nvmrc new file mode 100644 index 00000000..92f279e3 --- /dev/null +++ b/tools/.nvmrc @@ -0,0 +1 @@ +v22 \ No newline at end of file diff --git a/tools/github/generate-release-body.ts b/tools/github/generate-release-body.ts new file mode 100644 index 00000000..7f4f79a2 --- /dev/null +++ b/tools/github/generate-release-body.ts @@ -0,0 +1,82 @@ +import { Octokit } from "octokit"; +import yargs from "yargs"; +import { getCommitAndLabels, getCompareLink } from "./github-utils"; + +const BINARY_CHANGES_LABEL = "B5-clientnoteworthy"; +const BREAKING_CHANGES_LABEL = "breaking"; + +function capitalize(s) { + return s[0].toUpperCase() + s.slice(1); +} + +async function main() { + const argv = yargs(process.argv.slice(2)) + .usage("Usage: npm run ts-node github/generate-release-body.ts [args]") + .version("1.0.0") + .options({ + from: { + type: "string", + describe: "previous tag to retrieve commits from", + required: true, + }, + to: { + type: "string", + describe: "current tag being drafted", + required: true, + }, + owner: { + type: "string", + describe: "Repository owner (Ex: datahaven-xyz)", + required: true, + }, + repo: { + type: "string", + describe: "Repository name (Ex: datahaven)", + required: true, + }, + }) + .demandOption(["from", "to"]) + .help().argv; + + const octokit = new Octokit({ + auth: process.env.GITHUB_TOKEN || undefined, + }); + + const previousTag = argv.from; + const newTag = argv.to; + const moduleLinks = ["polkadot-sdk", "frontier"].map((repoName) => ({ + name: repoName, + link: getCompareLink(repoName, previousTag, newTag), + })); + + const { prByLabels } = await getCommitAndLabels( + octokit, + argv.owner, + argv.repo, + previousTag, + newTag + ); + const filteredPr = prByLabels[BINARY_CHANGES_LABEL] || []; + + const printPr = (pr) => { + if (pr.labels.includes(BREAKING_CHANGES_LABEL)) { + return "⚠️ " + pr.title + " (#" + pr.number + ")"; + } else { + return pr.title + " (#" + pr.number + ")"; + } + }; + + const template = ` +## Changes + +${filteredPr.map((pr) => `* ${printPr(pr)}`).join("\n")} + +## Dependency changes + +DataHaven: https://github.com/${argv.owner}/${argv.repo}/compare/${previousTag}...${newTag} +${moduleLinks.map((modules) => `${capitalize(modules.name)}: ${modules.link}`).join("\n")} +`; + console.log(template); +} + +main(); diff --git a/tools/github/github-utils.ts b/tools/github/github-utils.ts new file mode 100644 index 00000000..0bba9237 --- /dev/null +++ b/tools/github/github-utils.ts @@ -0,0 +1,111 @@ +import { Octokit } from "octokit"; +import { execSync } from "node:child_process"; + +// Typescript 4 will support it natively, but not yet :( +type Await = T extends PromiseLike ? U : T; +type Commits = Await>["data"]["commits"]; + +export function getCompareLink(packageName: string, previousTag: string, newTag: string) { + const previousPackage = execSync( + `git show ${previousTag}:../Cargo.lock | grep ${packageName}? | head -1 | grep -o '".*"'` + ).toString(); + const previousCommit = /#([0-9a-f]*)/g.exec(previousPackage)[1].slice(0, 8); + const previousRepo = /(https:\/\/.*)\?/g.exec(previousPackage)[1]; + + const newPackage = execSync( + `git show ${newTag}:../operator/Cargo.lock | grep ${packageName}? | head -1 | grep -o '".*"'` + ).toString(); + const newCommit = /#([0-9a-f]*)/g.exec(newPackage)[1].slice(0, 8); + const newRepo = /(https:\/\/.*)\?/g.exec(newPackage)[1]; + const newRepoOrganization = /github.com\/([^\/]*)/g.exec(newRepo)[1]; + + const diffLink = + previousRepo !== newRepo + ? `${previousRepo}/compare/${previousCommit}...${newRepoOrganization}:${newCommit}` + : `${previousRepo}/compare/${previousCommit}...${newCommit}`; + + return diffLink; +} + +export async function getCommitAndLabels( + octokit: Octokit, + owner: string, + repo: string, + previousTag: string, + newTag: string +): Promise<{ prByLabels: any; commits: any[] }> { + let commits: Commits = []; + let more = true; + let page = 0; + while (more) { + const compare = await octokit.rest.repos.compareCommitsWithBasehead({ + owner, + repo, + basehead: previousTag + "..." + newTag, + per_page: 200, + page, + }); + commits = commits.concat(compare.data.commits); + more = compare.data.commits.length === 200; + page++; + } + + // Determine commits to exclude + // - commits reverted in the same range + const excludedCommits: number[] = []; + const revertedCommits: number[] = []; + for (let i = commits.length - 1; i >= 0; i--) { + const commitMessageFirstLine = commits[i].commit.message.split("\n")[0].trim(); + + if (revertedCommits[commitMessageFirstLine] != null) { + excludedCommits.push(i); + excludedCommits.push(revertedCommits[commitMessageFirstLine]); + } else { + const foundRevertedCommitName = commitMessageFirstLine.match(/Revert \"(.*)\"/); + if (foundRevertedCommitName?.length > 0) { + revertedCommits[foundRevertedCommitName[1]] = i; + } + } + } + + const prByLabels = {}; + for (let i = 0; i < commits.length; i++) { + const commitMessageFirstLine = commits[i].commit.message.split("\n")[0].trim(); + if (!excludedCommits.includes(i)) { + const foundPrsNumbers = commitMessageFirstLine.match(/\(#([0-9]+)\)$/); + if (foundPrsNumbers && foundPrsNumbers.length > 1) { + // This will check current repo and if the PR is not found, will try the official repo + const repos = [ + { owner, repo }, + { owner: "datahaven-xyz", repo: "datahaven" }, + ]; + for (const { owner, repo } of repos) { + try { + const pr = await octokit.rest.pulls.get({ + owner, + repo, + pull_number: parseInt(foundPrsNumbers[1]), + }); + + if (pr.data.labels && pr.data.labels.length > 0) { + for (const label of pr.data.labels) { + prByLabels[label.name] = prByLabels[label.name] || []; + prByLabels[label.name].push(pr.data); + } + } else { + prByLabels[""] = prByLabels[""] || []; + prByLabels[""].push(pr); + } + break; + } catch (e) { + // PR not found... let's try the other repo + } + } + } + } + } + return { + prByLabels, + commits, + }; +} diff --git a/tools/package.json b/tools/package.json new file mode 100644 index 00000000..e0925ca8 --- /dev/null +++ b/tools/package.json @@ -0,0 +1,13 @@ +{ + "name": "datahaven-tools", + "version": "0.0.1", + "license": "GPL-3.0", + "dependencies": { + "octokit": "^1.0.6", + "ts-node": "^8.10.1", + "yargs": "^17.0.1" + }, + "devDependencies": { + "@types/yargs": "^15.0.12" + } +} diff --git a/tools/tsconfig.json b/tools/tsconfig.json new file mode 100644 index 00000000..bca570db --- /dev/null +++ b/tools/tsconfig.json @@ -0,0 +1,9 @@ +{ + "compilerOptions": { + "allowSyntheticDefaultImports": true, + "esModuleInterop": true, + "target": "es2020", + "module": "commonjs" + }, + "exclude": ["node_modules", "tests"] +}