feat: Implement gh workflow to publish a draft client release (#172)

.github/workflow-templates/build-prod-binary/action.yml

@@ -0,0 +1,57 @@
+name: Build Production Binary
+description: |
+  Builds production a DataHaven binary for a given CPU target
+
+inputs:
+  target:
+    description: The CPU target for the binary
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Download sources from artifacts
+      uses: actions/download-artifact@v4
+      with:
+        name: datahaven-sources
+        path: .
+    - name: Build production DataHaven
+      shell: bash
+      run: |
+        # Build DataHaven
+        # (we don't use volumes because of ownership/permissions issues)
+        docker build \
+          --tag prod --no-cache \
+          --build-arg="COMMIT=${{ github.event.inputs.sha }}" \
+          --build-arg="RUSTFLAGS=-C target-cpu=${{ inputs.target }}" \
+          --file ./docker/datahaven-production.Dockerfile
+          . # Use current directory as build context
+
+        # Copy DataHaven binary
+        docker rm -f dummy 2> /dev/null | true
+        docker create -ti --name dummy prod bash
+        docker cp dummy:/datahaven/datahaven-node datahaven-node
+        docker rm -f dummy
+
+        GLIBC_VERSION="$(objdump -T datahaven-node | grep "GLIBC_" | sed 's/.*GLIBC_\([.0-9]*\).*/\1/g' | sort -Vu | tail -1)"
+
+        if [[ $GLIBC_VERSION == "2.34" ]]; then
+          echo "✅ Using expected GLIBC version: ${GLIBC_VERSION}";
+        else
+          echo "❌ Unexpected GLIBC version: ${GLIBC_VERSION}";
+          exit 1;
+        fi
+
+        # Cleanup
+        docker rmi prod
+
+    - name: Save DataHaven node binary
+      shell: bash
+      run: |
+        mkdir -p build
+        cp datahaven-node build/datahaven-node-${{ inputs.target }}
+    - name: Upload binary
+      uses: actions/upload-artifact@v4
+      with:
+        name: datahavenbinaries-${{inputs.target}}
+        path: build/datahaven-node-${{inputs.target }}

.github/workflow-templates/publish-docker/action.yml

@@ -0,0 +1,69 @@
+name: Publish docker image
+description: |
+  Publish docker image tags to dockerhub
+
+inputs:
+  dockerhub_username:
+    description: "Dockerhub username"
+    required: true
+  dockerhub_password:
+    description: "Dockerhub password"
+    required: true
+  image_tags:
+    description: "Image tags"
+    required: true
+  image_title:
+    description: "Image title"
+    required: true
+  image_description:
+    description: "Image description"
+    required: true
+  image_url:
+    description: "Image url"
+    required: true
+  image_source:
+    description: "Image source"
+    required: true
+  image_created:
+    description: "Image creation timestamp"
+    required: true
+  image_revision:
+    description: "Image revision"
+    required: true
+  image_licenses:
+    description: "Image licenses"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3.8.0
+      with:
+        version: latest
+        driver-opts: |
+          image=moby/buildkit:master
+    - name: Login to DockerHub
+      uses: docker/login-action@v3
+      with:
+        username: ${{ inputs.dockerhub_username }}
+        password: ${{ inputs.dockerhub_password }}
+    - name: Build and push moonbeam
+      id: docker_build
+      uses: docker/build-push-action@v6
+      with:
+        context: .
+        file: ./docker/moonbeam.Dockerfile
+        platforms: linux/amd64
+        push: true
+        tags: ${{ inputs.image_tags }}
+        labels: |
+          org.opencontainers.image.title=${{ inputs.image_title }}
+          org.opencontainers.image.description=${{ inputs.image_title }}
+          org.opencontainers.image.url=${{ inputs.image_url }}
+          org.opencontainers.image.source=${{ inputs.image_source }}
+          org.opencontainers.image.created=${{ inputs.image_created }}
+          org.opencontainers.image.revision=${{ inputs.image_revision }}
+          org.opencontainers.image.licenses=${{ inputs.image_licenses }}

.github/workflows/task-publish-binary

@@ -0,0 +1,135 @@
+name: Publish Binary Draft
+
+# The code (like generate-release-body) will be taken from the tag version, not master
+on:
+  workflow_dispatch:
+    inputs:
+      from:
+        description: tag (ex. v0.1.0) to retrieve commit diff from
+        required: true
+      to:
+        description: tag (ex. v0.2.0) to generate release note and binaries from
+        required: true
+
+jobs:
+  build-binary:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    strategy:
+      matrix:
+        cpu: ["x86-64", "skylake", "znver3"]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+        with:
+          ref: ${{ github.event.inputs.to }}
+          fetch-depth: 0
+      - name: Upload sources as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: datahaven-sources
+          path: .
+          retention-days: 1
+      - name: Cargo build
+        uses: ./.github/workflow-templates/build-prod-binary
+        with:
+          target: ${{ matrix.cpu }}
+
+  ####### Prepare and publish the release draft #######
+
+  publish-draft-release:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    needs: ["build-binary"]
+    outputs:
+      release_url: ${{ steps.create-release.outputs.html_url }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+        with:
+          ref: ${{ github.event.inputs.to }}
+          fetch-depth: 0
+      - uses: actions/download-artifact@v5
+        with:
+          pattern: datahaven-binaries-*
+          merge-multiple: true
+          path: build
+      - name: Use Node.js
+        uses: actions/setup-node@v5
+        with:
+          node-version-file: "tools/.nvmrc"
+      - name: Prepare DataHaven binary for release body generation
+        working-directory: build
+        run: |
+          mv datahaven-node-x86-64 datahaven-node
+      - name: Generate release body
+        id: generate-release-body
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        working-directory: tools
+        run: |
+          yarn
+          yarn -s run ts-node github/generate-release-body.ts --owner "${{ github.repository_owner }}" --repo "$(basename ${{ github.repository }})" --from "${{ github.event.inputs.from }}" --to "${{ github.event.inputs.to }}" --srtool-report-folder '../build/' > ../body.md
+      - name: Prepare binary assets
+        working-directory: build
+        run: |
+          # Prepare binaries for upload
+          cp datahaven-node ../datahaven-node
+          cp datahaven-node-skylake ../datahaven-node-skylake
+          cp datahaven-node-znver3 ../datahaven-node-znver3
+      - name: Create draft release with binaries
+        id: create-release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: ${{ github.event.inputs.to }}
+          name: DataHaven ${{ github.event.inputs.to }}
+          body_path: body.md
+          draft: true
+          files: |
+            datahaven-node
+            datahaven-node-skylake
+            datahaven-node-znver3
+
+  ####### Publish Release Candidate Docker Image #######
+
+  docker-release-candidate:
+    runs-on: ubuntu-latest
+    needs: ["build-binary"]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+        with:
+          ref: ${{ github.event.inputs.to }}
+      - uses: actions/download-artifact@v5
+        with:
+          pattern: datahaven-binaries-*
+          merge-multiple: true
+          path: build
+      - name: Rename binary for Docker
+        working-directory: build
+        run: |
+          mv datahaven-node-x86-64 datahaven-node
+      - name: Prepare Docker metadata
+        id: prep
+        run: |
+          DOCKER_IMAGE=datahavenxyz/datahaven
+          VERSION="${{ github.event.inputs.to }}"
+          TAG="${VERSION}-rc"
+
+          echo "tags=${DOCKER_IMAGE}:${TAG}" >> $GITHUB_OUTPUT
+          echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
+      - name: Cargo build
+        uses: ./.github/workflow-templates/publish-docker
+        with:
+          dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
+          dockerhub_password: ${{ secrets.DOCKERHUB_TOKEN }}
+          image_tags: ${{ steps.prep.outputs.tags }}
+          image_title: ${{ github.event.repository.name }}
+          image_description: ${{ github.event.repository.description }}
+          image_url: ${{ github.event.repository.html_url }}
+          image_source: ${{ github.event.repository.clone_url }}
+          image_created: ${{ steps.prep.outputs.created }}
+          image_revision: ${{ github.sha }}
+          image_licenses: ${{ github.event.repository.license.spdx_id }}

docker/datahaven-production.Dockerfile

@@ -0,0 +1,67 @@
+# Production Node for DataHaven
+#
+# Requires to run from repository root and to copy the binary in the build folder (part of the release workflow)
+
+FROM docker.io/library/ubuntu:22.04 AS builder
+
+# Branch or tag to build DataHaven from
+ARG COMMIT="main"
+ARG RUSTFLAGS=""
+ENV RUSTFLAGS=$RUSTFLAGS
+ENV DEBIAN_FRONTEND=noninteractive
+ENV PROTOC_VER=21.12
+
+WORKDIR /
+
+RUN echo "*** Installing Basic dependencies ***"
+RUN apt-get update && apt-get install -y ca-certificates && update-ca-certificates
+RUN apt install --assume-yes git clang curl libpq-dev libssl-dev llvm libudev-dev make protobuf-compiler pkg-config
+RUN echo "Installing protoc v${PROTOC_VER}..." \
+RUN curl -Lo protoc.zip "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VER}/protoc-${PROTOC_VER}-linux-x86_64.zip" \
+    && unzip -q protoc.zip -d /usr/local/ \
+    && rm protoc.zip \
+
+RUN set -e
+
+RUN echo "*** Installing Rust environment ***"
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+ENV PATH="/root/.cargo/bin:$PATH"
+RUN rustup default stable
+# rustup version are pinned in the rust-toolchain file
+
+COPY ./operator /datahaven/datahaven
+WORKDIR /datahaven/datahaven
+
+# Print target cpu
+RUN rustc --print target-cpus
+
+RUN echo "*** Building DataHaven ***"
+RUN cargo build --profile=production --all
+
+FROM debian:stable-slim
+LABEL maintainer="steve@moonsonglabs.com"
+LABEL description="Production Binary for DataHaven Nodes"
+
+RUN useradd -m -u 1000 -U -s /bin/sh -d /datahaven datahaven && \
+	mkdir -p /datahaven/.local/share && \
+	mkdir /data && \
+	chown -R datahaven:datahaven /data && \
+	ln -s /data /datahaven/.local/share/datahaven && \
+	rm -rf /usr/sbin
+
+USER datahaven
+
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+COPY --from=builder --chown=datahaven /datahaven/target/production/datahaven-node /datahaven/datahaven-node
+
+RUN chmod uog+x /datahaven/datahaven-node
+
+# 30333 for parachain p2p
+# 30334 for relaychain p2p
+# 9944 for Websocket & RPC call
+# 9615 for Prometheus (metrics)
+EXPOSE 30333 30334 9944 9615
+
+VOLUME ["/data"]
+
+ENTRYPOINT ["/datahaven/datahaven-node"]

docker/datahaven.Dockerfile

@@ -0,0 +1,35 @@
+# DataHaven Binary
+#
+# Requires to run from repository root and to copy the binary in the build folder (part of the release workflow)
+
+FROM debian:stable AS builder
+
+RUN apt-get update && apt-get install -y ca-certificates && update-ca-certificates
+
+FROM debian:stable-slim
+LABEL maintainer="steve@moonsonglabs.com"
+LABEL description="DataHaven Binary"
+
+RUN useradd -m -u 1000 -U -s /bin/sh -d /datahaven datahaven && \
+	mkdir -p /datahaven/.local/share && \
+	mkdir /data && \
+	chown -R datahaven:datahaven /data && \
+	ln -s /data /datahaven/.local/share/datahaven && \
+	rm -rf /usr/sbin
+
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+
+USER datahaven
+
+COPY --chown=datahaven build/* /datahaven
+RUN chmod uog+x /datahaven/datahaven*
+
+# 30333 for parachain p2p
+# 30334 for relaychain p2p
+# 9944 for Websocket & RPC call
+# 9615 for Prometheus (metrics)
+EXPOSE 30333 30334 9944 9615
+
+VOLUME ["/data"]
+
+ENTRYPOINT ["/datahaven/datahaven-node"]

operator/Cargo.toml

@@ -283,3 +283,86 @@ shp-types = { git = "https://github.com/Moonsong-Labs/storage-hub.git", tag = "v
 cumulus-client-service = { git = "https://github.com/paritytech/polkadot-sdk.git", tag = "polkadot-stable2412-6", default-features = false }
 ## Precompiles
 pallet-evm-precompile-file-system = { git = "https://github.com/Moonsong-Labs/storage-hub.git", tag = "v0.0.2-alpha", default-features = false }
+
+# The list of dependencies below (which can be both direct and indirect dependencies) are crates
+# that are suspected to be CPU-intensive, and that are unlikely to require debugging (as some of
+# their debug info might be missing) or to require to be frequently recompiled. We compile these
+# dependencies with `opt-level=3` even in "dev" mode in order to make "dev" mode more usable.
+# The majority of these crates are cryptographic libraries.
+#
+# Note that this does **not** affect crates that depend on DataHaven. In other words, if you add
+# a dependency on DataHaven, you have to copy-paste this list in your own `Cargo.toml` (assuming
+# that you want the same list). This list is only relevant when running `cargo build` from within
+# the DataHaven workspace.
+#
+# If you see an error mentioning "profile package spec ... did not match any packages", it
+# probably concerns this list.
+#
+# This list is ordered alphabetically.
+[profile.dev.package]
+blake2 = { opt-level = 3 }
+blake2b_simd = { opt-level = 3 }
+chacha20poly1305 = { opt-level = 3 }
+cranelift-codegen = { opt-level = 3 }
+cranelift-wasm = { opt-level = 3 }
+crc32fast = { opt-level = 3 }
+crossbeam-deque = { opt-level = 3 }
+crypto-mac = { opt-level = 3 }
+curve25519-dalek = { opt-level = 3 }
+ed25519-zebra = { opt-level = 3 }
+futures-channel = { opt-level = 3 }
+hash-db = { opt-level = 3 }
+hashbrown = { opt-level = 3 }
+hmac = { opt-level = 3 }
+httparse = { opt-level = 3 }
+integer-sqrt = { opt-level = 3 }
+k256 = { opt-level = 3 }
+keccak = { opt-level = 3 }
+libm = { opt-level = 3 }
+librocksdb-sys = { opt-level = 3 }
+libsecp256k1 = { opt-level = 3 }
+libz-sys = { opt-level = 3 }
+mio = { opt-level = 3 }
+nalgebra = { opt-level = 3 }
+num-bigint = { opt-level = 3 }
+parking_lot = { opt-level = 3 }
+parking_lot_core = { opt-level = 3 }
+percent-encoding = { opt-level = 3 }
+primitive-types = { opt-level = 3 }
+ring = { opt-level = 3 }
+rustls = { opt-level = 3 }
+secp256k1 = { opt-level = 3 }
+sha2 = { opt-level = 3 }
+sha3 = { opt-level = 3 }
+smallvec = { opt-level = 3 }
+snow = { opt-level = 3 }
+twox-hash = { opt-level = 3 }
+uint = { opt-level = 3 }
+wasmi = { opt-level = 3 }
+x25519-dalek = { opt-level = 3 }
+yamux = { opt-level = 3 }
+zeroize = { opt-level = 3 }
+
+# make sure dev builds with backtrace do
+# not slow us down
+[profile.dev.package.backtrace]
+inherits = "release"
+
+[profile.production]
+inherits = "release"
+debug-assertions = false # Disable debug-assert! for production builds
+codegen-units = 1
+incremental = false
+lto = true
+
+[profile.release]
+debug-assertions = true # Enable debug-assert! for non-production profiles
+opt-level = 3
+# Moonbeam runtime requires unwinding.
+panic = "unwind"
+
+[profile.testnet]
+debug = 1               # debug symbols are useful for profilers
+debug-assertions = true # Enable debug-assert! for non-production profiles
+inherits = "release"
+overflow-checks = true

tools/.nvmrc

@@ -0,0 +1 @@
+v22

tools/github/generate-release-body.ts

@@ -0,0 +1,82 @@
+import { Octokit } from "octokit";
+import yargs from "yargs";
+import { getCommitAndLabels, getCompareLink } from "./github-utils";
+
+const BINARY_CHANGES_LABEL = "B5-clientnoteworthy";
+const BREAKING_CHANGES_LABEL = "breaking";
+
+function capitalize(s) {
+  return s[0].toUpperCase() + s.slice(1);
+}
+
+async function main() {
+  const argv = yargs(process.argv.slice(2))
+    .usage("Usage: npm run ts-node github/generate-release-body.ts [args]")
+    .version("1.0.0")
+    .options({
+      from: {
+        type: "string",
+        describe: "previous tag to retrieve commits from",
+        required: true,
+      },
+      to: {
+        type: "string",
+        describe: "current tag being drafted",
+        required: true,
+      },
+      owner: {
+        type: "string",
+        describe: "Repository owner (Ex: datahaven-xyz)",
+        required: true,
+      },
+      repo: {
+        type: "string",
+        describe: "Repository name (Ex: datahaven)",
+        required: true,
+      },
+    })
+    .demandOption(["from", "to"])
+    .help().argv;
+
+  const octokit = new Octokit({
+    auth: process.env.GITHUB_TOKEN || undefined,
+  });
+
+  const previousTag = argv.from;
+  const newTag = argv.to;
+  const moduleLinks = ["polkadot-sdk", "frontier"].map((repoName) => ({
+    name: repoName,
+    link: getCompareLink(repoName, previousTag, newTag),
+  }));
+
+  const { prByLabels } = await getCommitAndLabels(
+    octokit,
+    argv.owner,
+    argv.repo,
+    previousTag,
+    newTag
+  );
+  const filteredPr = prByLabels[BINARY_CHANGES_LABEL] || [];
+
+  const printPr = (pr) => {
+    if (pr.labels.includes(BREAKING_CHANGES_LABEL)) {
+      return "⚠️ " + pr.title + " (#" + pr.number + ")";
+    } else {
+      return pr.title + " (#" + pr.number + ")";
+    }
+  };
+
+  const template = `
+## Changes
+
+${filteredPr.map((pr) => `* ${printPr(pr)}`).join("\n")}
+
+## Dependency changes
+
+DataHaven: https://github.com/${argv.owner}/${argv.repo}/compare/${previousTag}...${newTag}
+${moduleLinks.map((modules) => `${capitalize(modules.name)}: ${modules.link}`).join("\n")}
+`;
+  console.log(template);
+}
+
+main();

tools/github/github-utils.ts

@@ -0,0 +1,111 @@
+import { Octokit } from "octokit";
+import { execSync } from "node:child_process";
+
+// Typescript 4 will support it natively, but not yet :(
+type Await<T> = T extends PromiseLike<infer U> ? U : T;
+type Commits = Await<ReturnType<Octokit["rest"]["repos"]["compareCommits"]>>["data"]["commits"];
+
+export function getCompareLink(packageName: string, previousTag: string, newTag: string) {
+  const previousPackage = execSync(
+    `git show ${previousTag}:../Cargo.lock | grep ${packageName}? | head -1 | grep -o '".*"'`
+  ).toString();
+  const previousCommit = /#([0-9a-f]*)/g.exec(previousPackage)[1].slice(0, 8);
+  const previousRepo = /(https:\/\/.*)\?/g.exec(previousPackage)[1];
+
+  const newPackage = execSync(
+    `git show ${newTag}:../operator/Cargo.lock | grep ${packageName}? | head -1 | grep -o '".*"'`
+  ).toString();
+  const newCommit = /#([0-9a-f]*)/g.exec(newPackage)[1].slice(0, 8);
+  const newRepo = /(https:\/\/.*)\?/g.exec(newPackage)[1];
+  const newRepoOrganization = /github.com\/([^\/]*)/g.exec(newRepo)[1];
+
+  const diffLink =
+    previousRepo !== newRepo
+      ? `${previousRepo}/compare/${previousCommit}...${newRepoOrganization}:${newCommit}`
+      : `${previousRepo}/compare/${previousCommit}...${newCommit}`;
+
+  return diffLink;
+}
+
+export async function getCommitAndLabels(
+  octokit: Octokit,
+  owner: string,
+  repo: string,
+  previousTag: string,
+  newTag: string
+): Promise<{ prByLabels: any; commits: any[] }> {
+  let commits: Commits = [];
+  let more = true;
+  let page = 0;
+  while (more) {
+    const compare = await octokit.rest.repos.compareCommitsWithBasehead({
+      owner,
+      repo,
+      basehead: previousTag + "..." + newTag,
+      per_page: 200,
+      page,
+    });
+    commits = commits.concat(compare.data.commits);
+    more = compare.data.commits.length === 200;
+    page++;
+  }
+
+  // Determine commits to exclude
+  // - commits reverted in the same range
+  const excludedCommits: number[] = [];
+  const revertedCommits: number[] = [];
+  for (let i = commits.length - 1; i >= 0; i--) {
+    const commitMessageFirstLine = commits[i].commit.message.split("\n")[0].trim();
+
+    if (revertedCommits[commitMessageFirstLine] != null) {
+      excludedCommits.push(i);
+      excludedCommits.push(revertedCommits[commitMessageFirstLine]);
+    } else {
+      const foundRevertedCommitName = commitMessageFirstLine.match(/Revert \"(.*)\"/);
+      if (foundRevertedCommitName?.length > 0) {
+        revertedCommits[foundRevertedCommitName[1]] = i;
+      }
+    }
+  }
+
+  const prByLabels = {};
+  for (let i = 0; i < commits.length; i++) {
+    const commitMessageFirstLine = commits[i].commit.message.split("\n")[0].trim();
+    if (!excludedCommits.includes(i)) {
+      const foundPrsNumbers = commitMessageFirstLine.match(/\(#([0-9]+)\)$/);
+      if (foundPrsNumbers && foundPrsNumbers.length > 1) {
+        // This will check current repo and if the PR is not found, will try the official repo
+        const repos = [
+          { owner, repo },
+          { owner: "datahaven-xyz", repo: "datahaven" },
+        ];
+        for (const { owner, repo } of repos) {
+          try {
+            const pr = await octokit.rest.pulls.get({
+              owner,
+              repo,
+              pull_number: parseInt(foundPrsNumbers[1]),
+            });
+
+            if (pr.data.labels && pr.data.labels.length > 0) {
+              for (const label of pr.data.labels) {
+                prByLabels[label.name] = prByLabels[label.name] || [];
+                prByLabels[label.name].push(pr.data);
+              }
+            } else {
+              prByLabels[""] = prByLabels[""] || [];
+              prByLabels[""].push(pr);
+            }
+            break;
+          } catch (e) {
+            // PR not found... let's try the other repo
+          }
+        }
+      }
+    }
+  }
+  return {
+    prByLabels,
+    commits,
+  };
+}

tools/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "datahaven-tools",
+  "version": "0.0.1",
+  "license": "GPL-3.0",
+  "dependencies": {
+    "octokit": "^1.0.6",
+    "ts-node": "^8.10.1",
+    "yargs": "^17.0.1"
+  },
+  "devDependencies": {
+    "@types/yargs": "^15.0.12"
+  }
+}

tools/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "compilerOptions": {
+    "allowSyntheticDefaultImports": true,
+    "esModuleInterop": true,
+    "target": "es2020",
+    "module": "commonjs"
+  },
+  "exclude": ["node_modules", "tests"]
+}