From d2bf185bcc51cccd921e86c83e3a0551a23aa62e Mon Sep 17 00:00:00 2001 From: Facundo Farall <37149322+ffarall@users.noreply.github.com> Date: Thu, 12 Jun 2025 05:24:03 -0300 Subject: [PATCH] =?UTF-8?q?feat:=20=F0=9F=9A=80=20Add=20`deploy`=20command?= =?UTF-8?q?=20to=20CLI=20(#87)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### New Features 1. Add the `deploy` command to our CLI. 1. Conditionally deploys kurtosis eth network if we're in `stagenet` environment. 2. Deploys DH nodes. 3. Deploys contracts (all of them). In `mainnet` and `testnet` it shouldn't deploy EL contracts, but for now that's not implemented. 4. Configures parameters, validators and relayers in the same way as `launch`. 5. Currently, it only deploys `beefy` and `beacon` relayers, `execution` and `solochain` relayers are pending for a subsequent PR. 2. Add `waitFor` utility function that receives a lambda. ### Refactors 1. Several common functionalities used both by the `launch` and `deploy` command have been moved to the `cli/handlers/common` directory, from where both commands use them. These include 1. Checks for installed dependencies. 2. Common constants. 3. The `LaunchedNetwork` class has been moved to this directory. 4. DataHaven nodes common functions. 5. Kurtosis common functions. 6. Relayer common functions. 7. Kubernetes functions (although only used by `deploy`, it seemed fitting to have it here still). 8. Remove CLI questions and separator prints from `deploy-contracts.ts` and `set-datahaven-parameters.ts` scripts. These things should be in the `cli/launch` folder, which consumes the functions in these scripts. 9. Remove `setParametersFromCollection` from `utils` folder and put it in `cli`. 10. Create base snowbridge relayer configs for `local` and `stagenet` as two separate directories. ### Fixes 1. Sets the default time of the `deploy` command to 6s as Lodestar is slower than Lighthouse. 2. In `runShellCommandWithLogger` only print `stderr` if the command fails. ### Additional Minor Changes 1. K8s secret key names changed from `dh-beefy-relay-eth-key` to `dh-beefy-relay-ethereum-key` and `dh-beacon-relay-sub-key` to `dh-beacon-relay-substrate-key`, for simplicity in the deployment script. 11. Update suggested configs for `.vscode` configs. --------- Co-authored-by: Steve Degosserie <723552+stiiifff@users.noreply.github.com> --- README.md | 6 +- deployment/README.md | 8 +- .../configs/beacon-relay.json | 52 +- .../configs/beefy-relay.json | 40 +- .../configs/execution-relay.json | 2 +- .../configs/solochain-relay.json | 49 ++ .../snowbridge/dh-beacon-relay.yaml | 19 +- .../snowbridge/dh-beefy-relay.yaml | 19 +- .../templates/deployment.yaml | 4 + test/cli/handlers/common/checks.ts | 113 +++++ test/cli/handlers/common/consts.ts | 44 ++ test/cli/handlers/common/datahaven.ts | 187 +++++++ test/cli/handlers/common/index.ts | 1 + test/cli/handlers/common/kubernetes.ts | 81 +++ test/cli/handlers/common/kurtosis.ts | 214 ++++++++ .../{launch => common}/launchedNetwork.ts | 88 +--- test/cli/handlers/common/relayer.ts | 324 ++++++++++++ test/cli/handlers/consts.ts | 13 - test/cli/handlers/deploy/cleanup.ts | 169 +++++++ test/cli/handlers/deploy/contracts.ts | 48 ++ test/cli/handlers/deploy/datahaven.ts | 176 +++++++ test/cli/handlers/deploy/index.ts | 113 +++++ test/cli/handlers/deploy/kurtosis.ts | 37 ++ test/cli/handlers/deploy/parameters.ts | 42 ++ test/cli/handlers/deploy/relayer.ts | 308 ++++++++++++ test/cli/handlers/deploy/validator.ts | 33 ++ test/cli/handlers/index.ts | 3 +- test/cli/handlers/launch/checks.ts | 59 --- test/cli/handlers/launch/contracts.ts | 66 +++ test/cli/handlers/launch/datahaven.ts | 220 +------- test/cli/handlers/launch/index.ts | 58 +-- test/cli/handlers/launch/kurtosis.ts | 116 +---- test/cli/handlers/launch/parameters.ts | 59 +++ test/cli/handlers/launch/relayer.ts | 473 +++++------------- test/cli/handlers/launch/summary.ts | 11 +- test/cli/handlers/stop/index.ts | 78 +-- test/cli/index.ts | 93 +++- test/configs/snowbridge/beefy-relay.json | 23 - test/configs/snowbridge/execution-relay.json | 40 -- .../snowbridge/{ => local}/beacon-relay.json | 2 +- .../configs/snowbridge/local/beefy-relay.json | 23 + .../snowbridge/local/execution-relay.json | 40 ++ .../snowbridge/local/solochain-relay.json | 49 ++ test/configs/snowbridge/solochain-relay.json | 49 -- .../snowbridge/stagenet/beacon-relay.json | 29 ++ .../snowbridge/stagenet/beefy-relay.json | 23 + .../snowbridge/stagenet/execution-relay.json | 41 ++ .../snowbridge/stagenet/solochain-relay.json | 49 ++ test/package.json | 4 +- test/scripts/deploy-contracts.ts | 126 ++--- test/scripts/set-datahaven-parameters.ts | 55 +- test/utils/kurtosis.ts | 2 +- test/utils/parameters.ts | 42 +- test/utils/parser.ts | 13 +- test/utils/shell.ts | 35 +- test/utils/waits.ts | 43 ++ 56 files changed, 2857 insertions(+), 1257 deletions(-) create mode 100644 deployment/charts/bridges-common-relay/configs/solochain-relay.json create mode 100644 test/cli/handlers/common/checks.ts create mode 100644 test/cli/handlers/common/consts.ts create mode 100644 test/cli/handlers/common/datahaven.ts create mode 100644 test/cli/handlers/common/index.ts create mode 100644 test/cli/handlers/common/kubernetes.ts create mode 100644 test/cli/handlers/common/kurtosis.ts rename test/cli/handlers/{launch => common}/launchedNetwork.ts (60%) create mode 100644 test/cli/handlers/common/relayer.ts delete mode 100644 test/cli/handlers/consts.ts create mode 100644 test/cli/handlers/deploy/cleanup.ts create mode 100644 test/cli/handlers/deploy/contracts.ts create mode 100644 test/cli/handlers/deploy/datahaven.ts create mode 100644 test/cli/handlers/deploy/index.ts create mode 100644 test/cli/handlers/deploy/kurtosis.ts create mode 100644 test/cli/handlers/deploy/parameters.ts create mode 100644 test/cli/handlers/deploy/relayer.ts create mode 100644 test/cli/handlers/deploy/validator.ts delete mode 100644 test/cli/handlers/launch/checks.ts create mode 100644 test/cli/handlers/launch/contracts.ts create mode 100644 test/cli/handlers/launch/parameters.ts delete mode 100644 test/configs/snowbridge/beefy-relay.json delete mode 100644 test/configs/snowbridge/execution-relay.json rename test/configs/snowbridge/{ => local}/beacon-relay.json (86%) create mode 100644 test/configs/snowbridge/local/beefy-relay.json create mode 100644 test/configs/snowbridge/local/execution-relay.json create mode 100644 test/configs/snowbridge/local/solochain-relay.json delete mode 100644 test/configs/snowbridge/solochain-relay.json create mode 100644 test/configs/snowbridge/stagenet/beacon-relay.json create mode 100644 test/configs/snowbridge/stagenet/beefy-relay.json create mode 100644 test/configs/snowbridge/stagenet/execution-relay.json create mode 100644 test/configs/snowbridge/stagenet/solochain-relay.json create mode 100644 test/utils/waits.ts diff --git a/README.md b/README.md index f48677c6..f9cc2717 100644 --- a/README.md +++ b/README.md @@ -101,11 +101,11 @@ These settings configure Solidity support: #### Typescript -This repo uses [Biome](https://github.com/biomejs/biome) for TypeScript linting and formatting. Bare in mind, that as of writing, it needs to be the pre-release version of the extension, that supports setting an inner folder as the project root. To make the extension work nicely with this repo, add the following to your `.vscode/settings.json` file: +This repo uses [Biome](https://github.com/biomejs/biome) for TypeScript linting and formatting. To make the extension work nicely with this repo, add the following to your `.vscode/settings.json` file: ```json { - "biome.projects": [{ "path": "test/" }], + "biome.lsp.bin": "test/node_modules/.bin/biome", "[typescript]": { "editor.defaultFormatter": "biomejs.biome", "editor.codeActionsOnSave": { @@ -115,7 +115,7 @@ This repo uses [Biome](https://github.com/biomejs/biome) for TypeScript linting } ``` -- Sets up Biome for JavaScript/TypeScript formatting in the test directory. +- Sets the Biome binary to the one in the `test/` folder. - Sets Biome as the default formatter for TypeScript. - Sets Biome to always organise imports on save. diff --git a/deployment/README.md b/deployment/README.md index aa22211f..7f29d755 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -36,9 +36,9 @@ kubectl delete pvc -l app.kubernetes.io/instance=dh-validator -n kt-datahaven-st ### Create secrets ```sh -kubectl create secret generic dh-beefy-relay-eth-key --from-literal=pvk="" -n kt-datahaven-stagenet -kubectl create secret generic dh-beacon-relay-sub-key --from-literal=pvk="" -n kt-datahaven-stagenet -kubectl create secret generic dh-execution-relay-sub-key --from-literal=pvk="" -n kt-datahaven-stagenet +kubectl create secret generic dh-beefy-relay-ethereum-key --from-literal=pvk="" -n kt-datahaven-stagenet +kubectl create secret generic dh-beacon-relay-substrate-key --from-literal=pvk="" -n kt-datahaven-stagenet +kubectl create secret generic dh-execution-relay-substrate-key --from-literal=pvk="" -n kt-datahaven-stagenet ``` ### Deploy @@ -57,7 +57,7 @@ helm uninstall dh-beefy-relay -n kt-datahaven-stagenet helm uninstall dh-execution-relay -n kt-datahaven-stagenet ``` -### Delete secrets +### Delete secrets ```sh kubectl delete secret -n kt-datahaven-stagenet diff --git a/deployment/charts/bridges-common-relay/configs/beacon-relay.json b/deployment/charts/bridges-common-relay/configs/beacon-relay.json index a298c388..f3bed46e 100644 --- a/deployment/charts/bridges-common-relay/configs/beacon-relay.json +++ b/deployment/charts/bridges-common-relay/configs/beacon-relay.json @@ -1,29 +1,29 @@ { - "source": { - "beacon": { - "endpoint": "http://cl-1-lodestar-reth:4000", - "stateEndpoint": "http://cl-1-lodestar-reth:4000", - "spec": { - "syncCommitteeSize": 512, - "slotsInEpoch": 32, - "epochsPerSyncCommitteePeriod": 256, - "forkVersions": { - "deneb": 0, - "electra": 0 + "source": { + "beacon": { + "endpoint": "http://cl-1-lodestar-reth:4000", + "stateEndpoint": "http://cl-1-lodestar-reth:4000", + "spec": { + "syncCommitteeSize": 512, + "slotsInEpoch": 32, + "epochsPerSyncCommitteePeriod": 256, + "forkVersions": { + "deneb": 0, + "electra": 0 + } + }, + "datastore": { + "location": "/data", + "maxEntries": 100 + } } - }, - "datastore": { - "location": "tmp/datastore", - "maxEntries": 100 - } - } - }, - "sink": { - "parachain": { - "endpoint": "ws://dh-validator-0:9944", - "maxWatchedExtrinsics": 8, - "headerRedundancy": 20 }, - "updateSlotInterval": 30 - } -} + "sink": { + "parachain": { + "endpoint": "ws://dh-validator-0:9944", + "maxWatchedExtrinsics": 8, + "headerRedundancy": 20 + }, + "updateSlotInterval": 30 + } +} \ No newline at end of file diff --git a/deployment/charts/bridges-common-relay/configs/beefy-relay.json b/deployment/charts/bridges-common-relay/configs/beefy-relay.json index 4fddf34c..2feeec15 100644 --- a/deployment/charts/bridges-common-relay/configs/beefy-relay.json +++ b/deployment/charts/bridges-common-relay/configs/beefy-relay.json @@ -1,23 +1,23 @@ { - "source": { - "polkadot": { - "endpoint": "ws://dh-validator-0:9944" - } - }, - "sink": { - "ethereum": { - "endpoint": "ws://el-1-reth-lodestar:8546", - "gas-limit": "" + "source": { + "polkadot": { + "endpoint": "ws://dh-validator-0:9944" + } }, - "descendants-until-final": 3, - "contracts": { - "BeefyClient": "0x4826533B4897376654Bb4d4AD88B7faFD0C98528", - "Gateway": "0x8f86403A4DE0BB5791fa46B8e795C547942fE4Cf" + "sink": { + "ethereum": { + "endpoint": "ws://el-1-reth-lodestar:8546", + "gas-limit": "" + }, + "descendants-until-final": 3, + "contracts": { + "BeefyClient": "0x4826533B4897376654Bb4d4AD88B7faFD0C98528", + "Gateway": "0x8f86403A4DE0BB5791fa46B8e795C547942fE4Cf" + } + }, + "on-demand-sync": { + "max-tokens": 5, + "refill-amount": 1, + "refill-period": 3600 } - }, - "on-demand-sync": { - "max-tokens": 5, - "refill-amount": 1, - "refill-period": 3600 - } -} +} \ No newline at end of file diff --git a/deployment/charts/bridges-common-relay/configs/execution-relay.json b/deployment/charts/bridges-common-relay/configs/execution-relay.json index 78194483..a712926d 100644 --- a/deployment/charts/bridges-common-relay/configs/execution-relay.json +++ b/deployment/charts/bridges-common-relay/configs/execution-relay.json @@ -34,7 +34,7 @@ }, "instantVerification": false, "schedule": { - "id": 1, + "id": null, "totalRelayerCount": 1, "sleepInterval": 1 } diff --git a/deployment/charts/bridges-common-relay/configs/solochain-relay.json b/deployment/charts/bridges-common-relay/configs/solochain-relay.json new file mode 100644 index 00000000..d8c0b8d4 --- /dev/null +++ b/deployment/charts/bridges-common-relay/configs/solochain-relay.json @@ -0,0 +1,49 @@ +{ + "source": { + "ethereum": { + "endpoint": "ws://el-1-reth-lodestar:8546" + }, + "solochain": { + "endpoint": "ws://dh-validator-0:9944" + }, + "contracts": { + "BeefyClient": "0x4826533B4897376654Bb4d4AD88B7faFD0C98528", + "Gateway": "0x8f86403A4DE0BB5791fa46B8e795C547942fE4Cf" + }, + "beacon": { + "endpoint": "http://cl-1-lodestar-reth:4000", + "stateEndpoint": "http://cl-1-lodestar-reth:4000", + "spec": { + "syncCommitteeSize": 512, + "slotsInEpoch": 32, + "epochsPerSyncCommitteePeriod": 256, + "forkVersions": { + "deneb": 0, + "electra": 0 + } + }, + "datastore": { + "location": "/data", + "maxEntries": 100 + } + } + }, + "sink": { + "contracts": { + "Gateway": "0x8f86403A4DE0BB5791fa46B8e795C547942fE4Cf" + }, + "ethereum": { + "endpoint": "ws://el-1-reth-lodestar:8546" + } + }, + "schedule": { + "id": 0, + "totalRelayerCount": 1, + "sleepInterval": 10 + }, + "reward-address": "0xd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d", + "ofac": { + "enabled": false, + "apiKey": "" + } +} diff --git a/deployment/charts/bridges-common-relay/snowbridge/dh-beacon-relay.yaml b/deployment/charts/bridges-common-relay/snowbridge/dh-beacon-relay.yaml index ba16d640..482e48c9 100644 --- a/deployment/charts/bridges-common-relay/snowbridge/dh-beacon-relay.yaml +++ b/deployment/charts/bridges-common-relay/snowbridge/dh-beacon-relay.yaml @@ -13,13 +13,14 @@ imagePullSecrets: config: name: "beacon-relay" -existingSecretName: "dh-beacon-relay-sub-key" +existingSecretName: "dh-beacon-relay-substrate-key" -extraArgs: [ - "run", - "beacon", - "--config", - "/configs/beacon-relay.json", - "--substrate.private-key-file", - "/secrets/dh-beacon-relay-sub-key", -] +extraArgs: + [ + "run", + "beacon", + "--config", + "/configs/beacon-relay.json", + "--substrate.private-key-file", + "/secrets/dh-beacon-relay-substrate-key", + ] diff --git a/deployment/charts/bridges-common-relay/snowbridge/dh-beefy-relay.yaml b/deployment/charts/bridges-common-relay/snowbridge/dh-beefy-relay.yaml index bc65553c..202be03a 100644 --- a/deployment/charts/bridges-common-relay/snowbridge/dh-beefy-relay.yaml +++ b/deployment/charts/bridges-common-relay/snowbridge/dh-beefy-relay.yaml @@ -13,13 +13,14 @@ imagePullSecrets: config: name: "beefy-relay" -existingSecretName: "dh-beefy-relay-eth-key" +existingSecretName: "dh-beefy-relay-ethereum-key" -extraArgs: [ - "run", - "beefy", - "--config", - "/configs/beefy-relay.json", - "--ethereum.private-key-file", - "/secrets/dh-beefy-relay-eth-key", -] +extraArgs: + [ + "run", + "beefy", + "--config", + "/configs/beefy-relay.json", + "--ethereum.private-key-file", + "/secrets/dh-beefy-relay-ethereum-key", + ] diff --git a/deployment/charts/bridges-common-relay/templates/deployment.yaml b/deployment/charts/bridges-common-relay/templates/deployment.yaml index 5e0e1dc5..bf3b173b 100644 --- a/deployment/charts/bridges-common-relay/templates/deployment.yaml +++ b/deployment/charts/bridges-common-relay/templates/deployment.yaml @@ -26,6 +26,10 @@ spec: serviceAccountName: {{ include "bridges-common-relay.serviceAccountName" . }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- if .Values.initContainers }} + initContainers: + {{- toYaml .Values.initContainers | nindent 8 }} + {{- end }} containers: - name: {{ .Chart.Name }} securityContext: diff --git a/test/cli/handlers/common/checks.ts b/test/cli/handlers/common/checks.ts new file mode 100644 index 00000000..0976f5c7 --- /dev/null +++ b/test/cli/handlers/common/checks.ts @@ -0,0 +1,113 @@ +import { $ } from "bun"; +import invariant from "tiny-invariant"; +import { logger, printDivider, printHeader } from "utils"; +import type { DeployOptions } from "../deploy"; +import type { LaunchedNetwork } from "./launchedNetwork"; + +// ===== Checks ===== +export const checkBaseDependencies = async (): Promise => { + printHeader("Base Dependencies Checks"); + + if (!(await checkKurtosisInstalled())) { + logger.error("Kurtosis CLI is required to be installed: https://docs.kurtosis.com/install"); + throw Error("❌ Kurtosis CLI application not found."); + } + + logger.success("Kurtosis CLI found"); + + if (!(await checkDockerRunning())) { + logger.error("Is Docker Running? Unable to make connection to docker daemon"); + throw Error("❌ Error connecting to Docker"); + } + + logger.success("Docker is running"); + + if (!(await checkForgeInstalled())) { + logger.error("Is foundry installed? https://book.getfoundry.sh/getting-started/installation"); + throw Error("❌ Forge binary not found in PATH"); + } + + logger.success("Forge is installed"); + + printDivider(); +}; + +export const deploymentChecks = async ( + options: DeployOptions, + launchedNetwork: LaunchedNetwork +): Promise => { + printHeader("Deploy Dependencies Checks"); + + if (!(await checkHelmInstalled())) { + logger.error("Is Helm installed? https://helm.sh/docs/intro/install/"); + throw Error("❌ Helm binary not found in PATH"); + } + + logger.success("Helm is installed"); + + switch (options.environment) { + case "stagenet": + launchedNetwork.kubeNamespace = `kt-${options.kurtosisEnclaveName}`; + break; + case "testnet": + case "mainnet": + launchedNetwork.kubeNamespace = options.kubeNamespace ?? `datahaven-${options.environment}`; + + invariant( + options.elRpcUrl !== undefined, + "❌ --el-rpc-url is required in testnet environment" + ); + invariant( + options.clEndpoint !== undefined, + "❌ --cl-endpoint is required in testnet environment" + ); + launchedNetwork.elRpcUrl = options.elRpcUrl; + launchedNetwork.clEndpoint = options.clEndpoint; + + break; + } + + logger.info(`ℹ️ Deploying to Kubernetes namespace: ${launchedNetwork.kubeNamespace}`); + + printDivider(); +}; + +const checkKurtosisInstalled = async (): Promise => { + const { exitCode, stderr, stdout } = await $`kurtosis version`.nothrow().quiet(); + if (exitCode !== 0) { + logger.error(stderr.toString()); + return false; + } + logger.debug(stdout.toString()); + return true; +}; + +const checkDockerRunning = async (): Promise => { + const { exitCode, stderr, stdout } = await $`docker system info`.nothrow().quiet(); + if (exitCode !== 0) { + logger.error(stderr.toString()); + return false; + } + logger.debug(stdout.toString()); + return true; +}; + +const checkForgeInstalled = async (): Promise => { + const { exitCode, stderr, stdout } = await $`forge --version`.nothrow().quiet(); + if (exitCode !== 0) { + logger.error(stderr.toString()); + return false; + } + logger.debug(stdout.toString()); + return true; +}; + +const checkHelmInstalled = async (): Promise => { + const { exitCode, stderr, stdout } = await $`helm version`.nothrow().quiet(); + if (exitCode !== 0) { + logger.error(stderr.toString()); + return false; + } + logger.debug(stdout.toString()); + return true; +}; diff --git a/test/cli/handlers/common/consts.ts b/test/cli/handlers/common/consts.ts new file mode 100644 index 00000000..29f32146 --- /dev/null +++ b/test/cli/handlers/common/consts.ts @@ -0,0 +1,44 @@ +export const ZERO_HASH = "0x0000000000000000000000000000000000000000000000000000000000000000"; + +/** + * The name of the Docker network that DataHaven nodes and + * Snowbridge relayers will be connected to, in a local deployment. + */ +export const DOCKER_NETWORK_NAME = "datahaven-net"; + +/** + * 33-byte compressed public keys for DataHaven next validator set + * These correspond to Alice & Bob + * These are the fallback keys if we can't fetch the next authorities directly from the network + */ +export const FALLBACK_DATAHAVEN_AUTHORITY_PUBLIC_KEYS: Record = { + alice: "0x020a1091341fe5664bfa1782d5e04779689068c916b04cb365ec3153755684d9a1", + bob: "0x0390084fdbf27d2b79d26a4f13f0ccd982cb755a661969143c37cbc49ef5b91f27" +} as const; + +/** + * The components (Docker containers) that can be launched and stopped. + */ +export const COMPONENTS = { + datahaven: { + imageName: "moonsonglabs/datahaven", + componentName: "Datahaven Network", + optionName: "datahaven" + }, + snowbridge: { + imageName: "snowbridge-relay", + componentName: "Snowbridge Relayers", + optionName: "relayer" + } +} as const; + +/** + * The base services that are always launched when Kurtosis is used. + */ +export const BASE_SERVICES = [ + "cl-1-lodestar-reth", + "cl-2-lodestar-reth", + "el-1-reth-lodestar", + "el-2-reth-lodestar", + "dora" +]; diff --git a/test/cli/handlers/common/datahaven.ts b/test/cli/handlers/common/datahaven.ts new file mode 100644 index 00000000..a669989c --- /dev/null +++ b/test/cli/handlers/common/datahaven.ts @@ -0,0 +1,187 @@ +import fs from "node:fs"; +import path from "node:path"; +import { secp256k1 } from "@noble/curves/secp256k1"; +import { createClient, type PolkadotClient } from "polkadot-api"; +import { withPolkadotSdkCompat } from "polkadot-api/polkadot-sdk-compat"; +import { getWsProvider } from "polkadot-api/ws-provider/web"; +import invariant from "tiny-invariant"; +import { createPapiConnectors, logger } from "utils"; +import { type Hex, keccak256, toHex } from "viem"; +import { publicKeyToAddress } from "viem/accounts"; +import type { LaunchedNetwork } from "./launchedNetwork"; + +/** + * Checks if the DataHaven network is ready by sending a POST request to the system_chain method. + * + * @param port - The port number to check. + * @param timeoutMs - The timeout in milliseconds for the attempt to connect to the network. + * @returns True if the network is ready, false otherwise. + */ +export const isNetworkReady = async (port: number, timeoutMs: number): Promise => { + const wsUrl = `ws://127.0.0.1:${port}`; + let client: PolkadotClient | undefined; + + // Temporarily capture and suppress error logs during connection attempts. + // This is to avoid the "Unable to connect to ws:" error logs from the `client._request` call. + const originalConsoleError = console.error; + console.error = () => {}; + + try { + // Use withPolkadotSdkCompat for consistency, though _request might not strictly need it. + client = createClient(withPolkadotSdkCompat(getWsProvider(wsUrl))); + + // Add timeout to the RPC call to prevent hanging. + const chainNamePromise = client._request("system_chain", []); + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => reject(new Error("RPC call timeout")), timeoutMs); + }); + + const chainName = await Promise.race([chainNamePromise, timeoutPromise]); + logger.debug(`isNetworkReady PAPI check successful for port ${port}, chain: ${chainName}`); + client.destroy(); + return !!chainName; // Ensure it's a boolean and chainName is truthy + } catch (error) { + logger.debug(`isNetworkReady PAPI check failed for port ${port}: ${error}`); + if (client) { + client.destroy(); + } + return false; + } finally { + // Restore original console methods. + console.error = originalConsoleError; + } +}; + +/** + * Converts a compressed secp256k1 public key to an Ethereum address. + * + * This function takes a compressed public key (33 bytes), decompresses it to get the full + * uncompressed public key (64 bytes of x and y coordinates), and then derives the + * corresponding Ethereum address using the standard Ethereum address derivation algorithm. + * + * @param compressedPubKey - The compressed public key as a hex string (with or without "0x" prefix) + * @returns The corresponding Ethereum address (checksummed, with "0x" prefix) + * + * @throws {Error} If the provided public key is invalid or cannot be decompressed + */ +export const compressedPubKeyToEthereumAddress = (compressedPubKey: string): string => { + // Ensure the input is a hex string and remove "0x" prefix + const compressedKeyHex = compressedPubKey.startsWith("0x") + ? compressedPubKey.substring(2) + : compressedPubKey; + + // Decompress the public key + const point = secp256k1.ProjectivePoint.fromHex(compressedKeyHex); + // toRawBytes(false) returns the uncompressed key (64 bytes, x and y coordinates) + const uncompressedPubKeyBytes = point.toRawBytes(false); + const uncompressedPubKeyHex = toHex(uncompressedPubKeyBytes); // Prefixes with "0x" + + // Compute the Ethereum address from the uncompressed public key + // publicKeyToAddress expects a 0x-prefixed hex string representing the 64-byte uncompressed public key + const address = publicKeyToAddress(uncompressedPubKeyHex); + return address; +}; + +/** + * Prepares the configuration for DataHaven authorities by fetching their BEEFY public keys, + * converting them to Ethereum addresses, and updating the network configuration file. + * + * This function performs the following steps: + * 1. Connects to the first available DataHaven node matching the container prefix + * 2. Fetches the BEEFY NextAuthorities from the node's runtime + * 3. Converts each compressed public key to an Ethereum address + * 4. Computes the keccak256 hash of each address (authority hash) + * 5. Updates the network configuration file with the authority hashes + * + * The configuration is saved to `../contracts/config/{NETWORK}.json` where NETWORK + * defaults to "anvil" if not specified in environment variables. + * + * @param launchedNetwork - The launched network instance containing container information + * @param containerNamePrefix - The prefix to filter DataHaven containers by (e.g., "datahaven-", "dh-validator-") + * + * @throws {Error} If no DataHaven nodes are found in the launched network + * @throws {Error} If BEEFY authorities cannot be fetched from the node + * @throws {Error} If public key conversion fails + * @throws {Error} If the configuration file cannot be read or written + */ +export const setupDataHavenValidatorConfig = async ( + launchedNetwork: LaunchedNetwork, + containerNamePrefix: string +): Promise => { + const networkName = process.env.NETWORK || "anvil"; + logger.info(`🔧 Preparing DataHaven authorities configuration for network: ${networkName}...`); + + let authorityPublicKeys: string[] = []; + const dhNodes = launchedNetwork.containers.filter((x) => x.name.startsWith(containerNamePrefix)); + + invariant(dhNodes.length > 0, "No DataHaven nodes found in launchedNetwork"); + + const firstNode = dhNodes[0]; + const wsUrl = `ws://127.0.0.1:${firstNode.publicPorts.ws}`; + const { client: papiClient, typedApi: dhApi } = createPapiConnectors(wsUrl); + + logger.info( + `📡 Attempting to fetch BEEFY next authorities from node ${firstNode.name} (port ${firstNode.publicPorts.ws})...` + ); + + // Fetch NextAuthorities + // Beefy.NextAuthorities returns a fixed-length array of bytes representing the authority public keys + const nextAuthoritiesRaw = await dhApi.query.Beefy.NextAuthorities.getValue({ at: "best" }); + + invariant(nextAuthoritiesRaw && nextAuthoritiesRaw.length > 0, "No BEEFY next authorities found"); + + authorityPublicKeys = nextAuthoritiesRaw.map((key) => key.asHex()); // .asHex() returns the hex string representation of the corresponding key + logger.success( + `Successfully fetched ${authorityPublicKeys.length} BEEFY next authorities directly.` + ); + + // Clean up PAPI client, otherwise it will hang around and prevent this process from exiting. + papiClient.destroy(); + + const authorityHashes: string[] = []; + for (const compressedKey of authorityPublicKeys) { + try { + const ethAddress = compressedPubKeyToEthereumAddress(compressedKey); + const authorityHash = keccak256(ethAddress as Hex); + authorityHashes.push(authorityHash); + logger.debug( + `Processed public key ${compressedKey} -> ETH address ${ethAddress} -> Authority hash ${authorityHash}` + ); + } catch (error) { + logger.error(`❌ Failed to process public key ${compressedKey}: ${error}`); + throw new Error(`Failed to process public key ${compressedKey}`); + } + } + + // process.cwd() is 'test/', so config is at '../contracts/config' + const configDir = path.join(process.cwd(), "../contracts/config"); + const configFilePath = path.join(configDir, `${networkName}.json`); + + try { + if (!fs.existsSync(configFilePath)) { + logger.warn( + `⚠️ Configuration file ${configFilePath} not found. Skipping update of validator sets.` + ); + // Optionally, create a default structure if it makes sense, or simply return. + // For now, if the base network config doesn't exist, we can't update it. + return; + } + + const configFileContent = fs.readFileSync(configFilePath, "utf-8"); + const configJson = JSON.parse(configFileContent); + + if (!configJson.snowbridge) { + logger.warn(`⚠️ "snowbridge" section not found in ${configFilePath}, creating it.`); + configJson.snowbridge = {}; + } + + configJson.snowbridge.initialValidators = authorityHashes; + configJson.snowbridge.nextValidators = authorityHashes; + + fs.writeFileSync(configFilePath, JSON.stringify(configJson, null, 2)); + logger.success(`DataHaven authority hashes updated in: ${configFilePath}`); + } catch (error) { + logger.error(`❌ Failed to read or update ${configFilePath}: ${error}`); + throw new Error(`Failed to update authority hashes in ${configFilePath}.`); + } +}; diff --git a/test/cli/handlers/common/index.ts b/test/cli/handlers/common/index.ts new file mode 100644 index 00000000..8f84876c --- /dev/null +++ b/test/cli/handlers/common/index.ts @@ -0,0 +1 @@ +export * from "./checks"; diff --git a/test/cli/handlers/common/kubernetes.ts b/test/cli/handlers/common/kubernetes.ts new file mode 100644 index 00000000..d1f3a8af --- /dev/null +++ b/test/cli/handlers/common/kubernetes.ts @@ -0,0 +1,81 @@ +import { logger } from "utils"; +import type { LaunchedNetwork } from "./launchedNetwork"; + +/** + * Forwards a port from a Kubernetes service to localhost and returns a cleanup function. + * + * @param serviceName - The name of the Kubernetes service to forward from + * @param localPort - The local port to bind to + * @param kubePort - The Kubernetes service port to forward from + * @param launchedNetwork - The launched network instance containing namespace info + * @param options - Optional configuration + * @returns Promise<{ cleanup: () => Promise }> - Object containing cleanup function + */ +export const forwardPort = async ( + serviceName: string, + localPort: number, + kubePort: number, + launchedNetwork: LaunchedNetwork +): Promise<{ cleanup: () => Promise }> => { + logger.info( + `🔗 Setting up port forward: localhost:${localPort} -> svc/dh-validator-0:${kubePort}` + ); + + // Start kubectl port-forward as a background process using Bun.spawn + const portForwardProcess = Bun.spawn( + [ + "kubectl", + "port-forward", + `svc/${serviceName}`, + "-n", + launchedNetwork.kubeNamespace, + `${localPort}:${kubePort}` + ], + { + stdout: "pipe", + stderr: "pipe" + } + ); + + // Check if the process is still running (didn't exit due to error) + if (portForwardProcess.exitCode !== null) { + const stderr = await new Response(portForwardProcess.stderr).text(); + throw new Error(`Port forward failed to start: ${stderr}`); + } + + logger.success( + `Port forward established: localhost:${localPort} -> svc/dh-validator-0:${kubePort}` + ); + + // Return cleanup function + const cleanup = async (): Promise => { + logger.info(`🧹 Cleaning up port forward for localhost:${localPort}`); + + if (!portForwardProcess.killed) { + portForwardProcess.kill(); + + // Wait for process to actually exit + try { + await portForwardProcess.exited; + } catch (error) { + // Process was killed, this is expected + logger.debug(`Port forward process killed: ${error}`); + } + } + + logger.success(`Port forward cleanup completed for localhost:${localPort}`); + }; + + // Add a cleanup handler that doesn't interfere with exit codes + const exitHandler = () => { + if (!portForwardProcess.killed) { + portForwardProcess.kill(); + } + }; + + process.on("exit", exitHandler); + process.on("SIGINT", exitHandler); + process.on("SIGTERM", exitHandler); + + return { cleanup }; +}; diff --git a/test/cli/handlers/common/kurtosis.ts b/test/cli/handlers/common/kurtosis.ts new file mode 100644 index 00000000..4b9dae87 --- /dev/null +++ b/test/cli/handlers/common/kurtosis.ts @@ -0,0 +1,214 @@ +import { $ } from "bun"; +import invariant from "tiny-invariant"; +import { + getPortFromKurtosis, + type KurtosisEnclaveInfo, + KurtosisEnclaveInfoSchema, + logger +} from "utils"; +import { parse, stringify } from "yaml"; +import type { LaunchedNetwork } from "./launchedNetwork"; + +/** + * Checks if a Kurtosis enclave with the specified name is currently running. + * + * @param enclaveName - The name of the Kurtosis enclave to check + * @returns True if the enclave is running, false otherwise + */ +export const checkKurtosisEnclaveRunning = async (enclaveName: string): Promise => { + const enclaves = await getRunningKurtosisEnclaves(); + return enclaves.some((enclave) => enclave.name === enclaveName); +}; + +/** + * Gets a list of currently running Kurtosis enclaves + * @returns Promise - Array of running enclave information + */ +export const getRunningKurtosisEnclaves = async (): Promise => { + logger.debug("🔎 Checking for running Kurtosis enclaves..."); + + const lines = (await Array.fromAsync($`kurtosis enclave ls`.lines())).filter( + (line) => line.length > 0 + ); + logger.trace(lines); + + // Remove header line + lines.shift(); + + const enclaves: KurtosisEnclaveInfo[] = []; + + if (lines.length === 0) { + logger.debug("🤷‍ No Kurtosis enclaves found running."); + return enclaves; + } + + logger.debug(`🔎 Found ${lines.length} Kurtosis enclave(s) running.`); + // Updated regex to match the actual format: "uuid name status creationTime" + const enclaveRegex = /^(\S+)\s+(\S+)\s+(\S+)\s+(.+)$/; + + for (const line of lines) { + const match = line.match(enclaveRegex); + if (match) { + const [, uuid, name, status, creationTime] = match; + const parseResult = KurtosisEnclaveInfoSchema.safeParse({ + uuid: uuid.trim(), + name: name.trim(), + status: status.trim(), + creationTime: creationTime.trim() + }); + + if (parseResult.success) { + enclaves.push(parseResult.data); + } else { + logger.warn( + `⚠️ Could not parse enclave line: "${line}". Error: ${parseResult.error.message}` + ); + } + } else { + logger.warn(`⚠️ Could not parse enclave line (regex mismatch): "${line}"`); + } + } + + if (lines.length > 0 && enclaves.length === 0) { + logger.warn("⚠️ Found enclave lines in output, but failed to parse any of them."); + } + + return enclaves; +}; + +/** + * Modifies a Kurtosis configuration file based on deployment options. + * + * This function reads a YAML configuration file, applies modifications based on the provided + * deployment options, and writes the modified configuration to a new file in the tmp/configs directory. + * + * @param options.blockscout - If true, adds "blockscout" to the additional_services array + * @param options.slotTime - If provided, sets the network_params.seconds_per_slot value + * @param options.kurtosisNetworkArgs - Space-separated key=value pairs to add to network_params + * @param configFile - Path to the original YAML configuration file to modify + * @returns Promise - Path to the modified configuration file in tmp/configs/ + * @throws Will throw an error if the config file is not found + */ +export const modifyConfig = async ( + options: { + blockscout?: boolean; + slotTime?: number; + kurtosisNetworkArgs?: string; + }, + configFile: string +) => { + const outputDir = "tmp/configs"; + logger.debug(`Ensuring output directory exists: ${outputDir}`); + await $`mkdir -p ${outputDir}`.quiet(); + + const file = Bun.file(configFile); + invariant(file, `❌ Config file ${configFile} not found`); + + const config = await file.text(); + logger.debug(`Parsing config at ${configFile}`); + logger.trace(config); + + const parsedConfig = parse(config); + + if (options.blockscout) { + parsedConfig.additional_services.push("blockscout"); + } + + if (options.slotTime) { + parsedConfig.network_params.seconds_per_slot = options.slotTime; + } + + if (options.kurtosisNetworkArgs) { + logger.debug(`Using custom Kurtosis network args: ${options.kurtosisNetworkArgs}`); + const args = options.kurtosisNetworkArgs.split(" "); + for (const arg of args) { + const [key, value] = arg.split("="); + parsedConfig.network_params[key] = value; + } + } + + logger.trace(parsedConfig); + const outputFile = `${outputDir}/modified-config.yaml`; + logger.debug(`Modified config saving to ${outputFile}`); + + await Bun.write(outputFile, stringify(parsedConfig)); + return outputFile; +}; + +/** + * Registers the Execution Layer (EL) and Consensus Layer (CL) service endpoints with the LaunchedNetwork instance. + * + * This function retrieves the public ports for the Ethereum network services from Kurtosis and configures + * the LaunchedNetwork instance with the appropriate RPC URLs and endpoints for client communication. + * + * Services registered: + * - Execution Layer (EL): Reth RPC endpoint via "el-1-reth-lodestar" service + * - Consensus Layer (CL): lodestar HTTP endpoint via "cl-1-lodestar-reth" service + * + * @param launchedNetwork - The LaunchedNetwork instance to populate with service endpoints + * @param enclaveName - The name of the Kurtosis enclave containing the services + * @throws Will log warnings if services cannot be found or ports cannot be determined, but won't fail + */ +export const registerServices = async (launchedNetwork: LaunchedNetwork, enclaveName: string) => { + logger.info("📝 Registering Kurtosis service endpoints..."); + + // Configure EL RPC URL + try { + const rethPublicPort = await getPortFromKurtosis("el-1-reth-lodestar", "rpc", enclaveName); + invariant(rethPublicPort && rethPublicPort > 0, "❌ Could not find EL RPC port"); + const elRpcUrl = `http://127.0.0.1:${rethPublicPort}`; + launchedNetwork.elRpcUrl = elRpcUrl; + logger.info(`📝 Execution Layer RPC URL configured: ${elRpcUrl}`); + + // Configure CL Endpoint + const lodestarPublicPort = await getPortFromKurtosis("cl-1-lodestar-reth", "http", enclaveName); + const clEndpoint = `http://127.0.0.1:${lodestarPublicPort}`; + invariant( + clEndpoint, + "❌ CL Endpoint could not be determined from Kurtosis service cl-1-lodestar-reth" + ); + launchedNetwork.clEndpoint = clEndpoint; + logger.info(`📝 Consensus Layer Endpoint configured: ${clEndpoint}`); + } catch (error) { + logger.warn(`⚠️ Kurtosis service endpoints could not be determined: ${error}`); + } +}; + +/** + * Runs a Kurtosis Ethereum network enclave with the specified configuration. + * + * This function handles the complete process of starting a Kurtosis enclave: + * 1. Modifies the configuration file based on the provided options + * 2. Executes the kurtosis run command with the modified configuration + * 3. Handles error cases and logs appropriate debug information + * + * @param options - Configuration options containing kurtosisEnclaveName and other settings + * @param configFilePath - Path to the base YAML configuration file to use + * @throws Will throw an error if the Kurtosis network fails to start properly + */ +export const runKurtosisEnclave = async ( + options: { + kurtosisEnclaveName: string; + blockscout?: boolean; + slotTime?: number; + kurtosisNetworkArgs?: string; + }, + configFilePath: string +): Promise => { + logger.info("🚀 Starting Kurtosis enclave..."); + + const configFile = await modifyConfig(options, configFilePath); + + logger.info(`⚙️ Using Kurtosis config file: ${configFile}`); + + const { stderr, stdout, exitCode } = + await $`kurtosis run github.com/ethpandaops/ethereum-package --args-file ${configFile} --enclave ${options.kurtosisEnclaveName}` + .nothrow() + .quiet(); + + if (exitCode !== 0) { + logger.error(stderr.toString()); + throw Error("❌ Kurtosis network has failed to start properly."); + } + logger.debug(stdout.toString()); +}; diff --git a/test/cli/handlers/launch/launchedNetwork.ts b/test/cli/handlers/common/launchedNetwork.ts similarity index 60% rename from test/cli/handlers/launch/launchedNetwork.ts rename to test/cli/handlers/common/launchedNetwork.ts index 3c062d11..90d95989 100644 --- a/test/cli/handlers/launch/launchedNetwork.ts +++ b/test/cli/handlers/common/launchedNetwork.ts @@ -1,9 +1,6 @@ -import fs from "node:fs"; import invariant from "tiny-invariant"; import { logger, type RelayerType } from "utils"; -type PipeOptions = number | "inherit" | "pipe" | "ignore"; -type BunProcess = Bun.Subprocess; type ContainerSpec = { name: string; publicPorts: Record }; /** @@ -12,28 +9,24 @@ type ContainerSpec = { name: string; publicPorts: Record }; */ export class LaunchedNetwork { protected runId: string; - protected processes: BunProcess[]; protected _containers: ContainerSpec[]; - protected fileDescriptors: number[]; protected _networkName: string; protected _activeRelayers: RelayerType[]; /** The RPC URL for the Ethereum Execution Layer (EL) client. */ protected _elRpcUrl?: string; /** The HTTP endpoint for the Ethereum Consensus Layer (CL) client. */ protected _clEndpoint?: string; - /** The RPC URL for the DataHaven node. */ - protected _dhRpcUrl?: string; + /** The Kubernetes namespace for the network. Used only for deploy commands. */ + protected _kubeNamespace?: string; constructor() { this.runId = crypto.randomUUID(); - this.processes = []; - this.fileDescriptors = []; this._containers = []; this._activeRelayers = []; this._networkName = ""; this._elRpcUrl = undefined; this._clEndpoint = undefined; - this._dhRpcUrl = undefined; + this._kubeNamespace = undefined; } public set networkName(name: string) { @@ -67,22 +60,6 @@ export class LaunchedNetwork { return container.publicPorts.ws ?? -1; } - /** - * Adds a file descriptor to be managed and cleaned up. - * @param fd - The file descriptor number. - */ - addFileDescriptor(fd: number) { - this.fileDescriptors.push(fd); - } - - /** - * Adds a running process to be managed and cleaned up. - * @param process - The Bun subprocess object. - */ - addProcess(process: BunProcess) { - this.processes.push(process); - } - addContainer(containerName: string, publicPorts: Record = {}) { this._containers.push({ name: containerName, publicPorts }); } @@ -96,38 +73,6 @@ export class LaunchedNetwork { return port; } - /** - * Updates the DataHaven RPC URL based on the current container public port - * This should be called after DataHaven containers are added to the network - */ - public updateDhRpcUrl(): void { - const port = this.getPublicWsPort(); - this._dhRpcUrl = `ws://127.0.0.1:${port}`; - logger.debug(`DataHaven RPC URL set to ${this._dhRpcUrl}`); - } - - /** - * Sets the RPC URL for the DataHaven node. - * @param url - The DataHaven RPC URL string. - */ - public set dhRpcUrl(url: string) { - this._dhRpcUrl = url; - } - - /** - * Gets the RPC URL for the DataHaven node. - * @returns The DataHaven RPC URL string. - * @throws If the DataHaven RPC URL has not been set. - */ - public get dhRpcUrl(): string { - if (!this._dhRpcUrl) { - // Try to generate the URL if not set - this.updateDhRpcUrl(); - } - invariant(this._dhRpcUrl, "❌ DataHaven RPC URL not set in LaunchedNetwork"); - return this._dhRpcUrl; - } - /** * Sets the RPC URL for the Ethereum Execution Layer (EL) client. * @param url - The EL RPC URL string. @@ -164,12 +109,6 @@ export class LaunchedNetwork { return this._clEndpoint; } - registerRelayerType(type: RelayerType): void { - if (!this._activeRelayers.includes(type)) { - this._activeRelayers.push(type); - } - } - public get containers(): ContainerSpec[] { return this._containers; } @@ -178,21 +117,12 @@ export class LaunchedNetwork { return [...this._activeRelayers]; } - async cleanup() { - logger.debug("Running cleanup"); - for (const process of this.processes) { - logger.debug(`Process is still running: ${process.pid}`); - process.unref(); - } + public set kubeNamespace(namespace: string) { + this._kubeNamespace = namespace; + } - for (const fd of this.fileDescriptors) { - try { - fs.closeSync(fd); - this.fileDescriptors = this.fileDescriptors.filter((x) => x !== fd); - logger.debug(`Closed file descriptor ${fd}`); - } catch (error) { - logger.error(`Error closing file descriptor ${fd}: ${error}`); - } - } + public get kubeNamespace(): string { + invariant(this._kubeNamespace, "❌ Kubernetes namespace not set in LaunchedNetwork"); + return this._kubeNamespace; } } diff --git a/test/cli/handlers/common/relayer.ts b/test/cli/handlers/common/relayer.ts new file mode 100644 index 00000000..91d7c0dc --- /dev/null +++ b/test/cli/handlers/common/relayer.ts @@ -0,0 +1,324 @@ +import path from "node:path"; +import { datahaven } from "@polkadot-api/descriptors"; +import { $ } from "bun"; +import { createClient } from "polkadot-api"; +import { withPolkadotSdkCompat } from "polkadot-api/polkadot-sdk-compat"; +import { getWsProvider } from "polkadot-api/ws-provider/web"; +import invariant from "tiny-invariant"; +import { getEvmEcdsaSigner, logger, parseRelayConfig, SUBSTRATE_FUNDED_ACCOUNTS } from "utils"; +import type { BeaconCheckpoint, FinalityCheckpointsResponse } from "utils/types"; +import { parseJsonToBeaconCheckpoint } from "utils/types"; +import { waitFor } from "utils/waits"; +import { ZERO_HASH } from "./consts"; +import type { LaunchedNetwork } from "./launchedNetwork"; + +export type BeaconConfig = { + type: "beacon"; + ethClEndpoint: string; + substrateWsEndpoint: string; +}; + +export type BeefyConfig = { + type: "beefy"; + ethElRpcEndpoint: string; + substrateWsEndpoint: string; + beefyClientAddress: string; + gatewayAddress: string; +}; + +export type ExecutionConfig = { + type: "execution"; + ethElRpcEndpoint: string; + ethClEndpoint: string; + substrateWsEndpoint: string; + gatewayAddress: string; +}; + +export type SolochainConfig = { + type: "solochain"; + ethElRpcEndpoint: string; + substrateWsEndpoint: string; + beefyClientAddress: string; + gatewayAddress: string; + ethClEndpoint: string; +}; + +export type RelayerConfigType = BeaconConfig | BeefyConfig | ExecutionConfig | SolochainConfig; + +export type RelayerSpec = { + name: string; + configFilePath: string; + templateFilePath?: string; + config: RelayerConfigType; + pk: { ethereum?: string; substrate?: string }; +}; + +export const INITIAL_CHECKPOINT_FILE = "dump-initial-checkpoint.json"; +export const INITIAL_CHECKPOINT_DIR = "tmp/beacon-checkpoint"; +export const INITIAL_CHECKPOINT_PATH = path.join(INITIAL_CHECKPOINT_DIR, INITIAL_CHECKPOINT_FILE); + +/** + * Generates configuration files for relayers. + * + * @param relayerSpec - The relayer specification containing name, type, and config path. + * @param environment - The environment to use for template files (e.g., "local", "stagenet", "testnet", "mainnet"). + * @param configDir - The directory where config files should be written. + */ +export const generateRelayerConfig = async ( + relayerSpec: RelayerSpec, + environment: string, + configDir: string +) => { + const { name, configFilePath, templateFilePath: _templateFilePath, config } = relayerSpec; + const { type } = config; + const configFileName = path.basename(configFilePath); + + logger.debug(`Creating config for ${name}`); + const templateFilePath = + _templateFilePath ?? `configs/snowbridge/${environment}/${configFileName}`; + const outputFilePath = path.resolve(configDir, configFileName); + logger.debug(`Reading config file ${templateFilePath}`); + const file = Bun.file(templateFilePath); + + if (!(await file.exists())) { + logger.error(`File ${templateFilePath} does not exist`); + throw new Error("Error reading snowbridge config file"); + } + const json = await file.json(); + + logger.debug(`Generating ${type} relayer configuration for ${name}`); + + switch (type) { + case "beacon": { + const cfg = parseRelayConfig(json, type); + cfg.source.beacon.endpoint = config.ethClEndpoint; + cfg.source.beacon.stateEndpoint = config.ethClEndpoint; + cfg.source.beacon.datastore.location = "/data"; + cfg.sink.parachain.endpoint = config.substrateWsEndpoint; + + await Bun.write(outputFilePath, JSON.stringify(cfg, null, 4)); + logger.success(`Updated beacon config written to ${outputFilePath}`); + break; + } + case "beefy": { + const cfg = parseRelayConfig(json, type); + cfg.source.polkadot.endpoint = config.substrateWsEndpoint; + cfg.sink.ethereum.endpoint = config.ethElRpcEndpoint; + cfg.sink.contracts.BeefyClient = config.beefyClientAddress; + cfg.sink.contracts.Gateway = config.gatewayAddress; + + await Bun.write(outputFilePath, JSON.stringify(cfg, null, 4)); + logger.success(`Updated beefy config written to ${outputFilePath}`); + break; + } + case "execution": { + const cfg = parseRelayConfig(json, type); + cfg.source.ethereum.endpoint = config.ethElRpcEndpoint; + cfg.source.beacon.endpoint = config.ethClEndpoint; + cfg.source.beacon.stateEndpoint = config.ethClEndpoint; + cfg.source.beacon.datastore.location = "/data"; + cfg.sink.parachain.endpoint = config.substrateWsEndpoint; + cfg.source.contracts.Gateway = config.gatewayAddress; + + await Bun.write(outputFilePath, JSON.stringify(cfg, null, 4)); + logger.success(`Updated execution config written to ${outputFilePath}`); + break; + } + case "solochain": { + const cfg = parseRelayConfig(json, type); + cfg.source.ethereum.endpoint = config.ethElRpcEndpoint; + cfg.source.solochain.endpoint = config.substrateWsEndpoint; + cfg.source.contracts.BeefyClient = config.beefyClientAddress; + cfg.source.contracts.Gateway = config.gatewayAddress; + cfg.source.beacon.endpoint = config.ethClEndpoint; + cfg.source.beacon.stateEndpoint = config.ethClEndpoint; + cfg.source.beacon.datastore.location = "/data"; + cfg.sink.ethereum.endpoint = config.ethElRpcEndpoint; + cfg.sink.contracts.Gateway = config.gatewayAddress; + + await Bun.write(outputFilePath, JSON.stringify(cfg, null, 4)); + logger.success(`Updated solochain config written to ${outputFilePath}`); + break; + } + default: + throw new Error(`Unsupported relayer type with config: \n${JSON.stringify(config)}`); + } +}; + +/** + * Waits for the beacon chain to be ready by polling its finality checkpoints. + * + * @param launchedNetwork - An instance of LaunchedNetwork to get the CL endpoint. + * @param pollIntervalMs - The interval in milliseconds to poll the beacon chain. + * @param timeoutMs - The total time in milliseconds to wait before timing out. + * @throws Error if the beacon chain is not ready within the timeout. + */ +export const waitBeaconChainReady = async ( + launchedNetwork: LaunchedNetwork, + pollIntervalMs: number, + timeoutMs: number +) => { + const iterations = Math.floor(timeoutMs / pollIntervalMs); + + logger.trace("Waiting for beacon chain to be ready..."); + + await waitFor({ + lambda: async () => { + try { + const response = await fetch( + `${launchedNetwork.clEndpoint}/eth/v1/beacon/states/head/finality_checkpoints` + ); + if (!response.ok) { + throw new Error(`HTTP error! Status: ${response.status}`); + } + + const data = (await response.json()) as FinalityCheckpointsResponse; + logger.debug(`Beacon chain state: ${JSON.stringify(data)}`); + + invariant(data.data, "❌ No data returned from beacon chain"); + invariant(data.data.finalized, "❌ No finalised block returned from beacon chain"); + invariant( + data.data.finalized.root, + "❌ No finalised block root returned from beacon chain" + ); + + const initialBeaconBlock = data.data.finalized.root; + + if (initialBeaconBlock && initialBeaconBlock !== ZERO_HASH) { + logger.info(`⏲️ Beacon chain is ready with finalised block: ${initialBeaconBlock}`); + return true; + } + + logger.info(`⌛️ Retrying beacon chain state fetch in ${pollIntervalMs / 1000}s...`); + return false; + } catch (error) { + logger.error(`Failed to fetch beacon chain state: ${error}`); + return false; + } + }, + iterations, + delay: pollIntervalMs, + errorMessage: "Beacon chain is not ready. Relayers cannot be launched." + }); +}; + +/** + * Initialises the Ethereum Beacon Client pallet on the Substrate chain. + * It waits for the beacon chain to be ready, generates an initial checkpoint, + * and submits this checkpoint to the Substrate runtime via a sudo call. + * + * @param beaconConfigHostPath - The host path to the beacon configuration file. + * @param relayerImageTag - The Docker image tag for the relayer. + * @param datastorePath - The path to the datastore directory. + * @param launchedNetwork - An instance of LaunchedNetwork to interact with the running network. + * @throws If there's an error generating the beacon checkpoint or submitting it to Substrate. + */ +export const initEthClientPallet = async ( + beaconConfigHostPath: string, + relayerImageTag: string, + datastorePath: string, + launchedNetwork: LaunchedNetwork +) => { + logger.debug("Initialising eth client pallet"); + // Poll the beacon chain until it's ready every 10 seconds for 10 minutes + await waitBeaconChainReady(launchedNetwork, 10000, 600000); + + const beaconConfigContainerPath = "/app/beacon-relay.json"; + const checkpointHostPath = path.resolve(INITIAL_CHECKPOINT_PATH); + const checkpointContainerPath = `/app/${INITIAL_CHECKPOINT_FILE}`; + + logger.debug("Generating beacon checkpoint"); + // Pre-create the checkpoint file so that Docker doesn't interpret it as a directory + await Bun.write(INITIAL_CHECKPOINT_PATH, ""); + + logger.debug("Removing 'generate-beacon-checkpoint' container if it exists"); + logger.debug(await $`docker rm -f generate-beacon-checkpoint`.text()); + + // When running in Linux, `host.docker.internal` is not pre-defined when running in a container. + // So we need to add the parameter `--add-host host.docker.internal:host-gateway` to the command. + // In Mac this is not needed and could cause issues. + const addHostParam = + process.platform === "linux" ? "--add-host host.docker.internal:host-gateway" : ""; + + logger.debug("Generating beacon checkpoint"); + const datastoreHostPath = path.resolve(datastorePath); + const command = `docker run \ + -v ${beaconConfigHostPath}:${beaconConfigContainerPath}:ro \ + -v ${checkpointHostPath}:${checkpointContainerPath} \ + -v ${datastoreHostPath}:/data \ + --name generate-beacon-checkpoint \ + --platform linux/amd64 \ + --pull always \ + --workdir /app \ + ${addHostParam} \ + ${launchedNetwork.networkName ? `--network ${launchedNetwork.networkName}` : ""} \ + ${relayerImageTag} \ + generate-beacon-checkpoint --config beacon-relay.json --export-json`; + logger.debug(`Running command: ${command}`); + logger.debug(await $`sh -c "${command}"`.text()); + + // Load the checkpoint into a JSON object and clean it up + const initialCheckpointFile = Bun.file(INITIAL_CHECKPOINT_PATH); + const initialCheckpointRaw = await initialCheckpointFile.text(); + const initialCheckpoint = parseJsonToBeaconCheckpoint(JSON.parse(initialCheckpointRaw)); + await initialCheckpointFile.delete(); + + logger.trace("Initial checkpoint:"); + logger.trace(initialCheckpoint.toJSON()); + + // Send the checkpoint to the Substrate runtime + const substrateRpcUrl = `http://127.0.0.1:${launchedNetwork.getPublicWsPort()}`; + await sendCheckpointToSubstrate(substrateRpcUrl, initialCheckpoint); + logger.success("Ethereum Beacon Client pallet initialised"); +}; + +/** + * Sends the beacon checkpoint to the Substrate runtime, waiting for the transaction to be finalised and successful. + * + * @param networkRpcUrl - The RPC URL of the Substrate network. + * @param checkpoint - The beacon checkpoint to send. + * @throws If the transaction signing fails, it becomes an invalid transaction, or the transaction is included but fails. + */ +const sendCheckpointToSubstrate = async (networkRpcUrl: string, checkpoint: BeaconCheckpoint) => { + logger.trace("Sending checkpoint to Substrate..."); + + const client = createClient(withPolkadotSdkCompat(getWsProvider(networkRpcUrl))); + const dhApi = client.getTypedApi(datahaven); + + logger.trace("Client created"); + + const signer = getEvmEcdsaSigner(SUBSTRATE_FUNDED_ACCOUNTS.ALITH.privateKey); + logger.trace("Signer created"); + + const forceCheckpointCall = dhApi.tx.EthereumBeaconClient.force_checkpoint({ + update: checkpoint + }); + + logger.debug("Force checkpoint call:"); + logger.debug(forceCheckpointCall.decodedCall); + + const tx = dhApi.tx.Sudo.sudo({ + call: forceCheckpointCall.decodedCall + }); + + logger.debug("Sudo call:"); + logger.debug(tx.decodedCall); + + try { + const txFinalisedPayload = await tx.signAndSubmit(signer); + + if (!txFinalisedPayload.ok) { + throw new Error("❌ Beacon checkpoint transaction failed"); + } + + logger.info( + `📪 "force_checkpoint" transaction with hash ${txFinalisedPayload.txHash} submitted successfully and finalised in block ${txFinalisedPayload.block.hash}` + ); + } catch (error) { + logger.error(`Failed to submit checkpoint transaction: ${error}`); + throw new Error(`Failed to submit checkpoint: ${error}`); + } finally { + client.destroy(); + logger.debug("Destroyed client"); + } +}; diff --git a/test/cli/handlers/consts.ts b/test/cli/handlers/consts.ts deleted file mode 100644 index 30ea2fe9..00000000 --- a/test/cli/handlers/consts.ts +++ /dev/null @@ -1,13 +0,0 @@ -export const DOCKER_NETWORK_NAME = "datahaven-net"; -export const COMPONENTS = { - datahaven: { - imageName: "moonsonglabs/datahaven", - componentName: "Datahaven Network", - optionName: "datahaven" - }, - snowbridge: { - imageName: "snowbridge-relay", - componentName: "Snowbridge Relayers", - optionName: "relayer" - } -} as const; diff --git a/test/cli/handlers/deploy/cleanup.ts b/test/cli/handlers/deploy/cleanup.ts new file mode 100644 index 00000000..39cfbc20 --- /dev/null +++ b/test/cli/handlers/deploy/cleanup.ts @@ -0,0 +1,169 @@ +import { $ } from "bun"; +import invariant from "tiny-invariant"; +import { logger, printDivider, printHeader } from "utils"; +import { waitFor } from "utils/waits"; +import { checkKurtosisEnclaveRunning } from "../common/kurtosis"; +import type { LaunchedNetwork } from "../common/launchedNetwork"; +import type { DeployOptions } from "."; + +export const cleanup = async ( + options: DeployOptions, + launchedNetwork: LaunchedNetwork +): Promise => { + printHeader("Cleaning up"); + + if (options.skipCleanup) { + logger.info("🏳️ Skipping cleanup"); + printDivider(); + return; + } + + if (options.environment === "stagenet") { + await checkAndCleanKurtosisDeployment(options); + } + + await checkAndCleanHelmReleases(launchedNetwork); + + printDivider(); +}; + +/** + * Checks for existing Kurtosis deployment and removes the specified enclave if found. + * + * This function performs a cleanup operation before deployment by: + * 1. Verifying that the Kurtosis gateway process is running (required for Kubernetes integration) + * 2. Listing all running Kurtosis enclaves + * 3. Checking if the specified enclave exists + * 4. Removing the enclave if found to ensure a clean deployment environment + * + * The function ensures that any existing Kurtosis enclave with the same name is properly + * cleaned up before starting a new deployment, preventing conflicts and stale resources. + * + * @param options - Deployment configuration options + * @param options.kurtosisEnclaveName - The name of the Kurtosis enclave to check for and remove. + * Must be defined in the options object. + * + * @returns Promise - Resolves when all cleanup operations are complete + * + * @throws {Error} Throws if: + * - The Kurtosis gateway process is not running (required for Kubernetes integration) + * - Kurtosis commands fail (e.g., Kurtosis not installed, insufficient permissions) + * - Network connectivity issues prevent Kurtosis API access + */ +const checkAndCleanKurtosisDeployment = async (options: DeployOptions): Promise => { + logger.info("☸️ Checking for existing Kurtosis deployment in Kubernetes..."); + + // Check if the Kurtosis gateway process is running. + const { exitCode, stdout } = await $`pgrep -f "kurtosis gateway"`.nothrow().quiet(); + if (exitCode !== 0) { + logger.error( + "❌ `kurtosis gateway` process not found running. This is required for Kurtosis to work with Kubernetes." + ); + throw new Error("Kurtosis gateway process not found running."); + } + logger.debug(`Kurtosis gateway process found running: ${stdout}`); + + // Check if Kurtosis enclave is running. + if (await checkKurtosisEnclaveRunning(options.kurtosisEnclaveName)) { + logger.info(`🔎 Found Kurtosis enclave ${options.kurtosisEnclaveName} running.`); + } else { + logger.info(`🤷‍ No Kurtosis enclave ${options.kurtosisEnclaveName} found running.`); + return; + } + + logger.info("🪓 Removing Kurtosis enclave..."); + logger.debug(await $`kurtosis enclave rm ${options.kurtosisEnclaveName} -f`.text()); + + // Wait for the underlying Kubernetes namespace to be fully deleted + const kubernetesNamespace = `kt-${options.kurtosisEnclaveName}`; + await waitForNamespaceDeletion(kubernetesNamespace); + + logger.success(`Kurtosis enclave ${options.kurtosisEnclaveName} removed successfully.`); +}; + +/** + * Checks for existing DataHaven Helm releases in the specified Kubernetes namespace and removes them. + * + * This function performs a cleanup operation before deployment by: + * 1. Listing all Helm releases in the target namespace + * 2. Identifying any existing DataHaven releases + * 3. Uninstalling each release individually + * 4. Logging the progress and results of each operation + * + * The function ensures a clean deployment environment by removing any conflicting + * or stale Helm releases that might interfere with the new deployment. + * + * @param options - Deployment configuration options + * @param options.kubeNamespace - The Kubernetes namespace to check for Helm releases. + * Must be defined or the function will throw an error. + * + * @returns Promise - Resolves when all cleanup operations are complete + * + * @throws {Error} Throws if: + * - The kubeNamespace is not defined in options + * - Helm commands fail (e.g., Helm not installed, insufficient permissions) + * - Network connectivity issues prevent Kubernetes API access + */ +const checkAndCleanHelmReleases = async (launchedNetwork: LaunchedNetwork): Promise => { + logger.info("☸️ Checking for existing DataHaven Helm releases in Kubernetes..."); + + invariant(launchedNetwork.kubeNamespace, "❌ Kubernetes namespace not defined"); + + try { + const releaseListOutput = await $`helm list -q -n ${launchedNetwork.kubeNamespace}`.text(); + const releases = releaseListOutput + .trim() + .split("\n") + .filter((r) => r.length > 0); + + if (releases.length > 0) { + logger.info( + `🔎 Found existing DataHaven Helm releases: ${releases.join(", ")}. Uninstalling...` + ); + for (const release of releases) { + logger.info(`🪓 Uninstalling Helm release: ${release} in namespace datahaven...`); + await $`helm uninstall ${release} -n ${launchedNetwork.kubeNamespace}`.text(); + logger.success(`Helm release ${release} uninstalled successfully.`); + } + } else { + logger.info("👍 No existing DataHaven Helm releases found in namespace datahaven."); + } + } catch (error) { + logger.error( + `❌ Failed to check or clean Kubernetes Helm releases: ${error}. This may be expected if Helm is not installed or not configured. Proceeding...` + ); + + throw error; + } +}; + +/** + * Waits for a Kubernetes namespace to be fully deleted. + * This is necessary because namespace deletion in Kubernetes is asynchronous + * and Kurtosis may fail to create a new enclave if the namespace is still being deleted. + * + * @param namespaceName - The name of the Kubernetes namespace to wait for deletion + * @returns Promise - Resolves when the namespace is fully deleted or doesn't exist + */ +const waitForNamespaceDeletion = async (namespaceName: string): Promise => { + logger.info(`⌛️ Waiting for Kubernetes namespace ${namespaceName} to be fully deleted...`); + + await waitFor({ + lambda: async () => { + try { + const { exitCode } = await $`kubectl get namespace ${namespaceName}`.nothrow().quiet(); + // If kubectl get namespace returns non-zero exit code, the namespace doesn't exist + return exitCode !== 0; + } catch (error) { + // If kubectl command fails, assume namespace is deleted or kubectl is not available + logger.debug(`kubectl command failed: ${error}. Assuming namespace is deleted.`); + return true; + } + }, + iterations: 120, // Wait up to 2 minutes + delay: 1000, // 1 second between checks + errorMessage: "Kubernetes namespace not deleted" + }); + + logger.success(`Kubernetes namespace ${namespaceName} fully deleted.`); +}; diff --git a/test/cli/handlers/deploy/contracts.ts b/test/cli/handlers/deploy/contracts.ts new file mode 100644 index 00000000..62f613ff --- /dev/null +++ b/test/cli/handlers/deploy/contracts.ts @@ -0,0 +1,48 @@ +import { + buildContracts, + constructDeployCommand, + executeDeployment, + validateDeploymentParams +} from "scripts/deploy-contracts"; +import { logger, printDivider, printHeader } from "utils"; +import type { ParameterCollection } from "utils/parameters"; + +interface DeployContractsOptions { + rpcUrl: string; + verified?: boolean; + blockscoutBackendUrl?: string; + parameterCollection?: ParameterCollection; + skipContracts: boolean; +} + +/** + * Deploys smart contracts to the specified RPC URL + * + * @param options - Configuration options for deployment + * @param options.rpcUrl - The RPC URL to deploy to + * @param options.verified - Whether to verify contracts (requires blockscoutBackendUrl) + * @param options.blockscoutBackendUrl - URL for the Blockscout API (required if verified is true) + * @param options.parameterCollection - Collection of parameters to update in the DataHaven runtime + * @returns Promise resolving to true if contracts were deployed successfully, false if skipped + */ +export const deployContracts = async (options: DeployContractsOptions) => { + printHeader("Deploying Smart Contracts"); + + if (options.skipContracts) { + logger.info("🏳️ Skipping contract deployment"); + printDivider(); + return; + } + + // Check if required parameters are provided + validateDeploymentParams(options); + + // Build contracts + await buildContracts(); + + // Construct and execute deployment + const deployCommand = constructDeployCommand(options); + await executeDeployment(deployCommand, options.parameterCollection); + + printDivider(); +}; diff --git a/test/cli/handlers/deploy/datahaven.ts b/test/cli/handlers/deploy/datahaven.ts new file mode 100644 index 00000000..77cd0986 --- /dev/null +++ b/test/cli/handlers/deploy/datahaven.ts @@ -0,0 +1,176 @@ +import path from "node:path"; +import { $ } from "bun"; +import invariant from "tiny-invariant"; +import { logger, printDivider, printHeader } from "utils"; +import { waitFor } from "utils/waits"; +import { isNetworkReady, setupDataHavenValidatorConfig } from "../common/datahaven"; +import { forwardPort } from "../common/kubernetes"; +import type { LaunchedNetwork } from "../common/launchedNetwork"; +import type { DeployOptions } from "."; + +const DEFAULT_PUBLIC_WS_PORT = 9944; + +/** + * Deploys a DataHaven solochain network in a Kubernetes namespace. + * + * @param options - Configuration options for launching the network. + * @param launchedNetwork - An instance of LaunchedNetwork to track the network's state. + * @returns A promise that resolves to a cleanup function for the validator port forwarding. + */ +export const deployDataHavenSolochain = async ( + options: DeployOptions, + launchedNetwork: LaunchedNetwork +): Promise<() => Promise> => { + if (options.skipDatahavenSolochain) { + logger.info("🏳️ Skipping DataHaven deployment"); + + // Forward port from validator to localhost, to interact with the network. + const { cleanup: validatorPortForwardCleanup } = await forwardPort( + "dh-validator-0", + DEFAULT_PUBLIC_WS_PORT, + DEFAULT_PUBLIC_WS_PORT, + launchedNetwork + ); + + await registerNodes(launchedNetwork); + await setupDataHavenValidatorConfig(launchedNetwork, "dh-validator-"); + + printDivider(); + return validatorPortForwardCleanup; + } + + printHeader("Deploying DataHaven Network"); + + invariant(options.datahavenImageTag, "❌ DataHaven image tag not defined"); + + await checkTagExists(options.datahavenImageTag); + + await checkOrCreateKubernetesNamespace(launchedNetwork.kubeNamespace); + + // Create secret for Docker Hub credentials, if they were provided. + if (options.dockerUsername && options.dockerPassword && options.dockerEmail) { + logger.info("🔐 Creating Docker Hub secret..."); + logger.debug( + await $`kubectl create secret docker-registry datahaven-dockerhub \ + --docker-username=${options.dockerUsername} \ + --docker-password=${options.dockerPassword} \ + --docker-email=${options.dockerEmail} \ + -n ${launchedNetwork.kubeNamespace}`.text() + ); + logger.success("Docker Hub secret created successfully"); + } + + // Deploy DataHaven bootnode and validators with helm chart. + logger.info("🚀 Deploying DataHaven bootnode with helm chart..."); + const bootnodeTimeout = "5m"; // 5 minutes + logger.debug( + await $`helm upgrade --install dh-bootnode . -f ./datahaven/dh-bootnode.yaml \ + -n ${launchedNetwork.kubeNamespace} \ + --wait \ + --timeout ${bootnodeTimeout}` + .cwd(path.join(process.cwd(), "../deployment/charts/node")) + .text() + ); + logger.success("DataHaven bootnode deployed successfully"); + + logger.info("🚀 Deploying DataHaven validators with helm chart..."); + const validatorTimeout = "5m"; // 5 minutes + logger.debug( + await $`helm upgrade --install dh-validator . -f ./datahaven/dh-validator.yaml \ + -n ${launchedNetwork.kubeNamespace} \ + --wait \ + --timeout ${validatorTimeout}` + .cwd(path.join(process.cwd(), "../deployment/charts/node")) + .text() + ); + logger.success("DataHaven validators deployed successfully"); + + // Forward port from validator to localhost, to interact with the network. + const { cleanup: validatorPortForwardCleanup } = await forwardPort( + "dh-validator-0", + DEFAULT_PUBLIC_WS_PORT, + DEFAULT_PUBLIC_WS_PORT, + launchedNetwork + ); + + // Wait for the network to start. + logger.info("⌛️ Waiting for DataHaven to start..."); + const timeoutMs = 5000; // 5 second timeout + const delayMs = 5000; // 5 second delay between iterations + await waitFor({ + lambda: async () => { + logger.info(`📡 Checking if DataHaven is ready (timeout: ${timeoutMs / 1000}s)...`); + const isReady = await isNetworkReady(DEFAULT_PUBLIC_WS_PORT, timeoutMs); + if (!isReady) { + logger.info(`⌛️ Node not ready, waiting ${delayMs / 1000}s to check again...`); + } + return isReady; + }, + iterations: 12, // 12 iterations of 5 + 5 = 2 minutes + delay: delayMs, // 5 second delay between iterations + errorMessage: "DataHaven network not ready" + }); + + logger.success( + `DataHaven network started, primary node accessible on port ${DEFAULT_PUBLIC_WS_PORT}` + ); + + await registerNodes(launchedNetwork); + await setupDataHavenValidatorConfig(launchedNetwork, "dh-validator-"); + + printDivider(); + return validatorPortForwardCleanup; +}; + +/** + * Checks if an image exists in Docker Hub. + * + * @param tag - The tag of the image to check. + * @returns A promise that resolves when the image is found. + */ +const checkTagExists = async (tag: string) => { + const cleanTag = tag.trim(); + logger.debug(`Checking if image ${cleanTag} is available on Docker Hub`); + const result = await $`docker manifest inspect ${cleanTag}`.nothrow().quiet(); + invariant( + result.exitCode === 0, + `❌ Image ${tag} not found.\n Does this image exist?\n Are you logged and have access to the repository?` + ); + + logger.success(`Image ${cleanTag} found on Docker Hub`); +}; + +/** + * Checks if a Kubernetes namespace exists and creates it if it doesn't. + * + * @param namespace - The name of the namespace to check or create. + * @returns A promise that resolves when the namespace exists or has been created. + */ +const checkOrCreateKubernetesNamespace = async (namespace: string) => { + logger.info(`🔍 Checking if Kubernetes namespace "${namespace}" exists...`); + + // Check if namespace exists + const checkResult = await $`kubectl get namespace ${namespace}`.nothrow().quiet(); + + if (checkResult.exitCode === 0) { + logger.success(`Namespace "${namespace}" already exists`); + return; + } + + logger.info(`📦 Creating Kubernetes namespace "${namespace}"...`); + const createResult = await $`kubectl create namespace ${namespace}`.nothrow(); + + if (createResult.exitCode !== 0) { + throw new Error(`Failed to create namespace "${namespace}": ${createResult.stderr}`); + } + + logger.success(`Successfully created namespace "${namespace}"`); +}; + +const registerNodes = async (launchedNetwork: LaunchedNetwork) => { + // Register the validator node, using the standard host WS port that we just forwarded. + launchedNetwork.addContainer("dh-validator-0", { + ws: DEFAULT_PUBLIC_WS_PORT + }); + logger.info("📝 Node dh-validator-0 successfully registered in launchedNetwork."); +}; diff --git a/test/cli/handlers/deploy/index.ts b/test/cli/handlers/deploy/index.ts new file mode 100644 index 00000000..b2e79b7e --- /dev/null +++ b/test/cli/handlers/deploy/index.ts @@ -0,0 +1,113 @@ +import type { Command } from "node_modules/@commander-js/extra-typings"; +import { type DeployEnvironment, logger } from "utils"; +import { createParameterCollection } from "utils/parameters"; +import { checkBaseDependencies, deploymentChecks } from "../common/checks"; +import { LaunchedNetwork } from "../common/launchedNetwork"; +import { cleanup } from "./cleanup"; +import { deployContracts } from "./contracts"; +import { deployDataHavenSolochain } from "./datahaven"; +import { deployKurtosis } from "./kurtosis"; +import { setParametersFromCollection } from "./parameters"; +import { deployRelayers } from "./relayer"; +import { performValidatorOperations } from "./validator"; + +// Non-optional properties determined by having default values +export interface DeployOptions { + environment: DeployEnvironment; + kubeNamespace?: string; + kurtosisEnclaveName: string; + slotTime: number; + kurtosisNetworkArgs?: string; + verified?: boolean; + blockscout?: boolean; + datahavenImageTag: string; + elRpcUrl?: string; + clEndpoint?: string; + relayerImageTag: string; + // TODO: This shouldn't be necessary once the repo is public + dockerUsername?: string; + // TODO: This shouldn't be necessary once the repo is public + dockerPassword?: string; + // TODO: This shouldn't be necessary once the repo is public + dockerEmail?: string; + skipCleanup: boolean; + skipKurtosis: boolean; + skipDatahavenSolochain: boolean; + skipContracts: boolean; + skipValidatorOperations: boolean; + skipSetParameters: boolean; + skipRelayers: boolean; +} + +const deployFunction = async (options: DeployOptions, launchedNetwork: LaunchedNetwork) => { + logger.debug("Running with options:"); + logger.debug(options); + + const timeStart = performance.now(); + + await checkBaseDependencies(); + await deploymentChecks(options, launchedNetwork); + + await cleanup(options, launchedNetwork); + + // Create parameter collection to be used throughout the launch process + const parameterCollection = await createParameterCollection(); + + await deployKurtosis(options, launchedNetwork); + + // Inside the deployDataHavenSolochain function, it will forward the port from the validator to the local machine. + // This is to allow the rest of the script to interact with the network. + // The cleanup function is returned to allow the script to clean up the port forwarding. + const validatorPortForwardCleanup = await deployDataHavenSolochain(options, launchedNetwork); + + // TODO: Handle Blockscout and verifier parameters to verify contracts if that is the intention. + const blockscoutBackendUrl = undefined; + + await deployContracts({ + rpcUrl: launchedNetwork.elRpcUrl, + verified: options.verified, + blockscoutBackendUrl, + parameterCollection, + skipContracts: options.skipContracts + }); + + await performValidatorOperations(options, launchedNetwork.elRpcUrl); + + await setParametersFromCollection({ + collection: parameterCollection, + skipSetParameters: options.skipSetParameters + }); + + await deployRelayers(options, launchedNetwork); + + // Cleaning up the port forwarding for the validator. + await validatorPortForwardCleanup(); + + const fullEnd = performance.now(); + const fullMinutes = ((fullEnd - timeStart) / (1000 * 60)).toFixed(1); + logger.success(`Deploy function completed successfully in ${fullMinutes} minutes`); +}; + +export const deploy = async (options: DeployOptions) => { + const run = new LaunchedNetwork(); + await deployFunction(options, run); +}; + +export const deployPreActionHook = ( + thisCmd: Command<[], DeployOptions & { [key: string]: any }> +) => { + const opts = thisCmd.opts(); + if (opts.verified && !opts.blockscout) { + thisCmd.error("--verified requires --blockscout to be set"); + } + + if (opts.environment === "stagenet" && opts.kubeNamespace !== undefined) { + logger.warn( + "⚠️ --kube-namespace is not allowed in stagenet environment. The Kurtosis namespace will be used instead." + ); + } + + if (opts.environment !== "stagenet" && opts.elRpcUrl === undefined) { + thisCmd.error("--eth-rpc-url is required in non-stagenet environment"); + } +}; diff --git a/test/cli/handlers/deploy/kurtosis.ts b/test/cli/handlers/deploy/kurtosis.ts new file mode 100644 index 00000000..84bbeba4 --- /dev/null +++ b/test/cli/handlers/deploy/kurtosis.ts @@ -0,0 +1,37 @@ +import type { DeployOptions } from "cli/handlers"; +import invariant from "tiny-invariant"; +import { logger, printDivider, printHeader } from "utils"; +import { registerServices, runKurtosisEnclave } from "../common/kurtosis"; +import type { LaunchedNetwork } from "../common/launchedNetwork"; + +/** + * Deploys a Kurtosis Ethereum network enclave for stagenet environment. + * + * @param options - Configuration options + * @param launchedNetwork - The LaunchedNetwork instance to store network details + */ +export const deployKurtosis = async ( + options: DeployOptions, + launchedNetwork: LaunchedNetwork +): Promise => { + if (options.skipKurtosis) { + logger.info("🏳️ Skipping Kurtosis deployment"); + + await registerServices(launchedNetwork, options.kurtosisEnclaveName); + printDivider(); + return; + } + + printHeader("Deploying Kurtosis Ethereum Network"); + + invariant( + options.environment === "stagenet", + "❌ Kurtosis should only be used in stagenet environment" + ); + + await runKurtosisEnclave(options, "configs/kurtosis/minimal.yaml"); + + await registerServices(launchedNetwork, options.kurtosisEnclaveName); + logger.success("Kurtosis network operations completed successfully."); + printDivider(); +}; diff --git a/test/cli/handlers/deploy/parameters.ts b/test/cli/handlers/deploy/parameters.ts new file mode 100644 index 00000000..a0aea647 --- /dev/null +++ b/test/cli/handlers/deploy/parameters.ts @@ -0,0 +1,42 @@ +import { setDataHavenParameters } from "scripts/set-datahaven-parameters"; +import { logger, printDivider, printHeader } from "utils"; +import type { ParameterCollection } from "utils/parameters"; + +// Standard ports for the substrate network +const DEFAULT_SUBSTRATE_WS_PORT = 9944; + +/** + * A helper function to set DataHaven parameters from a ParameterCollection + * + * @param options Options for setting parameters + * @param options.launchedNetwork The launched network instance + * @param options.collection The parameter collection + * @returns Promise resolving to true if parameters were set successfully + */ +export const setParametersFromCollection = async ({ + collection, + skipSetParameters +}: { + collection: ParameterCollection; + skipSetParameters: boolean; +}): Promise => { + printHeader("Setting DataHaven Runtime Parameters"); + + if (skipSetParameters) { + logger.info("🏳️ Skipping parameter setting"); + printDivider(); + return false; + } + + const parametersFilePath = await collection.generateParametersFile(); + + const rpcUrl = `ws://127.0.0.1:${DEFAULT_SUBSTRATE_WS_PORT}`; + + const parametersSet = await setDataHavenParameters({ + rpcUrl, + parametersFilePath + }); + + printDivider(); + return parametersSet; +}; diff --git a/test/cli/handlers/deploy/relayer.ts b/test/cli/handlers/deploy/relayer.ts new file mode 100644 index 00000000..8a6eea55 --- /dev/null +++ b/test/cli/handlers/deploy/relayer.ts @@ -0,0 +1,308 @@ +import path from "node:path"; +import { $ } from "bun"; +import { createClient, type PolkadotClient } from "polkadot-api"; +import { withPolkadotSdkCompat } from "polkadot-api/polkadot-sdk-compat"; +import { getWsProvider } from "polkadot-api/ws-provider/web"; +import invariant from "tiny-invariant"; +import { + ANVIL_FUNDED_ACCOUNTS, + logger, + parseDeploymentsFile, + printDivider, + printHeader, + SUBSTRATE_FUNDED_ACCOUNTS +} from "utils"; +import { waitFor } from "utils/waits"; +import { ZERO_HASH } from "../common/consts"; +import type { LaunchedNetwork } from "../common/launchedNetwork"; +import { generateRelayerConfig, initEthClientPallet, type RelayerSpec } from "../common/relayer"; +import type { DeployOptions } from "."; + +// Standard ports for the Ethereum network +const ETH_EL_RPC_PORT = 8546; +const ETH_CL_HTTP_PORT = 4000; + +const RELAYER_CONFIG_DIR = "../deployment/charts/bridges-common-relay/configs"; +const RELAYER_CONFIG_PATHS = { + BEACON: path.join(RELAYER_CONFIG_DIR, "beacon-relay.json"), + BEEFY: path.join(RELAYER_CONFIG_DIR, "beefy-relay.json"), + SOLOCHAIN: path.join(RELAYER_CONFIG_DIR, "solochain-relay.json") +}; + +/** + * Deploys Snowbridge relayers for the DataHaven network in a Kubernetes namespace. + * + * @param options - Configuration options for launching the relayers. + * @param launchedNetwork - An instance of LaunchedNetwork to track the network's state. + */ +export const deployRelayers = async (options: DeployOptions, launchedNetwork: LaunchedNetwork) => { + printHeader("Starting Snowbridge Relayers"); + + if (options.skipRelayers) { + logger.info("🏳️ Skipping relayer deployment"); + printDivider(); + return; + } + + // Get DataHaven node port + const dhNodes = launchedNetwork.containers.filter((container) => + container.name.includes("dh-validator") + ); + + invariant(dhNodes.length > 0, "❌ No DataHaven nodes found in launchedNetwork"); + const firstDhNode = dhNodes[0]; + const substrateWsPort = firstDhNode.publicPorts.ws; + const substrateNodeId = firstDhNode.name; + logger.info( + `🔌 Using DataHaven node ${substrateNodeId} on port ${substrateWsPort} for relayers and BEEFY check.` + ); + + invariant(options.relayerImageTag, "❌ relayerImageTag is required"); + + // Check if BEEFY is ready before proceeding + await waitBeefyReady(launchedNetwork, 2000, 60000); + + const anvilDeployments = await parseDeploymentsFile(); + const beefyClientAddress = anvilDeployments.BeefyClient; + const gatewayAddress = anvilDeployments.Gateway; + invariant(beefyClientAddress, "❌ BeefyClient address not found in anvil.json"); + invariant(gatewayAddress, "❌ Gateway address not found in anvil.json"); + + logger.debug(`Ensuring output directory exists: ${RELAYER_CONFIG_DIR}`); + await $`mkdir -p ${RELAYER_CONFIG_DIR}`.quiet(); + + const ethElRpcEndpoint = `ws://el-1-reth-lodestar:${ETH_EL_RPC_PORT}`; + const ethClEndpoint = `http://cl-1-lodestar-reth:${ETH_CL_HTTP_PORT}`; + const substrateWsEndpoint = `ws://${substrateNodeId}:${substrateWsPort}`; + + const relayersToStart: RelayerSpec[] = [ + { + name: "relayer-🥩", + configFilePath: RELAYER_CONFIG_PATHS.BEEFY, + config: { + type: "beefy", + ethElRpcEndpoint, + substrateWsEndpoint, + beefyClientAddress, + gatewayAddress + }, + pk: { + ethereum: ANVIL_FUNDED_ACCOUNTS[1].privateKey + } + }, + { + name: "relayer-🥓", + configFilePath: RELAYER_CONFIG_PATHS.BEACON, + config: { + type: "beacon", + ethClEndpoint, + substrateWsEndpoint + }, + pk: { + substrate: SUBSTRATE_FUNDED_ACCOUNTS.BALTATHAR.privateKey + } + } + // TODO: Add solochain relayer + // { + // name: "relayer-⛓️", + // configFilePath: RELAYER_CONFIG_PATHS.SOLOCHAIN, + // config: { + // type: "solochain", + // ethElRpcEndpoint, + // substrateWsEndpoint, + // beefyClientAddress, + // gatewayAddress, + // ethClEndpoint + // }, + // pk: { + // ethereum: ANVIL_FUNDED_ACCOUNTS[1].privateKey, + // substrate: SUBSTRATE_FUNDED_ACCOUNTS.CHARLETH.privateKey + // } + // }, + // TODO: Add execution relayer + // { + // name: "relayer-⚙️", + // configFilePath: RELAYER_CONFIG_PATHS.EXECUTION, + // config: { + // type: "execution", + // ethElRpcEndpoint, + // ethClEndpoint, + // substrateWsEndpoint, + // gatewayAddress + // }, + // pk: { + // substrate: SUBSTRATE_FUNDED_ACCOUNTS.DOROTHY.privateKey + // } + // } + ]; + + for (const relayerSpec of relayersToStart) { + await generateRelayerConfig(relayerSpec, options.environment, RELAYER_CONFIG_DIR); + } + + invariant(options.relayerImageTag, "❌ Relayer image tag not defined"); + + // Generating the relayer config file for running the beacon relayer locally, to generate the first checkpoint + const localBeaconConfigDir = "tmp/configs"; + const localBeaconConfigFilePath = path.join(localBeaconConfigDir, "beacon-relay-checkpoint.json"); + const localBeaconConfig: RelayerSpec = { + name: "relayer-🥓-local", + configFilePath: localBeaconConfigFilePath, + templateFilePath: "configs/snowbridge/local/beacon-relay.json", + config: { + type: "beacon", + ethClEndpoint: launchedNetwork.clEndpoint.replace("127.0.0.1", "host.docker.internal"), + substrateWsEndpoint: `ws://${substrateNodeId}:${substrateWsPort}` + }, + pk: { + substrate: SUBSTRATE_FUNDED_ACCOUNTS.BALTATHAR.privateKey + } + }; + await generateRelayerConfig(localBeaconConfig, options.environment, localBeaconConfigDir); + + await initEthClientPallet( + path.resolve(localBeaconConfigFilePath), + options.relayerImageTag, + "tmp/datastore", + launchedNetwork + ); + + for (const { name, config, pk } of relayersToStart) { + try { + const containerName = `dh-${config.type}-relay`; + logger.info(`🚀 Starting relayer ${containerName} ...`); + + // Adding secret key as Kubernetes secret + const secrets: { pk: string; name: string }[] = []; + switch (config.type) { + case "beacon": + invariant(pk.substrate, "❌ Substrate private key is required for beacon relayer"); + secrets.push({ + pk: pk.substrate, + name: `dh-${config.type}-relay-substrate-key` + }); + break; + case "beefy": + invariant(pk.ethereum, "❌ Ethereum private key is required for beefy relayer"); + secrets.push({ + pk: pk.ethereum, + name: `dh-${config.type}-relay-ethereum-key` + }); + break; + case "solochain": + invariant(pk.substrate, "❌ Substrate private key is required for solochain relayer"); + invariant(pk.ethereum, "❌ Ethereum private key is required for solochain relayer"); + secrets.push({ + pk: pk.substrate, + name: `dh-${config.type}-relay-substrate-key` + }); + secrets.push({ + pk: pk.ethereum, + name: `dh-${config.type}-relay-ethereum-key` + }); + break; + case "execution": + invariant(pk.substrate, "❌ Substrate private key is required for execution relayer"); + secrets.push({ + pk: pk.substrate, + name: `dh-${config.type}-relay-substrate-key` + }); + break; + } + + for (const secret of secrets) { + logger.debug( + await $`kubectl create secret generic ${secret.name} \ + --from-literal=pvk="${secret.pk}" \ + -n ${launchedNetwork.kubeNamespace}`.text() + ); + logger.success(`Secret key ${secret.name} added to Kubernetes`); + } + + // Deploying relayer with helm chart + const relayerTimeout = "2m"; // 2 minutes + logger.debug( + await $`helm upgrade --install ${containerName} . -f ./snowbridge/${containerName}.yaml \ + -n ${launchedNetwork.kubeNamespace} \ + --wait \ + --timeout ${relayerTimeout}` + .cwd(path.join(process.cwd(), "../deployment/charts/bridges-common-relay")) + .text() + ); + + logger.success(`Started relayer ${name}`); + } catch (e) { + logger.error(`Error starting relayer ${name}`); + logger.error(e); + } + } + + logger.success("Snowbridge relayers started"); + printDivider(); +}; + +/** + * Waits for the BEEFY protocol to be ready by polling its finalized head. + * + * @param launchedNetwork - An instance of LaunchedNetwork to get the node endpoint. + * @param pollIntervalMs - The interval in milliseconds to poll the BEEFY endpoint. + * @param timeoutMs - The total time in milliseconds to wait before timing out. + * @throws Error if BEEFY is not ready within the timeout. + */ +const waitBeefyReady = async ( + launchedNetwork: LaunchedNetwork, + pollIntervalMs: number, + timeoutMs: number +): Promise => { + const port = launchedNetwork.getPublicWsPort(); + const wsUrl = `ws://127.0.0.1:${port}`; + const iterations = Math.floor(timeoutMs / pollIntervalMs); + + logger.info(`⌛️ Waiting for BEEFY to be ready on port ${port}...`); + + let client: PolkadotClient | undefined; + const clientTimeoutMs = pollIntervalMs / 2; + const delayMs = pollIntervalMs / 2; + try { + client = createClient(withPolkadotSdkCompat(getWsProvider(wsUrl))); + + await waitFor({ + lambda: async () => { + try { + logger.debug("Attempting to to check beefy_getFinalizedHead"); + + // Add timeout to the RPC call to prevent hanging. + const finalisedHeadPromise = client?._request("beefy_getFinalizedHead", []); + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => reject(new Error("RPC call timeout")), clientTimeoutMs); + }); + + const finalisedHeadHex = await Promise.race([finalisedHeadPromise, timeoutPromise]); + + if (finalisedHeadHex && finalisedHeadHex !== ZERO_HASH) { + logger.info(`🥩 BEEFY is ready. Finalised head: ${finalisedHeadHex}.`); + return true; + } + + logger.debug( + `BEEFY not ready or finalised head is zero. Retrying in ${delayMs / 1000}s...` + ); + return false; + } catch (rpcError) { + logger.warn(`RPC error checking BEEFY status: ${rpcError}. Retrying...`); + return false; + } + }, + iterations, + delay: delayMs, + errorMessage: "BEEFY protocol not ready. Relayers cannot be launched." + }); + } catch (error) { + logger.error(`❌ Failed to connect to DataHaven node for BEEFY check: ${error}`); + throw new Error("BEEFY protocol not ready. Relayers cannot be launched."); + } finally { + if (client) { + client.destroy(); + } + } +}; diff --git a/test/cli/handlers/deploy/validator.ts b/test/cli/handlers/deploy/validator.ts new file mode 100644 index 00000000..de9c7328 --- /dev/null +++ b/test/cli/handlers/deploy/validator.ts @@ -0,0 +1,33 @@ +import { fundValidators } from "scripts/fund-validators"; +import { setupValidators } from "scripts/setup-validators"; +import { updateValidatorSet } from "scripts/update-validator-set"; +import { logger, printDivider } from "utils"; +import type { DeployOptions } from ".."; + +export const performValidatorOperations = async (options: DeployOptions, networkRpcUrl: string) => { + if (options.skipValidatorOperations) { + logger.info("🏳️ Skipping validator operations"); + printDivider(); + return; + } + + // If not specified, prompt for funding + const shouldFundValidators = options.environment === "stagenet"; + + if (shouldFundValidators) { + await fundValidators({ + rpcUrl: networkRpcUrl + }); + } else { + logger.info("👍 Skipping validator funding"); + printDivider(); + } + + await setupValidators({ + rpcUrl: networkRpcUrl + }); + + await updateValidatorSet({ + rpcUrl: networkRpcUrl + }); +}; diff --git a/test/cli/handlers/index.ts b/test/cli/handlers/index.ts index 988f2b98..3bc4b87f 100644 --- a/test/cli/handlers/index.ts +++ b/test/cli/handlers/index.ts @@ -1,4 +1,5 @@ -export * from "./consts"; +export * from "./common"; +export * from "./deploy"; export * from "./exec"; export * from "./launch"; export * from "./stop"; diff --git a/test/cli/handlers/launch/checks.ts b/test/cli/handlers/launch/checks.ts deleted file mode 100644 index 8b424f47..00000000 --- a/test/cli/handlers/launch/checks.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { $ } from "bun"; -import { logger, printDivider, printHeader } from "utils"; - -// ===== Checks ===== -export const checkDependencies = async (): Promise => { - printHeader("Environment Checks"); - - if (!(await checkKurtosisInstalled())) { - logger.error("Kurtosis CLI is required to be installed: https://docs.kurtosis.com/install"); - throw Error("❌ Kurtosis CLI application not found."); - } - - logger.success("Kurtosis CLI found"); - - if (!(await checkDockerRunning())) { - logger.error("Is Docker Running? Unable to make connection to docker daemon"); - throw Error("❌ Error connecting to Docker"); - } - - logger.success("Docker is running"); - - if (!(await checkForgeInstalled())) { - logger.error("Is foundry installed? https://book.getfoundry.sh/getting-started/installation"); - throw Error("❌ forge binary not found in PATH"); - } - - logger.success("Forge is installed"); - printDivider(); -}; - -const checkKurtosisInstalled = async (): Promise => { - const { exitCode, stderr, stdout } = await $`kurtosis version`.nothrow().quiet(); - if (exitCode !== 0) { - logger.error(stderr.toString()); - return false; - } - logger.debug(stdout.toString()); - return true; -}; - -const checkDockerRunning = async (): Promise => { - const { exitCode, stderr, stdout } = await $`docker system info`.nothrow().quiet(); - if (exitCode !== 0) { - logger.error(stderr.toString()); - return false; - } - logger.debug(stdout.toString()); - return true; -}; - -const checkForgeInstalled = async (): Promise => { - const { exitCode, stderr, stdout } = await $`forge --version`.nothrow().quiet(); - if (exitCode !== 0) { - logger.error(stderr.toString()); - return false; - } - logger.debug(stdout.toString()); - return true; -}; diff --git a/test/cli/handlers/launch/contracts.ts b/test/cli/handlers/launch/contracts.ts new file mode 100644 index 00000000..4072a937 --- /dev/null +++ b/test/cli/handlers/launch/contracts.ts @@ -0,0 +1,66 @@ +import { + buildContracts, + constructDeployCommand, + executeDeployment, + validateDeploymentParams +} from "scripts/deploy-contracts"; +import { confirmWithTimeout, logger, printDivider, printHeader } from "utils"; +import type { ParameterCollection } from "utils/parameters"; + +interface DeployContractsOptions { + rpcUrl: string; + verified?: boolean; + blockscoutBackendUrl?: string; + deployContracts?: boolean; + parameterCollection?: ParameterCollection; +} + +/** + * Deploys smart contracts to the specified RPC URL + * + * @param options - Configuration options for deployment + * @param options.rpcUrl - The RPC URL to deploy to + * @param options.verified - Whether to verify contracts (requires blockscoutBackendUrl) + * @param options.blockscoutBackendUrl - URL for the Blockscout API (required if verified is true) + * @param options.deployContracts - Flag to control deployment (if undefined, will prompt) + * @param options.parameterCollection - Collection of parameters to update in the DataHaven runtime + * @returns Promise resolving to true if contracts were deployed successfully, false if skipped + */ +export const deployContracts = async (options: DeployContractsOptions): Promise => { + printHeader("Deploying Smart Contracts"); + const { deployContracts } = options; + + // Check if deployContracts option was set via flags, or prompt if not + let shouldDeployContracts = deployContracts; + if (shouldDeployContracts === undefined) { + shouldDeployContracts = await confirmWithTimeout( + "Do you want to deploy the smart contracts?", + true, + 10 + ); + } else { + logger.info( + `🏳️ Using flag option: ${shouldDeployContracts ? "will deploy" : "will not deploy"} smart contracts` + ); + } + + if (!shouldDeployContracts) { + logger.info("👍 Skipping contract deployment. Done!"); + printDivider(); + + return false; + } + + // Check if required parameters are provided + validateDeploymentParams(options); + + // Build contracts + await buildContracts(); + + // Construct and execute deployment + const deployCommand = constructDeployCommand(options); + await executeDeployment(deployCommand, options.parameterCollection); + printDivider(); + + return true; +}; diff --git a/test/cli/handlers/launch/datahaven.ts b/test/cli/handlers/launch/datahaven.ts index fe389627..5e02ab3b 100644 --- a/test/cli/handlers/launch/datahaven.ts +++ b/test/cli/handlers/launch/datahaven.ts @@ -1,11 +1,4 @@ -import fs from "node:fs"; -import path from "node:path"; -import { secp256k1 } from "@noble/curves/secp256k1"; -import { datahaven } from "@polkadot-api/descriptors"; import { $ } from "bun"; -import { createClient, type PolkadotClient } from "polkadot-api"; -import { withPolkadotSdkCompat } from "polkadot-api/polkadot-sdk-compat"; -import { getWsProvider } from "polkadot-api/ws-provider/web"; import { cargoCrossbuild } from "scripts/cargo-crossbuild"; import invariant from "tiny-invariant"; import { @@ -16,11 +9,11 @@ import { printHeader, waitForContainerToStart } from "utils"; -import { type Hex, keccak256, toHex } from "viem"; -import { publicKeyToAddress } from "viem/accounts"; -import { DOCKER_NETWORK_NAME } from "../consts"; +import { waitFor } from "utils/waits"; +import { DOCKER_NETWORK_NAME } from "../common/consts"; +import { isNetworkReady, setupDataHavenValidatorConfig } from "../common/datahaven"; +import type { LaunchedNetwork } from "../common/launchedNetwork"; import type { LaunchOptions } from "."; -import type { LaunchedNetwork } from "./launchedNetwork"; const LOG_LEVEL = Bun.env.LOG_LEVEL || "info"; @@ -43,14 +36,6 @@ const DEFAULT_PUBLIC_WS_PORT = 9944; // /operator/runtime/stagenet/src/genesis_config_presets.rs#L98 const CLI_AUTHORITY_IDS = ["alice", "bob"] as const; -// 33-byte compressed public keys for DataHaven next validator set -// These correspond to Alice & Bob -// These are the fallback keys if we can't fetch the next authorities directly from the network -const FALLBACK_DATAHAVEN_AUTHORITY_PUBLIC_KEYS: Record = { - alice: "0x020a1091341fe5664bfa1782d5e04779689068c916b04cb365ec3153755684d9a1", - bob: "0x0390084fdbf27d2b79d26a4f13f0ccd982cb755a661969143c37cbc49ef5b91f27" -} as const; - /** * Launches a DataHaven solochain network for testing. * @@ -158,30 +143,30 @@ export const launchDataHavenSolochain = async ( // logger.debug(listeningLine); } - for (let i = 0; i < 30; i++) { - logger.info("⌛️ Waiting for datahaven to start..."); - if (await isNetworkReady(DEFAULT_PUBLIC_WS_PORT)) { - logger.success( - `DataHaven network started, primary node accessible on port ${DEFAULT_PUBLIC_WS_PORT}` - ); + logger.info("⌛️ Waiting for DataHaven to start..."); + const timeoutMs = 2000; // 2 second timeout + await waitFor({ + lambda: async () => { + const isReady = await isNetworkReady(DEFAULT_PUBLIC_WS_PORT, timeoutMs); + if (!isReady) { + logger.debug("Node not ready, waiting 1 second..."); + } + return isReady; + }, + iterations: 30, + delay: timeoutMs, + errorMessage: "DataHaven network not ready" + }); - await registerNodes(launchedNetwork); + logger.success( + `DataHaven network started, primary node accessible on port ${DEFAULT_PUBLIC_WS_PORT}` + ); - // Call setupDataHavenValidatorConfig now that nodes are up - logger.info("🔧 Proceeding with DataHaven validator configuration setup..."); - await setupDataHavenValidatorConfig(launchedNetwork); + await registerNodes(launchedNetwork); - // Set the DataHaven RPC URL in the LaunchedNetwork instance - launchedNetwork.dhRpcUrl = `ws://127.0.0.1:${DEFAULT_PUBLIC_WS_PORT}`; + await setupDataHavenValidatorConfig(launchedNetwork, "datahaven-"); - printDivider(); - return; - } - logger.debug("Node not ready, waiting 1 second..."); - await new Promise((resolve) => setTimeout(resolve, 1000)); - } - - throw new Error("DataHaven network failed to start after 30 seconds"); + printDivider(); }; /** @@ -241,31 +226,6 @@ const cleanDataHavenContainers = async (options: LaunchOptions): Promise = ); }; -/** - * Checks if the DataHaven network is ready by sending a POST request to the system_chain method. - * - * @param port - The port number to check. - * @returns True if the network is ready, false otherwise. - */ -export const isNetworkReady = async (port: number): Promise => { - const wsUrl = `ws://127.0.0.1:${port}`; - let client: PolkadotClient | undefined; - try { - // Use withPolkadotSdkCompat for consistency, though _request might not strictly need it. - client = createClient(withPolkadotSdkCompat(getWsProvider(wsUrl))); - const chainName = await client._request("system_chain", []); - logger.debug(`isNetworkReady PAPI check successful for port ${port}, chain: ${chainName}`); - client.destroy(); - return !!chainName; // Ensure it's a boolean and chainName is truthy - } catch (error) { - logger.debug(`isNetworkReady PAPI check failed for port ${port}: ${error}`); - if (client) { - client.destroy(); - } - return false; - } -}; - const buildLocalImage = async (options: LaunchOptions) => { let shouldBuildDataHaven = options.buildDatahaven; @@ -327,7 +287,7 @@ const registerNodes = async (launchedNetwork: LaunchedNetwork) => { launchedNetwork.networkName = DOCKER_NETWORK_NAME; const targetContainerName = "datahaven-alice"; - const aliceHostWsPort = 9944; // Standard host port for Alice's WS, as set during launch. + const aliceHostWsPort = DEFAULT_PUBLIC_WS_PORT; // Standard host port for Alice's WS, as set during launch. logger.debug(`Checking Docker status for container: ${targetContainerName}`); // Use ^ and $ for an exact name match in the filter. @@ -348,133 +308,3 @@ const registerNodes = async (launchedNetwork: LaunchedNetwork) => { launchedNetwork.addContainer(targetContainerName, { ws: aliceHostWsPort }); logger.info(`📝 Node ${targetContainerName} successfully registered in launchedNetwork.`); }; - -// Function to convert compressed public key to Ethereum address -export const compressedPubKeyToEthereumAddress = (compressedPubKey: string): string => { - // Ensure the input is a hex string and remove "0x" prefix - const compressedKeyHex = compressedPubKey.startsWith("0x") - ? compressedPubKey.substring(2) - : compressedPubKey; - - // Decompress the public key - const point = secp256k1.ProjectivePoint.fromHex(compressedKeyHex); - // toRawBytes(false) returns the uncompressed key (64 bytes, x and y coordinates) - const uncompressedPubKeyBytes = point.toRawBytes(false); - const uncompressedPubKeyHex = toHex(uncompressedPubKeyBytes); // Prefixes with "0x" - - // Compute the Ethereum address from the uncompressed public key - // publicKeyToAddress expects a 0x-prefixed hex string representing the 64-byte uncompressed public key - const address = publicKeyToAddress(uncompressedPubKeyHex); - return address; -}; - -/** - * Prepares the configuration for DataHaven authorities by converting their - * compressed public keys to Ethereum addresses and saving them to a JSON file. - */ -export async function setupDataHavenValidatorConfig( - launchedNetwork: LaunchedNetwork -): Promise { - const networkName = process.env.NETWORK || "anvil"; - logger.info(`🔧 Preparing DataHaven authorities configuration for network: ${networkName}...`); - - let authorityPublicKeys: string[] = []; - const dhNodes = launchedNetwork.containers.filter((x) => x.name.startsWith("datahaven-")); - - if (dhNodes.length === 0) { - logger.warn( - "⚠️ No DataHaven nodes found in launchedNetwork. Falling back to hardcoded authority set for validator config." - ); - authorityPublicKeys = Object.values(FALLBACK_DATAHAVEN_AUTHORITY_PUBLIC_KEYS); - } else { - const firstNode = dhNodes[0]; - const wsUrl = `ws://127.0.0.1:${firstNode.publicPorts.ws}`; - let papiClient: PolkadotClient | undefined; - try { - logger.info( - `📡 Attempting to fetch BEEFY next authorities from node ${firstNode.name} (port ${firstNode.publicPorts.ws})...` - ); - papiClient = createClient(withPolkadotSdkCompat(getWsProvider(wsUrl))); - const dhApi = papiClient.getTypedApi(datahaven); - - // Fetch NextAuthorities - // Beefy.NextAuthorities returns a fixed-length array of bytes representing the authority public keys - const nextAuthoritiesRaw = await dhApi.query.Beefy.NextAuthorities.getValue({ at: "best" }); - - if (nextAuthoritiesRaw && nextAuthoritiesRaw.length > 0) { - authorityPublicKeys = nextAuthoritiesRaw.map((key) => key.asHex()); // .asHex() returns the hex string representation of the corresponding key - logger.success( - `Successfully fetched ${authorityPublicKeys.length} BEEFY next authorities directly.` - ); - } else { - logger.warn( - "⚠️ Fetched BEEFY nextAuthorities is empty. Falling back to hardcoded authority set." - ); - authorityPublicKeys = Object.values(FALLBACK_DATAHAVEN_AUTHORITY_PUBLIC_KEYS); - } - papiClient.destroy(); - } catch (error) { - logger.error( - `❌ Error fetching BEEFY next authorities from node ${firstNode.name}: ${error}. Falling back to hardcoded authority set.` - ); - authorityPublicKeys = Object.values(FALLBACK_DATAHAVEN_AUTHORITY_PUBLIC_KEYS); - if (papiClient) { - papiClient.destroy(); - } - } - } - - if (authorityPublicKeys.length === 0) { - logger.error( - "❌ No authority public keys available (neither fetched nor hardcoded). Cannot prepare validator config." - ); - throw new Error("No DataHaven authority keys available."); - } - - const authorityHashes: string[] = []; - for (const compressedKey of authorityPublicKeys) { - try { - const ethAddress = compressedPubKeyToEthereumAddress(compressedKey); - const authorityHash = keccak256(ethAddress as Hex); - authorityHashes.push(authorityHash); - logger.debug( - `Processed public key ${compressedKey} -> ETH address ${ethAddress} -> Authority hash ${authorityHash}` - ); - } catch (error) { - logger.error(`❌ Failed to process public key ${compressedKey}: ${error}`); - throw new Error(`Failed to process public key ${compressedKey}`); - } - } - - // process.cwd() is 'test/', so config is at '../contracts/config' - const configDir = path.join(process.cwd(), "../contracts/config"); - const configFilePath = path.join(configDir, `${networkName}.json`); - - try { - if (!fs.existsSync(configFilePath)) { - logger.warn( - `⚠️ Configuration file ${configFilePath} not found. Skipping update of validator sets.` - ); - // Optionally, create a default structure if it makes sense, or simply return. - // For now, if the base network config doesn't exist, we can't update it. - return; - } - - const configFileContent = fs.readFileSync(configFilePath, "utf-8"); - const configJson = JSON.parse(configFileContent); - - if (!configJson.snowbridge) { - configJson.snowbridge = {}; - logger.warn(`"snowbridge" section not found in ${configFilePath}, created it.`); - } - - configJson.snowbridge.initialValidators = authorityHashes; - configJson.snowbridge.nextValidators = authorityHashes; - - fs.writeFileSync(configFilePath, JSON.stringify(configJson, null, 2)); - logger.success(`DataHaven authority hashes updated in: ${configFilePath}`); - } catch (error) { - logger.error(`❌ Failed to read or update ${configFilePath}: ${error}`); - throw new Error(`Failed to update authority hashes in ${configFilePath}.`); - } -} diff --git a/test/cli/handlers/launch/index.ts b/test/cli/handlers/launch/index.ts index de9317cd..06c8756e 100644 --- a/test/cli/handlers/launch/index.ts +++ b/test/cli/handlers/launch/index.ts @@ -1,46 +1,38 @@ import type { Command } from "@commander-js/extra-typings"; -import { deployContracts } from "scripts/deploy-contracts"; import { getPortFromKurtosis, logger } from "utils"; -import { createParameterCollection, setParametersFromCollection } from "utils/parameters"; -import { checkDependencies } from "./checks"; +import { createParameterCollection } from "utils/parameters"; +import { checkBaseDependencies } from "../common/checks"; +import { LaunchedNetwork } from "../common/launchedNetwork"; +import { deployContracts } from "./contracts"; import { launchDataHavenSolochain } from "./datahaven"; import { launchKurtosis } from "./kurtosis"; -import { LaunchedNetwork } from "./launchedNetwork"; +import { setParametersFromCollection } from "./parameters"; import { launchRelayers } from "./relayer"; import { performSummaryOperations } from "./summary"; import { performValidatorOperations, performValidatorSetUpdate } from "./validator"; -// Non-optional properties determined by having default values +// Non-optional properties should have default values set by the CLI export interface LaunchOptions { - verified?: boolean; + datahaven?: boolean; + buildDatahaven?: boolean; + datahavenBuildExtraArgs: string; + datahavenImageTag: string; launchKurtosis?: boolean; + kurtosisEnclaveName: string; + slotTime?: number; + kurtosisNetworkArgs?: string; + verified?: boolean; + blockscout?: boolean; deployContracts?: boolean; fundValidators?: boolean; setupValidators?: boolean; updateValidatorSet?: boolean; - kurtosisEnclaveName: string; - blockscout?: boolean; + setParameters?: boolean; relayer?: boolean; relayerImageTag: string; cleanNetwork?: boolean; - datahaven?: boolean; - buildDatahaven?: boolean; - datahavenImageTag: string; - datahavenBuildExtraArgs: string; - kurtosisNetworkArgs?: string; - // Kept as optional due to parse fn - slotTime?: number; - setParameters?: boolean; } -export const BASE_SERVICES = [ - "cl-1-lodestar-reth", - "cl-2-lodestar-reth", - "el-1-reth-lodestar", - "el-2-reth-lodestar", - "dora" -]; - // ===== Launch Handler Functions ===== const launchFunction = async (options: LaunchOptions, launchedNetwork: LaunchedNetwork) => { @@ -49,16 +41,16 @@ const launchFunction = async (options: LaunchOptions, launchedNetwork: LaunchedN const timeStart = performance.now(); - await checkDependencies(); + await checkBaseDependencies(); // Create parameter collection to be used throughout the launch process const parameterCollection = await createParameterCollection(); await launchDataHavenSolochain(options, launchedNetwork); - await launchKurtosis(launchedNetwork, options); + await launchKurtosis(options, launchedNetwork); - logger.trace("Deploy contracts using the extracted function"); + logger.trace("Checking if Blockscout is enabled..."); let blockscoutBackendUrl: string | undefined; if (options.blockscout === true) { @@ -85,14 +77,14 @@ const launchFunction = async (options: LaunchOptions, launchedNetwork: LaunchedN await performValidatorOperations(options, launchedNetwork.elRpcUrl, contractsDeployed); - await launchRelayers(options, launchedNetwork); - await setParametersFromCollection({ - rpcUrl: launchedNetwork.dhRpcUrl, + launchedNetwork, collection: parameterCollection, setParameters: options.setParameters }); + await launchRelayers(options, launchedNetwork); + await performValidatorSetUpdate(options, launchedNetwork.elRpcUrl, contractsDeployed); await performSummaryOperations(options, launchedNetwork); @@ -103,11 +95,7 @@ const launchFunction = async (options: LaunchOptions, launchedNetwork: LaunchedN export const launch = async (options: LaunchOptions) => { const run = new LaunchedNetwork(); - try { - await launchFunction(options, run); - } finally { - await run.cleanup(); - } + await launchFunction(options, run); }; export const launchPreActionHook = ( diff --git a/test/cli/handlers/launch/kurtosis.ts b/test/cli/handlers/launch/kurtosis.ts index 3aaf2bd3..b9d5990a 100644 --- a/test/cli/handlers/launch/kurtosis.ts +++ b/test/cli/handlers/launch/kurtosis.ts @@ -1,9 +1,12 @@ import { $ } from "bun"; import type { LaunchOptions } from "cli/handlers"; -import invariant from "tiny-invariant"; -import { confirmWithTimeout, getPortFromKurtosis, logger, printDivider, printHeader } from "utils"; -import { parse, stringify } from "yaml"; -import type { LaunchedNetwork } from "./launchedNetwork"; +import { confirmWithTimeout, logger, printDivider, printHeader } from "utils"; +import { + checkKurtosisEnclaveRunning, + registerServices, + runKurtosisEnclave +} from "../common/kurtosis"; +import type { LaunchedNetwork } from "../common/launchedNetwork"; /** * Launches a Kurtosis Ethereum network enclave for testing. @@ -12,10 +15,10 @@ import type { LaunchedNetwork } from "./launchedNetwork"; * @param options - Configuration options */ export const launchKurtosis = async ( - launchedNetwork: LaunchedNetwork, - options: LaunchOptions + options: LaunchOptions, + launchedNetwork: LaunchedNetwork ): Promise => { - printHeader("Starting Kurtosis EthereumNetwork"); + printHeader("Starting Kurtosis Ethereum Network"); let shouldLaunchKurtosis = options.launchKurtosis; @@ -35,7 +38,7 @@ export const launchKurtosis = async ( return; } - if (await checkKurtosisRunning(options.kurtosisEnclaveName)) { + if (await checkKurtosisEnclaveRunning(options.kurtosisEnclaveName)) { logger.info("ℹ️ Kurtosis Ethereum network is already running."); // If the user wants to launch the Kurtosis network, we ask them if they want @@ -76,104 +79,9 @@ export const launchKurtosis = async ( ); } - logger.info("🚀 Starting Kurtosis enclave..."); - - const configFile = await modifyConfig(options, "configs/kurtosis/minimal.yaml"); - - logger.info(`⚙️ Using Kurtosis config file: ${configFile}`); - - const { stderr, stdout, exitCode } = - await $`kurtosis run github.com/ethpandaops/ethereum-package --args-file ${configFile} --enclave ${options.kurtosisEnclaveName}` - .nothrow() - .quiet(); - - if (exitCode !== 0) { - logger.error(stderr.toString()); - throw Error("❌ Kurtosis network has failed to start properly."); - } - logger.debug(stdout.toString()); + await runKurtosisEnclave(options, "configs/kurtosis/minimal.yaml"); await registerServices(launchedNetwork, options.kurtosisEnclaveName); logger.success("Kurtosis network operations completed successfully."); printDivider(); }; - -/** - * Checks if a Kurtosis enclave with the specified name is currently running. - * - * @param enclaveName - The name of the Kurtosis enclave to check - * @returns True if the enclave is running, false otherwise - */ -const checkKurtosisRunning = async (enclaveName: string): Promise => { - const text = await $`kurtosis enclave ls | grep "${enclaveName}" | grep RUNNING`.text(); - return text.length > 0; -}; - -const modifyConfig = async (options: LaunchOptions, configFile: string) => { - const outputDir = "tmp/configs"; - logger.debug(`Ensuring output directory exists: ${outputDir}`); - await $`mkdir -p ${outputDir}`.quiet(); - - const file = Bun.file(configFile); - invariant(file, `❌ Config file ${configFile} not found`); - - const config = await file.text(); - logger.debug(`Parsing config at ${configFile}`); - logger.trace(config); - - const parsedConfig = parse(config); - - if (options.blockscout) { - parsedConfig.additional_services.push("blockscout"); - } - - if (options.slotTime) { - parsedConfig.network_params.seconds_per_slot = options.slotTime; - } - - if (options.kurtosisNetworkArgs) { - logger.debug(`Using custom Kurtosis network args: ${options.kurtosisNetworkArgs}`); - const args = options.kurtosisNetworkArgs.split(" "); - for (const arg of args) { - const [key, value] = arg.split("="); - parsedConfig.network_params[key] = value; - } - } - - logger.trace(parsedConfig); - const outputFile = `${outputDir}/modified-config.yaml`; - logger.debug(`Modified config saving to ${outputFile}`); - - await Bun.write(outputFile, stringify(parsedConfig)); - return outputFile; -}; - -/** - * Registers the EL and CL service endpoints with the LaunchedNetwork instance. - * - * @param launchedNetwork - The LaunchedNetwork instance to store network details. - */ -const registerServices = async (launchedNetwork: LaunchedNetwork, enclaveName: string) => { - logger.info("📝 Registering Kurtosis service endpoints..."); - - // Configure EL RPC URL - try { - const rethPublicPort = await getPortFromKurtosis("el-1-reth-lodestar", "rpc", enclaveName); - invariant(rethPublicPort && rethPublicPort > 0, "❌ Could not find EL RPC port"); - const elRpcUrl = `http://127.0.0.1:${rethPublicPort}`; - launchedNetwork.elRpcUrl = elRpcUrl; - logger.info(`📝 Execution Layer RPC URL configured: ${elRpcUrl}`); - - // Configure CL Endpoint - const lodestarPublicPort = await getPortFromKurtosis("cl-1-lodestar-reth", "http", enclaveName); - const clEndpoint = `http://127.0.0.1:${lodestarPublicPort}`; - invariant( - clEndpoint, - "❌ CL Endpoint could not be determined from Kurtosis service cl-1-lodestar-reth" - ); - launchedNetwork.clEndpoint = clEndpoint; - logger.info(`📝 Consensus Layer Endpoint configured: ${clEndpoint}`); - } catch (error) { - logger.warn(`⚠️ Kurtosis service endpoints could not be determined: ${error}`); - } -}; diff --git a/test/cli/handlers/launch/parameters.ts b/test/cli/handlers/launch/parameters.ts new file mode 100644 index 00000000..969e6773 --- /dev/null +++ b/test/cli/handlers/launch/parameters.ts @@ -0,0 +1,59 @@ +import { setDataHavenParameters } from "scripts/set-datahaven-parameters"; +import { logger, printDivider, printHeader } from "utils"; +import { confirmWithTimeout } from "utils/input"; +import type { ParameterCollection } from "utils/parameters"; +import type { LaunchedNetwork } from "../common/launchedNetwork"; + +/** + * A helper function to set DataHaven parameters from a ParameterCollection + * + * @param options Options for setting parameters + * @param options.launchedNetwork The launched network instance + * @param options.collection The parameter collection + * @param options.setParameters Flag to control execution + * @returns Promise resolving to true if parameters were set successfully + */ +export const setParametersFromCollection = async ({ + launchedNetwork, + collection, + setParameters +}: { + launchedNetwork: LaunchedNetwork; + collection: ParameterCollection; + setParameters?: boolean; +}): Promise => { + printHeader("Setting DataHaven Runtime Parameters"); + + const parametersFilePath = await collection.generateParametersFile(); + const rpcUrl = `ws://127.0.0.1:${launchedNetwork.getPublicWsPort()}`; + + // Check if setParameters option was set via flags, or prompt if not + let shouldSetParameters = setParameters; + if (shouldSetParameters === undefined) { + shouldSetParameters = await confirmWithTimeout( + "Do you want to set the DataHaven runtime parameters?", + true, + 10 + ); + } else { + logger.info( + `🏳️ Using flag option: ${ + shouldSetParameters ? "will set" : "will not set" + } DataHaven parameters` + ); + } + + if (!shouldSetParameters) { + logger.info("👍 Skipping DataHaven parameter setting. Done!"); + printDivider(); + return false; + } + + const parametersSet = await setDataHavenParameters({ + rpcUrl, + parametersFilePath + }); + + printDivider(); + return parametersSet; +}; diff --git a/test/cli/handlers/launch/relayer.ts b/test/cli/handlers/launch/relayer.ts index 1ccecc57..786b1586 100644 --- a/test/cli/handlers/launch/relayer.ts +++ b/test/cli/handlers/launch/relayer.ts @@ -1,5 +1,4 @@ import path from "node:path"; -import { datahaven } from "@polkadot-api/descriptors"; import { $ } from "bun"; import { createClient, type PolkadotClient } from "polkadot-api"; import { withPolkadotSdkCompat } from "polkadot-api/polkadot-sdk-compat"; @@ -8,33 +7,21 @@ import invariant from "tiny-invariant"; import { ANVIL_FUNDED_ACCOUNTS, confirmWithTimeout, - getEvmEcdsaSigner, getPortFromKurtosis, killExistingContainers, logger, parseDeploymentsFile, - parseRelayConfig, printDivider, printHeader, - type RelayerType, runShellCommandWithLogger, SUBSTRATE_FUNDED_ACCOUNTS, waitForContainerToStart } from "utils"; -import type { BeaconCheckpoint, FinalityCheckpointsResponse } from "utils/types"; -import { parseJsonToBeaconCheckpoint } from "utils/types"; +import { waitFor } from "utils/waits"; +import { ZERO_HASH } from "../common/consts"; +import type { LaunchedNetwork } from "../common/launchedNetwork"; +import { generateRelayerConfig, initEthClientPallet, type RelayerSpec } from "../common/relayer"; import type { LaunchOptions } from "."; -import type { LaunchedNetwork } from "./launchedNetwork"; - -const ZERO_HASH = "0x0000000000000000000000000000000000000000000000000000000000000000"; - -type RelayerSpec = { - name: string; - type: RelayerType; - config: string; - pk: { type: "ethereum" | "substrate"; value: string }; - secondaryPk?: { type: "ethereum" | "substrate"; value: string }; -}; const RELAYER_CONFIG_DIR = "tmp/configs"; const RELAYER_CONFIG_PATHS = { @@ -43,9 +30,6 @@ const RELAYER_CONFIG_PATHS = { EXECUTION: path.join(RELAYER_CONFIG_DIR, "execution-relay.json"), SOLOCHAIN: path.join(RELAYER_CONFIG_DIR, "solochain-relay.json") }; -const INITIAL_CHECKPOINT_FILE = "dump-initial-checkpoint.json"; -const INITIAL_CHECKPOINT_DIR = "tmp/beacon-checkpoint"; -const INITIAL_CHECKPOINT_PATH = path.join(INITIAL_CHECKPOINT_DIR, INITIAL_CHECKPOINT_FILE); /** * Launches Snowbridge relayers for the DataHaven network. @@ -116,141 +100,104 @@ export const launchRelayers = async (options: LaunchOptions, launchedNetwork: La logger.debug(`Ensuring datastore directory exists: ${datastorePath}`); await $`mkdir -p ${datastorePath}`.quiet(); + const ethWsPort = await getPortFromKurtosis( + "el-1-reth-lodestar", + "ws", + options.kurtosisEnclaveName + ); + const ethHttpPort = await getPortFromKurtosis( + "cl-1-lodestar-reth", + "http", + options.kurtosisEnclaveName + ); + + const ethElRpcEndpoint = `ws://host.docker.internal:${ethWsPort}`; + const ethClEndpoint = `http://host.docker.internal:${ethHttpPort}`; + const substrateWsEndpoint = `ws://${substrateNodeId}:${substrateWsPort}`; + const relayersToStart: RelayerSpec[] = [ { name: "relayer-🥩", - type: "beefy", - config: RELAYER_CONFIG_PATHS.BEEFY, + configFilePath: RELAYER_CONFIG_PATHS.BEEFY, + config: { + type: "beefy", + ethElRpcEndpoint, + substrateWsEndpoint, + beefyClientAddress, + gatewayAddress + }, pk: { - type: "ethereum", - value: ANVIL_FUNDED_ACCOUNTS[1].privateKey + ethereum: ANVIL_FUNDED_ACCOUNTS[1].privateKey } }, { name: "relayer-🥓", - type: "beacon", - config: RELAYER_CONFIG_PATHS.BEACON, + configFilePath: RELAYER_CONFIG_PATHS.BEACON, + config: { + type: "beacon", + ethClEndpoint, + substrateWsEndpoint + }, pk: { - type: "substrate", - value: SUBSTRATE_FUNDED_ACCOUNTS.BALTATHAR.privateKey + substrate: SUBSTRATE_FUNDED_ACCOUNTS.BALTATHAR.privateKey } }, { name: "relayer-⛓️", - type: "solochain", - config: RELAYER_CONFIG_PATHS.SOLOCHAIN, - pk: { - type: "ethereum", - value: ANVIL_FUNDED_ACCOUNTS[1].privateKey + configFilePath: RELAYER_CONFIG_PATHS.SOLOCHAIN, + config: { + type: "solochain", + ethElRpcEndpoint, + substrateWsEndpoint, + beefyClientAddress, + gatewayAddress, + ethClEndpoint }, - secondaryPk: { - type: "substrate", - value: SUBSTRATE_FUNDED_ACCOUNTS.CHARLETH.privateKey + pk: { + ethereum: ANVIL_FUNDED_ACCOUNTS[1].privateKey, + substrate: SUBSTRATE_FUNDED_ACCOUNTS.CHARLETH.privateKey } }, { name: "relayer-⚙️", - type: "execution", - config: RELAYER_CONFIG_PATHS.EXECUTION, + configFilePath: RELAYER_CONFIG_PATHS.EXECUTION, + config: { + type: "execution", + ethElRpcEndpoint, + ethClEndpoint, + substrateWsEndpoint, + gatewayAddress + }, pk: { - type: "substrate", - value: SUBSTRATE_FUNDED_ACCOUNTS.BALTATHAR.privateKey + substrate: SUBSTRATE_FUNDED_ACCOUNTS.DOROTHY.privateKey } } ]; - for (const { config, type, name } of relayersToStart) { - const configFileName = path.basename(config); - - logger.debug(`Creating config for ${name}`); - const templateFilePath = `configs/snowbridge/${configFileName}`; - const outputFilePath = path.resolve(RELAYER_CONFIG_DIR, configFileName); - logger.debug(`Reading config file ${templateFilePath}`); - const file = Bun.file(templateFilePath); - - if (!(await file.exists())) { - logger.error(`File ${templateFilePath} does not exist`); - throw new Error("Error reading snowbridge config file"); - } - const json = await file.json(); - - const ethWsPort = await getPortFromKurtosis( - "el-1-reth-lodestar", - "ws", - options.kurtosisEnclaveName - ); - const ethHttpPort = await getPortFromKurtosis( - "cl-1-lodestar-reth", - "http", - options.kurtosisEnclaveName - ); - logger.debug( - `Fetched ports: ETH WS=${ethWsPort}, ETH HTTP=${ethHttpPort}, Substrate WS=${substrateWsPort} (from DataHaven node)` - ); - - switch (type) { - case "beacon": { - const cfg = parseRelayConfig(json, type); - cfg.source.beacon.endpoint = `http://host.docker.internal:${ethHttpPort}`; - cfg.source.beacon.stateEndpoint = `http://host.docker.internal:${ethHttpPort}`; - cfg.source.beacon.datastore.location = "/data"; - cfg.sink.parachain.endpoint = `ws://${substrateNodeId}:${substrateWsPort}`; - - await Bun.write(outputFilePath, JSON.stringify(cfg, null, 4)); - logger.success(`Updated beacon config written to ${outputFilePath}`); - break; - } - case "beefy": { - const cfg = parseRelayConfig(json, type); - cfg.source.polkadot.endpoint = `ws://${substrateNodeId}:${substrateWsPort}`; - cfg.sink.ethereum.endpoint = `ws://host.docker.internal:${ethWsPort}`; - cfg.sink.contracts.BeefyClient = beefyClientAddress; - cfg.sink.contracts.Gateway = gatewayAddress; - await Bun.write(outputFilePath, JSON.stringify(cfg, null, 4)); - logger.success(`Updated beefy config written to ${outputFilePath}`); - break; - } - case "execution": { - const cfg = parseRelayConfig(json, type); - cfg.source.ethereum.endpoint = `ws://host.docker.internal:${ethWsPort}`; - cfg.source.beacon.endpoint = `http://host.docker.internal:${ethHttpPort}`; - cfg.source.beacon.stateEndpoint = `http://host.docker.internal:${ethHttpPort}`; - cfg.source.beacon.datastore.location = "/data"; - cfg.sink.parachain.endpoint = `ws://${substrateNodeId}:${substrateWsPort}`; - cfg.source.contracts.Gateway = gatewayAddress; - await Bun.write(outputFilePath, JSON.stringify(cfg, null, 4)); - logger.success(`Updated execution config written to ${outputFilePath}`); - break; - } - case "solochain": { - const cfg = parseRelayConfig(json, type); - cfg.source.ethereum.endpoint = `ws://host.docker.internal:${ethWsPort}`; - cfg.source.solochain.endpoint = `ws://${substrateNodeId}:${substrateWsPort}`; - cfg.source.contracts.BeefyClient = beefyClientAddress; - cfg.source.contracts.Gateway = gatewayAddress; - cfg.source.beacon.endpoint = `http://host.docker.internal:${ethHttpPort}`; - cfg.source.beacon.stateEndpoint = `http://host.docker.internal:${ethHttpPort}`; - cfg.source.beacon.datastore.location = datastorePath; - cfg.sink.ethereum.endpoint = `ws://host.docker.internal:${ethWsPort}`; - cfg.sink.contracts.Gateway = gatewayAddress; - await Bun.write(outputFilePath, JSON.stringify(cfg, null, 4)); - logger.success(`Updated solochain config written to ${outputFilePath}`); - break; - } - } + for (const relayerSpec of relayersToStart) { + await generateRelayerConfig(relayerSpec, "local", RELAYER_CONFIG_DIR); } invariant(options.relayerImageTag, "❌ Relayer image tag not defined"); + invariant( + launchedNetwork.networkName, + "❌ Docker network name not found in LaunchedNetwork instance" + ); - await initEthClientPallet(options, launchedNetwork, datastorePath); + await initEthClientPallet( + path.resolve(RELAYER_CONFIG_PATHS.BEACON), + options.relayerImageTag, + datastorePath, + launchedNetwork + ); - for (const { config, name, type, pk, secondaryPk } of relayersToStart) { + for (const { configFilePath, name, config, pk } of relayersToStart) { try { - const containerName = `snowbridge-${type}-relay`; + const containerName = `snowbridge-${config.type}-relay`; logger.info(`🚀 Starting relayer ${containerName} ...`); - const hostConfigFilePath = path.resolve(config); - const containerConfigFilePath = `/${config}`; + const hostConfigFilePath = path.resolve(configFilePath); + const containerConfigFilePath = `/${configFilePath}`; const networkName = launchedNetwork.networkName; invariant(networkName, "❌ Docker network name not found in LaunchedNetwork instance"); @@ -258,6 +205,8 @@ export const launchRelayers = async (options: LaunchOptions, launchedNetwork: La "docker", "run", "-d", + "--pull", + "always", "--platform", "linux/amd64", "--add-host", @@ -269,24 +218,39 @@ export const launchRelayers = async (options: LaunchOptions, launchedNetwork: La ]; const volumeMounts: string[] = ["-v", `${hostConfigFilePath}:${containerConfigFilePath}`]; - const hostDatastorePath = path.resolve(datastorePath); - const containerDatastorePath = "/data"; - if (type === "beacon" || type === "execution") { + if (config.type === "beacon" || config.type === "execution") { + const hostDatastorePath = path.resolve(datastorePath); + const containerDatastorePath = "/data"; volumeMounts.push("-v", `${hostDatastorePath}:${containerDatastorePath}`); } - const relayerCommandArgs: string[] = [ - "run", - type, - "--config", - config, - `--${pk.type}.private-key`, - pk.value - ]; + const relayerCommandArgs: string[] = ["run", config.type, "--config", configFilePath]; - if (type === "solochain" && secondaryPk) { - relayerCommandArgs.push("--substrate.private-key", secondaryPk.value); + switch (config.type) { + case "beacon": + invariant(pk.substrate, "❌ Substrate private key is required for beacon relayer"); + relayerCommandArgs.push("--substrate.private-key", pk.substrate); + break; + case "beefy": + invariant(pk.ethereum, "❌ Ethereum private key is required for beefy relayer"); + relayerCommandArgs.push("--ethereum.private-key", pk.ethereum); + break; + case "solochain": + invariant(pk.ethereum, "❌ Ethereum private key is required for solochain relayer"); + relayerCommandArgs.push("--ethereum.private-key", pk.ethereum); + if (pk.substrate) { + relayerCommandArgs.push("--substrate.private-key", pk.substrate); + } else { + logger.warn( + "⚠️ No substrate private key provided for solochain relayer. This might be an issue depending on the configuration." + ); + } + break; + case "execution": + invariant(pk.substrate, "❌ Substrate private key is required for execution relayer"); + relayerCommandArgs.push("--substrate.private-key", pk.substrate); + break; } const command: string[] = [ @@ -311,7 +275,7 @@ export const launchRelayers = async (options: LaunchOptions, launchedNetwork: La // tail: 1 // }); - logger.debug(`Started relayer ${name} with process ${process.pid}`); + logger.success(`Started relayer ${name} with process ${process.pid}`); } catch (e) { logger.error(`Error starting relayer ${name}`); logger.error(e); @@ -337,218 +301,53 @@ const waitBeefyReady = async ( ): Promise => { const port = launchedNetwork.getPublicWsPort(); const wsUrl = `ws://127.0.0.1:${port}`; - const maxAttempts = Math.floor(timeoutMs / pollIntervalMs); + const iterations = Math.floor(timeoutMs / pollIntervalMs); logger.info(`⌛️ Waiting for BEEFY to be ready on port ${port}...`); let client: PolkadotClient | undefined; + const clientTimeoutMs = pollIntervalMs / 2; + const delayMs = pollIntervalMs / 2; try { client = createClient(withPolkadotSdkCompat(getWsProvider(wsUrl))); - for (let attempt = 1; attempt <= maxAttempts; attempt++) { - try { - logger.debug(`Attempt ${attempt}/${maxAttempts} to check beefy_getFinalizedHead`); - const finalizedHeadHex = await client._request("beefy_getFinalizedHead", []); + await waitFor({ + lambda: async () => { + try { + logger.debug("Attempting to to check beefy_getFinalizedHead"); - if (finalizedHeadHex && finalizedHeadHex !== ZERO_HASH) { - logger.info(`🥩 BEEFY is ready. Finalized head: ${finalizedHeadHex}`); - client.destroy(); - return; + // Add timeout to the RPC call to prevent hanging. + const finalisedHeadPromise = client?._request("beefy_getFinalizedHead", []); + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => reject(new Error("RPC call timeout")), clientTimeoutMs); + }); + + const finalisedHeadHex = await Promise.race([finalisedHeadPromise, timeoutPromise]); + + if (finalisedHeadHex && finalisedHeadHex !== ZERO_HASH) { + logger.info(`🥩 BEEFY is ready. Finalised head: ${finalisedHeadHex}.`); + return true; + } + + logger.debug( + `BEEFY not ready or finalised head is zero. Retrying in ${delayMs / 1000}s...` + ); + return false; + } catch (rpcError) { + logger.warn(`RPC error checking BEEFY status: ${rpcError}. Retrying...`); + return false; } - - logger.debug( - `BEEFY not ready or finalized head is zero. Retrying in ${pollIntervalMs / 1000}s...` - ); - } catch (rpcError) { - logger.warn(`RPC error checking BEEFY status: ${rpcError}. Retrying...`); - } - - await new Promise((resolve) => setTimeout(resolve, pollIntervalMs)); - } - - logger.error(`❌ BEEFY failed to become ready after ${timeoutMs / 1000} seconds`); - if (client) client.destroy(); - throw new Error("BEEFY protocol not ready. Relayers cannot be launched."); + }, + iterations, + delay: delayMs, + errorMessage: "BEEFY protocol not ready. Relayers cannot be launched." + }); } catch (error) { logger.error(`❌ Failed to connect to DataHaven node for BEEFY check: ${error}`); + throw new Error("BEEFY protocol not ready. Relayers cannot be launched."); + } finally { if (client) { client.destroy(); } - throw new Error("BEEFY protocol not ready. Relayers cannot be launched."); - } -}; - -/** - * Initialises the Ethereum Beacon Client pallet on the Substrate chain. - * It waits for the beacon chain to be ready, generates an initial checkpoint, - * and submits this checkpoint to the Substrate runtime via a sudo call. - * - * @param options - Launch options containing the relayer binary path. - * @param launchedNetwork - An instance of LaunchedNetwork to interact with the running network. - * @param datastorePath - The path to the datastore directory. - * @throws If there's an error generating the beacon checkpoint or submitting it to Substrate. - */ -export const initEthClientPallet = async ( - options: LaunchOptions, - launchedNetwork: LaunchedNetwork, - datastorePath: string -) => { - logger.debug("Initialising eth client pallet"); - // Poll the beacon chain until it's ready every 10 seconds for 10 minutes - await waitBeaconChainReady(launchedNetwork, 10000, 600000); - - const beaconConfigHostPath = path.resolve(RELAYER_CONFIG_PATHS.BEACON); - const beaconConfigContainerPath = `/app/${RELAYER_CONFIG_PATHS.BEACON}`; - const checkpointHostPath = path.resolve(INITIAL_CHECKPOINT_PATH); - const checkpointContainerPath = `/app/${INITIAL_CHECKPOINT_FILE}`; - - logger.debug("Generating beacon checkpoint"); - // Pre-create the checkpoint file so that Docker doesn't interpret it as a directory - await Bun.write(INITIAL_CHECKPOINT_PATH, ""); - - logger.debug("Removing 'generate-beacon-checkpoint' container if it exists"); - logger.debug(await $`docker rm -f generate-beacon-checkpoint`.text()); - - logger.debug("Generating beacon checkpoint"); - invariant( - launchedNetwork.networkName, - "❌ Docker network name not found in LaunchedNetwork instance" - ); - const datastoreHostPath = path.resolve(datastorePath); - const command = `docker run \ - -v ${beaconConfigHostPath}:${beaconConfigContainerPath}:ro \ - -v ${checkpointHostPath}:${checkpointContainerPath} \ - -v ${datastoreHostPath}:/data \ - --name generate-beacon-checkpoint \ - --workdir /app \ - --add-host host.docker.internal:host-gateway \ - --network ${launchedNetwork.networkName} \ - ${options.relayerImageTag} \ - generate-beacon-checkpoint --config ${RELAYER_CONFIG_PATHS.BEACON} --export-json`; - logger.debug(`Running command: ${command}`); - logger.debug(await $`sh -c "${command}"`.text()); - - // Load the checkpoint into a JSON object and clean it up - const initialCheckpointFile = Bun.file(INITIAL_CHECKPOINT_PATH); - const initialCheckpointRaw = await initialCheckpointFile.text(); - const initialCheckpoint = parseJsonToBeaconCheckpoint(JSON.parse(initialCheckpointRaw)); - if (initialCheckpointFile.delete) { - await initialCheckpointFile.delete(); - } - - logger.trace("Initial checkpoint:"); - logger.trace(initialCheckpoint.toJSON()); - - // Send the checkpoint to the Substrate runtime - const substrateRpcUrl = `http://127.0.0.1:${launchedNetwork.getPublicWsPort()}`; - await sendCheckpointToSubstrate(substrateRpcUrl, initialCheckpoint); - logger.success("Ethereum Beacon Client pallet initialised"); -}; - -/** - * Waits for the beacon chain to be ready by polling its finality checkpoints. - * - * @param launchedNetwork - An instance of LaunchedNetwork to get the CL endpoint. - * @param pollIntervalMs - The interval in milliseconds to poll the beacon chain. - * @param timeoutMs - The total time in milliseconds to wait before timing out. - * @throws Error if the beacon chain is not ready within the timeout. - */ -const waitBeaconChainReady = async ( - launchedNetwork: LaunchedNetwork, - pollIntervalMs: number, - timeoutMs: number -) => { - let initialBeaconBlock = ZERO_HASH; - let attempts = 0; - let keepPolling = true; - const maxAttempts = timeoutMs / pollIntervalMs; - - logger.trace("Waiting for beacon chain to be ready..."); - - while (keepPolling) { - try { - const response = await fetch( - `${launchedNetwork.clEndpoint}/eth/v1/beacon/states/head/finality_checkpoints` - ); - if (!response.ok) { - throw new Error(`HTTP error! Status: ${response.status}`); - } - - const data = (await response.json()) as FinalityCheckpointsResponse; - logger.debug(`Beacon chain state: ${JSON.stringify(data)}`); - - invariant(data.data, "❌ No data returned from beacon chain"); - invariant(data.data.finalized, "❌ No finalised block returned from beacon chain"); - invariant(data.data.finalized.root, "❌ No finalised block root returned from beacon chain"); - initialBeaconBlock = data.data.finalized.root; - } catch (error) { - logger.error(`Failed to fetch beacon chain state: ${error}`); - } - - if (initialBeaconBlock === ZERO_HASH) { - attempts++; - - if (attempts >= maxAttempts) { - throw new Error(`Beacon chain is not ready after ${maxAttempts} attempts`); - } - - logger.info(`⌛️ Retrying beacon chain state fetch in ${pollIntervalMs / 1000}s...`); - await new Promise((resolve) => setTimeout(resolve, pollIntervalMs)); - } else { - keepPolling = false; - } - } - - logger.info(`⏲️ Beacon chain is ready with finalised block: ${initialBeaconBlock}`); -}; - -/** - * Sends the beacon checkpoint to the Substrate runtime, waiting for the transaction to be finalised and successful. - * - * @param networkRpcUrl - The RPC URL of the Substrate network. - * @param checkpoint - The beacon checkpoint to send. - * @throws If the transaction signing fails, it becomes an invalid transaction, or the transaction is included but fails. - */ -const sendCheckpointToSubstrate = async (networkRpcUrl: string, checkpoint: BeaconCheckpoint) => { - logger.trace("Sending checkpoint to Substrate..."); - - const client = createClient(withPolkadotSdkCompat(getWsProvider(networkRpcUrl))); - const dhApi = client.getTypedApi(datahaven); - - logger.trace("Client created"); - - const signer = getEvmEcdsaSigner(SUBSTRATE_FUNDED_ACCOUNTS.ALITH.privateKey); - logger.trace("Signer created"); - - const forceCheckpointCall = dhApi.tx.EthereumBeaconClient.force_checkpoint({ - update: checkpoint - }); - - logger.debug("Force checkpoint call:"); - logger.debug(forceCheckpointCall.decodedCall); - - const tx = dhApi.tx.Sudo.sudo({ - call: forceCheckpointCall.decodedCall - }); - - logger.debug("Sudo call:"); - logger.debug(tx.decodedCall); - - try { - const txFinalisedPayload = await tx.signAndSubmit(signer); - - if (!txFinalisedPayload.ok) { - throw new Error("❌ Beacon checkpoint transaction failed"); - } - - logger.info( - `📪 "force_checkpoint" transaction with hash ${txFinalisedPayload.txHash} submitted successfully and finalised in block ${txFinalisedPayload.block.hash}` - ); - } catch (error) { - logger.error(`Failed to submit checkpoint transaction: ${error}`); - throw new Error(`Failed to submit checkpoint: ${error}`); - } finally { - client.destroy(); - logger.debug("Destroyed client"); } }; diff --git a/test/cli/handlers/launch/summary.ts b/test/cli/handlers/launch/summary.ts index c225d6d4..9def2668 100644 --- a/test/cli/handlers/launch/summary.ts +++ b/test/cli/handlers/launch/summary.ts @@ -1,7 +1,8 @@ import invariant from "tiny-invariant"; import { getServiceFromKurtosis, logger, printHeader } from "utils"; -import { BASE_SERVICES, type LaunchOptions } from "."; -import type { LaunchedNetwork } from "./launchedNetwork"; +import { BASE_SERVICES } from "../common/consts"; +import type { LaunchedNetwork } from "../common/launchedNetwork"; +import type { LaunchOptions } from "."; export const performSummaryOperations = async ( options: LaunchOptions, @@ -21,7 +22,11 @@ export const performSummaryOperations = async ( logger.trace("Services to display", servicesToDisplay); - const displayData: { service: string; ports: Record; url: string }[] = []; + const displayData: { + service: string; + ports: Record; + url: string; + }[] = []; for (const service of servicesToDisplay) { logger.debug(`Checking service: ${service}`); diff --git a/test/cli/handlers/stop/index.ts b/test/cli/handlers/stop/index.ts index f42b290d..8dab7954 100644 --- a/test/cli/handlers/stop/index.ts +++ b/test/cli/handlers/stop/index.ts @@ -9,9 +9,9 @@ import { printHeader, runShellCommandWithLogger } from "utils"; -import { z } from "zod"; -import { COMPONENTS, DOCKER_NETWORK_NAME } from "../consts"; -import { checkDependencies } from "../launch/checks"; +import { checkBaseDependencies } from "../common/checks"; +import { COMPONENTS, DOCKER_NETWORK_NAME } from "../common/consts"; +import { getRunningKurtosisEnclaves } from "../common/kurtosis"; export interface StopOptions { all?: boolean; @@ -33,7 +33,7 @@ export const stop = async (options: StopOptions) => { logger.info("🛑 Stopping network components..."); logger.debug(`Stop options: ${JSON.stringify(options)}`); - await checkDependencies(); + await checkBaseDependencies(); printHeader("Snowbridge Relayers"); await stopDockerComponents("snowbridge", options); @@ -143,67 +143,23 @@ const stopAllEnclaves = async (options: StopOptions) => { return; } - const lines = (await Array.fromAsync($`kurtosis enclave ls`.lines())).filter( - (line) => line.length > 0 - ); - logger.trace(lines); + const enclaves = await getRunningKurtosisEnclaves(); - lines.shift(); - const enclaveCount = lines.length; - const KurtosisEnclaveInfoSchema = z.object({ - uuid: z.string().min(1), - name: z.string().min(1), - status: z.string().min(1), - creationTime: z.string().min(1) - }); - - type KurtosisEnclaveInfo = z.infer; - const enclaves: KurtosisEnclaveInfo[] = []; - - if (enclaveCount > 0) { - logger.info(`🔎 Found ${enclaveCount} Kurtosis enclave(s) running.`); - // Updated regex to match the actual format: "uuid name status creationTime" - const enclaveRegex = /^(\S+)\s+(\S+)\s+(\S+)\s+(.+)$/; - - for (const line of lines) { - const match = line.match(enclaveRegex); - if (match) { - const [, uuid, name, status, creationTime] = match; - const parseResult = KurtosisEnclaveInfoSchema.safeParse({ - uuid: uuid.trim(), - name: name.trim(), - status: status.trim(), - creationTime: creationTime.trim() - }); - - if (parseResult.success) { - enclaves.push(parseResult.data); - } else { - logger.warn( - `⚠️ Could not parse enclave line: "${line}". Error: ${parseResult.error.message}` - ); - } - } else { - logger.warn(`⚠️ Could not parse enclave line (regex mismatch): "${line}"`); - } - } - - if (enclaves.length > 0) { - logger.debug("Parsed enclave details:"); - - for (const { creationTime, name, status, uuid } of enclaves) { - logger.debug(`UUID: ${uuid}, Name: ${name}, Status: ${status}, Created: ${creationTime}`); - logger.info(`🗑️ Removing enclave ${name}`); - logger.debug(await $`kurtosis enclave rm ${uuid} -f`.text()); - } - } else if (lines.length > 0 && enclaves.length === 0) { - logger.warn("Found enclave lines in output, but failed to parse any of them."); - } - } else { + if (enclaves.length === 0) { logger.info("🤷‍ No Kurtosis enclaves found running."); return; } - logger.info(`🪓 ${lines.length} enclaves cleaned`); + + logger.info(`🔎 Found ${enclaves.length} Kurtosis enclave(s) running.`); + logger.debug("Parsed enclave details:"); + + for (const { creationTime, name, status, uuid } of enclaves) { + logger.debug(`UUID: ${uuid}, Name: ${name}, Status: ${status}, Created: ${creationTime}`); + logger.info(`🗑️ Removing enclave ${name}`); + logger.debug(await $`kurtosis enclave rm ${uuid} -f`.text()); + } + + logger.info(`🪓 ${enclaves.length} enclaves cleaned`); }; export const stopKurtosisEngine = async (options: StopOptions) => { diff --git a/test/cli/index.ts b/test/cli/index.ts index e7914d86..214cacfa 100644 --- a/test/cli/index.ts +++ b/test/cli/index.ts @@ -1,6 +1,14 @@ #!/usr/bin/env bun import { Command, InvalidArgumentError } from "@commander-js/extra-typings"; -import { launch, launchPreActionHook, stop, stopPreActionHook } from "./handlers"; +import type { DeployEnvironment } from "utils"; +import { + deploy, + deployPreActionHook, + launch, + launchPreActionHook, + stop, + stopPreActionHook +} from "./handlers"; // Function to parse integer function parseIntValue(value: string): number { @@ -11,6 +19,16 @@ function parseIntValue(value: string): number { return parsedValue; } +// Function to parse and validate DeployEnvironment +function parseDeployEnvironment(value: string): DeployEnvironment { + if (value === "stagenet" || value === "testnet" || value === "mainnet") { + return value; + } + throw new InvalidArgumentError( + "Invalid environment. Must be one of 'stagenet', 'testnet', or 'mainnet'." + ); +} + // ===== Program ===== const program = new Command() .version("0.2.0") @@ -18,6 +36,70 @@ const program = new Command() .summary("🫎 DataHaven CLI: Network Toolbox") .usage("[options]"); +// ===== Deploy ====== +program + .command("deploy") + .addHelpText( + "before", + `🫎 DataHaven: Network Deployer CLI for deploying a full DataHaven network stack to a Kubernetes cluster + It will deploy: + - DataHaven solochain validators (all envs), + - Storage providers (all envs) (TODO), + - Kurtosis Ethereum private network (stagenet env), + - Snowbridge Relayers (all envs) + ` + ) + .description("Deploy a full DataHaven network stack to a Kubernetes cluster") + .option( + "--e, --environment ", + "Environment to deploy to", + parseDeployEnvironment, + "stagenet" + ) + .option( + "--k, --kube-namespace ", + "Kubernetes namespace to deploy to. In 'stagenet' this parameter is ignored and the Kurtosis namespace is used instead. Default will be `datahaven-`." + ) + .option( + "--ke, --kurtosis-enclave-name ", + "Name of the Kurtosis enclave", + "datahaven-stagenet" + ) + .option("--st, --slot-time ", "Set slot time in seconds", parseIntValue, 12) + .option("--kn, --kurtosis-network-args ", "CustomKurtosis network args") + .option("--v, --verified", "Verify smart contracts with Blockscout") + .option("--b, --blockscout", "Enable Blockscout") + .option( + "--dit, --datahaven-image-tag ", + "Tag of the datahaven image to use", + "moonsonglabs/datahaven:main" + ) + .option( + "--el-rpc-url ", + "URL of the Ethereum Execution Layer (EL) RPC endpoint to use. In stagenet environment, the Kurtosis Ethereum network will be used. In testnet and mainnet environment, this parameter is required." + ) + .option( + "--cl-endpoint ", + "URL of the Ethereum Consensus Layer (CL) endpoint to use. In stagenet environment, the Kurtosis Ethereum network will be used. In testnet and mainnet environment, this parameter is required." + ) + .option( + "--rit, --relayer-image-tag ", + "Tag of the relayer image to use", + "moonsonglabs/snowbridge-relay:latest" + ) + .option("--docker-username ", "Docker Hub username") + .option("--docker-password ", "Docker Hub password") + .option("--docker-email ", "Docker Hub email") + .option("--skip-cleanup", "Skip cleaning up the network", false) + .option("--skip-kurtosis", "Skip deploying Kurtosis Ethereum private network", false) + .option("--skip-datahaven-solochain", "Skip deploying DataHaven solochain validators", false) + .option("--skip-contracts", "Skip deploying smart contracts", false) + .option("--skip-validator-operations", "Skip performing validator operations", false) + .option("--skip-set-parameters", "Skip setting DataHaven runtime parameters", false) + .option("--skip-relayers", "Skip deploying Snowbridge Relayers", false) + .hook("preAction", deployPreActionHook) + .action(deploy); + // ===== Launch ====== program .command("launch") @@ -26,9 +108,10 @@ program `🫎 DataHaven: Network Launcher CLI for launching a full DataHaven network. Complete with: - Solo-chain validators, - - Storage providers, + - Storage providers (TODO), + - Ethereum Private network, - Snowbridge Relayers - - Ethereum Private network` + ` ) .description("Launch a full E2E DataHaven & Ethereum network and more") .option("--d, --datahaven", "(Re)Launch DataHaven network") @@ -45,13 +128,13 @@ program .option("--nsv, --no-setup-validators", "Skip setup validators") .option("--uv, --update-validator-set", "Update validator set") .option("--nuv, --no-update-validator-set", "Skip update validator set") + .option("--sp, --set-parameters", "Set DataHaven runtime parameters") + .option("--nsp, --no-set-parameters", "Skip setting DataHaven runtime parameters") .option("--r, --relayer", "Launch Snowbridge Relayers") .option("--nr, --no-relayer", "Skip Snowbridge Relayers") .option("--b, --blockscout", "Enable Blockscout") .option("--slot-time ", "Set slot time in seconds", parseIntValue) .option("--cn, --clean-network", "Always clean Kurtosis enclave and Docker containers") - .option("--sp, --set-parameters", "Set DataHaven runtime parameters") - .option("--nsp, --no-set-parameters", "Skip setting DataHaven runtime parameters") .option( "--datahaven-build-extra-args ", "Extra args for DataHaven node Cargo build (the plain command is `cargo build --release` for linux, `cargo zigbuild --target x86_64-unknown-linux-gnu --release` for mac)", diff --git a/test/configs/snowbridge/beefy-relay.json b/test/configs/snowbridge/beefy-relay.json deleted file mode 100644 index f0c7cb71..00000000 --- a/test/configs/snowbridge/beefy-relay.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "source": { - "polkadot": { - "endpoint": "ws://127.0.0.1:9944" - } - }, - "sink": { - "ethereum": { - "endpoint": "ws://127.0.0.1:32806", - "gas-limit": "" - }, - "descendants-until-final": 3, - "contracts": { - "BeefyClient": "0x9d4454B023096f34B160D6B654540c56A1F81688", - "Gateway": "0x36C02dA8a0983159322a80FFE9F24b1acfF8B570" - } - }, - "on-demand-sync": { - "max-tokens": 5, - "refill-amount": 1, - "refill-period": 3600 - } -} \ No newline at end of file diff --git a/test/configs/snowbridge/execution-relay.json b/test/configs/snowbridge/execution-relay.json deleted file mode 100644 index 7d401057..00000000 --- a/test/configs/snowbridge/execution-relay.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "source": { - "ethereum": { - "endpoint": "ws://127.0.0.1:8546" - }, - "contracts": { - "Gateway": "" - }, - "beacon": { - "endpoint": "http://127.0.0.1:9596", - "stateEndpoint": "http://127.0.0.1:9596", - "spec": { - "syncCommitteeSize": 512, - "slotsInEpoch": 32, - "epochsPerSyncCommitteePeriod": 256, - "forkVersions": { - "deneb": 0, - "electra": 0 - } - }, - "datastore": { - "location": "", - "maxEntries": 100 - } - } - }, - "sink": { - "parachain": { - "endpoint": "", - "maxWatchedExtrinsics": 8, - "headerRedundancy": 20 - } - }, - "instantVerification": false, - "schedule": { - "id": null, - "totalRelayerCount": 1, - "sleepInterval": 1 - } -} diff --git a/test/configs/snowbridge/beacon-relay.json b/test/configs/snowbridge/local/beacon-relay.json similarity index 86% rename from test/configs/snowbridge/beacon-relay.json rename to test/configs/snowbridge/local/beacon-relay.json index ec4c05e9..62a078e5 100644 --- a/test/configs/snowbridge/beacon-relay.json +++ b/test/configs/snowbridge/local/beacon-relay.json @@ -13,7 +13,7 @@ } }, "datastore": { - "location": "/Users/facundofarall/Desktop/Moonsong/datahaven/test/tmp/facu-test", + "location": "/path/to/datastore", "maxEntries": 100 } } diff --git a/test/configs/snowbridge/local/beefy-relay.json b/test/configs/snowbridge/local/beefy-relay.json new file mode 100644 index 00000000..ba1c094e --- /dev/null +++ b/test/configs/snowbridge/local/beefy-relay.json @@ -0,0 +1,23 @@ +{ + "source": { + "polkadot": { + "endpoint": "ws://127.0.0.1:9944" + } + }, + "sink": { + "ethereum": { + "endpoint": "ws://127.0.0.1:32806", + "gas-limit": "" + }, + "descendants-until-final": 3, + "contracts": { + "BeefyClient": "0x9d4454B023096f34B160D6B654540c56A1F81688", + "Gateway": "0x36C02dA8a0983159322a80FFE9F24b1acfF8B570" + } + }, + "on-demand-sync": { + "max-tokens": 5, + "refill-amount": 1, + "refill-period": 3600 + } +} diff --git a/test/configs/snowbridge/local/execution-relay.json b/test/configs/snowbridge/local/execution-relay.json new file mode 100644 index 00000000..8c4a44c1 --- /dev/null +++ b/test/configs/snowbridge/local/execution-relay.json @@ -0,0 +1,40 @@ +{ + "source": { + "ethereum": { + "endpoint": "ws://127.0.0.1:8546" + }, + "contracts": { + "Gateway": "" + }, + "beacon": { + "endpoint": "http://127.0.0.1:9596", + "stateEndpoint": "http://127.0.0.1:9596", + "spec": { + "syncCommitteeSize": 512, + "slotsInEpoch": 32, + "epochsPerSyncCommitteePeriod": 256, + "forkVersions": { + "deneb": 0, + "electra": 0 + } + }, + "datastore": { + "location": "", + "maxEntries": 100 + } + } + }, + "sink": { + "parachain": { + "endpoint": "", + "maxWatchedExtrinsics": 8, + "headerRedundancy": 20 + } + }, + "instantVerification": false, + "schedule": { + "id": null, + "totalRelayerCount": 1, + "sleepInterval": 1 + } +} diff --git a/test/configs/snowbridge/local/solochain-relay.json b/test/configs/snowbridge/local/solochain-relay.json new file mode 100644 index 00000000..b08de875 --- /dev/null +++ b/test/configs/snowbridge/local/solochain-relay.json @@ -0,0 +1,49 @@ +{ + "source": { + "ethereum": { + "endpoint": "" + }, + "solochain": { + "endpoint": "" + }, + "contracts": { + "BeefyClient": "", + "Gateway": "" + }, + "beacon": { + "endpoint": "http://127.0.0.1:33030", + "stateEndpoint": "http://127.0.0.1:33030", + "spec": { + "syncCommitteeSize": 512, + "slotsInEpoch": 32, + "epochsPerSyncCommitteePeriod": 256, + "forkVersions": { + "deneb": 0, + "electra": 0 + } + }, + "datastore": { + "location": "/path/to/datastore", + "maxEntries": 100 + } + } + }, + "sink": { + "ethereum": { + "endpoint": "" + }, + "contracts": { + "Gateway": "" + } + }, + "schedule": { + "id": 0, + "totalRelayerCount": 1, + "sleepInterval": 10 + }, + "reward-address": "0xd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d", + "ofac": { + "enabled": false, + "apiKey": "" + } +} diff --git a/test/configs/snowbridge/solochain-relay.json b/test/configs/snowbridge/solochain-relay.json deleted file mode 100644 index 8716653b..00000000 --- a/test/configs/snowbridge/solochain-relay.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "source": { - "ethereum": { - "endpoint": "" - }, - "solochain": { - "endpoint": "" - }, - "contracts": { - "BeefyClient": "", - "Gateway": "" - }, - "beacon": { - "endpoint": "http://127.0.0.1:33030", - "stateEndpoint": "http://127.0.0.1:33030", - "spec": { - "syncCommitteeSize": 512, - "slotsInEpoch": 32, - "epochsPerSyncCommitteePeriod": 256, - "forkVersions": { - "deneb": 0, - "electra": 0 - } - }, - "datastore": { - "location": "/Users/tdemeco/Desktop/Moonsong/datahaven/test/tmp/tobi-test", - "maxEntries": 100 - } - } - }, - "sink": { - "ethereum": { - "endpoint": "" - }, - "contracts": { - "Gateway": "" - } - }, - "schedule": { - "id": 0, - "totalRelayerCount": 1, - "sleepInterval": 10 - }, - "reward-address": "0xd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d", - "ofac": { - "enabled": false, - "apiKey": "" - } -} \ No newline at end of file diff --git a/test/configs/snowbridge/stagenet/beacon-relay.json b/test/configs/snowbridge/stagenet/beacon-relay.json new file mode 100644 index 00000000..62a078e5 --- /dev/null +++ b/test/configs/snowbridge/stagenet/beacon-relay.json @@ -0,0 +1,29 @@ +{ + "source": { + "beacon": { + "endpoint": "http://127.0.0.1:33030", + "stateEndpoint": "http://127.0.0.1:33030", + "spec": { + "syncCommitteeSize": 512, + "slotsInEpoch": 32, + "epochsPerSyncCommitteePeriod": 256, + "forkVersions": { + "deneb": 0, + "electra": 0 + } + }, + "datastore": { + "location": "/path/to/datastore", + "maxEntries": 100 + } + } + }, + "sink": { + "parachain": { + "endpoint": "ws://127.0.0.1:9944", + "maxWatchedExtrinsics": 8, + "headerRedundancy": 20 + }, + "updateSlotInterval": 30 + } +} diff --git a/test/configs/snowbridge/stagenet/beefy-relay.json b/test/configs/snowbridge/stagenet/beefy-relay.json new file mode 100644 index 00000000..ba1c094e --- /dev/null +++ b/test/configs/snowbridge/stagenet/beefy-relay.json @@ -0,0 +1,23 @@ +{ + "source": { + "polkadot": { + "endpoint": "ws://127.0.0.1:9944" + } + }, + "sink": { + "ethereum": { + "endpoint": "ws://127.0.0.1:32806", + "gas-limit": "" + }, + "descendants-until-final": 3, + "contracts": { + "BeefyClient": "0x9d4454B023096f34B160D6B654540c56A1F81688", + "Gateway": "0x36C02dA8a0983159322a80FFE9F24b1acfF8B570" + } + }, + "on-demand-sync": { + "max-tokens": 5, + "refill-amount": 1, + "refill-period": 3600 + } +} diff --git a/test/configs/snowbridge/stagenet/execution-relay.json b/test/configs/snowbridge/stagenet/execution-relay.json new file mode 100644 index 00000000..37d0e2e3 --- /dev/null +++ b/test/configs/snowbridge/stagenet/execution-relay.json @@ -0,0 +1,41 @@ +{ + "source": { + "ethereum": { + "endpoint": "ws://127.0.0.1:8546" + }, + "contracts": { + "Gateway": "" + }, + "channel-id": "", + "beacon": { + "endpoint": "http://127.0.0.1:9596", + "stateEndpoint": "http://127.0.0.1:9596", + "spec": { + "syncCommitteeSize": 512, + "slotsInEpoch": 32, + "epochsPerSyncCommitteePeriod": 256, + "forkVersions": { + "deneb": 0, + "electra": 0 + } + }, + "datastore": { + "location": "", + "maxEntries": 100 + } + } + }, + "sink": { + "parachain": { + "endpoint": "", + "maxWatchedExtrinsics": 8, + "headerRedundancy": 20 + } + }, + "instantVerification": false, + "schedule": { + "id": null, + "totalRelayerCount": 1, + "sleepInterval": 1 + } +} diff --git a/test/configs/snowbridge/stagenet/solochain-relay.json b/test/configs/snowbridge/stagenet/solochain-relay.json new file mode 100644 index 00000000..b08de875 --- /dev/null +++ b/test/configs/snowbridge/stagenet/solochain-relay.json @@ -0,0 +1,49 @@ +{ + "source": { + "ethereum": { + "endpoint": "" + }, + "solochain": { + "endpoint": "" + }, + "contracts": { + "BeefyClient": "", + "Gateway": "" + }, + "beacon": { + "endpoint": "http://127.0.0.1:33030", + "stateEndpoint": "http://127.0.0.1:33030", + "spec": { + "syncCommitteeSize": 512, + "slotsInEpoch": 32, + "epochsPerSyncCommitteePeriod": 256, + "forkVersions": { + "deneb": 0, + "electra": 0 + } + }, + "datastore": { + "location": "/path/to/datastore", + "maxEntries": 100 + } + } + }, + "sink": { + "ethereum": { + "endpoint": "" + }, + "contracts": { + "Gateway": "" + } + }, + "schedule": { + "id": 0, + "totalRelayerCount": 1, + "sleepInterval": 10 + }, + "reward-address": "0xd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d", + "ofac": { + "enabled": false, + "apiKey": "" + } +} diff --git a/test/package.json b/test/package.json index dd5e04e3..a917bfef 100644 --- a/test/package.json +++ b/test/package.json @@ -15,7 +15,7 @@ "start:e2e:verified": "bun cli launch --verified --blockscout --deploy-contracts --setup-validators --update-validator-set --fund-validators", "start:e2e:verified:relayers": "bun cli launch --verified --blockscout --deploy-contracts --setup-validators --update-validator-set --fund-validators --relayer --datahaven", "start:e2e:local": "LOG_LEVEL=debug bun start:e2e:ci --bd", - "start:e2e:ci": "bun cli launch --datahaven --no-build-datahaven --launch-kurtosis --deploy-contracts --fund-validators --setup-validators --update-validator-set --relayer --clean-network --slot-time 2", + "start:e2e:ci": "bun cli launch --datahaven --no-build-datahaven --launch-kurtosis --deploy-contracts --fund-validators --setup-validators --update-validator-set --set-parameters --relayer --clean-network --slot-time 2", "start:e2e:minrelayer": "bun cli launch --relayer --deploy-contracts --no-setup-validators --no-update-validator-set --no-fund-validators --datahaven", "start:all": "bun cli launch --datahaven --build-datahaven --launch-kurtosis --deploy-contracts --fund-validators --setup-validators --update-validator-set --relayer --blockscout --verified --clean-network --set-parameters", "stop:docker:datahaven": "docker rm -f $(docker ps -aq --filter name='^datahaven-') 2>/dev/null || true; docker network rm datahaven-net || true", @@ -74,4 +74,4 @@ "ssh2", "utf-8-validate" ] -} \ No newline at end of file +} diff --git a/test/scripts/deploy-contracts.ts b/test/scripts/deploy-contracts.ts index 4ce1ba5b..e0436fa3 100644 --- a/test/scripts/deploy-contracts.ts +++ b/test/scripts/deploy-contracts.ts @@ -1,73 +1,30 @@ import { $ } from "bun"; import invariant from "tiny-invariant"; -import { - confirmWithTimeout, - logger, - parseDeploymentsFile, - printDivider, - printHeader, - runShellCommandWithLogger -} from "utils"; +import { logger, parseDeploymentsFile, runShellCommandWithLogger } from "utils"; import type { ParameterCollection } from "utils/parameters"; -interface DeployContractsOptions { +interface ContractDeploymentOptions { rpcUrl: string; verified?: boolean; blockscoutBackendUrl?: string; - deployContracts?: boolean; - parameterCollection?: ParameterCollection; } /** - * Deploys smart contracts to the specified RPC URL - * - * @param options - Configuration options for deployment - * @param options.rpcUrl - The RPC URL to deploy to - * @param options.verified - Whether to verify contracts (requires blockscoutBackendUrl) - * @param options.blockscoutBackendUrl - URL for the Blockscout API (required if verified is true) - * @param options.deployContracts - Flag to control deployment (if undefined, will prompt) - * @param options.parameterCollection - Collection of parameters to update in the DataHaven runtime - * @returns Promise resolving to true if contracts were deployed successfully, false if skipped + * Validates deployment parameters */ -export const deployContracts = async (options: DeployContractsOptions): Promise => { - const { - rpcUrl, - verified = false, - blockscoutBackendUrl, - deployContracts, - parameterCollection - } = options; +export const validateDeploymentParams = (options: ContractDeploymentOptions) => { + const { rpcUrl, verified, blockscoutBackendUrl } = options; - // Check if deployContracts option was set via flags, or prompt if not - let shouldDeployContracts = deployContracts; - if (shouldDeployContracts === undefined) { - shouldDeployContracts = await confirmWithTimeout( - "Do you want to deploy the smart contracts?", - true, - 10 - ); - } else { - logger.info( - `🏳️ Using flag option: ${shouldDeployContracts ? "will deploy" : "will not deploy"} smart contracts` - ); - } - - if (!shouldDeployContracts) { - logger.info("👍 Skipping contract deployment. Done!"); - printDivider(); - - return false; - } - - // Check if required parameters are provided invariant(rpcUrl, "❌ RPC URL is required"); if (verified) { invariant(blockscoutBackendUrl, "❌ Blockscout backend URL is required for verification"); } +}; - printHeader("Deploying Smart Contracts"); - - // Build contracts +/** + * Builds smart contracts using forge + */ +export const buildContracts = async () => { logger.info("🛳️ Building contracts..."); const { exitCode: buildExitCode, @@ -80,14 +37,32 @@ export const deployContracts = async (options: DeployContractsOptions): Promise< throw Error("❌ Contracts have failed to build properly."); } logger.debug(buildStdout.toString()); +}; + +/** + * Constructs the deployment command + */ +export const constructDeployCommand = (options: ContractDeploymentOptions): string => { + const { rpcUrl, verified, blockscoutBackendUrl } = options; let deployCommand = `forge script script/deploy/DeployLocal.s.sol --rpc-url ${rpcUrl} --color never -vv --no-rpc-rate-limit --non-interactive --broadcast`; if (verified && blockscoutBackendUrl) { + // TODO: Allow for other verifiers like Etherscan. deployCommand += ` --verify --verifier blockscout --verifier-url ${blockscoutBackendUrl}/api/ --delay 0`; logger.info("🔍 Contract verification enabled"); } + return deployCommand; +}; + +/** + * Executes contract deployment + */ +export const executeDeployment = async ( + deployCommand: string, + parameterCollection?: ParameterCollection +) => { logger.info("⌛️ Deploying contracts (this might take a few minutes)..."); // Using custom shell command to improve logging with forge's stdoutput @@ -115,33 +90,25 @@ export const deployContracts = async (options: DeployContractsOptions): Promise< } logger.success("Contracts deployed successfully"); - printDivider(); - - return true; }; // Allow script to be run directly with CLI arguments if (import.meta.main) { const args = process.argv.slice(2); - const options: { - rpcUrl?: string; - verified: boolean; - blockscoutBackendUrl?: string; - deployContracts?: boolean; - } = { - verified: args.includes("--verified"), - deployContracts: args.includes("--deploy-contracts") - ? true - : args.includes("--no-deploy-contracts") - ? false - : undefined - }; // Extract RPC URL const rpcUrlIndex = args.indexOf("--rpc-url"); - if (rpcUrlIndex !== -1 && rpcUrlIndex + 1 < args.length) { - options.rpcUrl = args[rpcUrlIndex + 1]; - } + invariant(rpcUrlIndex !== -1, "❌ --rpc-url flag is required"); + invariant(rpcUrlIndex + 1 < args.length, "❌ --rpc-url flag requires an argument"); + + const options: { + rpcUrl: string; + verified: boolean; + blockscoutBackendUrl?: string; + } = { + rpcUrl: args[rpcUrlIndex + 1], + verified: args.includes("--verified") + }; // Extract Blockscout URL if verification is enabled if (options.verified) { @@ -161,13 +128,10 @@ if (import.meta.main) { process.exit(1); } - deployContracts({ - rpcUrl: options.rpcUrl, - verified: options.verified, - blockscoutBackendUrl: options.blockscoutBackendUrl, - deployContracts: options.deployContracts - }).catch((error) => { - console.error("Deployment failed:", error); - process.exit(1); - }); + validateDeploymentParams(options); + + await buildContracts(); + + const deployCommand = constructDeployCommand(options); + await executeDeployment(deployCommand); } diff --git a/test/scripts/set-datahaven-parameters.ts b/test/scripts/set-datahaven-parameters.ts index a86f4a92..5d1479fa 100644 --- a/test/scripts/set-datahaven-parameters.ts +++ b/test/scripts/set-datahaven-parameters.ts @@ -3,22 +3,13 @@ import { datahaven } from "@polkadot-api/descriptors"; import { createClient } from "polkadot-api"; import { withPolkadotSdkCompat } from "polkadot-api/polkadot-sdk-compat"; import { getWsProvider } from "polkadot-api/ws-provider/web"; -import invariant from "tiny-invariant"; -import { - confirmWithTimeout, - getEvmEcdsaSigner, - logger, - printDivider, - printHeader, - SUBSTRATE_FUNDED_ACCOUNTS -} from "utils"; +import { getEvmEcdsaSigner, logger, SUBSTRATE_FUNDED_ACCOUNTS } from "utils"; import { type ParsedDataHavenParameter, parseJsonToParameters } from "utils/types"; // Interface for the options object of setDataHavenParameters interface SetDataHavenParametersOptions { rpcUrl: string; parametersFilePath: string; - setParameters?: boolean; } /** @@ -27,39 +18,12 @@ interface SetDataHavenParametersOptions { * @param options - Configuration options for setting parameters * @param options.rpcUrl - The RPC URL of the DataHaven node * @param options.parametersFilePath - Path to the JSON file containing an array of parameters to set - * @param options.setParameters - Flag to control execution (if undefined, will prompt) * @returns Promise resolving to true if parameters were set successfully, false if skipped */ export const setDataHavenParameters = async ( options: SetDataHavenParametersOptions ): Promise => { - const { rpcUrl, parametersFilePath, setParameters } = options; - - // Check if setParameters option was set via flags, or prompt if not - let shouldSetParameters = setParameters; - if (shouldSetParameters === undefined) { - shouldSetParameters = await confirmWithTimeout( - "Do you want to set the DataHaven runtime parameters?", - true, - 10 - ); - } else { - logger.info( - `🏳️ Using flag option: ${ - shouldSetParameters ? "will set" : "will not set" - } DataHaven parameters` - ); - } - - if (!shouldSetParameters) { - logger.info("👍 Skipping DataHaven parameter setting. Done!"); - printDivider(); - return false; - } - - // Check if required parameters are provided - invariant(rpcUrl, "❌ RPC URL is required"); - invariant(parametersFilePath, "❌ Parameters file path is required"); + const { rpcUrl, parametersFilePath } = options; // Load parameters from the JSON file let parameters: ParsedDataHavenParameter[]; @@ -71,7 +35,6 @@ export const setDataHavenParameters = async ( if (parameters.length === 0) { logger.warn("⚠️ The parameters file is empty. No parameters to set."); - printDivider(); return false; } } catch (error: any) { @@ -81,8 +44,6 @@ export const setDataHavenParameters = async ( throw error; } - printHeader("Setting DataHaven Runtime Parameters"); - const client = createClient(withPolkadotSdkCompat(getWsProvider(rpcUrl))); const dhApi = client.getTypedApi(datahaven); logger.trace("Substrate client created"); @@ -95,7 +56,9 @@ export const setDataHavenParameters = async ( try { for (const param of parameters) { // TODO: Add a graceful way to print the value of the parameter, since it won't always be representable as a hex string - logger.info(`Attempting to set parameter: ${String(param.name)} = ${param.value.asHex()}`); + logger.info( + `🔧 Attempting to set parameter: ${param.name.toString()} = ${param.value.asHex()}` + ); const setParameterArgs: any = { key_value: { @@ -144,7 +107,6 @@ export const setDataHavenParameters = async ( } else { logger.warn("Some DataHaven parameters could not be set. Please check logs."); } - printDivider(); return allSuccessful; }; @@ -161,10 +123,6 @@ if (import.meta.main) { parametersFile: { type: "string", short: "f" - }, - setParameters: { - type: "boolean", - short: "p" } }, strict: true @@ -182,8 +140,7 @@ if (import.meta.main) { setDataHavenParameters({ rpcUrl: values.rpcUrl, - parametersFilePath: values.parametersFile, - setParameters: values.setParameters + parametersFilePath: values.parametersFile }).catch((error: Error) => { console.error("Setting DataHaven parameters failed:", error.message || error); process.exit(1); diff --git a/test/utils/kurtosis.ts b/test/utils/kurtosis.ts index d26a3ca5..42f2aa43 100644 --- a/test/utils/kurtosis.ts +++ b/test/utils/kurtosis.ts @@ -71,7 +71,7 @@ const serviceSchema = z.object({ files: z.record(z.string(), z.array(z.string())).optional(), entrypoint: z.array(z.string()).optional(), cmd: z.array(z.string()), - env_vars: z.record(z.string(), z.string()), + env_vars: z.record(z.string(), z.string()).optional(), labels: z.record(z.string(), z.string()).optional(), tini_enabled: z.boolean() }); diff --git a/test/utils/parameters.ts b/test/utils/parameters.ts index 7ed17b8b..f3e1b9ce 100644 --- a/test/utils/parameters.ts +++ b/test/utils/parameters.ts @@ -1,6 +1,6 @@ import path from "node:path"; import { $ } from "bun"; -import { logger, printDivider } from "utils"; +import { logger } from "utils"; import type { ParsedDataHavenParameter } from "utils/types"; // Constants for paths @@ -71,7 +71,7 @@ export class ParameterCollection { /** * Creates a new ParameterCollection, pre-loaded with template parameters if available */ -export async function createParameterCollection(): Promise { +export const createParameterCollection = async (): Promise => { const collection = new ParameterCollection(); const templateFile = Bun.file(PARAMETERS_TEMPLATE_PATH); @@ -83,40 +83,4 @@ export async function createParameterCollection(): Promise } return collection; -} - -/** - * A helper function to set DataHaven parameters from a ParameterCollection - * - * @param options Options for setting parameters - * @param options.rpcUrl The RPC URL of the DataHaven node - * @param options.collection The parameter collection - * @param options.setParameters Flag to control execution - * @returns Promise resolving to true if parameters were set successfully - */ -export async function setParametersFromCollection({ - rpcUrl, - collection, - setParameters -}: { - rpcUrl: string; - collection: ParameterCollection; - setParameters?: boolean; -}): Promise { - const parametersFilePath = await collection.generateParametersFile(); - - try { - // Import the setDataHavenParameters function dynamically - const { setDataHavenParameters } = await import("../scripts/set-datahaven-parameters"); - - return await setDataHavenParameters({ - rpcUrl, - parametersFilePath, - setParameters - }); - } catch (error) { - logger.error(`Failed to set DataHaven parameters: ${error}`); - printDivider(); - return false; - } -} +}; diff --git a/test/utils/parser.ts b/test/utils/parser.ts index 692a297e..1d494a4d 100644 --- a/test/utils/parser.ts +++ b/test/utils/parser.ts @@ -1,5 +1,13 @@ import { z } from "zod"; +export const KurtosisEnclaveInfoSchema = z.object({ + uuid: z.string().min(1), + name: z.string().min(1), + status: z.string().min(1), + creationTime: z.string().min(1) +}); +export type KurtosisEnclaveInfo = z.infer; + export const BeaconRelayConfigSchema = z.object({ source: z.object({ beacon: z.object({ @@ -105,7 +113,6 @@ export const SolochainRelayConfigSchema = z.object({ apiKey: z.string() }) }); - export type SolochainRelayConfig = z.infer; export const ExecutionRelayConfigSchema = z.object({ @@ -226,6 +233,8 @@ export function parseRelayConfig( case "solochain": return parseSolochainConfig(config); default: - throw new Error(`Unknown relayer type: ${type}`); + throw new Error(`Unsupported relayer type with config: \n${JSON.stringify(config)}`); } } + +export type DeployEnvironment = "stagenet" | "testnet" | "mainnet"; diff --git a/test/utils/shell.ts b/test/utils/shell.ts index 67613bf1..20b27a45 100644 --- a/test/utils/shell.ts +++ b/test/utils/shell.ts @@ -34,8 +34,10 @@ export const runShellCommandWithLogger = async ( const stdoutReader = proc.stdout.getReader(); const stderrReader = proc.stderr.getReader(); + let stderrBuffer = ""; + const readStream = async ( - reader: typeof stdoutReader | typeof stderrReader, + reader: typeof stdoutReader, streamName: string, logLevel: LogLevel ) => { @@ -58,16 +60,37 @@ export const runShellCommandWithLogger = async ( } }; - await Promise.all([ - readStream(stdoutReader, "stdout", logLevel), - readStream(stderrReader, "stderr", "error") - ]); + const readStderr = async () => { + try { + while (true) { + const { done, value } = await stderrReader.read(); + if (done) break; + stderrBuffer += new TextDecoder().decode(value); + } + } catch (err) { + logger.error("Error reading from stderr stream:", err); + } finally { + stderrReader.releaseLock(); + } + }; + + await Promise.all([readStream(stdoutReader, "stdout", logLevel), readStderr()]); if (options?.waitFor) { await options.waitFor(); } - await proc.exited; + const exitCode = await proc.exited; + + // Only log stderr if the command failed + if (exitCode !== 0) { + const trimmedStderr = stderrBuffer.trim(); + if (trimmedStderr) { + logger.error( + trimmedStderr.includes("\n") ? `>_ \n${trimmedStderr}` : `>_ ${trimmedStderr}` + ); + } + } } catch (err) { logger.error("❌ Error running shell command:", command, "in", cwd); logger.error(err); diff --git a/test/utils/waits.ts b/test/utils/waits.ts new file mode 100644 index 00000000..a85ef02f --- /dev/null +++ b/test/utils/waits.ts @@ -0,0 +1,43 @@ +import { sleep } from "bun"; +import { logger } from "./logger"; + +/** + * Options for the `waitFor` function. + * @param lambda - The condition to wait for. + * @param iterations - The number of iterations to wait for the condition to be true. + * @param delay - The delay between iterations. + */ +export interface WaitForOptions { + lambda: () => Promise; + iterations?: number; + delay?: number; + errorMessage?: string; +} + +/** + * Waits for an arbitrary condition to be true. It keeps polling the condition until it is true or + * a timeout is reached. + */ +export const waitFor = async (options: WaitForOptions) => { + const { lambda, iterations = 100, delay = 100, errorMessage } = options; + + for (let i = 0; i < iterations; i++) { + try { + const result = await lambda(); + if (result) { + return; + } + } catch (e: unknown) { + logger.debug(`Try ${i + 1} of ${iterations} failed: ${e}`); + } + + // Only sleep if there are more iterations remaining + if (i < iterations - 1) { + await sleep(delay); + } + } + + throw new Error( + `Failed after ${(iterations * delay) / 1000}s: ${errorMessage || "No error message provided"}` + ); +};