From 98428ed3017ec432b42ec0c2bbbb153776047ed3 Mon Sep 17 00:00:00 2001 From: Facundo Farall <37149322+ffarall@users.noreply.github.com> Date: Thu, 15 May 2025 18:56:36 -0300 Subject: [PATCH] =?UTF-8?q?feat(CLI):=20=E2=9C=A8=20Run=20beacon=20relay?= =?UTF-8?q?=20in=20CLI=20(#70)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - **New Features** - Initialise Ethereum client pallet with a beacon chain checkpoint before starting relayers. - **Improvements** - Store Ethereum node RPC endpoints in `launchedNetwork` for later retrieval. - Standardised CLI options with explicit paired flags for enabling and disabling features, improving usability. - Increased slot frequency and number of validator keys per node in test network configurations. - Expanded and clarified test environment setup documentation and added a new CLI usage section in the main README. - **Bug Fixes** - Updated runtime fork version constants for testing environments, to match with Kurtosis'. ## Summary by CodeRabbit - **New Features** - Enhanced CLI with explicit enable/disable flags for network components and relayers. - Added initialization of the Ethereum Beacon Client pallet, ensuring the beacon chain is ready and submitting an initial checkpoint before starting relayers. - **Improvements** - Streamlined network setup by centralizing service endpoint registration and simplifying RPC URL handling. - Expanded and clarified CLI and test documentation with detailed setup instructions and option descriptions. - **Configuration Updates** - Updated network and beacon relay configurations for improved slot timing, validator key allocation, and sync committee period. - Adjusted Ethereum fork version constants to ensure compatibility. - **Bug Fixes** - Improved error handling and validation during network and relayer initialization. - **Documentation** - Added an "E2E CLI" section to the main README. - Enhanced test environment documentation with clearer steps and tips. --- README.md | 17 +- operator/runtime/stagenet/src/configs/mod.rs | 15 +- test/README.md | 25 ++- test/cli/handlers/launch/index.ts | 19 +- test/cli/handlers/launch/kurtosis.ts | 56 +++-- test/cli/handlers/launch/launchedNetwork.ts | 73 ++++++ test/cli/handlers/launch/relayer.ts | 183 ++++++++++++++- test/cli/index.ts | 36 +-- test/configs/kurtosis/minimal.yaml | 4 +- test/configs/snowbridge/beacon-relay.json | 4 +- test/package.json | 6 +- test/utils/types.ts | 221 +++++++++++++++++++ 12 files changed, 574 insertions(+), 85 deletions(-) create mode 100644 test/utils/types.ts diff --git a/README.md b/README.md index e0228a51..6073f106 100644 --- a/README.md +++ b/README.md @@ -14,11 +14,26 @@ datahaven/ └── README.md ``` +## E2E CLI + +This repo comes with a CLI for launching a local DataHaven network, packaged with: + +1. A full Ethereum network with: + - 2 x Execution Layer clients (e.g., reth) + - 2 x Consensus Layer clients (e.g., lighthouse) + - Blockscout Explorer services for EL (if enabled with --blockscout) + - Dora Explorer service for CL + - Contracts deployed and configured for the DataHaven network. +2. A DataHaven solochain. +3. Snowbridge relayers for cross-chain communication. + +To launch the network, follow the instructions in the [test README](./test/README.md). + ## Docker This repo publishes images to [DockerHub](https://hub.docker.com/r/moonsonglabs/datahaven). -> [!TIP] +> [!TIP] > > If you cannot see this repo you must be added to the permission list for the private repo. diff --git a/operator/runtime/stagenet/src/configs/mod.rs b/operator/runtime/stagenet/src/configs/mod.rs index cd50848e..1a9ea2c3 100644 --- a/operator/runtime/stagenet/src/configs/mod.rs +++ b/operator/runtime/stagenet/src/configs/mod.rs @@ -691,6 +691,9 @@ impl snowbridge_pallet_system_v2::Config for Runtime { } // For tests, benchmarks and fast-runtime configurations we use the mocked fork versions +// The version numbers are taken from looking at the Dora explorer when launching the +// kurtosis Ethereum network. Hovering over the fork names, shows the version numbers. +// These version numbers need to match, otherwise the aggregated signature verification will fail. #[cfg(any( feature = "std", feature = "fast-runtime", @@ -700,27 +703,27 @@ impl snowbridge_pallet_system_v2::Config for Runtime { parameter_types! { pub const ChainForkVersions: ForkVersions = ForkVersions { genesis: Fork { - version: [0, 0, 0, 0], // 0x00000000 + version: [16, 0, 0, 56], // 0x10000038 epoch: 0, }, altair: Fork { - version: [1, 0, 0, 0], // 0x01000000 + version: [32, 0, 0, 56], // 0x20000038 epoch: 0, }, bellatrix: Fork { - version: [2, 0, 0, 0], // 0x02000000 + version: [48, 0, 0, 56], // 0x30000038 epoch: 0, }, capella: Fork { - version: [3, 0, 0, 0], // 0x03000000 + version: [64, 0, 0, 56], // 0x40000038 epoch: 0, }, deneb: Fork { - version: [4, 0, 0, 0], // 0x04000000 + version: [80, 0, 0, 56], // 0x50000038 epoch: 0, }, electra: Fork { - version: [5, 0, 0, 0], // 0x05000000 + version: [96, 0, 0, 56], // 0x60000038 epoch: 0, }, }; diff --git a/test/README.md b/test/README.md index 49694756..4a8e5d90 100644 --- a/test/README.md +++ b/test/README.md @@ -36,15 +36,18 @@ Follow these steps to set up and interact with your test environment: This script will: - 1. Start a Kurtosis network with (among other things): - - 2 x Ethereum Execution Layer clients (reth) - - 2 x Ethereum Consensus Layer clients (lighthouse) - - 1 x Blockscout frontend - - 1 x Blockscout backend - 2. Send a test transaction to the network using the [send-txn.ts](./scripts/send-txn.ts) script. - 3. Deploy all DataHaven smart contracts needed for a local deployment, using the [DeployLocal.s.sol](../contracts/script/deploy/DeployLocal.s.sol) script. + 1. Check for required dependencies. + 2. Launch a DataHaven solochain. + 3. Start a Kurtosis network which includes: + - 2 Ethereum Execution Layer clients (reth) + - 2 Ethereum Consensus Layer clients (lighthouse) + - Blockscout Explorer services for EL (if enabled with --blockscout) + - Dora Explorer service for CL + 4. Deploy DataHaven smart contracts to the Ethereum network. This can optionally include verification on Blockscout if the `--verified` flag is used (requires Blockscout to be enabled). + 5. Perform validator setup and funding operations. + 6. Launch Snowbridge relayers. - > ℹ️ NOTE + > [!NOTE] > > If you want to also have the contracts verified on blockscout, you can run `bun start:e2e:verified` instead. This will do all the previous steps, but also verify the contracts on blockscout. However, note that this takes some time to complete. @@ -79,7 +82,7 @@ You can also access the backend via REST API, documented here: [http://127.0.0.1 ### E2E Tests -> 🧙‍♂️ TIP +> [!TIP] > > Remember to run the network with `bun cli` before running the tests. @@ -87,7 +90,7 @@ You can also access the backend via REST API, documented here: [http://127.0.0.1 bun test:e2e ``` -> ℹ️ NOTE +> [!NOTE] > > You can increase the logging level by setting `LOG_LEVEL=debug` before running the tests. @@ -175,7 +178,7 @@ This script will: 1. Compile the runtime using `cargo build --release` in the `../operator` directory. 2. Re-generate the Polkadot-API types using the newly built WASM binary. -> ℹ️ NOTE +> [!NOTE] > > The script uses the `--release` flag by default, meaning it uses the WASM binary from `./operator/target/release`. If you need to use a different build target, you may need to adjust the script or run the steps manually. diff --git a/test/cli/handlers/launch/index.ts b/test/cli/handlers/launch/index.ts index c2122c3d..33a0dd7d 100644 --- a/test/cli/handlers/launch/index.ts +++ b/test/cli/handlers/launch/index.ts @@ -1,13 +1,6 @@ import type { Command } from "@commander-js/extra-typings"; import { deployContracts } from "scripts/deploy-contracts"; -import { sendDataHavenTxn, sendEthTxn } from "scripts/send-txn"; -import invariant from "tiny-invariant"; -import { - ANVIL_FUNDED_ACCOUNTS, - SUBSTRATE_FUNDED_ACCOUNTS, - getPortFromKurtosis, - logger -} from "utils"; +import { getPortFromKurtosis, logger } from "utils"; import { checkDependencies } from "./checks"; import { launchDataHavenSolochain } from "./datahaven"; import { launchKurtosis } from "./kurtosis"; @@ -54,11 +47,7 @@ const launchFunction = async (options: LaunchOptions, launchedNetwork: LaunchedN await launchDataHavenSolochain(options, launchedNetwork); - await launchKurtosis(options); - - const rethPublicPort = await getPortFromKurtosis("el-1-reth-lighthouse", "rpc"); - const elRpcUrl = `http://127.0.0.1:${rethPublicPort}`; - invariant(elRpcUrl, "❌ Network RPC URL not found"); + await launchKurtosis(launchedNetwork, options); let blockscoutBackendUrl: string | undefined = undefined; @@ -73,13 +62,13 @@ const launchFunction = async (options: LaunchOptions, launchedNetwork: LaunchedN } const contractsDeployed = await deployContracts({ - rpcUrl: elRpcUrl, + rpcUrl: launchedNetwork.getElRpcUrl(), verified: options.verified, blockscoutBackendUrl, deployContracts: options.deployContracts }); - await performValidatorOperations(options, elRpcUrl, contractsDeployed); + await performValidatorOperations(options, launchedNetwork.getElRpcUrl(), contractsDeployed); await launchRelayers(options, launchedNetwork); diff --git a/test/cli/handlers/launch/kurtosis.ts b/test/cli/handlers/launch/kurtosis.ts index 5cd4525f..15cbab18 100644 --- a/test/cli/handlers/launch/kurtosis.ts +++ b/test/cli/handlers/launch/kurtosis.ts @@ -1,25 +1,20 @@ import { $ } from "bun"; import type { LaunchOptions } from "cli/handlers"; import invariant from "tiny-invariant"; -import { - type KurtosisService, - confirmWithTimeout, - getServicesFromKurtosis, - logger, - printDivider, - printHeader -} from "utils"; +import { confirmWithTimeout, getPortFromKurtosis, logger, printDivider, printHeader } from "utils"; import { parse, stringify } from "yaml"; +import type { LaunchedNetwork } from "./launchedNetwork"; /** * Launches a Kurtosis Ethereum network enclave for testing. * + * @param launchedNetwork - The LaunchedNetwork instance to store network details * @param options - Configuration options - * @returns Object containing success status and Docker services information */ export const launchKurtosis = async ( + launchedNetwork: LaunchedNetwork, options: LaunchOptions = {} -): Promise> => { +): Promise => { printHeader("Starting Kurtosis Network"); if ((await checkKurtosisRunning()) && !options.alwaysClean) { @@ -28,14 +23,14 @@ export const launchKurtosis = async ( logger.trace("Checking if launchKurtosis option was set via flags"); if (options.launchKurtosis === false) { logger.info("Keeping existing Kurtosis enclave."); + await registerServices(launchedNetwork); printDivider(); - return getServicesFromKurtosis(); + return; } if (options.launchKurtosis === true) { logger.info("Proceeding to clean and relaunch the Kurtosis enclave..."); } else { - // Use confirmWithTimeout if launchKurtosis is undefined const shouldRelaunch = await confirmWithTimeout( "Do you want to clean and relaunch the Kurtosis enclave?", true, @@ -44,8 +39,9 @@ export const launchKurtosis = async ( if (!shouldRelaunch) { logger.info("Keeping existing Kurtosis enclave."); + await registerServices(launchedNetwork); printDivider(); - return getServicesFromKurtosis(); + return; } logger.info("Proceeding to clean and relaunch the Kurtosis enclave..."); @@ -84,13 +80,9 @@ export const launchKurtosis = async ( } logger.debug(stdout.toString()); - logger.info("🔍 Gathering Kurtosis public ports..."); - const services = await getServicesFromKurtosis(); - - logger.success("Kurtosis network started successfully"); + await registerServices(launchedNetwork); + logger.success("Kurtosis network operations completed successfully."); printDivider(); - - return services; }; /** @@ -141,3 +133,29 @@ const modifyConfig = async (options: LaunchOptions, configFile: string) => { await Bun.write(outputFile, stringify(parsedConfig)); return outputFile; }; + +/** + * Registers the EL and CL service endpoints with the LaunchedNetwork instance. + * + * @param launchedNetwork - The LaunchedNetwork instance to store network details. + */ +const registerServices = async (launchedNetwork: LaunchedNetwork) => { + logger.info("⚙️ Registering Kurtosis service endpoints..."); + + // Configure EL RPC URL + const rethPublicPort = await getPortFromKurtosis("el-1-reth-lighthouse", "rpc"); + invariant(rethPublicPort && rethPublicPort > 0, "❌ Could not find EL RPC port"); + const elRpcUrl = `http://127.0.0.1:${rethPublicPort}`; + launchedNetwork.setElRpcUrl(elRpcUrl); + logger.info(`👍 Execution Layer RPC URL configured: ${elRpcUrl}`); + + // Configure CL Endpoint + const lighthousePublicPort = await getPortFromKurtosis("cl-1-lighthouse-reth", "http"); + const clEndpoint = `http://127.0.0.1:${lighthousePublicPort}`; + invariant( + clEndpoint, + "❌ CL Endpoint could not be determined from Kurtosis service cl-1-lighthouse-reth" + ); + launchedNetwork.setClEndpoint(clEndpoint); + logger.info(`👍 Consensus Layer Endpoint configured: ${clEndpoint}`); +}; diff --git a/test/cli/handlers/launch/launchedNetwork.ts b/test/cli/handlers/launch/launchedNetwork.ts index ee3bc2be..efdc076a 100644 --- a/test/cli/handlers/launch/launchedNetwork.ts +++ b/test/cli/handlers/launch/launchedNetwork.ts @@ -2,45 +2,118 @@ import fs from "node:fs"; import invariant from "tiny-invariant"; import { logger } from "utils"; +/** + * Represents the state and associated resources of a launched network environment, + * including DataHaven nodes, Kurtosis services, and related process/file descriptors. + */ export class LaunchedNetwork { protected runId: string; protected processes: Bun.Subprocess<"inherit" | "pipe" | "ignore", number, number>[]; protected fileDescriptors: number[]; protected DHNodes: { id: string; port: number }[]; + /** The RPC URL for the Ethereum Execution Layer (EL) client. */ + protected elRpcUrl?: string; + /** The HTTP endpoint for the Ethereum Consensus Layer (CL) client. */ + protected clEndpoint?: string; constructor() { this.runId = crypto.randomUUID(); this.processes = []; this.fileDescriptors = []; this.DHNodes = []; + this.elRpcUrl = undefined; + this.clEndpoint = undefined; } + /** + * Gets the unique ID for this run of the launched network. + * @returns The run ID string. + */ getRunId(): string { return this.runId; } + /** + * Gets the list of launched DataHaven (DH) nodes. + * @returns An array of DH node objects, each with an id and port. + */ getDHNodes(): { id: string; port: number }[] { return [...this.DHNodes]; } + /** + * Gets the port for a specific DataHaven (DH) node by its ID. + * @param id - The ID of the DH node. + * @returns The port number of the DH node. + * @throws If the node with the given ID is not found. + */ getDHPort(id: string): number { const node = this.DHNodes.find((x) => x.id === id); invariant(node, `❌ Datahaven node ${id} not found`); return node.port; } + /** + * Adds a file descriptor to be managed and cleaned up. + * @param fd - The file descriptor number. + */ addFileDescriptor(fd: number) { this.fileDescriptors.push(fd); } + /** + * Adds a running process to be managed and cleaned up. + * @param process - The Bun subprocess object. + */ addProcess(process: Bun.Subprocess<"inherit" | "pipe" | "ignore", number, number>) { this.processes.push(process); } + /** + * Adds a DataHaven (DH) node to the list of launched nodes. + * @param id - The ID of the DH node. + * @param port - The port number the DH node is running on. + */ addDHNode(id: string, port: number) { this.DHNodes.push({ id, port }); } + /** + * Sets the RPC URL for the Ethereum Execution Layer (EL) client. + * @param url - The EL RPC URL string. + */ + setElRpcUrl(url: string) { + this.elRpcUrl = url; + } + + /** + * Gets the RPC URL for the Ethereum Execution Layer (EL) client. + * @returns The EL RPC URL string. + * @throws If the EL RPC URL has not been set. + */ + getElRpcUrl(): string { + invariant(this.elRpcUrl, "❌ EL RPC URL not set in LaunchedNetwork"); + return this.elRpcUrl; + } + + /** + * Sets the HTTP endpoint for the Ethereum Consensus Layer (CL) client. + * @param url - The CL HTTP endpoint string. + */ + setClEndpoint(url: string) { + this.clEndpoint = url; + } + + /** + * Gets the HTTP endpoint for the Ethereum Consensus Layer (CL) client. + * @returns The CL HTTP endpoint string. + * @throws If the CL HTTP endpoint has not been set. + */ + getClEndpoint(): string { + invariant(this.clEndpoint, "❌ CL HTTP Endpoint not set in LaunchedNetwork"); + return this.clEndpoint; + } + async cleanup() { for (const process of this.processes) { logger.debug(`Process is still running: ${process.pid}`); diff --git a/test/cli/handlers/launch/relayer.ts b/test/cli/handlers/launch/relayer.ts index acd4f1b6..5a064b18 100644 --- a/test/cli/handlers/launch/relayer.ts +++ b/test/cli/handlers/launch/relayer.ts @@ -1,12 +1,17 @@ import fs from "node:fs"; import path from "node:path"; +import { datahaven } from "@polkadot-api/descriptors"; import { $ } from "bun"; +import { createClient } from "polkadot-api"; +import { withPolkadotSdkCompat } from "polkadot-api/polkadot-sdk-compat"; +import { getWsProvider } from "polkadot-api/ws-provider/web"; import invariant from "tiny-invariant"; import { ANVIL_FUNDED_ACCOUNTS, type RelayerType, SUBSTRATE_FUNDED_ACCOUNTS, confirmWithTimeout, + getEvmEcdsaSigner, getPortFromKurtosis, logger, parseDeploymentsFile, @@ -14,9 +19,13 @@ import { printDivider, printHeader } from "utils"; +import type { BeaconCheckpoint, FinalityCheckpointsResponse } from "utils/types"; +import { parseJsonToBeaconCheckpoint } from "utils/types"; import type { LaunchOptions } from "."; import type { LaunchedNetwork } from "./launchedNetwork"; +const ZERO_HASH = "0x0000000000000000000000000000000000000000000000000000000000000000"; + type RelayerSpec = { name: string; type: RelayerType; @@ -24,6 +33,13 @@ type RelayerSpec = { pk: { type: "ethereum" | "substrate"; value: string }; }; +const RELAYER_CONFIG_DIR = "tmp/configs"; +const RELAYER_CONFIG_PATHS = { + BEACON: path.join(RELAYER_CONFIG_DIR, "beacon-relay.json"), + BEEFY: path.join(RELAYER_CONFIG_DIR, "beefy-relay.json") +}; +const INITIAL_CHECKPOINT_PATH = "./dump-initial-checkpoint.json"; + /** * Launches Snowbridge relayers for the DataHaven network. * @@ -61,9 +77,8 @@ export const launchRelayers = async (options: LaunchOptions, launchedNetwork: La invariant(beefyClientAddress, "❌ BeefyClient address not found in anvil.json"); invariant(gatewayAddress, "❌ Gateway address not found in anvil.json"); - const outputDir = "tmp/configs"; - logger.debug(`Ensuring output directory exists: ${outputDir}`); - await $`mkdir -p ${outputDir}`.quiet(); + logger.debug(`Ensuring output directory exists: ${RELAYER_CONFIG_DIR}`); + await $`mkdir -p ${RELAYER_CONFIG_DIR}`.quiet(); const datastorePath = "tmp/datastore"; logger.debug(`Ensuring datastore directory exists: ${datastorePath}`); @@ -77,7 +92,7 @@ export const launchRelayers = async (options: LaunchOptions, launchedNetwork: La { name: "relayer-🥩", type: "beefy", - config: "beefy-relay.json", + config: RELAYER_CONFIG_PATHS.BEEFY, pk: { type: "ethereum", value: ANVIL_FUNDED_ACCOUNTS[1].privateKey @@ -86,15 +101,17 @@ export const launchRelayers = async (options: LaunchOptions, launchedNetwork: La { name: "relayer-🥓", type: "beacon", - config: "beacon-relay.json", + config: RELAYER_CONFIG_PATHS.BEACON, pk: { type: "substrate", - value: SUBSTRATE_FUNDED_ACCOUNTS.GOLIATH.privateKey + value: SUBSTRATE_FUNDED_ACCOUNTS.ALITH.privateKey } } ]; - for (const { config: configFileName, type, name } of relayersToStart) { + for (const { config, type, name } of relayersToStart) { + const configFileName = path.basename(config); + logger.debug(`Creating config for ${name}`); const templateFilePath = `configs/snowbridge/${configFileName}`; const outputFilePath = `tmp/configs/${configFileName}`; @@ -143,6 +160,8 @@ export const launchRelayers = async (options: LaunchOptions, launchedNetwork: La `❌ Relayer binary does not exist at ${options.relayerBinPath}` ); + await initEthClientPallet(options, launchedNetwork); + for (const { config, name, type, pk } of relayersToStart) { try { logger.info(`Starting relayer ${name} ...`); @@ -157,7 +176,7 @@ export const launchRelayers = async (options: LaunchOptions, launchedNetwork: La "run", type, "--config", - path.join("tmp/configs", config), + config, type === "beacon" ? "--substrate.private-key" : "--ethereum.private-key", pk.value ]; @@ -183,3 +202,151 @@ export const launchRelayers = async (options: LaunchOptions, launchedNetwork: La logger.success("Snowbridge relayers started"); printDivider(); }; + +/** + * Initialises the Ethereum Beacon Client pallet on the Substrate chain. + * It waits for the beacon chain to be ready, generates an initial checkpoint, + * and submits this checkpoint to the Substrate runtime via a sudo call. + * + * @param options - Launch options containing the relayer binary path. + * @param launchedNetwork - An instance of LaunchedNetwork to interact with the running network. + * @throws If there's an error generating the beacon checkpoint or submitting it to Substrate. + */ +export const initEthClientPallet = async ( + options: LaunchOptions, + launchedNetwork: LaunchedNetwork +) => { + // Poll the beacon chain until it's ready every 10 seconds for 5 minutes + await waitBeaconChainReady(launchedNetwork, 10000, 300000); + + // Generate the initial checkpoint for the CL client in Substrate + const { stdout, stderr, exitCode } = + await $`${options.relayerBinPath} generate-beacon-checkpoint --config ${RELAYER_CONFIG_PATHS.BEACON} --export-json` + .nothrow() + .quiet(); + if (exitCode !== 0) { + logger.error(stderr); + throw new Error("Error generating beacon checkpoint"); + } + logger.trace(`Beacon checkpoint stdout: ${stdout}`); + + // Load the checkpoint into a JSON object and clean it up + const initialCheckpointRaw = fs.readFileSync(INITIAL_CHECKPOINT_PATH, "utf-8"); + const initialCheckpoint = parseJsonToBeaconCheckpoint(JSON.parse(initialCheckpointRaw)); + fs.unlinkSync(INITIAL_CHECKPOINT_PATH); + + logger.trace("Initial checkpoint:"); + logger.trace(initialCheckpoint.toJSON()); + + // Send the checkpoint to the Substrate runtime + const substrateRpcUrl = `http://127.0.0.1:${launchedNetwork.getDHNodes()[0].port}`; + await sendCheckpointToSubstrate(substrateRpcUrl, initialCheckpoint); +}; + +/** + * Waits for the beacon chain to be ready by polling its finality checkpoints. + * + * @param launchedNetwork - An instance of LaunchedNetwork to get the CL endpoint. + * @param pollIntervalMs - The interval in milliseconds to poll the beacon chain. + * @param timeoutMs - The total time in milliseconds to wait before timing out. + * @throws Error if the beacon chain is not ready within the timeout. + */ +const waitBeaconChainReady = async ( + launchedNetwork: LaunchedNetwork, + pollIntervalMs: number, + timeoutMs: number +) => { + let initialBeaconBlock = ZERO_HASH; + let attempts = 0; + let keepPolling = true; + const maxAttempts = timeoutMs / pollIntervalMs; + + logger.trace("Waiting for beacon chain to be ready..."); + + while (keepPolling) { + try { + const response = await fetch( + `${launchedNetwork.getClEndpoint()}/eth/v1/beacon/states/head/finality_checkpoints` + ); + if (!response.ok) { + throw new Error(`HTTP error! Status: ${response.status}`); + } + + const data = (await response.json()) as FinalityCheckpointsResponse; + logger.debug(`Beacon chain state: ${JSON.stringify(data)}`); + + invariant(data.data, "❌ No data returned from beacon chain"); + invariant(data.data.finalized, "❌ No finalised block returned from beacon chain"); + invariant(data.data.finalized.root, "❌ No finalised block root returned from beacon chain"); + initialBeaconBlock = data.data.finalized.root; + } catch (error) { + logger.error(`Failed to fetch beacon chain state: ${error}`); + } + + if (initialBeaconBlock === ZERO_HASH) { + attempts++; + + if (attempts >= maxAttempts) { + throw new Error(`Beacon chain is not ready after ${maxAttempts} attempts`); + } + + logger.info(`⌛️ Retrying beacon chain state fetch in ${pollIntervalMs / 1000}s...`); + await new Promise((resolve) => setTimeout(resolve, pollIntervalMs)); + } else { + keepPolling = false; + } + } + + logger.info(`⏲️ Beacon chain is ready with finalised block: ${initialBeaconBlock}`); +}; + +/** + * Sends the beacon checkpoint to the Substrate runtime, waiting for the transaction to be finalised and successful. + * + * @param networkRpcUrl - The RPC URL of the Substrate network. + * @param checkpoint - The beacon checkpoint to send. + * @throws If the transaction signing fails, it becomes an invalid transaction, or the transaction is included but fails. + */ +const sendCheckpointToSubstrate = async (networkRpcUrl: string, checkpoint: BeaconCheckpoint) => { + logger.trace("Sending checkpoint to Substrate..."); + + const client = createClient(withPolkadotSdkCompat(getWsProvider(networkRpcUrl))); + const dhApi = client.getTypedApi(datahaven); + + logger.trace("Client created"); + + const signer = getEvmEcdsaSigner(SUBSTRATE_FUNDED_ACCOUNTS.ALITH.privateKey); + logger.trace("Signer created"); + + const forceCheckpointCall = dhApi.tx.EthereumBeaconClient.force_checkpoint({ + update: checkpoint + }); + + logger.debug("Force checkpoint call:"); + logger.debug(forceCheckpointCall.decodedCall); + + const tx = dhApi.tx.Sudo.sudo({ + call: forceCheckpointCall.decodedCall + }); + + logger.debug("Sudo call:"); + logger.debug(tx.decodedCall); + + try { + const txFinalisedPayload = await tx.signAndSubmit(signer); + + if (!txFinalisedPayload.ok) { + throw new Error("❌ Beacon checkpoint transaction failed"); + } + + logger.info( + `📪 "force_checkpoint" transaction with hash ${txFinalisedPayload.txHash} submitted successfully and finalised in block ${txFinalisedPayload.block.hash}` + ); + } catch (error) { + logger.error(`Failed to submit checkpoint transaction: ${error}`); + throw new Error(`Failed to submit checkpoint: ${error}`); + } finally { + client.destroy(); + logger.debug("Destroyed client"); + } +}; diff --git a/test/cli/index.ts b/test/cli/index.ts index c7bc17ac..c0e81a98 100644 --- a/test/cli/index.ts +++ b/test/cli/index.ts @@ -14,32 +14,32 @@ function parseIntValue(value: string): number { // So far we only have the launch command // we can expand this to more commands in the future const program = new Command() - .option("--datahaven", "Enable Datahaven network to be launched") - .option("-l, --launch-kurtosis", "Launch Kurtosis") - .option("-d, --deploy-contracts", "Deploy smart contracts") - .option("-f, --fund-validators", "Fund validators") - .option("-n, --no-fund-validators", "Skip funding validators") - .option("-s, --setup-validators", "Setup validators") - .option("--no-setup-validators", "Skip setup validators") - .option("-u, --update-validator-set", "Update validator set") - .option("--no-update-validator-set", "Skip update validator set") - .option("-b, --blockscout", "Enable Blockscout") + .option("--d, --datahaven", "(Re)Launch Datahaven network") + .option("--nd, --no-datahaven", "Skip launching Datahaven network") + .option("--lk, --launch-kurtosis", "Launch Kurtosis Ethereum network with EL and CL clients") + .option("--nlk, --no-launch-kurtosis", "Skip launching Kurtosis Ethereum network") + .option("--dc, --deploy-contracts", "Deploy smart contracts") + .option("--ndc, --no-deploy-contracts", "Skip deploying smart contracts") + .option("--fv, --fund-validators", "Fund validators") + .option("--nfv, --no-fund-validators", "Skip funding validators") + .option("--sv, --setup-validators", "Setup validators") + .option("--nsv, --no-setup-validators", "Skip setup validators") + .option("--uv, --update-validator-set", "Update validator set") + .option("--nuv, --no-update-validator-set", "Skip update validator set") + .option("--r, --relayer", "Launch Snowbridge Relayers") + .option("--nr, --no-relayer", "Skip Snowbridge Relayers") + .option("--b, --blockscout", "Enable Blockscout") .option("--slot-time ", "Set slot time in seconds", parseIntValue) .option("--kurtosis-network-args ", "CustomKurtosis network args") - .option("-v, --verified", "Verify smart contracts with Blockscout") + .option("--verified", "Verify smart contracts with Blockscout") .option("--always-clean", "Always clean Kurtosis", false) - .option("-q, --skip-cleaning", "Skip cleaning Kurtosis") - .option("-r, --relayer", "Enable Relayer") + .option("--skip-cleaning", "Skip cleaning Kurtosis") .option( "--datahaven-bin-path ", "Path to the datahaven binary", "../operator/target/release/datahaven-node" ) - .option( - "-p, --relayer-bin-path ", - "Path to the relayer binary", - "tmp/bin/snowbridge-relay" - ) + .option("--relayer-bin-path ", "Path to the relayer binary", "tmp/bin/snowbridge-relay") .hook("preAction", launchPreActionHook) .action(launch); diff --git a/test/configs/kurtosis/minimal.yaml b/test/configs/kurtosis/minimal.yaml index 4e15b696..844ad013 100644 --- a/test/configs/kurtosis/minimal.yaml +++ b/test/configs/kurtosis/minimal.yaml @@ -8,8 +8,8 @@ additional_services: network_params: preset: mainnet - seconds_per_slot: 2 - num_validator_keys_per_node: 128 + seconds_per_slot: 1 + num_validator_keys_per_node: 256 prefunded_accounts: '{ "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266": {"balance": "10ETH"}, "0x9965507D1a55bcC2695C58ba16FB37d819B0A4dc": {"balance": "10ETH"}, diff --git a/test/configs/snowbridge/beacon-relay.json b/test/configs/snowbridge/beacon-relay.json index eb7000a3..ec4c05e9 100644 --- a/test/configs/snowbridge/beacon-relay.json +++ b/test/configs/snowbridge/beacon-relay.json @@ -6,10 +6,10 @@ "spec": { "syncCommitteeSize": 512, "slotsInEpoch": 32, - "epochsPerSyncCommitteePeriod": 4, + "epochsPerSyncCommitteePeriod": 256, "forkVersions": { "deneb": 0, - "electra": 18446744073709551615 + "electra": 0 } }, "datastore": { diff --git a/test/package.json b/test/package.json index 9b51431b..0188e3f4 100644 --- a/test/package.json +++ b/test/package.json @@ -12,9 +12,9 @@ "generate:wagmi": "wagmi generate", "generate:snowbridge-cfgs": "bun -e \"import {generateSnowbridgeConfigs} from './scripts/gen-snowbridge-cfgs.ts'; await generateSnowbridgeConfigs()\"", "generate:types": "(cd ../operator && cargo build --release) && bun x papi add --wasm \"../operator/target/release/wbuild/datahaven-stagenet-runtime/datahaven_stagenet_runtime.wasm\" datahaven", - "start:e2e:verified": "bun cli --verified --blockscout --deploy-contracts --setup-validators --update-validator-set --fund-validators --slot-time 1", - "start:e2e:ci": "bun cli -d --setup-validators --update-validator-set --fund-validators --always-clean --slot-time 2 --datahaven --relayer", - "start:e2e:minrelayer": "bun cli --relayer -d --no-setup-validators --no-update-validator-set --no-fund-validators --datahaven", + "start:e2e:verified": "bun cli --verified --blockscout --deploy-contracts --setup-validators --update-validator-set --fund-validators", + "start:e2e:ci": "bun cli --datahaven --launch-kurtosis --deploy-contracts --fund-validators --setup-validators --update-validator-set --relayer --always-clean", + "start:e2e:minrelayer": "bun cli --relayer --deploy-contracts --no-setup-validators --no-update-validator-set --no-fund-validators --datahaven", "stop:e2e": "pkill datahaven ; pkill snowbridge-relay ; kurtosis enclave stop datahaven-ethereum && kurtosis clean && kurtosis engine stop && docker container prune -f", "stop:e2e:verified": "bun stop:e2e", "stop:e2e:quick": "kurtosis enclave stop datahaven-ethereum", diff --git a/test/utils/types.ts b/test/utils/types.ts new file mode 100644 index 00000000..6a51b50a --- /dev/null +++ b/test/utils/types.ts @@ -0,0 +1,221 @@ +import { type FixedSizeArray, FixedSizeBinary } from "polkadot-api"; + +/** + * The type of the response from the `/eth/v1/beacon/states/head/finality_checkpoints` + * RPC method from the Beacon Chain. + */ +export interface FinalityCheckpointsResponse { + execution_optimistic: boolean; + finalized: boolean; + data: { + previous_justified: { + epoch: string; + root: string; + }; + current_justified: { + epoch: string; + root: string; + }; + finalized: { + epoch: string; + root: string; + }; + }; +} + +/** + * The type of the argument of the `force_checkpoint` extrinsic from the Ethereum + * Beacon Client pallet. + * + * Represents the structure of the BeaconCheckpoint as it should be after type + * coercions (e.g., to BigInt). + */ +export interface BeaconCheckpoint { + header: { + slot: bigint; + proposer_index: bigint; + parent_root: FixedSizeBinary<32>; + state_root: FixedSizeBinary<32>; + body_root: FixedSizeBinary<32>; + }; + current_sync_committee: { + pubkeys: FixedSizeArray<512, FixedSizeBinary<48>>; + aggregate_pubkey: FixedSizeBinary<48>; + }; + current_sync_committee_branch: FixedSizeBinary<32>[]; + validators_root: FixedSizeBinary<32>; + block_roots_root: FixedSizeBinary<32>; + block_roots_branch: FixedSizeBinary<32>[]; + toJSON: () => JsonBeaconCheckpoint; +} + +/** + * Represents the structure of the BeaconCheckpoint as it might be after JSON.parse + * before specific type coercions (e.g., to BigInt). + */ +interface RawBeaconCheckpoint { + header: { + slot: number | string | bigint; // JSON.parse will yield number or string for big numbers + proposer_index: number | string | bigint; // Same as above + parent_root: string; // Assuming hex string + state_root: string; // Assuming hex string + body_root: string; // Assuming hex string + }; + current_sync_committee: { + pubkeys: string[]; // Assuming array of hex strings + aggregate_pubkey: string; // Assuming hex string + }; + current_sync_committee_branch: string[]; // Assuming array of hex strings + validators_root: string; // Assuming hex string + block_roots_root: string; // Assuming hex string + block_roots_branch: string[]; // Assuming array of hex strings +} + +/** + * Represents the structure of a BeaconCheckpoint when serialized to JSON. + * BigInts are converted to strings, and FixedSizeBinary types are converted to hex strings. + */ +interface JsonBeaconCheckpoint { + header: { + slot: string; + proposer_index: string; + parent_root: string; + state_root: string; + body_root: string; + }; + current_sync_committee: { + pubkeys: string[]; + aggregate_pubkey: string; + }; + current_sync_committee_branch: string[]; + validators_root: string; + block_roots_root: string; + block_roots_branch: string[]; +} + +/** + * Parses a JSON object into a BeaconCheckpoint. + * + * @param jsonInput - The JSON object to parse. + * @returns The parsed BeaconCheckpoint. + */ +export const parseJsonToBeaconCheckpoint = (jsonInput: any): BeaconCheckpoint => { + const raw = jsonInput as RawBeaconCheckpoint; + + // Basic validation + if (!raw || typeof raw.header !== "object" || raw.header === null) { + throw new Error("Invalid JSON structure for BeaconCheckpoint: missing or invalid header"); + } + if (typeof raw.header.slot === "undefined" || typeof raw.header.proposer_index === "undefined") { + throw new Error( + "Invalid JSON structure for BeaconCheckpoint: header missing slot or proposer_index" + ); + } + + if ( + !raw.current_sync_committee?.pubkeys || + !raw.current_sync_committee.aggregate_pubkey || + !Array.isArray(raw.current_sync_committee.pubkeys) || + !Array.isArray(raw.current_sync_committee_branch) || + !raw.validators_root || + !raw.block_roots_root || + !Array.isArray(raw.block_roots_branch) + ) { + throw new Error( + "Invalid JSON structure for BeaconCheckpoint: missing sync-committee or root fields" + ); + } + + if (raw.current_sync_committee.pubkeys.length !== 512) { + throw new Error( + `Invalid sync-committee size. Expected 512 pubkeys, got ${raw.current_sync_committee.pubkeys.length}` + ); + } + + // Map pubkeys to FixedSizeBinary<48> + const pubkeys = new Array>(512); + for (let i = 0; i < raw.current_sync_committee.pubkeys.length; i++) { + pubkeys[i] = new FixedSizeBinary<48>(hexToUint8Array(raw.current_sync_committee.pubkeys[i])); + } + + const checkpointData: Omit = { + header: { + slot: BigInt(raw.header.slot), + proposer_index: BigInt(raw.header.proposer_index), + parent_root: new FixedSizeBinary<32>(hexToUint8Array(raw.header.parent_root)), + state_root: new FixedSizeBinary<32>(hexToUint8Array(raw.header.state_root)), + body_root: new FixedSizeBinary<32>(hexToUint8Array(raw.header.body_root)) + }, + current_sync_committee: { + pubkeys: asFixedSizeArray(pubkeys, 512), + aggregate_pubkey: new FixedSizeBinary<48>( + hexToUint8Array(raw.current_sync_committee.aggregate_pubkey) + ) + }, + current_sync_committee_branch: raw.current_sync_committee_branch.map( + (branch) => new FixedSizeBinary<32>(hexToUint8Array(branch)) + ), + validators_root: new FixedSizeBinary<32>(hexToUint8Array(raw.validators_root)), + block_roots_root: new FixedSizeBinary<32>(hexToUint8Array(raw.block_roots_root)), + block_roots_branch: raw.block_roots_branch.map( + (branch) => new FixedSizeBinary<32>(hexToUint8Array(branch)) + ) + }; + + return { + ...checkpointData, + toJSON: function (this: BeaconCheckpoint): JsonBeaconCheckpoint { + return { + header: { + slot: this.header.slot.toString(), + proposer_index: this.header.proposer_index.toString(), + parent_root: this.header.parent_root.asHex(), + state_root: this.header.state_root.asHex(), + body_root: this.header.body_root.asHex() + }, + current_sync_committee: { + pubkeys: this.current_sync_committee.pubkeys.map((pk) => pk.asHex()), + aggregate_pubkey: this.current_sync_committee.aggregate_pubkey.asHex() + }, + current_sync_committee_branch: this.current_sync_committee_branch.map((branch) => + branch.asHex() + ), + validators_root: this.validators_root.asHex(), + block_roots_root: this.block_roots_root.asHex(), + block_roots_branch: this.block_roots_branch.map((branch) => branch.asHex()) + }; + } + }; +}; + +/** + * Converts an array to a FixedSizeArray of the specified length. + * Throws an error if the array length does not match the expected length. + * + * @param arr - The array to convert. + * @param expectedLength - The expected length of the FixedSizeArray. + * @returns The array as a FixedSizeArray of the specified length. + */ +export const asFixedSizeArray = ( + arr: T[], + expectedLength: L +): FixedSizeArray => { + if (arr.length !== expectedLength) { + throw new Error(`Array length mismatch. Expected ${expectedLength}, got ${arr.length}.`); + } + return arr as FixedSizeArray; +}; + +/** + * Converts a hex string to a Uint8Array. + * + * @param hex - The hex string to convert. + * @returns The Uint8Array representation of the hex string. + */ +const hexToUint8Array = (hex: string): Uint8Array => { + let hexString = hex; + if (hexString.startsWith("0x")) { + hexString = hexString.slice(2); + } + return Buffer.from(hexString, "hex"); +};