From 9b311e00ef9447c791c41cf37fc1749fbf4a9a0c Mon Sep 17 00:00:00 2001 From: Facundo Farall <37149322+ffarall@users.noreply.github.com> Date: Wed, 16 Jul 2025 13:51:07 -0300 Subject: [PATCH] =?UTF-8?q?test:=20=F0=9F=8F=97=EF=B8=8F=20Setup=20e2e=20t?= =?UTF-8?q?esting=20framework=20(#104)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Implement E2E Testing Framework with Isolated Networks ### Summary Refactors the existing E2E testing infrastructure to provide isolated test environments with parallel execution support. Each test suite now runs in its own network namespace, preventing resource conflicts. ### Key Changes - **New Testing Framework** (`test/framework/`): Base classes for test lifecycle management with automatic setup/teardown - **Launcher Module** (`test/launcher/`): Extracted network orchestration logic from CLI handlers for reusability - **Parallel Execution**: Added `test-parallel.ts` script with concurrency limits to prevent resource exhaustion - **Test Isolation**: Each suite gets unique network IDs (format: `suiteName-timestamp`) and Docker networks - **Improved Test Organization**: Migrated tests to new framework, deprecated old test structure ### Test Improvements - Added 4 new test suites demonstrating framework usage. : - `contracts.test.ts` - Smart contract deployment/interaction - `datahaven-substrate.test.ts` - Substrate API operations - `cross-chain.test.ts` - Snowbridge cross-chain messaging - `ethereum-basic.test.ts` - Ethereum network operations > [!WARNING] The test suites themselves are bad and shouldn't be consider examples of good tests. They were AI generated just to test the concurrency of test runners ### Documentation - Added comprehensive framework overview (`E2E_FRAMEWORK_OVERVIEW.md`) - Updated README with parallel testing commands - Added test patterns and best practices ### Breaking Changes - Old test suites moved to `e2e - DEPRECATED/` directory - Test execution now requires extending `BaseTestSuite` class ### Testing Run tests with: `bun test:e2e` or `bun test:e2e:parallel` (with concurrency limits) ### TODO - [ ] Implement good test examples. - [ ] Implement useful test utils (like waiting for an event to show up in DataHaven or Ethereum). - [ ] Enforce tests with CI (currently cannot be done due to intermittent error when sending a transaction with PAPI). --------- Co-authored-by: Claude Co-authored-by: undercover-cactus --- .github/workflows/task-e2e.yml | 21 +- operator/runtime/stagenet/src/lib.rs | 2 +- test/.papi/descriptors/package.json | 2 +- test/.papi/metadata/datahaven.scale | Bin 369411 -> 369411 bytes test/README.md | 106 +-- test/cli/handlers/common/checks.ts | 135 +--- test/cli/handlers/common/datahaven.ts | 187 ----- test/cli/handlers/common/kubernetes.ts | 2 +- test/cli/handlers/common/kurtosis.ts | 214 ------ test/cli/handlers/common/relayer.ts | 329 -------- test/cli/handlers/deploy/cleanup.ts | 4 +- test/cli/handlers/deploy/datahaven.ts | 4 +- test/cli/handlers/deploy/index.ts | 2 +- test/cli/handlers/deploy/kurtosis.ts | 4 +- test/cli/handlers/deploy/relayer.ts | 11 +- test/cli/handlers/launch/contracts.ts | 22 +- test/cli/handlers/launch/datahaven.ts | 245 +----- test/cli/handlers/launch/index.ts | 14 +- test/cli/handlers/launch/kurtosis.ts | 41 +- test/cli/handlers/launch/parameters.ts | 15 +- test/cli/handlers/launch/relayer.ts | 332 +------- test/cli/handlers/launch/summary.ts | 4 +- test/cli/handlers/launch/validator.ts | 27 +- test/cli/handlers/stop/index.ts | 76 +- test/docs/E2E_FRAMEWORK_OVERVIEW.md | 171 +++++ test/framework/connectors.ts | 103 +++ test/framework/index.ts | 3 + test/framework/manager.ts | 83 ++ test/framework/suite.ts | 140 ++++ test/launcher/contracts.ts | 55 ++ test/launcher/datahaven.ts | 496 ++++++++++++ test/launcher/index.ts | 7 + test/launcher/kurtosis.ts | 346 +++++++++ test/launcher/network/index.ts | 284 +++++++ test/launcher/parameters.ts | 54 ++ test/launcher/relayers.ts | 708 ++++++++++++++++++ test/launcher/types/index.ts | 28 + .../types}/launchedNetwork.ts | 19 + test/launcher/utils/checks.ts | 150 ++++ .../consts.ts => launcher/utils/constants.ts} | 30 +- test/launcher/utils/crypto.ts | 42 ++ test/launcher/utils/index.ts | 3 + test/launcher/validators.ts | 81 ++ test/package.json | 5 +- test/scripts/cargo-crossbuild.ts | 18 +- test/scripts/fund-validators.ts | 7 +- test/scripts/setup-validators.ts | 6 +- test/scripts/test-parallel.ts | 357 +++++++++ test/scripts/update-validator-set.ts | 6 +- test/suites/contracts.test.ts | 105 +++ test/suites/cross-chain.test.ts | 109 +++ test/suites/datahaven-substrate.test.ts | 71 ++ test/suites/e2e/basic.test.ts | 58 -- test/suites/e2e/beefy-client.test.ts | 46 -- test/suites/e2e/datahaven-basic.test.ts | 74 -- test/suites/e2e/service-manager.test.ts | 18 - test/suites/ethereum-basic.test.ts | 173 +++++ test/tsconfig.json | 4 +- test/utils/docker.ts | 21 +- test/utils/index.ts | 1 + test/utils/parameters.ts | 4 +- test/utils/shell.ts | 2 + 62 files changed, 3846 insertions(+), 1841 deletions(-) delete mode 100644 test/cli/handlers/common/datahaven.ts delete mode 100644 test/cli/handlers/common/kurtosis.ts delete mode 100644 test/cli/handlers/common/relayer.ts create mode 100644 test/docs/E2E_FRAMEWORK_OVERVIEW.md create mode 100644 test/framework/connectors.ts create mode 100644 test/framework/index.ts create mode 100644 test/framework/manager.ts create mode 100644 test/framework/suite.ts create mode 100644 test/launcher/contracts.ts create mode 100644 test/launcher/datahaven.ts create mode 100644 test/launcher/index.ts create mode 100644 test/launcher/kurtosis.ts create mode 100644 test/launcher/network/index.ts create mode 100644 test/launcher/parameters.ts create mode 100644 test/launcher/relayers.ts create mode 100644 test/launcher/types/index.ts rename test/{cli/handlers/common => launcher/types}/launchedNetwork.ts (87%) create mode 100644 test/launcher/utils/checks.ts rename test/{cli/handlers/common/consts.ts => launcher/utils/constants.ts} (62%) create mode 100644 test/launcher/utils/crypto.ts create mode 100644 test/launcher/utils/index.ts create mode 100644 test/launcher/validators.ts create mode 100644 test/scripts/test-parallel.ts create mode 100644 test/suites/contracts.test.ts create mode 100644 test/suites/cross-chain.test.ts create mode 100644 test/suites/datahaven-substrate.test.ts delete mode 100644 test/suites/e2e/basic.test.ts delete mode 100644 test/suites/e2e/beefy-client.test.ts delete mode 100644 test/suites/e2e/datahaven-basic.test.ts delete mode 100644 test/suites/e2e/service-manager.test.ts create mode 100644 test/suites/ethereum-basic.test.ts diff --git a/.github/workflows/task-e2e.yml b/.github/workflows/task-e2e.yml index 3a7b985d..d563038d 100644 --- a/.github/workflows/task-e2e.yml +++ b/.github/workflows/task-e2e.yml @@ -85,10 +85,21 @@ jobs: docker rm temp - run: tmp/bin/snowbridge-relay --help - run: docker pull ${{ inputs.image-tag }} + - run: | + docker tag ${{ inputs.image-tag }} moonsonglabs/datahaven:local + docker images - run: bun install - - run: bun start:e2e:ci --datahaven-image-tag ${{ inputs.image-tag }} - - name: Check network - run: | - kurtosis enclave inspect datahaven-ethereum - docker container ls - run: bun test:e2e + # Try to collect all docker logs and upload it + - name: Collect docker logs + if: always() + run: | + mkdir ./logs + for name in `docker ps -a --format '{{.Names}}'`; do docker logs $name > ./logs/$name.log 2>&1; done + - name: Upload logs to GitHub + if: always() + uses: actions/upload-artifact@v4 + with: + name: logs + path: logs/ + retention-days: 1 diff --git a/operator/runtime/stagenet/src/lib.rs b/operator/runtime/stagenet/src/lib.rs index 2bc1c909..7dfbe10c 100644 --- a/operator/runtime/stagenet/src/lib.rs +++ b/operator/runtime/stagenet/src/lib.rs @@ -379,7 +379,7 @@ mod runtime { // `on_initialize` hook and the latter clears up messages in // its `on_initialize` hook, so otherwise messages will be cleared // up before they are processed. - #[runtime::pallet_index(70)] + #[runtime::pallet_index(59)] pub type MessageQueue = pallet_message_queue; // ╚════════════ Polkadot SDK Utility Pallets - Block 2 ═════════════╝ diff --git a/test/.papi/descriptors/package.json b/test/.papi/descriptors/package.json index 09d3d298..fc74d13c 100644 --- a/test/.papi/descriptors/package.json +++ b/test/.papi/descriptors/package.json @@ -1,5 +1,5 @@ { - "version": "0.1.0-autogenerated.8839746171427835598", + "version": "0.1.0-autogenerated.9968121159155506533", "name": "@polkadot-api/descriptors", "files": [ "dist" diff --git a/test/.papi/metadata/datahaven.scale b/test/.papi/metadata/datahaven.scale index c2228a6e98e0ad5874d9106d247c994d391e1c00..27fc01fcd801c4f67b74e3456946efa6d14fb9ea 100644 GIT binary patch delta 4967 zcmbVQ4Nz29mcHNZ_jru~Y!na_CXI@UhAf~Mjej8w8b&4jh=M?{g(vie{-L2&1O|!0 z#BqrsH|CC8AsJ_4MopM#QJFWX3R_weOxV(y5MvUZ(P(YCyY7@p*o>9R)Mm2hb+_`D zaxzl|eZPCox#!+{-nrj9_w6|ox93dUiGS6i)7R^bjgDHq(69S-g*hQo!{=g|%(ft& zw?R&hYl(+ntZqp)I}I&MsS{b7MsCFGEeDl0L_8^@BYCoTzvuuu#FpZ< z^!s}8I&}7ZT|5N=QJk1rlEwEyetsVJi}J>B@oY%~y84cne0JO5jZH1+5(`Sb=oT-Q zrt@xy=vUHX4?-zsOkszC_>Fq?@(n8)J4j{=#p<$0aY(#SmWrOfQ)L@Lodu(a-Q^9CNL}ts;cp|jNFIm;&)XuxXQ(!s;-i&vs+Tg)&Fi;Wjjeu ze&lyoH#t4-M#fHxY)2M<6C&J`xZ(ovNVczfdHHRFW%kUp=v$4}q;ve;zJ64npASo6q#q(6|_4@CGEW|Fmiwa*cVq7@NtkL!*irbairV(qqM+XP|jwwa#s!jMOW zu?sY-g$KiN(ZrS@zHDOBn_;?UVl{}bn>Z|pZ@S66+q9bRj$gj0dzXjOK z{fieZvoES-?3QU``b=kE3vcZs3ExogRy%jH^ zJ^0`{Pqo8*Ds*(L8?zMF3zBDzlG8qENVxM6I=f@iHj<02&iiIXbJ%a^Dm@ZwX{MEp zk+EoT(m6G0oNV+cvbg4)+J0AyvG${TwD8ibnDKLtw)lR!b~6epIa?h6{RWu!-`_Hf zJU@_l7yI_V!`WTjFpVZ!V!xZkck9*$PSQWT2+H+nZj0Di?XqrPD5AZ|As&epL*5{d z1d+)jCb~P1JeWYqHik!H`Dg+m^>_kF-~j?j5M8cYND_P8t8B?*Gd5X#;LbvduzHfp zv-sV3cIiazD4tz0QM-u9G?0i2%NDymNyrs1(Qlr3+mkeR{Y3i^`KHjC1LT{^S?Y7% z+uWOWCuTup#>_;2BW4-nW0p21W~3e;vy5QOGM@8J7ysAr9~-mG_0T_~^(-~9$$o@a znAl`0!r3M^*^h9piB0w+oM&Q_{Rpo&vB`df3%NY;G3La(zGS`O57=~G}ZNG zw|;3se_z(_KhL0M{)_HO80Z_o)%tJ?iO`WffiZJTzf8!+mt~l{i5`A={zG&*W z`lwmOz`<*%kiQJWv{0F(w5hlr5PqS+Do4Nhkl6^OqG94gcXB^mV_?aKtQHOY}55Yqt}i#2#E~qOsoub zVvO=xI6@tR9K4>1W5%yGAxb4D{S_e2?l*=XLl39^0SBJA2g7pMjj7ahGzFRmT}Z>Q zQRl+bRAzj)a*YQwOyw#cX3n@n{aEgD?5v{2Re5z^Rjs3OoKH!^95SZ?zYor~An4)S zF}+54r4c_LC$jsoJ%;8PpI5{3xF3mTRrzqRR*aE(&!anpjW3=@Kc}ZNQW0wMb3cWp zL&k2$GdLt)*^ZwCD?*vO1N%uav;#Tll(9Q85BucGop^0oZbhZ- zFil?UtaUq@{63wr!+{AWj}|HHh&<7XghfZpnYILGI;M^$xn6ZbV0+1ys)pX+ie|SV zJs6Ccaz`8H^OLIhOYs8vyEaV6oAO2*1;(tj<7DfPv0rX!rwFCBBVoo{Dk9c-Y8hg@ zK96TxzlxXw6HvJ#9qrgg=}S$DyxER;oR(p`kzpTDu_Sk=UhOx_g^=}WT8?6E<87P` z1Vh@oo2KrRayJEOQ2u5&7FdQ<$QvD)DXkr_&lw!E#d8z4c+T9SJl%nLa(f5XV<_mD zJftZXRFujuc4OK-jK%y=ck5fJCYH;ez-(MF98X}enfyQ9gCfgC)h54fMUwP(Ak45m zNu)rkd7aqJFRSMD;yyD`Y-6j{Oe=(b7SIuZUi6M4OQG%I!}Jug*@Dpsm~yl*7NK$$g;*a zHoLZZ>I1!8KCH^qKc{1JOEqb8wp?`p1vq3pdjL2CG7)b-DS(9=YbsQFNyBH1m(C_ptTmGFMJ1)L>gpwiZVJ4PhteMa9p(ofXu_39?yhy^PQ#vuM(EKo|c5nc^98s3H5f->Xkzf*Wn zCZo^dAg|PjC(>lnhX^;WoJEX6{Qu?q*g(xALlBd6S~7QP%&Ae*X2`e?@FTdS;{&Xs z#(XAm4O;NN?bk-`+YLeMg!5Pv?l(O&XslU&cAi-0z3Wv%U8r zJ|=Si*##_%YM>isfy3vZdn4=A0=IBE=_4E^hU@zXotAxC;I=Ned`v0XWxVk*okG%` z{{){Hr#^)>mRM`>&)7?E(1Kfd#}+-3<1ucJyILPMQo?u)Vn^h&^=;cIlFsd<%h&i( z)i$oINRdxQ@p++o0wppkk{6&-Rz>n6rA-TcXpx`Lw-_S5ZzFk*(xr|1!V)K`Xqaxi z7sXvU_{LaV#2ZujFl51x`Am7Pgq*9^gnCC99LS1P|(30d& zD%5!@RX0u;Muj>k&29z zUYxgfva-!KDl?CVo$a)i?uuo_F)J%I+Bmzmc{Xf0MVqdO8pwW{(dL zBiDLhlb5$(jCgWwDs1BH+J(l=2z)?fm}2F_jZj7Iy3en&+_gTHY^Ynm1T{zAUw;{X zdbl`$cMWPpMq!G$uOJ;B(O!_mJrE(!B!$(%xQsC)YXZVoYDKe`r!!XPW3_L{LxXsI zLjoF)T-vY_gk%&tc_ZZFPD~Kz3uj)}1jcwBSTp4J7cgFA6s<#xc)Vyi_KKb&XHX{) zH^0S_;r5YBm<(lIYx+%6lO!TdT!G zC8Z>}wp~ z1EQ85XCNM<$61I!R7Tx=4!9%JSu}gyRY7Uy8TskV#MplF%Rpat0PKLM+cZvuRfVHf zOsTqwAB5Ob6(+8)nB)i3u2Qz&GYf4Zx_bEyuh3b}a}?L)6q9%57N_f|FmEzv9wwV} zHCL6xxrtT+7;7aJBZApM7{eEr>z){m%RyYG;9Q4YBlW?{95#xY!^B6Q!(_>)rZ#<3 zID18ul+7oTUMMTYQ4wmpjo^j0@ZeS;&^|Mk8a# zNri#?-EY@ngC9Pn!?dm+ONS0Ce)x3hhWXd?{~~8X|GWP=R>}~F zmzDNZ>SARch_B1mj=&kYVC zAy;jtvKk^f+^IZ_i~Xfhf>(|f@0N~4f|y$wjYN@8zsX`N{ice&^gB~{E2D3o!+)x% zhLJj7QH|g%LUdLSDyoR?_gRN#_u;?jbIfp`qlfrR%fo$+@%tP@`md~r*DaziBa0Gr zSg&&gC+e_Xc?eF{VZF`~oT|fmog;Xr4(oM};5j<1*ExbS#jRUTqfz{@WrP3j`pl5U z1Lf{^go!^dE#Ns^9T717zmw<{0zvx&?4YpaoUiWgQV@I_pFxOygE!o}P*Ek`nIzc!%tNb0@c zMABo@q2 zrV+D@2NiVtunR`aF`iS<I+=fp1vVvw~kAhAg@%j-6j$N5@i@Cqk$za2}qFx2< zKHkFyj77Hh^%0mn2CWkD7%jh#f(dP2H5yIifRCmGZq&3k??E#bppWD=#Ui~A3H7cU zj{}5wNx+LKXz{+Z3ZoPfa&$eWa-#R#jeVT(-4^T^i7we=$7p)gxqOdVHYB3UTWZ4t z)QkN1;sOUE^~JeP#76dCBbaToR2PtO1?8Hvpu|!!Os1r57CC=2PW!b@_DlF-2v#9K zt-#a6NM<)GLrKr@cIjy7YkA4tN%AP$BVmBy+ z%n#uOUn@UOe*O@S(aP_d@NZCUl&viok4xS+cB284x`w!N@mg0d`dzuGh|I!?@`+~T z;FA2J842i?iyuL%DZHY_R_G}6VOhLek;k9LJl>;-q%Cps`$s_2jCZlX{kl~jJ&G6l zOo)_Ew!k#Vo{koL21BpnFWT~g8;O18cZ@VwVmBdPD&c3i_n^e(3FOh^uVFoI7^ni_ zcDJoibNZ8C&)UAm)Jzqjz+IsgcX^YK;X8v2(U05m7FDhg`P5rjfiU?kVQ~>EZtqXI zJf}u8gjUR07~MC%)QeDAgk1eLQbt9qh+5!uxXTwLGZrmBc^eUv1GS>^z^wUsj0M72 z9izgOoyTCT>4&!TK`ch!5I;6vMc_Ou#cyD>R*^`El?DhF?~^t?u+UoKTV+NVs}fX9 zTQ|SdZQqn;&Rv+5OKF~W*&xcOPR}Jbq)~X z3bnE_i_2-P9-u`PS;}?l07ai+HhGu1%f5bdQQy2^k# z-rjd9wfP)XHgCm^GV4858Z*^F6SPb^kCEQ5-$SSoS<>_omh&uC#?W^2fLnOJwE8n9G-_`Vc*4h3X%o*Qf(S^a?-slyg`Rutt}#LS^}K+c|6* z*+voJ)@K-I(FqfLbG&ZFi=@BvJU)UZ+b$q2XwVhUtn$hQyhSzW%}*#_Y%;eKZoXNi z=$;_|(TV>O>5EVCG2F7Lof10z1jgxMyY>WfQ6pPVASaO%K^gbN(z?9yBX}sn2Iz6x z#*GvR)=FB-Rrzy@uT|kQRne_Zkd7dpz%x~mZn{|>58{)!MHS7ai85>yAJ46-_|z0F zSB>KF*epGx_%;93G^u_6)X3-{9)fI1O-ZEp%~9MoAKl*ROL;{C?}FUClgG-At^7Rl z^wua;cJAOk2=bn;=WC#Zs7rsV+@IXQy~z!RGo>==dTx~bvSG%LuY|5ro}t(H$|@x( zlo~E8Oi6m;tkj$pbWKzw%PXst=XL*LOcC-=tCg?%nDu^av;2IG@<(!5o(|&UML#yy^zN?C}X8# zgOW?v@M{~CrN&O`=47Zvd1Aa(oy+bR=rfnqRJb(TO8TpTdc91}uvyKJ&(`uON)C6c zE*VnKZejw` sR2gleOUAHIxkQideTpNf3vqh!EVJ6It}bbPLMgkJl [!IMPORTANT] > If you are running this on a Mac, `zig` is a pre-requisite for crossbuilding the node. Instructions for installation can be found [here](https://ziglang.org/learn/getting-started/). -## QuickStart - -Run: +## Quick Start ```bash +# Install dependencies bun i -bun cli + +# Interactive CLI to launch a full local DataHaven network +bun cli launch + +# Run all the e2e tests +bun test:e2e + +# Run all the e2e tests with limited concurrency +bun test:e2e:parallel + +# Run a specific test suite +bun test suites/some-test.test.ts + ``` -## Manual Deployment +For more information on the E2E testing framework, see the [E2E Testing Framework Overview](./docs/E2E_FRAMEWORK_OVERVIEW.md). -Follow these steps to set up and interact with your test environment: +## Other Common Commands + +| Command | Description | +| ------------------------- | ----------------------------------------------------------------------------------------------------------- | +| `bun cli stop` | Stop all local DataHaven networks (interactive, will ask for confirmation on each component of the network) | +| `bun cli deploy` | Deploy the DataHaven network to a remote Kubernetes cluster | +| `bun generate:wagmi` | Generate contract TypeScript bindings for the contracts in the `contracts` directory | +| `bun generate:types` | Generate Polkadot API types | +| `bun generate:types:fast` | Generate Polkadot API types with the `--fast-runtime` feature enabled | + +## Local Network Deployment + +Follow these steps to set up and interact with your local network: 1. **Deploy a minimal test environment** ```bash - bun cli + bun cli launch ``` This script will: @@ -50,72 +68,26 @@ Follow these steps to set up and interact with your test environment: - Dora Explorer service for CL 4. Deploy DataHaven smart contracts to the Ethereum network. This can optionally include verification on Blockscout if the `--verified` flag is used (requires Blockscout to be enabled). 5. Perform validator setup and funding operations. - 6. Launch Snowbridge relayers. + 6. Set parameters in the DataHaven chain. + 7. Launch Snowbridge relayers. + 8. Perform validator set update. > [!NOTE] > - > If you want to also have the contracts verified on blockscout, you can run `bun start:e2e:verified` instead. This will do all the previous steps, but also verify the contracts on blockscout. However, note that this takes some time to complete. + > If you want to also have the contracts verified on Blockscout, you can pass the `--verified` flag to the `bun cli launch` command, along with the `--blockscout` flag. This will do all the previous, but also verify the contracts on Blockscout. However, note that this takes some time to complete. 2. **Explore the network** - Block Explorer: [http://127.0.0.1:3000](http://127.0.0.1:3000). - Kurtosis Dashboard: Run `kurtosis web` to access. From it you can see all the services running in the network, as well as their ports, status and logs. -## Network Management - -- **Stop the test environment** - - ```bash - bun stop:e2e - ``` - -- **Stop the Kurtosis engine completely** - - ```bash - bun stop:kurtosis-engine - ``` - -## Blockscout - -Can be accessed at: [http://127.0.0.1:3000](http://127.0.0.1:3000). - -You can also access the backend via REST API, documented here: [http://127.0.0.1:3000/api-docs](http://127.0.0.1:3000/api-docs) - -![API DOCS](../resources/swagger.png) - -## Testing - -### E2E Tests - -> [!TIP] -> -> Remember to run the network with `bun cli` before running the tests. - -```bash -bun test:e2e -``` - -> [!NOTE] -> -> You can increase the logging level by setting `LOG_LEVEL=debug` before running the tests. - -### Wagmi Bindings Generation - -To ensure contract bindings are up-to-date, run the following command after modifying smart contracts or updating ABIs: - -```bash -bun generate:wagmi -``` - -This command generates TypeScript bindings for interacting with the deployed smart contracts using Wagmi. - ## Troubleshooting ### E2E Network Launch doesn't work #### Script halts unexpectedly -When running `bun cli` the script appears to halt after the following: +When running `bun cli launch` the script appears to halt after the following: ```shell ## Setting up 1 EVM. diff --git a/test/cli/handlers/common/checks.ts b/test/cli/handlers/common/checks.ts index e33ee0b4..b5432b0a 100644 --- a/test/cli/handlers/common/checks.ts +++ b/test/cli/handlers/common/checks.ts @@ -1,43 +1,15 @@ import { $ } from "bun"; import invariant from "tiny-invariant"; import { logger, printDivider, printHeader } from "utils"; +import type { LaunchedNetwork } from "../../../launcher/types/launchedNetwork"; +import { checkBaseDependencies as checkBaseDependenciesFunc } from "../../../launcher/utils"; import type { DeployOptions } from "../deploy"; -import { MIN_BUN_VERSION } from "./consts"; -import type { LaunchedNetwork } from "./launchedNetwork"; // ===== Checks ===== export const checkBaseDependencies = async (): Promise => { printHeader("Base Dependencies Checks"); - if (!(await checkKurtosisInstalled())) { - logger.error("Kurtosis CLI is required to be installed: https://docs.kurtosis.com/install"); - throw Error("❌ Kurtosis CLI application not found."); - } - - logger.success("Kurtosis CLI found"); - - if (!(await checkBunVersion())) { - logger.error( - `Bun version must be ${MIN_BUN_VERSION.major}.${MIN_BUN_VERSION.minor} or higher: https://bun.sh/docs/installation#upgrading` - ); - throw Error("❌ Bun version is too old."); - } - - logger.success("Bun is installed and up to date"); - - if (!(await checkDockerRunning())) { - logger.error("Is Docker Running? Unable to make connection to docker daemon"); - throw Error("❌ Error connecting to Docker"); - } - - logger.success("Docker is running"); - - if (!(await checkForgeInstalled())) { - logger.error("Is foundry installed? https://book.getfoundry.sh/getting-started/installation"); - throw Error("❌ Forge binary not found in PATH"); - } - - logger.success("Forge is installed"); + await checkBaseDependenciesFunc(); printDivider(); }; @@ -83,104 +55,15 @@ export const deploymentChecks = async ( printDivider(); }; -const checkBunVersion = async (): Promise => { - const bunVersion = Bun.version; - const [major, minor] = bunVersion.split(".").map(Number); - - // Check if version meets minimum requirements - const isVersionValid = - major > MIN_BUN_VERSION.major || - (major === MIN_BUN_VERSION.major && minor >= MIN_BUN_VERSION.minor); - - if (!isVersionValid) { - logger.debug(`Bun version: ${bunVersion} (too old)`); - return false; - } - - logger.debug(`Bun version: ${bunVersion}`); - return true; -}; - -const checkKurtosisInstalled = async (): Promise => { - const { exitCode, stderr, stdout } = await $`kurtosis version`.nothrow().quiet(); - if (exitCode !== 0) { - logger.error(stderr.toString()); - return false; - } - logger.debug(stdout.toString()); - return true; -}; - -export const checkKurtosisCluster = async (kubernetes?: boolean): Promise => { - // First check if kurtosis cluster get works - const { exitCode, stderr, stdout } = await $`kurtosis cluster get`.nothrow().quiet(); - - if (exitCode !== 0) { - logger.warn(`⚠️ Kurtosis cluster get failed: ${stderr.toString()}`); - logger.info("ℹ️ Assuming local launch mode and continuing."); - return true; - } - - const currentCluster = stdout.toString().trim(); - logger.debug(`Current Kurtosis cluster: ${currentCluster}`); - - // Try to get the cluster type from config, but don't fail if config path is not reachable - const clusterTypeResult = - await $`CURRENT_CLUSTER=${currentCluster} && sed -n "/^ $CURRENT_CLUSTER:$/,/^ [^ ]/p" "$(kurtosis config path)" | grep "type:" | sed 's/.*type: "\(.*\)"/\1/'` - .nothrow() - .quiet(); - - if (clusterTypeResult.exitCode !== 0) { - logger.warn("⚠️ Failed to read Kurtosis cluster type from config"); - logger.debug(clusterTypeResult.stderr.toString()); - logger.info("ℹ️ Assuming local launch mode and continuing gracefully"); - return true; // Continue gracefully for local launch - } - - const clusterType = clusterTypeResult.stdout.toString().trim(); - logger.debug(`Kurtosis cluster type: ${clusterType}`); - - // Validate cluster type against expected type - if (kubernetes && clusterType !== "kubernetes") { - logger.error(`❌ Kurtosis cluster type is "${clusterType}" but kubernetes is required`); - return false; - } - - if (!kubernetes && clusterType !== "docker") { - logger.error(`❌ Kurtosis cluster type is "${clusterType}" but docker is required`); - return false; - } - - logger.success(`Kurtosis cluster type "${clusterType}" is compatible`); - return true; -}; - -const checkDockerRunning = async (): Promise => { - const { exitCode, stderr, stdout } = await $`docker system info`.nothrow().quiet(); - if (exitCode !== 0) { - logger.error(stderr.toString()); - return false; - } - logger.debug(stdout.toString()); - return true; -}; - -const checkForgeInstalled = async (): Promise => { - const { exitCode, stderr, stdout } = await $`forge --version`.nothrow().quiet(); - if (exitCode !== 0) { - logger.error(stderr.toString()); - return false; - } - logger.debug(stdout.toString()); - return true; -}; - -const checkHelmInstalled = async (): Promise => { +/** + * Checks if Helm is installed (only needed for deployment) + */ +export const checkHelmInstalled = async (): Promise => { const { exitCode, stderr, stdout } = await $`helm version`.nothrow().quiet(); if (exitCode !== 0) { - logger.error(stderr.toString()); + logger.debug(`Helm check failed: ${stderr.toString()}`); return false; } - logger.debug(stdout.toString()); + logger.debug(`Helm version: ${stdout.toString().trim()}`); return true; }; diff --git a/test/cli/handlers/common/datahaven.ts b/test/cli/handlers/common/datahaven.ts deleted file mode 100644 index a41376d6..00000000 --- a/test/cli/handlers/common/datahaven.ts +++ /dev/null @@ -1,187 +0,0 @@ -import fs from "node:fs"; -import path from "node:path"; -import { secp256k1 } from "@noble/curves/secp256k1"; -import { createClient, type PolkadotClient } from "polkadot-api"; -import { withPolkadotSdkCompat } from "polkadot-api/polkadot-sdk-compat"; -import { getWsProvider } from "polkadot-api/ws-provider/web"; -import invariant from "tiny-invariant"; -import { createPapiConnectors, logger } from "utils"; -import { type Hex, keccak256, toHex } from "viem"; -import { publicKeyToAddress } from "viem/accounts"; -import type { LaunchedNetwork } from "./launchedNetwork"; - -/** - * Checks if the DataHaven network is ready by sending a POST request to the system_chain method. - * - * @param port - The port number to check. - * @param timeoutMs - The timeout in milliseconds for the attempt to connect to the network. - * @returns True if the network is ready, false otherwise. - */ -export const isNetworkReady = async (port: number, timeoutMs: number): Promise => { - const wsUrl = `ws://127.0.0.1:${port}`; - let client: PolkadotClient | undefined; - - // Temporarily capture and suppress error logs during connection attempts. - // This is to avoid the "Unable to connect to ws:" error logs from the `client._request` call. - const originalConsoleError = console.error; - console.error = () => {}; - - try { - // Use withPolkadotSdkCompat for consistency, though _request might not strictly need it. - client = createClient(withPolkadotSdkCompat(getWsProvider(wsUrl))); - - // Add timeout to the RPC call to prevent hanging. - const chainNamePromise = client._request("system_chain", []); - const timeoutPromise = new Promise((_, reject) => { - setTimeout(() => reject(new Error("RPC call timeout")), timeoutMs); - }); - - const chainName = await Promise.race([chainNamePromise, timeoutPromise]); - logger.debug(`isNetworkReady PAPI check successful for port ${port}, chain: ${chainName}`); - client.destroy(); - return !!chainName; // Ensure it's a boolean and chainName is truthy - } catch (error) { - logger.debug(`isNetworkReady PAPI check failed for port ${port}: ${error}`); - if (client) { - client.destroy(); - } - return false; - } finally { - // Restore original console methods. - console.error = originalConsoleError; - } -}; - -/** - * Converts a compressed secp256k1 public key to an Ethereum address. - * - * This function takes a compressed public key (33 bytes), decompresses it to get the full - * uncompressed public key (64 bytes of x and y coordinates), and then derives the - * corresponding Ethereum address using the standard Ethereum address derivation algorithm. - * - * @param compressedPubKey - The compressed public key as a hex string (with or without "0x" prefix) - * @returns The corresponding Ethereum address (checksummed, with "0x" prefix) - * - * @throws {Error} If the provided public key is invalid or cannot be decompressed - */ -export const compressedPubKeyToEthereumAddress = (compressedPubKey: string): string => { - // Ensure the input is a hex string and remove "0x" prefix - const compressedKeyHex = compressedPubKey.startsWith("0x") - ? compressedPubKey.substring(2) - : compressedPubKey; - - // Decompress the public key - const point = secp256k1.ProjectivePoint.fromHex(compressedKeyHex); - // toRawBytes(false) returns the uncompressed key (64 bytes, x and y coordinates) - const uncompressedPubKeyBytes = point.toRawBytes(false); - const uncompressedPubKeyHex = toHex(uncompressedPubKeyBytes); // Prefixes with "0x" - - // Compute the Ethereum address from the uncompressed public key - // publicKeyToAddress expects a 0x-prefixed hex string representing the 64-byte uncompressed public key - const address = publicKeyToAddress(uncompressedPubKeyHex); - return address; -}; - -/** - * Prepares the configuration for DataHaven authorities by fetching their BEEFY public keys, - * converting them to Ethereum addresses, and updating the network configuration file. - * - * This function performs the following steps: - * 1. Connects to the first available DataHaven node matching the container prefix - * 2. Fetches the BEEFY NextAuthorities from the node's runtime - * 3. Converts each compressed public key to an Ethereum address - * 4. Computes the keccak256 hash of each address (authority hash) - * 5. Updates the network configuration file with the authority hashes - * - * The configuration is saved to `../contracts/config/{NETWORK}.json` where NETWORK - * defaults to "anvil" if not specified in environment variables. - * - * @param launchedNetwork - The launched network instance containing container information - * @param containerNamePrefix - The prefix to filter DataHaven containers by (e.g., "datahaven-", "dh-validator-") - * - * @throws {Error} If no DataHaven nodes are found in the launched network - * @throws {Error} If BEEFY authorities cannot be fetched from the node - * @throws {Error} If public key conversion fails - * @throws {Error} If the configuration file cannot be read or written - */ -export const setupDataHavenValidatorConfig = async ( - launchedNetwork: LaunchedNetwork, - containerNamePrefix: string -): Promise => { - const networkName = process.env.NETWORK || "anvil"; - logger.info(`🔧 Preparing DataHaven authorities configuration for network: ${networkName}...`); - - let authorityPublicKeys: string[] = []; - const dhNodes = launchedNetwork.containers.filter((x) => x.name.startsWith(containerNamePrefix)); - - invariant(dhNodes.length > 0, "No DataHaven nodes found in launchedNetwork"); - - const firstNode = dhNodes[0]; - const wsUrl = `ws://127.0.0.1:${firstNode.publicPorts.ws}`; - const { client: papiClient, typedApi: dhApi } = createPapiConnectors(wsUrl); - - logger.info( - `📡 Attempting to fetch BEEFY next authorities from node ${firstNode.name} (port ${firstNode.publicPorts.ws})...` - ); - - // Fetch NextAuthorities - // Beefy.NextAuthorities returns a fixed-length array of bytes representing the authority public keys - const nextAuthoritiesRaw = await dhApi.query.Beefy.NextAuthorities.getValue({ at: "best" }); - - invariant(nextAuthoritiesRaw && nextAuthoritiesRaw.length > 0, "No BEEFY next authorities found"); - - authorityPublicKeys = nextAuthoritiesRaw.map((key) => key.asHex()); // .asHex() returns the hex string representation of the corresponding key - logger.success( - `Successfully fetched ${authorityPublicKeys.length} BEEFY next authorities directly.` - ); - - // Clean up PAPI client, otherwise it will hang around and prevent this process from exiting. - papiClient.destroy(); - - const authorityHashes: string[] = []; - for (const compressedKey of authorityPublicKeys) { - try { - const ethAddress = compressedPubKeyToEthereumAddress(compressedKey); - const authorityHash = keccak256(ethAddress as Hex); - authorityHashes.push(authorityHash); - logger.debug( - `Processed public key ${compressedKey} -> ETH address ${ethAddress} -> Authority hash ${authorityHash}` - ); - } catch (error) { - logger.error(`❌ Failed to process public key ${compressedKey}: ${error}`); - throw new Error(`Failed to process public key ${compressedKey}`); - } - } - - // process.cwd() is 'test/', so config is at '../contracts/config' - const configDir = path.join(process.cwd(), "../contracts/config"); - const configFilePath = path.join(configDir, `${networkName}.json`); - - try { - if (!fs.existsSync(configFilePath)) { - logger.warn( - `⚠️ Configuration file ${configFilePath} not found. Skipping update of validator sets.` - ); - // Optionally, create a default structure if it makes sense, or simply return. - // For now, if the base network config doesn't exist, we can't update it. - return; - } - - const configFileContent = fs.readFileSync(configFilePath, "utf-8"); - const configJson = JSON.parse(configFileContent); - - if (!configJson.snowbridge) { - logger.warn(`⚠️ "snowbridge" section not found in ${configFilePath}, creating it.`); - configJson.snowbridge = {}; - } - - configJson.snowbridge.initialValidatorHashes = authorityHashes; - configJson.snowbridge.nextValidatorHashes = authorityHashes; - - fs.writeFileSync(configFilePath, JSON.stringify(configJson, null, 2)); - logger.success(`DataHaven authority hashes updated in: ${configFilePath}`); - } catch (error) { - logger.error(`❌ Failed to read or update ${configFilePath}: ${error}`); - throw new Error(`Failed to update authority hashes in ${configFilePath}.`); - } -}; diff --git a/test/cli/handlers/common/kubernetes.ts b/test/cli/handlers/common/kubernetes.ts index d1f3a8af..05293e97 100644 --- a/test/cli/handlers/common/kubernetes.ts +++ b/test/cli/handlers/common/kubernetes.ts @@ -1,5 +1,5 @@ import { logger } from "utils"; -import type { LaunchedNetwork } from "./launchedNetwork"; +import type { LaunchedNetwork } from "../../../launcher/types/launchedNetwork"; /** * Forwards a port from a Kubernetes service to localhost and returns a cleanup function. diff --git a/test/cli/handlers/common/kurtosis.ts b/test/cli/handlers/common/kurtosis.ts deleted file mode 100644 index 4b9dae87..00000000 --- a/test/cli/handlers/common/kurtosis.ts +++ /dev/null @@ -1,214 +0,0 @@ -import { $ } from "bun"; -import invariant from "tiny-invariant"; -import { - getPortFromKurtosis, - type KurtosisEnclaveInfo, - KurtosisEnclaveInfoSchema, - logger -} from "utils"; -import { parse, stringify } from "yaml"; -import type { LaunchedNetwork } from "./launchedNetwork"; - -/** - * Checks if a Kurtosis enclave with the specified name is currently running. - * - * @param enclaveName - The name of the Kurtosis enclave to check - * @returns True if the enclave is running, false otherwise - */ -export const checkKurtosisEnclaveRunning = async (enclaveName: string): Promise => { - const enclaves = await getRunningKurtosisEnclaves(); - return enclaves.some((enclave) => enclave.name === enclaveName); -}; - -/** - * Gets a list of currently running Kurtosis enclaves - * @returns Promise - Array of running enclave information - */ -export const getRunningKurtosisEnclaves = async (): Promise => { - logger.debug("🔎 Checking for running Kurtosis enclaves..."); - - const lines = (await Array.fromAsync($`kurtosis enclave ls`.lines())).filter( - (line) => line.length > 0 - ); - logger.trace(lines); - - // Remove header line - lines.shift(); - - const enclaves: KurtosisEnclaveInfo[] = []; - - if (lines.length === 0) { - logger.debug("🤷‍ No Kurtosis enclaves found running."); - return enclaves; - } - - logger.debug(`🔎 Found ${lines.length} Kurtosis enclave(s) running.`); - // Updated regex to match the actual format: "uuid name status creationTime" - const enclaveRegex = /^(\S+)\s+(\S+)\s+(\S+)\s+(.+)$/; - - for (const line of lines) { - const match = line.match(enclaveRegex); - if (match) { - const [, uuid, name, status, creationTime] = match; - const parseResult = KurtosisEnclaveInfoSchema.safeParse({ - uuid: uuid.trim(), - name: name.trim(), - status: status.trim(), - creationTime: creationTime.trim() - }); - - if (parseResult.success) { - enclaves.push(parseResult.data); - } else { - logger.warn( - `⚠️ Could not parse enclave line: "${line}". Error: ${parseResult.error.message}` - ); - } - } else { - logger.warn(`⚠️ Could not parse enclave line (regex mismatch): "${line}"`); - } - } - - if (lines.length > 0 && enclaves.length === 0) { - logger.warn("⚠️ Found enclave lines in output, but failed to parse any of them."); - } - - return enclaves; -}; - -/** - * Modifies a Kurtosis configuration file based on deployment options. - * - * This function reads a YAML configuration file, applies modifications based on the provided - * deployment options, and writes the modified configuration to a new file in the tmp/configs directory. - * - * @param options.blockscout - If true, adds "blockscout" to the additional_services array - * @param options.slotTime - If provided, sets the network_params.seconds_per_slot value - * @param options.kurtosisNetworkArgs - Space-separated key=value pairs to add to network_params - * @param configFile - Path to the original YAML configuration file to modify - * @returns Promise - Path to the modified configuration file in tmp/configs/ - * @throws Will throw an error if the config file is not found - */ -export const modifyConfig = async ( - options: { - blockscout?: boolean; - slotTime?: number; - kurtosisNetworkArgs?: string; - }, - configFile: string -) => { - const outputDir = "tmp/configs"; - logger.debug(`Ensuring output directory exists: ${outputDir}`); - await $`mkdir -p ${outputDir}`.quiet(); - - const file = Bun.file(configFile); - invariant(file, `❌ Config file ${configFile} not found`); - - const config = await file.text(); - logger.debug(`Parsing config at ${configFile}`); - logger.trace(config); - - const parsedConfig = parse(config); - - if (options.blockscout) { - parsedConfig.additional_services.push("blockscout"); - } - - if (options.slotTime) { - parsedConfig.network_params.seconds_per_slot = options.slotTime; - } - - if (options.kurtosisNetworkArgs) { - logger.debug(`Using custom Kurtosis network args: ${options.kurtosisNetworkArgs}`); - const args = options.kurtosisNetworkArgs.split(" "); - for (const arg of args) { - const [key, value] = arg.split("="); - parsedConfig.network_params[key] = value; - } - } - - logger.trace(parsedConfig); - const outputFile = `${outputDir}/modified-config.yaml`; - logger.debug(`Modified config saving to ${outputFile}`); - - await Bun.write(outputFile, stringify(parsedConfig)); - return outputFile; -}; - -/** - * Registers the Execution Layer (EL) and Consensus Layer (CL) service endpoints with the LaunchedNetwork instance. - * - * This function retrieves the public ports for the Ethereum network services from Kurtosis and configures - * the LaunchedNetwork instance with the appropriate RPC URLs and endpoints for client communication. - * - * Services registered: - * - Execution Layer (EL): Reth RPC endpoint via "el-1-reth-lodestar" service - * - Consensus Layer (CL): lodestar HTTP endpoint via "cl-1-lodestar-reth" service - * - * @param launchedNetwork - The LaunchedNetwork instance to populate with service endpoints - * @param enclaveName - The name of the Kurtosis enclave containing the services - * @throws Will log warnings if services cannot be found or ports cannot be determined, but won't fail - */ -export const registerServices = async (launchedNetwork: LaunchedNetwork, enclaveName: string) => { - logger.info("📝 Registering Kurtosis service endpoints..."); - - // Configure EL RPC URL - try { - const rethPublicPort = await getPortFromKurtosis("el-1-reth-lodestar", "rpc", enclaveName); - invariant(rethPublicPort && rethPublicPort > 0, "❌ Could not find EL RPC port"); - const elRpcUrl = `http://127.0.0.1:${rethPublicPort}`; - launchedNetwork.elRpcUrl = elRpcUrl; - logger.info(`📝 Execution Layer RPC URL configured: ${elRpcUrl}`); - - // Configure CL Endpoint - const lodestarPublicPort = await getPortFromKurtosis("cl-1-lodestar-reth", "http", enclaveName); - const clEndpoint = `http://127.0.0.1:${lodestarPublicPort}`; - invariant( - clEndpoint, - "❌ CL Endpoint could not be determined from Kurtosis service cl-1-lodestar-reth" - ); - launchedNetwork.clEndpoint = clEndpoint; - logger.info(`📝 Consensus Layer Endpoint configured: ${clEndpoint}`); - } catch (error) { - logger.warn(`⚠️ Kurtosis service endpoints could not be determined: ${error}`); - } -}; - -/** - * Runs a Kurtosis Ethereum network enclave with the specified configuration. - * - * This function handles the complete process of starting a Kurtosis enclave: - * 1. Modifies the configuration file based on the provided options - * 2. Executes the kurtosis run command with the modified configuration - * 3. Handles error cases and logs appropriate debug information - * - * @param options - Configuration options containing kurtosisEnclaveName and other settings - * @param configFilePath - Path to the base YAML configuration file to use - * @throws Will throw an error if the Kurtosis network fails to start properly - */ -export const runKurtosisEnclave = async ( - options: { - kurtosisEnclaveName: string; - blockscout?: boolean; - slotTime?: number; - kurtosisNetworkArgs?: string; - }, - configFilePath: string -): Promise => { - logger.info("🚀 Starting Kurtosis enclave..."); - - const configFile = await modifyConfig(options, configFilePath); - - logger.info(`⚙️ Using Kurtosis config file: ${configFile}`); - - const { stderr, stdout, exitCode } = - await $`kurtosis run github.com/ethpandaops/ethereum-package --args-file ${configFile} --enclave ${options.kurtosisEnclaveName}` - .nothrow() - .quiet(); - - if (exitCode !== 0) { - logger.error(stderr.toString()); - throw Error("❌ Kurtosis network has failed to start properly."); - } - logger.debug(stdout.toString()); -}; diff --git a/test/cli/handlers/common/relayer.ts b/test/cli/handlers/common/relayer.ts deleted file mode 100644 index 1dcd766b..00000000 --- a/test/cli/handlers/common/relayer.ts +++ /dev/null @@ -1,329 +0,0 @@ -import path from "node:path"; -import { datahaven } from "@polkadot-api/descriptors"; -import { $ } from "bun"; -import { createClient } from "polkadot-api"; -import { withPolkadotSdkCompat } from "polkadot-api/polkadot-sdk-compat"; -import { getWsProvider } from "polkadot-api/ws-provider/web"; -import invariant from "tiny-invariant"; -import { getEvmEcdsaSigner, logger, parseRelayConfig, SUBSTRATE_FUNDED_ACCOUNTS } from "utils"; -import type { BeaconCheckpoint, FinalityCheckpointsResponse } from "utils/types"; -import { parseJsonToBeaconCheckpoint } from "utils/types"; -import { waitFor } from "utils/waits"; -import { ZERO_HASH } from "./consts"; -import type { LaunchedNetwork } from "./launchedNetwork"; - -export type BeaconConfig = { - type: "beacon"; - ethClEndpoint: string; - substrateWsEndpoint: string; -}; - -export type BeefyConfig = { - type: "beefy"; - ethElRpcEndpoint: string; - substrateWsEndpoint: string; - beefyClientAddress: string; - gatewayAddress: string; -}; - -export type ExecutionConfig = { - type: "execution"; - ethElRpcEndpoint: string; - ethClEndpoint: string; - substrateWsEndpoint: string; - gatewayAddress: string; -}; - -export type SolochainConfig = { - type: "solochain"; - ethElRpcEndpoint: string; - substrateWsEndpoint: string; - beefyClientAddress: string; - gatewayAddress: string; - rewardRegistryAddress: string; - ethClEndpoint: string; -}; - -export type RelayerConfigType = BeaconConfig | BeefyConfig | ExecutionConfig | SolochainConfig; - -export type RelayerSpec = { - name: string; - configFilePath: string; - templateFilePath?: string; - config: RelayerConfigType; - pk: { ethereum?: string; substrate?: string }; -}; - -export const INITIAL_CHECKPOINT_FILE = "dump-initial-checkpoint.json"; -export const INITIAL_CHECKPOINT_DIR = "tmp/beacon-checkpoint"; -export const INITIAL_CHECKPOINT_PATH = path.join(INITIAL_CHECKPOINT_DIR, INITIAL_CHECKPOINT_FILE); - -/** - * Generates configuration files for relayers. - * - * @param relayerSpec - The relayer specification containing name, type, and config path. - * @param environment - The environment to use for template files (e.g., "local", "stagenet", "testnet", "mainnet"). - * @param configDir - The directory where config files should be written. - */ -export const generateRelayerConfig = async ( - relayerSpec: RelayerSpec, - environment: string, - configDir: string -) => { - const { name, configFilePath, templateFilePath: _templateFilePath, config } = relayerSpec; - const { type } = config; - const configFileName = path.basename(configFilePath); - - logger.debug(`Creating config for ${name}`); - const templateFilePath = - _templateFilePath ?? `configs/snowbridge/${environment}/${configFileName}`; - const outputFilePath = path.resolve(configDir, configFileName); - logger.debug(`Reading config file ${templateFilePath}`); - const file = Bun.file(templateFilePath); - - if (!(await file.exists())) { - logger.error(`File ${templateFilePath} does not exist`); - throw new Error("Error reading snowbridge config file"); - } - const json = await file.json(); - - logger.debug(`Generating ${type} relayer configuration for ${name}`); - - switch (type) { - case "beacon": { - const cfg = parseRelayConfig(json, type); - cfg.source.beacon.endpoint = config.ethClEndpoint; - cfg.source.beacon.stateEndpoint = config.ethClEndpoint; - cfg.source.beacon.datastore.location = "/data"; - cfg.sink.parachain.endpoint = config.substrateWsEndpoint; - - await Bun.write(outputFilePath, JSON.stringify(cfg, null, 4)); - logger.success(`Updated beacon config written to ${outputFilePath}`); - break; - } - case "beefy": { - const cfg = parseRelayConfig(json, type); - cfg.source.polkadot.endpoint = config.substrateWsEndpoint; - cfg.sink.ethereum.endpoint = config.ethElRpcEndpoint; - cfg.sink.contracts.BeefyClient = config.beefyClientAddress; - cfg.sink.contracts.Gateway = config.gatewayAddress; - - await Bun.write(outputFilePath, JSON.stringify(cfg, null, 4)); - logger.success(`Updated beefy config written to ${outputFilePath}`); - break; - } - case "execution": { - const cfg = parseRelayConfig(json, type); - cfg.source.ethereum.endpoint = config.ethElRpcEndpoint; - cfg.source.beacon.endpoint = config.ethClEndpoint; - cfg.source.beacon.stateEndpoint = config.ethClEndpoint; - cfg.source.beacon.datastore.location = "/data"; - cfg.sink.parachain.endpoint = config.substrateWsEndpoint; - cfg.source.contracts.Gateway = config.gatewayAddress; - - await Bun.write(outputFilePath, JSON.stringify(cfg, null, 4)); - logger.success(`Updated execution config written to ${outputFilePath}`); - break; - } - case "solochain": { - const cfg = parseRelayConfig(json, type); - cfg.source.ethereum.endpoint = config.ethElRpcEndpoint; - cfg.source.solochain.endpoint = config.substrateWsEndpoint; - cfg.source.contracts.BeefyClient = config.beefyClientAddress; - cfg.source.contracts.Gateway = config.gatewayAddress; - cfg.source.beacon.endpoint = config.ethClEndpoint; - cfg.source.beacon.stateEndpoint = config.ethClEndpoint; - cfg.source.beacon.datastore.location = "/data"; - cfg.sink.ethereum.endpoint = config.ethElRpcEndpoint; - cfg.sink.contracts.Gateway = config.gatewayAddress; - cfg["reward-address"] = config.rewardRegistryAddress; - - await Bun.write(outputFilePath, JSON.stringify(cfg, null, 4)); - logger.success(`Updated solochain config written to ${outputFilePath}`); - break; - } - default: - throw new Error(`Unsupported relayer type with config: \n${JSON.stringify(config)}`); - } -}; - -/** - * Waits for the beacon chain to be ready by polling its finality checkpoints. - * - * @param launchedNetwork - An instance of LaunchedNetwork to get the CL endpoint. - * @param pollIntervalMs - The interval in milliseconds to poll the beacon chain. - * @param timeoutMs - The total time in milliseconds to wait before timing out. - * @throws Error if the beacon chain is not ready within the timeout. - */ -export const waitBeaconChainReady = async ( - launchedNetwork: LaunchedNetwork, - pollIntervalMs: number, - timeoutMs: number -) => { - const iterations = Math.floor(timeoutMs / pollIntervalMs); - - logger.trace("Waiting for beacon chain to be ready..."); - - await waitFor({ - lambda: async () => { - try { - const response = await fetch( - `${launchedNetwork.clEndpoint}/eth/v1/beacon/states/head/finality_checkpoints` - ); - if (!response.ok) { - throw new Error(`HTTP error! Status: ${response.status}`); - } - - const data = (await response.json()) as FinalityCheckpointsResponse; - logger.debug(`Beacon chain state: ${JSON.stringify(data)}`); - - invariant(data.data, "❌ No data returned from beacon chain"); - invariant(data.data.finalized, "❌ No finalised block returned from beacon chain"); - invariant( - data.data.finalized.root, - "❌ No finalised block root returned from beacon chain" - ); - - const initialBeaconBlock = data.data.finalized.root; - - if (initialBeaconBlock && initialBeaconBlock !== ZERO_HASH) { - logger.info(`⏲️ Beacon chain is ready with finalised block: ${initialBeaconBlock}`); - return true; - } - - logger.info(`⌛️ Retrying beacon chain state fetch in ${pollIntervalMs / 1000}s...`); - return false; - } catch (error) { - logger.error(`Failed to fetch beacon chain state: ${error}`); - return false; - } - }, - iterations, - delay: pollIntervalMs, - errorMessage: "Beacon chain is not ready. Relayers cannot be launched." - }); -}; - -/** - * Initialises the Ethereum Beacon Client pallet on the Substrate chain. - * It waits for the beacon chain to be ready, generates an initial checkpoint, - * and submits this checkpoint to the Substrate runtime via a sudo call. - * - * @param beaconConfigHostPath - The host path to the beacon configuration file. - * @param relayerImageTag - The Docker image tag for the relayer. - * @param datastorePath - The path to the datastore directory. - * @param launchedNetwork - An instance of LaunchedNetwork to interact with the running network. - * @throws If there's an error generating the beacon checkpoint or submitting it to Substrate. - */ -export const initEthClientPallet = async ( - beaconConfigHostPath: string, - relayerImageTag: string, - datastorePath: string, - launchedNetwork: LaunchedNetwork -) => { - logger.debug("Initialising eth client pallet"); - // Poll the beacon chain until it's ready every 10 seconds for 10 minutes - await waitBeaconChainReady(launchedNetwork, 10000, 600000); - - const beaconConfigContainerPath = "/app/beacon-relay.json"; - const checkpointHostPath = path.resolve(INITIAL_CHECKPOINT_PATH); - const checkpointContainerPath = `/app/${INITIAL_CHECKPOINT_FILE}`; - - logger.debug("Generating beacon checkpoint"); - // Pre-create the checkpoint file so that Docker doesn't interpret it as a directory - await Bun.write(INITIAL_CHECKPOINT_PATH, ""); - - logger.debug("Removing 'generate-beacon-checkpoint' container if it exists"); - logger.debug(await $`docker rm -f generate-beacon-checkpoint`.text()); - - // When running in Linux, `host.docker.internal` is not pre-defined when running in a container. - // So we need to add the parameter `--add-host host.docker.internal:host-gateway` to the command. - // In Mac this is not needed and could cause issues. - const addHostParam = - process.platform === "linux" ? "--add-host host.docker.internal:host-gateway" : ""; - - // Opportunistic pull - pull the image from Docker Hub only if it's not a local image - const isLocal = relayerImageTag.endsWith(":local"); - - logger.debug("Generating beacon checkpoint"); - const datastoreHostPath = path.resolve(datastorePath); - const command = `docker run \ - -v ${beaconConfigHostPath}:${beaconConfigContainerPath}:ro \ - -v ${checkpointHostPath}:${checkpointContainerPath} \ - -v ${datastoreHostPath}:/data \ - --name generate-beacon-checkpoint \ - --platform linux/amd64 \ - --workdir /app \ - ${addHostParam} \ - ${launchedNetwork.networkName ? `--network ${launchedNetwork.networkName}` : ""} \ - ${isLocal ? "" : "--pull always"} \ - ${relayerImageTag} \ - generate-beacon-checkpoint --config beacon-relay.json --export-json`; - logger.debug(`Running command: ${command}`); - logger.debug(await $`sh -c "${command}"`.text()); - - // Load the checkpoint into a JSON object and clean it up - const initialCheckpointFile = Bun.file(INITIAL_CHECKPOINT_PATH); - const initialCheckpointRaw = await initialCheckpointFile.text(); - const initialCheckpoint = parseJsonToBeaconCheckpoint(JSON.parse(initialCheckpointRaw)); - await initialCheckpointFile.delete(); - - logger.trace("Initial checkpoint:"); - logger.trace(initialCheckpoint.toJSON()); - - // Send the checkpoint to the Substrate runtime - const substrateRpcUrl = `http://127.0.0.1:${launchedNetwork.getPublicWsPort()}`; - await sendCheckpointToSubstrate(substrateRpcUrl, initialCheckpoint); - logger.success("Ethereum Beacon Client pallet initialised"); -}; - -/** - * Sends the beacon checkpoint to the Substrate runtime, waiting for the transaction to be finalised and successful. - * - * @param networkRpcUrl - The RPC URL of the Substrate network. - * @param checkpoint - The beacon checkpoint to send. - * @throws If the transaction signing fails, it becomes an invalid transaction, or the transaction is included but fails. - */ -const sendCheckpointToSubstrate = async (networkRpcUrl: string, checkpoint: BeaconCheckpoint) => { - logger.trace("Sending checkpoint to Substrate..."); - - const client = createClient(withPolkadotSdkCompat(getWsProvider(networkRpcUrl))); - const dhApi = client.getTypedApi(datahaven); - - logger.trace("Client created"); - - const signer = getEvmEcdsaSigner(SUBSTRATE_FUNDED_ACCOUNTS.ALITH.privateKey); - logger.trace("Signer created"); - - const forceCheckpointCall = dhApi.tx.EthereumBeaconClient.force_checkpoint({ - update: checkpoint - }); - - logger.debug("Force checkpoint call:"); - logger.debug(forceCheckpointCall.decodedCall); - - const tx = dhApi.tx.Sudo.sudo({ - call: forceCheckpointCall.decodedCall - }); - - logger.debug("Sudo call:"); - logger.debug(tx.decodedCall); - - try { - const txFinalisedPayload = await tx.signAndSubmit(signer); - - if (!txFinalisedPayload.ok) { - throw new Error("❌ Beacon checkpoint transaction failed"); - } - - logger.info( - `📪 "force_checkpoint" transaction with hash ${txFinalisedPayload.txHash} submitted successfully and finalised in block ${txFinalisedPayload.block.hash}` - ); - } catch (error) { - logger.error(`Failed to submit checkpoint transaction: ${error}`); - throw new Error(`Failed to submit checkpoint: ${error}`); - } finally { - client.destroy(); - logger.debug("Destroyed client"); - } -}; diff --git a/test/cli/handlers/deploy/cleanup.ts b/test/cli/handlers/deploy/cleanup.ts index efff0fbc..4d0d2e0c 100644 --- a/test/cli/handlers/deploy/cleanup.ts +++ b/test/cli/handlers/deploy/cleanup.ts @@ -2,8 +2,8 @@ import { $ } from "bun"; import invariant from "tiny-invariant"; import { logger, printDivider, printHeader } from "utils"; import { waitFor } from "utils/waits"; -import { checkKurtosisEnclaveRunning } from "../common/kurtosis"; -import type { LaunchedNetwork } from "../common/launchedNetwork"; +import { checkKurtosisEnclaveRunning } from "../../../launcher/kurtosis"; +import type { LaunchedNetwork } from "../../../launcher/types/launchedNetwork"; import type { DeployOptions } from "."; export const cleanup = async ( diff --git a/test/cli/handlers/deploy/datahaven.ts b/test/cli/handlers/deploy/datahaven.ts index e0bdee5c..75dbfad9 100644 --- a/test/cli/handlers/deploy/datahaven.ts +++ b/test/cli/handlers/deploy/datahaven.ts @@ -3,9 +3,9 @@ import { $ } from "bun"; import invariant from "tiny-invariant"; import { logger, printDivider, printHeader } from "utils"; import { waitFor } from "utils/waits"; -import { isNetworkReady, setupDataHavenValidatorConfig } from "../common/datahaven"; +import { isNetworkReady, setupDataHavenValidatorConfig } from "../../../launcher/datahaven"; +import type { LaunchedNetwork } from "../../../launcher/types/launchedNetwork"; import { forwardPort } from "../common/kubernetes"; -import type { LaunchedNetwork } from "../common/launchedNetwork"; import type { DeployOptions } from "."; const DEFAULT_PUBLIC_WS_PORT = 9944; diff --git a/test/cli/handlers/deploy/index.ts b/test/cli/handlers/deploy/index.ts index e5ff5213..4fc45472 100644 --- a/test/cli/handlers/deploy/index.ts +++ b/test/cli/handlers/deploy/index.ts @@ -1,8 +1,8 @@ import type { Command } from "node_modules/@commander-js/extra-typings"; import { type DeployEnvironment, logger } from "utils"; import { createParameterCollection } from "utils/parameters"; +import { LaunchedNetwork } from "../../../launcher/types/launchedNetwork"; import { checkBaseDependencies, deploymentChecks } from "../common/checks"; -import { LaunchedNetwork } from "../common/launchedNetwork"; import { cleanup } from "./cleanup"; import { deployContracts } from "./contracts"; import { deployDataHavenSolochain } from "./datahaven"; diff --git a/test/cli/handlers/deploy/kurtosis.ts b/test/cli/handlers/deploy/kurtosis.ts index e1cc8841..cc994500 100644 --- a/test/cli/handlers/deploy/kurtosis.ts +++ b/test/cli/handlers/deploy/kurtosis.ts @@ -1,8 +1,8 @@ import type { DeployOptions } from "cli/handlers"; import invariant from "tiny-invariant"; import { logger, printDivider, printHeader } from "utils"; -import { registerServices, runKurtosisEnclave } from "../common/kurtosis"; -import type { LaunchedNetwork } from "../common/launchedNetwork"; +import { registerServices, runKurtosisEnclave } from "../../../launcher/kurtosis"; +import type { LaunchedNetwork } from "../../../launcher/types/launchedNetwork"; /** * Deploys a Kurtosis Ethereum network enclave for stagenet environment. diff --git a/test/cli/handlers/deploy/relayer.ts b/test/cli/handlers/deploy/relayer.ts index 26de2665..8f77eb84 100644 --- a/test/cli/handlers/deploy/relayer.ts +++ b/test/cli/handlers/deploy/relayer.ts @@ -13,9 +13,13 @@ import { SUBSTRATE_FUNDED_ACCOUNTS } from "utils"; import { waitFor } from "utils/waits"; -import { ZERO_HASH } from "../common/consts"; -import type { LaunchedNetwork } from "../common/launchedNetwork"; -import { generateRelayerConfig, initEthClientPallet, type RelayerSpec } from "../common/relayer"; +import { + generateRelayerConfig, + initEthClientPallet, + type RelayerSpec +} from "../../../launcher/relayers"; +import type { LaunchedNetwork } from "../../../launcher/types/launchedNetwork"; +import { ZERO_HASH } from "../../../launcher/utils/constants"; import type { DeployOptions } from "."; // Standard ports for the Ethereum network @@ -161,6 +165,7 @@ export const deployRelayers = async (options: DeployOptions, launchedNetwork: La await generateRelayerConfig(localBeaconConfig, options.environment, localBeaconConfigDir); await initEthClientPallet( + "cli-deploy", path.resolve(localBeaconConfigFilePath), options.relayerImageTag, "tmp/datastore", diff --git a/test/cli/handlers/launch/contracts.ts b/test/cli/handlers/launch/contracts.ts index 4072a937..6380412c 100644 --- a/test/cli/handlers/launch/contracts.ts +++ b/test/cli/handlers/launch/contracts.ts @@ -1,11 +1,6 @@ -import { - buildContracts, - constructDeployCommand, - executeDeployment, - validateDeploymentParams -} from "scripts/deploy-contracts"; import { confirmWithTimeout, logger, printDivider, printHeader } from "utils"; import type { ParameterCollection } from "utils/parameters"; +import { deployContracts as deployContractsCore } from "../../../launcher/contracts"; interface DeployContractsOptions { rpcUrl: string; @@ -51,16 +46,13 @@ export const deployContracts = async (options: DeployContractsOptions): Promise< return false; } - // Check if required parameters are provided - validateDeploymentParams(options); + await deployContractsCore({ + rpcUrl: options.rpcUrl, + verified: options.verified, + blockscoutBackendUrl: options.blockscoutBackendUrl, + parameterCollection: options.parameterCollection + }); - // Build contracts - await buildContracts(); - - // Construct and execute deployment - const deployCommand = constructDeployCommand(options); - await executeDeployment(deployCommand, options.parameterCollection); printDivider(); - return true; }; diff --git a/test/cli/handlers/launch/datahaven.ts b/test/cli/handlers/launch/datahaven.ts index 5e02ab3b..ed685376 100644 --- a/test/cli/handlers/launch/datahaven.ts +++ b/test/cli/handlers/launch/datahaven.ts @@ -1,36 +1,13 @@ -import { $ } from "bun"; -import { cargoCrossbuild } from "scripts/cargo-crossbuild"; import invariant from "tiny-invariant"; +import { confirmWithTimeout, logger, printDivider, printHeader } from "utils"; import { - confirmWithTimeout, - killExistingContainers, - logger, - printDivider, - printHeader, - waitForContainerToStart -} from "utils"; -import { waitFor } from "utils/waits"; -import { DOCKER_NETWORK_NAME } from "../common/consts"; -import { isNetworkReady, setupDataHavenValidatorConfig } from "../common/datahaven"; -import type { LaunchedNetwork } from "../common/launchedNetwork"; -import type { LaunchOptions } from "."; - -const LOG_LEVEL = Bun.env.LOG_LEVEL || "info"; - -const COMMON_LAUNCH_ARGS = [ - "--unsafe-force-node-key-generation", - "--tmp", - "--validator", - "--discover-local", - "--no-prometheus", - "--unsafe-rpc-external", - "--rpc-cors=all", - "--force-authoring", - "--no-telemetry", - "--enable-offchain-indexing=true" -]; - -const DEFAULT_PUBLIC_WS_PORT = 9944; + checkDataHavenRunning, + cleanDataHavenContainers, + launchLocalDataHavenSolochain, + registerNodes +} from "../../../launcher/datahaven"; +import type { LaunchedNetwork } from "../../../launcher/types/launchedNetwork"; +import { type LaunchOptions, NETWORK_ID } from "."; // 2 validators (Alice and Bob) are used for local & CI testing // /operator/runtime/stagenet/src/genesis_config_presets.rs#L98 @@ -65,7 +42,7 @@ export const launchDataHavenSolochain = async ( if (!shouldLaunchDataHaven) { logger.info("👍 Skipping DataHaven network launch. Done!"); - await registerNodes(launchedNetwork); + await registerNodes(NETWORK_ID, launchedNetwork); printDivider(); return; } @@ -89,146 +66,19 @@ export const launchDataHavenSolochain = async ( if (!shouldRelaunch) { logger.info("👍 Keeping existing DataHaven containers/network."); - await registerNodes(launchedNetwork); + await registerNodes(NETWORK_ID, launchedNetwork); printDivider(); return; } // Case: User wants to clean and relaunch the DataHaven containers - await cleanDataHavenContainers(options); + await cleanDataHavenContainers(NETWORK_ID); } } - logger.info(`⛓️‍💥 Creating Docker network: ${DOCKER_NETWORK_NAME}`); - logger.debug(await $`docker network rm ${DOCKER_NETWORK_NAME} -f`.text()); - logger.debug(await $`docker network create ${DOCKER_NETWORK_NAME}`.text()); - invariant(options.datahavenImageTag, "❌ DataHaven image tag not defined"); - await buildLocalImage(options); - await checkTagExists(options.datahavenImageTag); - - logger.success(`DataHaven nodes will use Docker network: ${DOCKER_NETWORK_NAME}`); - - for (const id of CLI_AUTHORITY_IDS) { - logger.info(`🚀 Starting ${id}...`); - const containerName = `datahaven-${id}`; - - const command: string[] = [ - "docker", - "run", - "-d", - "--name", - containerName, - "--network", - DOCKER_NETWORK_NAME, - ...(id === "alice" ? ["-p", `${DEFAULT_PUBLIC_WS_PORT}:9944`] : []), - options.datahavenImageTag, - `--${id}`, - ...COMMON_LAUNCH_ARGS - ]; - - logger.debug(await $`sh -c "${command.join(" ")}"`.text()); - - await waitForContainerToStart(containerName); - - // TODO: Un-comment this when it doesn't stop process from hanging - // This is working on SH, but not here so probably a Bun defect - // - // const listeningLine = await waitForLog({ - // search: "Running JSON-RPC server: addr=0.0.0.0:", - // containerName, - // timeoutSeconds: 30 - // }); - // logger.debug(listeningLine); - } - - logger.info("⌛️ Waiting for DataHaven to start..."); - const timeoutMs = 2000; // 2 second timeout - await waitFor({ - lambda: async () => { - const isReady = await isNetworkReady(DEFAULT_PUBLIC_WS_PORT, timeoutMs); - if (!isReady) { - logger.debug("Node not ready, waiting 1 second..."); - } - return isReady; - }, - iterations: 30, - delay: timeoutMs, - errorMessage: "DataHaven network not ready" - }); - - logger.success( - `DataHaven network started, primary node accessible on port ${DEFAULT_PUBLIC_WS_PORT}` - ); - - await registerNodes(launchedNetwork); - - await setupDataHavenValidatorConfig(launchedNetwork, "datahaven-"); - - printDivider(); -}; - -/** - * Checks if any DataHaven containers are currently running. - * - * @returns True if any DataHaven containers are running, false otherwise. - */ -const checkDataHavenRunning = async (): Promise => { - // Check for any container whose name starts with "datahaven-" - const containerIds = await $`docker ps --format "{{.Names}}" --filter "name=^datahaven-"`.text(); - const networkOutput = - await $`docker network ls --filter "name=^${DOCKER_NETWORK_NAME}$" --format "{{.Name}}"`.text(); - - // Check if containerIds has any actual IDs (not just whitespace) - const containersExist = containerIds.trim().length > 0; - if (containersExist) { - logger.info(`ℹ️ DataHaven containers already running: \n${containerIds}`); - } - - // Check if networkOutput has any network names (not just whitespace or empty lines) - const networksExist = - networkOutput - .trim() - .split("\n") - .filter((line) => line.trim().length > 0).length > 0; - if (networksExist) { - logger.info(`ℹ️ DataHaven network already running: ${networkOutput}`); - } - - return containersExist || networksExist; -}; - -/** - * Stops and removes all DataHaven containers. - */ -const cleanDataHavenContainers = async (options: LaunchOptions): Promise => { - logger.info("🧹 Stopping and removing existing DataHaven containers..."); - - invariant(options.datahavenImageTag, "❌ DataHaven image tag not defined"); - await killExistingContainers(options.datahavenImageTag); - - if (options.relayerImageTag) { - logger.info( - "🧹 Stopping and removing existing relayer containers (relayers depend on DataHaven nodes)..." - ); - await killExistingContainers(options.relayerImageTag); - } - - logger.info("✅ Existing DataHaven containers stopped and removed."); - - logger.debug(await $`docker network rm -f ${DOCKER_NETWORK_NAME}`.text()); - logger.info("✅ DataHaven Docker network removed."); - - invariant( - (await checkDataHavenRunning()) === false, - "❌ DataHaven containers were not stopped and removed" - ); -}; - -const buildLocalImage = async (options: LaunchOptions) => { let shouldBuildDataHaven = options.buildDatahaven; - if (shouldBuildDataHaven === undefined) { shouldBuildDataHaven = await confirmWithTimeout( "Do you want to build the DataHaven node local Docker image?", @@ -243,68 +93,19 @@ const buildLocalImage = async (options: LaunchOptions) => { if (!shouldBuildDataHaven) { logger.info("👍 Skipping DataHaven node local Docker image build. Done!"); - return; } - await cargoCrossbuild({ - datahavenBuildExtraArgs: options.datahavenBuildExtraArgs - }); - - logger.info("🐳 Building DataHaven node local Docker image..."); - if (LOG_LEVEL === "trace") { - await $`bun build:docker:operator`; - } else { - await $`bun build:docker:operator`.quiet(); - } - logger.success("DataHaven node local Docker image build completed successfully"); -}; - -/** - * Checks if an image exists locally or on Docker Hub. - * - * @param tag - The tag of the image to check. - * @returns A promise that resolves when the image is found. - */ -const checkTagExists = async (tag: string) => { - const cleaned = tag.trim(); - logger.debug(`Checking if image ${cleaned} is available locally`); - const { exitCode: localExists } = await $`docker image inspect ${cleaned}`.nothrow().quiet(); - - if (localExists !== 0) { - logger.debug(`Checking if image ${cleaned} is available on docker hub`); - const result = await $`docker manifest inspect ${cleaned}`.nothrow().quiet(); - invariant( - result.exitCode === 0, - `❌ Image ${tag} not found.\n Does this image exist?\n Are you logged and have access to the repository?` - ); - } - - logger.success(`Image ${tag} found locally`); -}; - -const registerNodes = async (launchedNetwork: LaunchedNetwork) => { - // Registering DataHaven nodes Docker network. - launchedNetwork.networkName = DOCKER_NETWORK_NAME; - - const targetContainerName = "datahaven-alice"; - const aliceHostWsPort = DEFAULT_PUBLIC_WS_PORT; // Standard host port for Alice's WS, as set during launch. - - logger.debug(`Checking Docker status for container: ${targetContainerName}`); - // Use ^ and $ for an exact name match in the filter. - const dockerPsOutput = await $`docker ps -q --filter "name=^${targetContainerName}$"`.text(); - const isContainerRunning = dockerPsOutput.trim().length > 0; - - if (!isContainerRunning) { - // If the target Docker container is not running, we cannot register it. - logger.warn(`⚠️ Docker container ${targetContainerName} is not running. Cannot register node.`); - return; - } - - // If the Docker container is running, proceed to register it in launchedNetwork. - // We use the standard host WS port that "datahaven-alice" is expected to use. - logger.debug( - `Docker container ${targetContainerName} is running. Registering with WS port ${aliceHostWsPort}.` + await launchLocalDataHavenSolochain( + { + networkId: NETWORK_ID, + datahavenImageTag: options.datahavenImageTag, + relayerImageTag: options.relayerImageTag, + authorityIds: CLI_AUTHORITY_IDS, + buildDatahaven: shouldBuildDataHaven, + datahavenBuildExtraArgs: options.datahavenBuildExtraArgs + }, + launchedNetwork ); - launchedNetwork.addContainer(targetContainerName, { ws: aliceHostWsPort }); - logger.info(`📝 Node ${targetContainerName} successfully registered in launchedNetwork.`); + + printDivider(); }; diff --git a/test/cli/handlers/launch/index.ts b/test/cli/handlers/launch/index.ts index 0d7c5a2e..151ecfdf 100644 --- a/test/cli/handlers/launch/index.ts +++ b/test/cli/handlers/launch/index.ts @@ -1,8 +1,9 @@ import type { Command } from "@commander-js/extra-typings"; -import { getPortFromKurtosis, logger } from "utils"; +import { logger } from "utils"; import { createParameterCollection } from "utils/parameters"; +import { getBlockscoutUrl } from "../../../launcher/kurtosis"; +import { LaunchedNetwork } from "../../../launcher/types/launchedNetwork"; import { checkBaseDependencies } from "../common/checks"; -import { LaunchedNetwork } from "../common/launchedNetwork"; import { deployContracts } from "./contracts"; import { launchDataHavenSolochain } from "./datahaven"; import { launchKurtosis } from "./kurtosis"; @@ -11,6 +12,8 @@ import { launchRelayers } from "./relayer"; import { performSummaryOperations } from "./summary"; import { performValidatorOperations, performValidatorSetUpdate } from "./validator"; +export const NETWORK_ID = "cli-launch"; + // Non-optional properties should have default values set by the CLI export interface LaunchOptions { all?: boolean; @@ -55,12 +58,7 @@ const launchFunction = async (options: LaunchOptions, launchedNetwork: LaunchedN let blockscoutBackendUrl: string | undefined; if (options.blockscout === true) { - const blockscoutPublicPort = await getPortFromKurtosis( - "blockscout", - "http", - options.kurtosisEnclaveName - ); - blockscoutBackendUrl = `http://127.0.0.1:${blockscoutPublicPort}`; + blockscoutBackendUrl = await getBlockscoutUrl(options.kurtosisEnclaveName); logger.trace("Blockscout backend URL:", blockscoutBackendUrl); } else if (options.verified) { logger.warn( diff --git a/test/cli/handlers/launch/kurtosis.ts b/test/cli/handlers/launch/kurtosis.ts index b73278a5..07de893f 100644 --- a/test/cli/handlers/launch/kurtosis.ts +++ b/test/cli/handlers/launch/kurtosis.ts @@ -1,12 +1,13 @@ -import { $ } from "bun"; -import { checkKurtosisCluster, type LaunchOptions } from "cli/handlers"; -import { confirmWithTimeout, logger, printDivider, printHeader } from "utils"; +import type { LaunchOptions } from "cli/handlers"; import { checkKurtosisEnclaveRunning, - registerServices, - runKurtosisEnclave -} from "../common/kurtosis"; -import type { LaunchedNetwork } from "../common/launchedNetwork"; + cleanKurtosisEnclave, + launchKurtosisNetwork, + registerServices +} from "launcher/kurtosis"; +import type { LaunchedNetwork } from "launcher/types/launchedNetwork"; +import { checkKurtosisCluster } from "launcher/utils/checks"; +import { confirmWithTimeout, logger, printDivider, printHeader } from "utils"; /** * Launches a Kurtosis Ethereum network enclave for testing. @@ -71,24 +72,18 @@ export const launchKurtosis = async ( } // Case: User wants to clean and relaunch the enclave - logger.info("🧹 Cleaning up Docker and Kurtosis environments..."); - logger.debug(await $`kurtosis enclave stop ${options.kurtosisEnclaveName}`.nothrow().text()); - logger.debug(await $`kurtosis clean`.text()); - logger.debug(await $`kurtosis engine stop`.nothrow().text()); - logger.debug(await $`docker system prune -f`.nothrow().text()); + await cleanKurtosisEnclave(options.kurtosisEnclaveName); } } - if (process.platform === "darwin") { - logger.debug("Detected macOS, pulling container images with linux/amd64 platform..."); - logger.debug( - await $`docker pull ghcr.io/blockscout/smart-contract-verifier:latest --platform linux/amd64`.text() - ); - } - - await runKurtosisEnclave(options, "configs/kurtosis/minimal.yaml"); - - await registerServices(launchedNetwork, options.kurtosisEnclaveName); - logger.success("Kurtosis network operations completed successfully."); + await launchKurtosisNetwork( + { + kurtosisEnclaveName: options.kurtosisEnclaveName, + blockscout: options.blockscout, + slotTime: options.slotTime, + kurtosisNetworkArgs: options.kurtosisNetworkArgs + }, + launchedNetwork + ); printDivider(); }; diff --git a/test/cli/handlers/launch/parameters.ts b/test/cli/handlers/launch/parameters.ts index 969e6773..60b7bbb1 100644 --- a/test/cli/handlers/launch/parameters.ts +++ b/test/cli/handlers/launch/parameters.ts @@ -1,8 +1,8 @@ -import { setDataHavenParameters } from "scripts/set-datahaven-parameters"; import { logger, printDivider, printHeader } from "utils"; import { confirmWithTimeout } from "utils/input"; import type { ParameterCollection } from "utils/parameters"; -import type { LaunchedNetwork } from "../common/launchedNetwork"; +import { setDataHavenParameters } from "../../../launcher/parameters"; +import type { LaunchedNetwork } from "../../../launcher/types/launchedNetwork"; /** * A helper function to set DataHaven parameters from a ParameterCollection @@ -24,9 +24,6 @@ export const setParametersFromCollection = async ({ }): Promise => { printHeader("Setting DataHaven Runtime Parameters"); - const parametersFilePath = await collection.generateParametersFile(); - const rpcUrl = `ws://127.0.0.1:${launchedNetwork.getPublicWsPort()}`; - // Check if setParameters option was set via flags, or prompt if not let shouldSetParameters = setParameters; if (shouldSetParameters === undefined) { @@ -49,11 +46,11 @@ export const setParametersFromCollection = async ({ return false; } - const parametersSet = await setDataHavenParameters({ - rpcUrl, - parametersFilePath + await setDataHavenParameters({ + launchedNetwork, + collection }); printDivider(); - return parametersSet; + return true; }; diff --git a/test/cli/handlers/launch/relayer.ts b/test/cli/handlers/launch/relayer.ts index d3670497..8a6f2338 100644 --- a/test/cli/handlers/launch/relayer.ts +++ b/test/cli/handlers/launch/relayer.ts @@ -1,35 +1,7 @@ -import path from "node:path"; -import { $ } from "bun"; -import { createClient, type PolkadotClient } from "polkadot-api"; -import { withPolkadotSdkCompat } from "polkadot-api/polkadot-sdk-compat"; -import { getWsProvider } from "polkadot-api/ws-provider/web"; -import invariant from "tiny-invariant"; -import { - ANVIL_FUNDED_ACCOUNTS, - confirmWithTimeout, - getPortFromKurtosis, - killExistingContainers, - logger, - parseDeploymentsFile, - printDivider, - printHeader, - runShellCommandWithLogger, - SUBSTRATE_FUNDED_ACCOUNTS, - waitForContainerToStart -} from "utils"; -import { waitFor } from "utils/waits"; -import { ZERO_HASH } from "../common/consts"; -import type { LaunchedNetwork } from "../common/launchedNetwork"; -import { generateRelayerConfig, initEthClientPallet, type RelayerSpec } from "../common/relayer"; -import type { LaunchOptions } from "."; - -const RELAYER_CONFIG_DIR = "tmp/configs"; -const RELAYER_CONFIG_PATHS = { - BEACON: path.join(RELAYER_CONFIG_DIR, "beacon-relay.json"), - BEEFY: path.join(RELAYER_CONFIG_DIR, "beefy-relay.json"), - EXECUTION: path.join(RELAYER_CONFIG_DIR, "execution-relay.json"), - SOLOCHAIN: path.join(RELAYER_CONFIG_DIR, "solochain-relay.json") -}; +import { confirmWithTimeout, logger, printDivider, printHeader } from "utils"; +import { launchRelayers as launchRelayersCore } from "../../../launcher/relayers"; +import type { LaunchedNetwork } from "../../../launcher/types/launchedNetwork"; +import { type LaunchOptions, NETWORK_ID } from "."; /** * Launches Snowbridge relayers for the DataHaven network. @@ -54,305 +26,19 @@ export const launchRelayers = async (options: LaunchOptions, launchedNetwork: La } if (!shouldLaunchRelayers) { - logger.info("👍 Snowbridge relayers launch. Done!"); + logger.info("👍 Skipping Snowbridge relayers launch. Done!"); printDivider(); return; } - // Get DataHaven node port - const dhNodes = launchedNetwork.containers.filter((container) => - container.name.includes("datahaven") - ); - let substrateWsPort: number; - let substrateNodeId: string; - - if (dhNodes.length === 0) { - logger.warn( - "⚠️ No DataHaven nodes found in launchedNetwork. Assuming DataHaven is running and defaulting to port 9944 for relayers." - ); - substrateWsPort = 9944; - substrateNodeId = "default (assumed)"; - } else { - const firstDhNode = dhNodes[0]; - substrateWsPort = firstDhNode.publicPorts.ws; - substrateNodeId = firstDhNode.name; - logger.info( - `🔌 Using DataHaven node ${substrateNodeId} on port ${substrateWsPort} for relayers and BEEFY check.` - ); - } - - invariant(options.relayerImageTag, "❌ relayerImageTag is required"); - await killExistingContainers(options.relayerImageTag); - - // Check if BEEFY is ready before proceeding - await waitBeefyReady(launchedNetwork, 2000, 60000); - - const anvilDeployments = await parseDeploymentsFile(); - const beefyClientAddress = anvilDeployments.BeefyClient; - const gatewayAddress = anvilDeployments.Gateway; - const rewardRegistryAddress = anvilDeployments.RewardsRegistry; - invariant(beefyClientAddress, "❌ BeefyClient address not found in anvil.json"); - invariant(gatewayAddress, "❌ Gateway address not found in anvil.json"); - invariant(rewardRegistryAddress, "❌ RewardsRegistry address not found in anvil.json"); - - logger.debug(`Ensuring output directory exists: ${RELAYER_CONFIG_DIR}`); - await $`mkdir -p ${RELAYER_CONFIG_DIR}`.quiet(); - - const datastorePath = "tmp/datastore"; - logger.debug(`Ensuring datastore directory exists: ${datastorePath}`); - await $`mkdir -p ${datastorePath}`.quiet(); - - const ethWsPort = await getPortFromKurtosis( - "el-1-reth-lodestar", - "ws", - options.kurtosisEnclaveName - ); - const ethHttpPort = await getPortFromKurtosis( - "cl-1-lodestar-reth", - "http", - options.kurtosisEnclaveName - ); - - const ethElRpcEndpoint = `ws://host.docker.internal:${ethWsPort}`; - const ethClEndpoint = `http://host.docker.internal:${ethHttpPort}`; - const substrateWsEndpoint = `ws://${substrateNodeId}:${substrateWsPort}`; - - const relayersToStart: RelayerSpec[] = [ + await launchRelayersCore( { - name: "relayer-🥩", - configFilePath: RELAYER_CONFIG_PATHS.BEEFY, - config: { - type: "beefy", - ethElRpcEndpoint, - substrateWsEndpoint, - beefyClientAddress, - gatewayAddress - }, - pk: { - ethereum: ANVIL_FUNDED_ACCOUNTS[1].privateKey - } + networkId: NETWORK_ID, + relayerImageTag: options.relayerImageTag, + kurtosisEnclaveName: options.kurtosisEnclaveName }, - { - name: "relayer-🥓", - configFilePath: RELAYER_CONFIG_PATHS.BEACON, - config: { - type: "beacon", - ethClEndpoint, - substrateWsEndpoint - }, - pk: { - substrate: SUBSTRATE_FUNDED_ACCOUNTS.BALTATHAR.privateKey - } - }, - { - name: "relayer-⛓️", - configFilePath: RELAYER_CONFIG_PATHS.SOLOCHAIN, - config: { - type: "solochain", - ethElRpcEndpoint, - substrateWsEndpoint, - beefyClientAddress, - gatewayAddress, - rewardRegistryAddress, - ethClEndpoint - }, - pk: { - ethereum: ANVIL_FUNDED_ACCOUNTS[1].privateKey, - substrate: SUBSTRATE_FUNDED_ACCOUNTS.CHARLETH.privateKey - } - }, - { - name: "relayer-⚙️", - configFilePath: RELAYER_CONFIG_PATHS.EXECUTION, - config: { - type: "execution", - ethElRpcEndpoint, - ethClEndpoint, - substrateWsEndpoint, - gatewayAddress - }, - pk: { - substrate: SUBSTRATE_FUNDED_ACCOUNTS.DOROTHY.privateKey - } - } - ]; - - for (const relayerSpec of relayersToStart) { - await generateRelayerConfig(relayerSpec, "local", RELAYER_CONFIG_DIR); - } - - invariant(options.relayerImageTag, "❌ Relayer image tag not defined"); - invariant( - launchedNetwork.networkName, - "❌ Docker network name not found in LaunchedNetwork instance" - ); - - await initEthClientPallet( - path.resolve(RELAYER_CONFIG_PATHS.BEACON), - options.relayerImageTag, - datastorePath, launchedNetwork ); - // Opportunistic pull - pull the image from Docker Hub only if it's not a local image - const isLocal = options.relayerImageTag.endsWith(":local"); - - for (const { configFilePath, name, config, pk } of relayersToStart) { - try { - const containerName = `snowbridge-${config.type}-relay`; - logger.info(`🚀 Starting relayer ${containerName} ...`); - - const hostConfigFilePath = path.resolve(configFilePath); - const containerConfigFilePath = `/${configFilePath}`; - const networkName = launchedNetwork.networkName; - invariant(networkName, "❌ Docker network name not found in LaunchedNetwork instance"); - - const commandBase: string[] = [ - "docker", - "run", - "-d", - "--platform", - "linux/amd64", - "--add-host", - "host.docker.internal:host-gateway", - "--name", - containerName, - "--network", - networkName, - ...(isLocal ? [] : ["--pull", "always"]) - ]; - - const volumeMounts: string[] = ["-v", `${hostConfigFilePath}:${containerConfigFilePath}`]; - - if (config.type === "beacon" || config.type === "execution") { - const hostDatastorePath = path.resolve(datastorePath); - const containerDatastorePath = "/data"; - volumeMounts.push("-v", `${hostDatastorePath}:${containerDatastorePath}`); - } - - const relayerCommandArgs: string[] = ["run", config.type, "--config", configFilePath]; - - switch (config.type) { - case "beacon": - invariant(pk.substrate, "❌ Substrate private key is required for beacon relayer"); - relayerCommandArgs.push("--substrate.private-key", pk.substrate); - break; - case "beefy": - invariant(pk.ethereum, "❌ Ethereum private key is required for beefy relayer"); - relayerCommandArgs.push("--ethereum.private-key", pk.ethereum); - break; - case "solochain": - invariant(pk.ethereum, "❌ Ethereum private key is required for solochain relayer"); - relayerCommandArgs.push("--ethereum.private-key", pk.ethereum); - if (pk.substrate) { - relayerCommandArgs.push("--substrate.private-key", pk.substrate); - } else { - logger.warn( - "⚠️ No substrate private key provided for solochain relayer. This might be an issue depending on the configuration." - ); - } - break; - case "execution": - invariant(pk.substrate, "❌ Substrate private key is required for execution relayer"); - relayerCommandArgs.push("--substrate.private-key", pk.substrate); - break; - } - - const command: string[] = [ - ...commandBase, - ...volumeMounts, - options.relayerImageTag, - ...relayerCommandArgs - ]; - - logger.debug(`Running command: ${command.join(" ")}`); - await runShellCommandWithLogger(command.join(" "), { logLevel: "debug" }); - - launchedNetwork.addContainer(containerName); - - await waitForContainerToStart(containerName); - - // TODO: Re-enable when we know what we want to tail for - // await waitForLog({ - // searchString: "", - // containerName, - // timeoutSeconds: 30, - // tail: 1 - // }); - - logger.success(`Started relayer ${name} with process ${process.pid}`); - } catch (e) { - logger.error(`Error starting relayer ${name}`); - logger.error(e); - } - } - - logger.success("Snowbridge relayers started"); printDivider(); }; - -/** - * Waits for the BEEFY protocol to be ready by polling its finalized head. - * - * @param launchedNetwork - An instance of LaunchedNetwork to get the node endpoint. - * @param pollIntervalMs - The interval in milliseconds to poll the BEEFY endpoint. - * @param timeoutMs - The total time in milliseconds to wait before timing out. - * @throws Error if BEEFY is not ready within the timeout. - */ -const waitBeefyReady = async ( - launchedNetwork: LaunchedNetwork, - pollIntervalMs: number, - timeoutMs: number -): Promise => { - const port = launchedNetwork.getPublicWsPort(); - const wsUrl = `ws://127.0.0.1:${port}`; - const iterations = Math.floor(timeoutMs / pollIntervalMs); - - logger.info(`⌛️ Waiting for BEEFY to be ready on port ${port}...`); - - let client: PolkadotClient | undefined; - const clientTimeoutMs = pollIntervalMs / 2; - const delayMs = pollIntervalMs / 2; - try { - client = createClient(withPolkadotSdkCompat(getWsProvider(wsUrl))); - - await waitFor({ - lambda: async () => { - try { - logger.debug("Attempting to to check beefy_getFinalizedHead"); - - // Add timeout to the RPC call to prevent hanging. - const finalisedHeadPromise = client?._request("beefy_getFinalizedHead", []); - const timeoutPromise = new Promise((_, reject) => { - setTimeout(() => reject(new Error("RPC call timeout")), clientTimeoutMs); - }); - - const finalisedHeadHex = await Promise.race([finalisedHeadPromise, timeoutPromise]); - - if (finalisedHeadHex && finalisedHeadHex !== ZERO_HASH) { - logger.info(`🥩 BEEFY is ready. Finalised head: ${finalisedHeadHex}.`); - return true; - } - - logger.debug( - `BEEFY not ready or finalised head is zero. Retrying in ${delayMs / 1000}s...` - ); - return false; - } catch (rpcError) { - logger.warn(`RPC error checking BEEFY status: ${rpcError}. Retrying...`); - return false; - } - }, - iterations, - delay: delayMs, - errorMessage: "BEEFY protocol not ready. Relayers cannot be launched." - }); - } catch (error) { - logger.error(`❌ Failed to connect to DataHaven node for BEEFY check: ${error}`); - throw new Error("BEEFY protocol not ready. Relayers cannot be launched."); - } finally { - if (client) { - client.destroy(); - } - } -}; diff --git a/test/cli/handlers/launch/summary.ts b/test/cli/handlers/launch/summary.ts index 9def2668..9f6e2b4a 100644 --- a/test/cli/handlers/launch/summary.ts +++ b/test/cli/handlers/launch/summary.ts @@ -1,7 +1,7 @@ import invariant from "tiny-invariant"; import { getServiceFromKurtosis, logger, printHeader } from "utils"; -import { BASE_SERVICES } from "../common/consts"; -import type { LaunchedNetwork } from "../common/launchedNetwork"; +import type { LaunchedNetwork } from "../../../launcher/types/launchedNetwork"; +import { BASE_SERVICES } from "../../../launcher/utils/constants"; import type { LaunchOptions } from "."; export const performSummaryOperations = async ( diff --git a/test/cli/handlers/launch/validator.ts b/test/cli/handlers/launch/validator.ts index 4e59ca4a..3057850b 100644 --- a/test/cli/handlers/launch/validator.ts +++ b/test/cli/handlers/launch/validator.ts @@ -1,7 +1,5 @@ -import { fundValidators } from "scripts/fund-validators"; -import { setupValidators } from "scripts/setup-validators"; -import { updateValidatorSet } from "scripts/update-validator-set"; -import { confirmWithTimeout, logger, printDivider } from "utils"; +import { confirmWithTimeout, logger, printDivider, printHeader } from "utils"; +import { fundValidators, setupValidators, updateValidatorSet } from "../../../launcher/validators"; import type { LaunchOptions } from ".."; export const performValidatorOperations = async ( @@ -9,6 +7,8 @@ export const performValidatorOperations = async ( networkRpcUrl: string, contractsDeployed: boolean ) => { + printHeader("Funding DataHaven Validators"); + // If not specified, prompt for funding let shouldFundValidators = options.fundValidators; if (shouldFundValidators === undefined) { @@ -30,14 +30,15 @@ export const performValidatorOperations = async ( ); } - await fundValidators({ - rpcUrl: networkRpcUrl - }); + await fundValidators({ rpcUrl: networkRpcUrl }); + printDivider(); } else { logger.info("👍 Skipping validator funding"); printDivider(); } + printHeader("Setting Up DataHaven Validators"); + // If not specified, prompt for setup let shouldSetupValidators = options.setupValidators; if (shouldSetupValidators === undefined) { @@ -59,9 +60,8 @@ export const performValidatorOperations = async ( ); } - await setupValidators({ - rpcUrl: networkRpcUrl - }); + await setupValidators({ rpcUrl: networkRpcUrl }); + printDivider(); } }; @@ -79,6 +79,8 @@ export const performValidatorSetUpdate = async ( networkRpcUrl: string, contractsDeployed: boolean ) => { + printHeader("Updating DataHaven Validator Set"); + // If not specified, prompt for update let shouldUpdateValidatorSet = options.updateValidatorSet; if (shouldUpdateValidatorSet === undefined) { @@ -100,9 +102,8 @@ export const performValidatorSetUpdate = async ( ); } - await updateValidatorSet({ - rpcUrl: networkRpcUrl - }); + await updateValidatorSet({ rpcUrl: networkRpcUrl }); + printDivider(); } else { logger.info("👍 Skipping validator set update"); printDivider(); diff --git a/test/cli/handlers/stop/index.ts b/test/cli/handlers/stop/index.ts index 8dab7954..8cf02849 100644 --- a/test/cli/handlers/stop/index.ts +++ b/test/cli/handlers/stop/index.ts @@ -3,15 +3,16 @@ import { $ } from "bun"; import invariant from "tiny-invariant"; import { confirmWithTimeout, + getContainersByPrefix, getContainersMatchingImage, killExistingContainers, logger, printHeader, runShellCommandWithLogger } from "utils"; +import { getRunningKurtosisEnclaves } from "../../../launcher/kurtosis"; +import { COMPONENTS } from "../../../launcher/utils/constants"; import { checkBaseDependencies } from "../common/checks"; -import { COMPONENTS, DOCKER_NETWORK_NAME } from "../common/consts"; -import { getRunningKurtosisEnclaves } from "../common/kurtosis"; export interface StopOptions { all?: boolean; @@ -39,7 +40,7 @@ export const stop = async (options: StopOptions) => { await stopDockerComponents("snowbridge", options); printHeader("Datahaven Network"); await stopDockerComponents("datahaven", options); - await removeDockerNetwork(DOCKER_NETWORK_NAME, options); + await removeDataHavenNetworks(options); printHeader("Ethereum Network"); await stopAllEnclaves(options); printHeader("Kurtosis Engine"); @@ -48,9 +49,8 @@ export const stop = async (options: StopOptions) => { export const stopDockerComponents = async (type: keyof typeof COMPONENTS, options: StopOptions) => { const name = COMPONENTS[type].componentName; - const imageName = COMPONENTS[type].imageName; logger.debug(`Checking currently running ${name} ...`); - const components = await getContainersMatchingImage(imageName); + const components = await getContainersByPrefix(type); logger.info(`🔎 Found ${components.length} containers(s) running the ${name}`); if (components.length === 0) { logger.info(`🤷‍ No ${name} containers found running`); @@ -59,7 +59,7 @@ export const stopDockerComponents = async (type: keyof typeof COMPONENTS, option let shouldStopComponent = options.all || options[COMPONENTS[type].optionName]; if (shouldStopComponent === undefined) { shouldStopComponent = await confirmWithTimeout( - `Do you want to stop the ${imageName} containers?`, + `Do you want to stop the ${type} containers?`, true, 10 ); @@ -74,8 +74,8 @@ export const stopDockerComponents = async (type: keyof typeof COMPONENTS, option return; } - await killExistingContainers(imageName); - const remaining = await getContainersMatchingImage(imageName); + await killExistingContainers(type); + const remaining = await getContainersByPrefix(type); invariant( remaining.length === 0, `❌ ${remaining.length} containers are still running and have not been stopped.` @@ -83,42 +83,54 @@ export const stopDockerComponents = async (type: keyof typeof COMPONENTS, option logger.info(`🪓 ${components.length} ${name} containers stopped successfully`); }; -const removeDockerNetwork = async (networkName: string, options: StopOptions) => { - logger.debug(`Checking if Docker network ${networkName} exists...`); - const networkOutput = - await $`docker network ls --filter "name=^${DOCKER_NETWORK_NAME}$" --format "{{.Name}}"`.text(); +const removeDataHavenNetworks = async (options: StopOptions) => { + logger.debug(`Checking for Docker networks with 'datahaven-' prefix...`); - // Check if networkOutput has any network names (not just whitespace or empty lines) - const networksExist = - networkOutput - .trim() - .split("\n") - .filter((line) => line.trim().length > 0).length > 0; - if (!networksExist) { - logger.info(`🤷‍ Docker network ${networkName} does not exist, skipping`); + // Find all networks that start with "datahaven-" + const networkOutput = + await $`docker network ls --filter "name=^datahaven-" --format "{{.Name}}"`.text(); + + // Parse the output to get network names + const networks = networkOutput + .trim() + .split("\n") + .filter((line) => line.trim().length > 0); + + if (networks.length === 0) { + logger.info("🤷‍ No DataHaven Docker networks found, skipping"); return; } - let shouldRemoveNetwork = options.all || options.datahaven; - if (shouldRemoveNetwork === undefined) { - shouldRemoveNetwork = await confirmWithTimeout( - `Do you want to remove the Docker network ${networkName}?`, + logger.info(`🔎 Found ${networks.length} DataHaven Docker network(s): ${networks.join(", ")}`); + + let shouldRemoveNetworks = options.all || options.datahaven; + if (shouldRemoveNetworks === undefined) { + shouldRemoveNetworks = await confirmWithTimeout( + `Do you want to remove ${networks.length} DataHaven Docker network(s)?`, true, 10 ); } - if (!shouldRemoveNetwork) { - logger.info(`👍 Skipping removing Docker network ${networkName} due to flag option`); + if (!shouldRemoveNetworks) { + logger.info("👍 Skipping removing DataHaven Docker networks due to flag option"); return; } - logger.info(`⛓️‍💥 Removing Docker network: ${networkName}`); - const { exitCode, stderr } = await $`docker network rm -f ${networkName}`.nothrow().quiet(); - if (exitCode !== 0) { - logger.warn(`⚠️ Failed to remove Docker network: ${stderr}`); - } else { - logger.info(`🪓 Docker network ${networkName} removed successfully`); + // Remove each network + let successCount = 0; + for (const networkName of networks) { + logger.info(`⛓️‍💥 Removing Docker network: ${networkName}`); + const { exitCode, stderr } = await $`docker network rm -f ${networkName}`.nothrow().quiet(); + if (exitCode !== 0) { + logger.warn(`⚠️ Failed to remove Docker network ${networkName}: ${stderr}`); + } else { + successCount++; + } + } + + if (successCount > 0) { + logger.info(`🪓 ${successCount} DataHaven Docker network(s) removed successfully`); } }; diff --git a/test/docs/E2E_FRAMEWORK_OVERVIEW.md b/test/docs/E2E_FRAMEWORK_OVERVIEW.md new file mode 100644 index 00000000..eae34074 --- /dev/null +++ b/test/docs/E2E_FRAMEWORK_OVERVIEW.md @@ -0,0 +1,171 @@ +# E2E Testing Framework Overview + +This document provides a concise overview of the DataHaven E2E testing framework architecture and usage. + +## Architecture + +The E2E testing framework creates isolated test environments for comprehensive integration testing of the DataHaven network, including EigenLayer AVS integration, EVM compatibility, and cross-chain functionality. + +### Directory Structure + +``` +test/ +├── suites/ # Test files (*.test.ts) +├── framework/ # Base classes and test utilities +├── launcher/ # Network orchestration code +├── utils/ # Common helpers and utilities +├── configs/ # Component configuration files +├── scripts/ # Automation scripts +└── cli/ # Interactive network management +``` + +### Test Isolation + +- Each test suite extends `BaseTestSuite` for lifecycle management +- Unique network IDs prevent resource conflicts (format: `suiteName-timestamp`) +- Automatic setup/teardown via `beforeAll`/`afterAll` hooks +- Independent Docker networks per test suite + +## Infrastructure Stack + +### Core Components + +1. **Kurtosis**: Orchestrates Ethereum test networks + + - Runs EL (reth) and CL (lodestar) clients + - Configurable parameters (slot time, validators) + - Optional Blockscout explorer integration + +2. **Docker**: Containerizes all components + + - DataHaven validator nodes + - Snowbridge relayers + - Test infrastructure + - Cross-platform support (Linux/macOS) + +3. **Bun**: TypeScript runtime and test runner + - Parallel test execution + - Resource management + - Interactive CLI tooling + +## Network Launch Sequence + +The `launchNetwork` function orchestrates the following steps: + +1. **Validation**: Check dependencies, create unique network ID +2. **DataHaven Launch**: Start validator nodes (Alice, Bob) in Docker +3. **Ethereum Network**: Spin up via Kurtosis with fast slot times +4. **Contract Deployment**: Deploy EigenLayer AVS contracts via Forge +5. **Configuration**: Fund accounts, setup validators, set parameters +6. **Snowbridge**: Launch relayers for cross-chain messaging +7. **Cleanup**: Automatic teardown on completion/failure + +## Test Development + +### Basic Test Structure + +```typescript +import { BaseTestSuite } from "../framework/base-test-suite"; + +class MyTestSuite extends BaseTestSuite { + constructor() { + super({ suiteName: "my-test" }); + this.setupHooks(); // Manages lifecycle + } +} + +const suite = new MyTestSuite(); + +describe("My Test Suite", () => { + test("should do something", async () => { + const connectors = suite.getTestConnectors(); + // Use connectors.publicClient, walletClient, dhApi, papiClient + }); +}); +``` + +### Available Connectors + +- `publicClient`: Viem public client for Ethereum reads +- `walletClient`: Viem wallet client for transactions +- `dhApi`: DataHaven Substrate API +- `papiClient`: Polkadot-API client + +## Key Tools & Dependencies + +### Blockchain Interaction + +- **Viem**: Ethereum client library +- **Wagmi**: Contract TypeScript bindings +- **Polkadot-API**: Substrate chain interactions +- **Forge**: Smart contract toolchain + +### Development Tools + +- **TypeScript**: Type safety +- **Biome**: Code formatting/linting +- **Zod**: Runtime validation +- **Commander**: CLI framework + +## Common Commands + +```bash +# Install dependencies +bun i + +# Launch interactive network manager +bun cli + +# Run all E2E tests +bun test:e2e + +# Run tests with concurrency limit +bun test:e2e:parallel + +# Run specific test suite +bun test suites/contracts.test.ts + +# Generate contract bindings +bun generate:wagmi + +# Generate Polkadot types +bun generate:types + +# Format code +bun fmt:fix + +# Type checking +bun typecheck +``` + +## Network Configuration + +### Default Test Network + +- **DataHaven**: 2 validator nodes (Alice, Bob) +- **Ethereum**: 2 EL/CL pairs, 1-second slots +- **Contracts**: Full EigenLayer AVS deployment +- **Snowbridge**: Beacon and Ethereum relayers + +### Customization Options + +- Build local Docker images +- Enable Blockscout verification +- Adjust slot times +- Configure validator counts + +## Troubleshooting + +1. **Dependency Issues**: Ensure Docker, Kurtosis, and Bun are installed +2. **Port Conflicts**: Check for existing services on required ports +3. **Resource Limits**: Adjust test concurrency if running out of resources +4. **Cleanup Failures**: Use `bun cli stop --A` to manually clean up networks + +## Best Practices + +1. Always extend `BaseTestSuite` for proper lifecycle management +2. Use unique suite names to avoid conflicts +3. Keep tests isolated and independent +4. Clean up resources in test teardown +5. Use the interactive CLI for debugging network issues +6. Regenerate types after contract or runtime changes diff --git a/test/framework/connectors.ts b/test/framework/connectors.ts new file mode 100644 index 00000000..1a20adf2 --- /dev/null +++ b/test/framework/connectors.ts @@ -0,0 +1,103 @@ +import { datahaven } from "@polkadot-api/descriptors"; +import { createClient as createPapiClient, type PolkadotClient } from "polkadot-api"; +import { withPolkadotSdkCompat } from "polkadot-api/polkadot-sdk-compat"; +import { getWsProvider } from "polkadot-api/ws-provider/web"; +import { ANVIL_FUNDED_ACCOUNTS, type DataHavenApi, logger } from "utils"; +import { + type Account, + createPublicClient, + createWalletClient, + http, + type PublicClient, + type WalletClient +} from "viem"; +import { privateKeyToAccount } from "viem/accounts"; +import { anvil } from "viem/chains"; +import type { LaunchNetworkResult } from "../launcher"; + +export interface TestConnectors { + // Ethereum connectors + publicClient: PublicClient; + walletClient: WalletClient; + + // DataHaven connectors + papiClient: PolkadotClient; + dhApi: DataHavenApi; + + // Raw URLs + elRpcUrl: string; + dhRpcUrl: string; +} + +export class ConnectorFactory { + private connectors: LaunchNetworkResult; + + constructor(connectors: LaunchNetworkResult) { + this.connectors = connectors; + } + + /** + * Create test connectors for interacting with the launched networks + */ + async createTestConnectors(): Promise { + logger.debug("Creating test connectors..."); + + // Create Ethereum clients + const publicClient = createPublicClient({ + chain: anvil, + transport: http(this.connectors.ethereumRpcUrl) + }); + + const account = privateKeyToAccount(ANVIL_FUNDED_ACCOUNTS[0].privateKey); + const walletClient = createWalletClient({ + account, + chain: anvil, + transport: http(this.connectors.ethereumRpcUrl) + }); + + // Create DataHaven/Substrate clients + // Note: polkadot-api can handle HTTP RPC URLs even when passed to getWsProvider + const wsProvider = getWsProvider(this.connectors.dataHavenRpcUrl); + const papiClient = createPapiClient(withPolkadotSdkCompat(wsProvider)); + + // Get typed API + const dhApi = papiClient.getTypedApi(datahaven); + + logger.debug("Test connectors created successfully"); + + return { + publicClient, + walletClient, + papiClient, + dhApi, + elRpcUrl: this.connectors.ethereumRpcUrl, + dhRpcUrl: this.connectors.dataHavenRpcUrl + }; + } + + /** + * Create a wallet client with a specific account + */ + createWalletClient(privateKey: `0x${string}`): WalletClient { + const account = privateKeyToAccount(privateKey); + return createWalletClient({ + account, + chain: anvil, + transport: http(this.connectors.ethereumRpcUrl) + }); + } + + /** + * Clean up connections + */ + async cleanup(connectors: TestConnectors): Promise { + logger.debug("Cleaning up test connectors..."); + + // Destroy PAPI client + if (connectors.papiClient) { + connectors.papiClient.destroy(); + } + + logger.debug("Test connectors cleaned up"); + } +} diff --git a/test/framework/index.ts b/test/framework/index.ts new file mode 100644 index 00000000..5dbdfd8c --- /dev/null +++ b/test/framework/index.ts @@ -0,0 +1,3 @@ +export * from "./connectors"; +export * from "./manager"; +export * from "./suite"; diff --git a/test/framework/manager.ts b/test/framework/manager.ts new file mode 100644 index 00000000..4b356ada --- /dev/null +++ b/test/framework/manager.ts @@ -0,0 +1,83 @@ +import { logger } from "utils"; + +export interface TestSuiteRegistry { + suiteId: string; + networkId: string; + startTime: number; + status: "running" | "completed" | "failed"; +} + +/** + * Manager for tracking running test suites and ensuring cleanup + */ +export class TestSuiteManager { + private static instance: TestSuiteManager; + private suites: Map = new Map(); + + private constructor() { + // Set up process exit handlers to ensure cleanup + process.on("exit", () => this.cleanupAll()); + process.on("SIGINT", () => this.cleanupAll()); + process.on("SIGTERM", () => this.cleanupAll()); + } + + static getInstance(): TestSuiteManager { + if (!TestSuiteManager.instance) { + TestSuiteManager.instance = new TestSuiteManager(); + } + return TestSuiteManager.instance; + } + + registerSuite(suiteId: string, networkId: string): void { + if (this.suites.has(suiteId)) { + throw new Error(`Test suite ${suiteId} is already registered`); + } + + this.suites.set(suiteId, { + suiteId, + networkId, + startTime: Date.now(), + status: "running" + }); + + logger.debug(`Registered test suite: ${suiteId} with network: ${networkId}`); + } + + completeSuite(suiteId: string): void { + const suite = this.suites.get(suiteId); + if (suite) { + suite.status = "completed"; + const duration = ((Date.now() - suite.startTime) / 1000).toFixed(1); + logger.debug(`Test suite ${suiteId} completed in ${duration}s`); + } + } + + failSuite(suiteId: string): void { + const suite = this.suites.get(suiteId); + if (suite) { + suite.status = "failed"; + logger.debug(`Test suite ${suiteId} failed`); + } + } + + getRunningCount(): number { + return Array.from(this.suites.values()).filter((s) => s.status === "running").length; + } + + getRunningNetworkIds(): string[] { + return Array.from(this.suites.values()) + .filter((s) => s.status === "running") + .map((s) => s.networkId); + } + + private cleanupAll(): void { + const runningSuites = Array.from(this.suites.values()).filter((s) => s.status === "running"); + + if (runningSuites.length > 0) { + logger.warn(`⚠️ Process exiting with ${runningSuites.length} test suite(s) still running`); + runningSuites.forEach((suite) => { + logger.warn(` - ${suite.suiteId} (network: ${suite.networkId})`); + }); + } + } +} diff --git a/test/framework/suite.ts b/test/framework/suite.ts new file mode 100644 index 00000000..92405209 --- /dev/null +++ b/test/framework/suite.ts @@ -0,0 +1,140 @@ +import { afterAll, beforeAll } from "bun:test"; +import { logger } from "utils"; +import { launchNetwork } from "../launcher"; +import type { LaunchNetworkResult } from "../launcher/types"; +import { ConnectorFactory, type TestConnectors } from "./connectors"; +import { TestSuiteManager } from "./manager"; + +export interface TestSuiteOptions { + suiteName: string; + networkOptions?: { + slotTime?: number; + blockscout?: boolean; + buildDatahaven?: boolean; + datahavenImageTag?: string; + relayerImageTag?: string; + }; +} + +export abstract class BaseTestSuite { + protected networkId: string; + protected connectors?: LaunchNetworkResult; + protected testConnectors?: TestConnectors; + private connectorFactory?: ConnectorFactory; + private options: TestSuiteOptions; + private manager: TestSuiteManager; + + constructor(options: TestSuiteOptions) { + this.options = options; + // Generate unique network ID using suite name and timestamp + this.networkId = `${options.suiteName}-${Date.now()}`.toLowerCase().replace(/[^a-z0-9-]/g, "-"); + this.manager = TestSuiteManager.getInstance(); + } + + protected setupHooks(): void { + beforeAll(async () => { + logger.info(`🧪 Setting up test suite: ${this.options.suiteName}`); + logger.info(`📝 Network ID: ${this.networkId}`); + + try { + // Register suite with manager + this.manager.registerSuite(this.options.suiteName, this.networkId); + + // Launch the network + this.connectors = await launchNetwork({ + networkId: this.networkId, + datahavenImageTag: + this.options.networkOptions?.datahavenImageTag || "moonsonglabs/datahaven:local", + relayerImageTag: + this.options.networkOptions?.relayerImageTag || "moonsonglabs/snowbridge-relay:latest", + buildDatahaven: false, // default to false in the test suite so we can speed up the CI + ...this.options.networkOptions + }); + + // Create test connectors + this.connectorFactory = new ConnectorFactory(this.connectors); + this.testConnectors = await this.connectorFactory.createTestConnectors(); + + // Allow derived classes to perform additional setup + await this.onSetup(); + + logger.success(`Test suite setup complete: ${this.options.suiteName}`); + } catch (error) { + logger.error(`Failed to setup test suite: ${this.options.suiteName}`, error); + this.manager.failSuite(this.options.suiteName); + throw error; + } + }); + + afterAll(async () => { + logger.info(`🧹 Tearing down test suite: ${this.options.suiteName}`); + + try { + // Allow derived classes to perform cleanup + await this.onTeardown(); + + // Cleanup test connectors + if (this.testConnectors && this.connectorFactory) { + await this.connectorFactory.cleanup(this.testConnectors); + } + + // Cleanup the network + if (this.connectors?.cleanup) { + await this.connectors.cleanup(); + } + + // Mark suite as completed + this.manager.completeSuite(this.options.suiteName); + + logger.success(`Test suite teardown complete: ${this.options.suiteName}`); + } catch (error) { + logger.error(`Error during test suite teardown: ${this.options.suiteName}`, error); + this.manager.failSuite(this.options.suiteName); + } + }); + } + + /** + * Override this method to perform additional setup after network launch + */ + protected async onSetup(): Promise { + // Default implementation does nothing + } + + /** + * Override this method to perform cleanup before network teardown + */ + protected async onTeardown(): Promise { + // Default implementation does nothing + } + + /** + * Get network connectors - throws if not initialized + */ + protected getConnectors(): LaunchNetworkResult { + if (!this.connectors) { + throw new Error("Network connectors not initialized. Did you call setupHooks()?"); + } + return this.connectors; + } + + /** + * Get test connectors - throws if not initialized + */ + public getTestConnectors(): TestConnectors { + if (!this.testConnectors) { + throw new Error("Test connectors not initialized. Did you call setupHooks()?"); + } + return this.testConnectors; + } + + /** + * Get connector factory - throws if not initialized + */ + public getConnectorFactory(): ConnectorFactory { + if (!this.connectorFactory) { + throw new Error("Connector factory not initialized. Did you call setupHooks()?"); + } + return this.connectorFactory; + } +} diff --git a/test/launcher/contracts.ts b/test/launcher/contracts.ts new file mode 100644 index 00000000..3f7b908e --- /dev/null +++ b/test/launcher/contracts.ts @@ -0,0 +1,55 @@ +import { + buildContracts, + constructDeployCommand, + executeDeployment, + validateDeploymentParams +} from "scripts/deploy-contracts"; +import { logger } from "utils"; +import type { ParameterCollection } from "utils/parameters"; + +/** + * Configuration options for contract deployment. + */ +export interface ContractsOptions { + rpcUrl: string; + verified?: boolean; + blockscoutBackendUrl?: string; + parameterCollection?: ParameterCollection; +} + +/** + * Deploys smart contracts to the specified network. + * + * This function handles the complete contract deployment process including: + * - Validating deployment parameters + * - Building contracts from source + * - Constructing deployment commands + * - Executing the deployment + * - Optionally verifying contracts on Blockscout + * - Automatically adding deployed contract addresses to parameter collection if provided + * + * @param options - Configuration options for deployment + * @param options.rpcUrl - The RPC URL of the target network + * @param options.verified - Whether to verify contracts on Blockscout (requires blockscoutBackendUrl) + * @param options.blockscoutBackendUrl - URL for the Blockscout API (required if verified is true) + * @param options.parameterCollection - Collection of parameters to update with deployed contract addresses + * + * @throws {Error} If deployment parameters are invalid + * @throws {Error} If contract building fails + * @throws {Error} If deployment execution fails + */ +export const deployContracts = async (options: ContractsOptions): Promise => { + logger.info("🚀 Deploying smart contracts..."); + + // Validate required parameters + validateDeploymentParams(options); + + // Build contracts + await buildContracts(); + + // Construct and execute deployment + const deployCommand = constructDeployCommand(options); + await executeDeployment(deployCommand, options.parameterCollection); + + logger.success("Smart contracts deployed successfully"); +}; diff --git a/test/launcher/datahaven.ts b/test/launcher/datahaven.ts new file mode 100644 index 00000000..a36c93f9 --- /dev/null +++ b/test/launcher/datahaven.ts @@ -0,0 +1,496 @@ +import { secp256k1 } from "@noble/curves/secp256k1"; +import { $ } from "bun"; +import { createClient, type PolkadotClient } from "polkadot-api"; +import { withPolkadotSdkCompat } from "polkadot-api/polkadot-sdk-compat"; +import { getWsProvider } from "polkadot-api/ws-provider/web"; +import { cargoCrossbuild } from "scripts/cargo-crossbuild"; +import invariant from "tiny-invariant"; +import { + createPapiConnectors, + getPublicPort, + killExistingContainers, + logger, + waitForContainerToStart +} from "utils"; +import { waitFor } from "utils/waits"; +import { type Hex, keccak256, toHex } from "viem"; +import { publicKeyToAddress } from "viem/accounts"; +import type { LaunchedNetwork } from "./types/launchedNetwork"; + +/** + * Options for DataHaven-related operations. + */ +export interface DataHavenOptions { + networkId: string; + datahavenImageTag: string; + relayerImageTag: string; + buildDatahaven: boolean; + authorityIds: readonly string[]; + datahavenBuildExtraArgs?: string; +} + +/** + * Launches a local DataHaven solochain network for testing. + * + * This function handles the complete setup of a local DataHaven test network including: + * - Building the local Docker image if requested + * - Verifying the Docker image exists + * - Creating a Docker network for node communication + * - Starting authority nodes based on the provided authority IDs + * - Waiting for nodes to become ready + * - Registering nodes in the launched network + * - Setting up validator configuration with BEEFY authorities + * + * @param options - Configuration options for launching the network + * @param options.networkId - The network ID to use for the docker network name (will be `datahaven-${networkId}`) + * @param options.datahavenImageTag - Docker image tag for DataHaven nodes + * @param options.relayerImageTag - Docker image tag for relayer nodes + * @param options.buildDatahaven - Whether to build the local Docker image before launching + * @param options.authorityIds - Array of authority IDs to launch (e.g., ["alice", "bob"]) + * @param options.datahavenBuildExtraArgs - Extra arguments for building DataHaven (e.g., "--features=fast-runtime") + * @param launchedNetwork - The launched network instance to track the network's state + * + * @throws {Error} If the DataHaven image tag is not provided + * @throws {Error} If the network fails to start within the timeout period + * @throws {Error} If container startup fails for any node + * @throws {Error} If the Docker image cannot be found locally or on Docker Hub + */ +export const launchLocalDataHavenSolochain = async ( + options: DataHavenOptions, + launchedNetwork: LaunchedNetwork +): Promise => { + logger.info("🚀 Launching DataHaven network..."); + + invariant(options.datahavenImageTag, "❌ DataHaven image tag not defined"); + + if (options.buildDatahaven) { + await buildLocalImage(options); + } + await checkTagExists(options.datahavenImageTag); + + const COMMON_LAUNCH_ARGS = [ + "--unsafe-force-node-key-generation", + "--tmp", + "--validator", + "--discover-local", + "--no-prometheus", + "--unsafe-rpc-external", + "--rpc-cors=all", + "--force-authoring", + "--no-telemetry", + "--enable-offchain-indexing=true" + ]; + + // Create a unique Docker network name using the network ID + const dockerNetworkName = `datahaven-${options.networkId}`; + + logger.info(`⛓️‍💥 Creating Docker network: ${dockerNetworkName}`); + logger.debug(await $`docker network rm ${dockerNetworkName} -f`.text()); + logger.debug(await $`docker network create ${dockerNetworkName}`.text()); + launchedNetwork.networkName = dockerNetworkName; + + logger.success(`DataHaven nodes will use Docker network: ${dockerNetworkName}`); + + for (const id of options.authorityIds) { + logger.info(`🚀 Starting ${id}...`); + const containerName = `datahaven-${id}-${options.networkId}`; + + const command: string[] = [ + "docker", + "run", + "-d", + "--name", + containerName, + "--network", + dockerNetworkName, + ...(id === "alice" ? ["-p", "9944"] : []), + options.datahavenImageTag, + `--${id}`, + ...COMMON_LAUNCH_ARGS + ]; + + logger.debug(await $`sh -c "${command.join(" ")}"`.text()); + + await waitForContainerToStart(containerName); + + // TODO: Un-comment this when it doesn't stop process from hanging + // This is working on SH, but not here so probably a Bun defect + // + // const listeningLine = await waitForLog({ + // search: "Running JSON-RPC server: addr=0.0.0.0:", + // containerName, + // timeoutSeconds: 30 + // }); + // logger.debug(listeningLine); + } + + // Register Alice node after all containers are started + await registerNodes(options.networkId, launchedNetwork); + + logger.info("⌛️ Waiting for DataHaven to start..."); + const timeoutMs = 2000; // 2 second timeout + + // Get the dynamic port from the launched network + const aliceContainerName = `datahaven-alice-${options.networkId}`; + const alicePort = launchedNetwork.getContainerPort(aliceContainerName); + + await waitFor({ + lambda: async () => { + const isReady = await isNetworkReady(alicePort, timeoutMs); + if (!isReady) { + logger.debug("Node not ready, waiting 1 second..."); + } + return isReady; + }, + iterations: 30, + delay: timeoutMs, + errorMessage: "DataHaven network not ready" + }); + + await setupDataHavenValidatorConfig(launchedNetwork, "datahaven-"); + + logger.success(`DataHaven network started, primary node accessible on port ${alicePort}`); +}; + +/** + * Checks if the DataHaven network is ready by connecting via WebSocket and calling the system_chain RPC method. + * + * This function suppresses console errors during connection attempts to avoid noise in the logs. + * It uses the Polkadot API to connect to the node and verify it's responding to RPC calls. + * + * @param port - The port number to check for WebSocket connectivity + * @param timeoutMs - The timeout in milliseconds for the RPC call + * @returns True if the network is ready and responding, false otherwise + */ +export const isNetworkReady = async (port: number, timeoutMs: number): Promise => { + const wsUrl = `ws://127.0.0.1:${port}`; + let client: PolkadotClient | undefined; + + // Temporarily capture and suppress error logs during connection attempts. + // This is to avoid the "Unable to connect to ws:" error logs from the `client._request` call. + const originalConsoleError = console.error; + console.error = () => {}; + + try { + // Use withPolkadotSdkCompat for consistency, though _request might not strictly need it. + client = createClient(withPolkadotSdkCompat(getWsProvider(wsUrl))); + + // Add timeout to the RPC call to prevent hanging. + const chainNamePromise = client._request("system_chain", []); + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => reject(new Error("RPC call timeout")), timeoutMs); + }); + + const chainName = await Promise.race([chainNamePromise, timeoutPromise]); + logger.debug(`isNetworkReady PAPI check successful for port ${port}, chain: ${chainName}`); + client.destroy(); + return !!chainName; // Ensure it's a boolean and chainName is truthy + } catch (error) { + logger.debug(`isNetworkReady PAPI check failed for port ${port}: ${error}`); + if (client) { + client.destroy(); + } + return false; + } finally { + // Restore original console methods. + console.error = originalConsoleError; + } +}; + +/** + * Converts a compressed secp256k1 public key to an Ethereum address. + * + * This function takes a compressed public key (33 bytes), decompresses it to get the full + * uncompressed public key (64 bytes of x and y coordinates), and then derives the + * corresponding Ethereum address using the standard Ethereum address derivation algorithm. + * + * @param compressedPubKey - The compressed public key as a hex string (with or without "0x" prefix) + * @returns The corresponding Ethereum address (checksummed, with "0x" prefix) + * + * @throws {Error} If the provided public key is invalid or cannot be decompressed + */ +export const compressedPubKeyToEthereumAddress = (compressedPubKey: string): string => { + // Ensure the input is a hex string and remove "0x" prefix + const compressedKeyHex = compressedPubKey.startsWith("0x") + ? compressedPubKey.substring(2) + : compressedPubKey; + + // Decompress the public key + const point = secp256k1.ProjectivePoint.fromHex(compressedKeyHex); + // toRawBytes(false) returns the uncompressed key (64 bytes, x and y coordinates) + const uncompressedPubKeyBytes = point.toRawBytes(false); + const uncompressedPubKeyHex = toHex(uncompressedPubKeyBytes); // Prefixes with "0x" + + // Compute the Ethereum address from the uncompressed public key + // publicKeyToAddress expects a 0x-prefixed hex string representing the 64-byte uncompressed public key + const address = publicKeyToAddress(uncompressedPubKeyHex); + return address; +}; + +/** + * Prepares the configuration for DataHaven authorities by fetching their BEEFY public keys, + * converting them to Ethereum addresses, and updating the network configuration file. + * + * This function performs the following steps: + * 1. Connects to the first available DataHaven node matching the container prefix + * 2. Fetches the BEEFY NextAuthorities from the node's runtime + * 3. Converts each compressed public key to an Ethereum address + * 4. Computes the keccak256 hash of each address (authority hash) + * 5. Updates the network configuration file with the authority hashes + * + * The configuration is saved to `../contracts/config/{NETWORK}.json` where NETWORK + * defaults to "anvil" if not specified in environment variables. + * + * @param launchedNetwork - The launched network instance containing container information + * @param containerNamePrefix - The prefix to filter DataHaven containers by (e.g., "datahaven-", "dh-validator-") + * + * @throws {Error} If no DataHaven nodes are found in the launched network + * @throws {Error} If BEEFY authorities cannot be fetched from the node + * @throws {Error} If public key conversion fails + * @throws {Error} If the configuration file cannot be read or written + */ +export const setupDataHavenValidatorConfig = async ( + launchedNetwork: LaunchedNetwork, + containerNamePrefix: string +): Promise => { + const networkName = process.env.NETWORK || "anvil"; + logger.info(`🔧 Preparing DataHaven authorities configuration for network: ${networkName}...`); + + let authorityPublicKeys: string[] = []; + const dhNodes = launchedNetwork.containers.filter((x) => x.name.startsWith(containerNamePrefix)); + + invariant(dhNodes.length > 0, "No DataHaven nodes found in launchedNetwork"); + + const firstNode = dhNodes[0]; + const wsUrl = `ws://127.0.0.1:${firstNode.publicPorts.ws}`; + const { client: papiClient, typedApi: dhApi } = createPapiConnectors(wsUrl); + + logger.info( + `📡 Attempting to fetch BEEFY next authorities from node ${firstNode.name} (port ${firstNode.publicPorts.ws})...` + ); + + // Fetch NextAuthorities + // Beefy.NextAuthorities returns a fixed-length array of bytes representing the authority public keys + const nextAuthoritiesRaw = await dhApi.query.Beefy.NextAuthorities.getValue({ + at: "best" + }); + + invariant(nextAuthoritiesRaw && nextAuthoritiesRaw.length > 0, "No BEEFY next authorities found"); + + authorityPublicKeys = nextAuthoritiesRaw.map((key) => key.asHex()); // .asHex() returns the hex string representation of the corresponding key + logger.success( + `Successfully fetched ${authorityPublicKeys.length} BEEFY next authorities directly.` + ); + + // Clean up PAPI client, otherwise it will hang around and prevent this process from exiting. + papiClient.destroy(); + + const authorityHashes: string[] = []; + for (const compressedKey of authorityPublicKeys) { + try { + const ethAddress = compressedPubKeyToEthereumAddress(compressedKey); + const authorityHash = keccak256(ethAddress as Hex); + authorityHashes.push(authorityHash); + logger.debug( + `Processed public key ${compressedKey} -> ETH address ${ethAddress} -> Authority hash ${authorityHash}` + ); + } catch (error) { + logger.error(`❌ Failed to process public key ${compressedKey}: ${error}`); + throw new Error(`Failed to process public key ${compressedKey}`); + } + } + + // process.cwd() is 'test/', so config is at '../contracts/config' + const configDir = `${process.cwd()}/../contracts/config`; + const configFilePath = `${configDir}/${networkName}.json`; + + try { + const configFile = Bun.file(configFilePath); + if (!(await configFile.exists())) { + logger.warn( + `⚠️ Configuration file ${configFilePath} not found. Skipping update of validator sets.` + ); + // Optionally, create a default structure if it makes sense, or simply return. + // For now, if the base network config doesn't exist, we can't update it. + return; + } + + const configFileContent = await configFile.text(); + const configJson = JSON.parse(configFileContent); + + if (!configJson.snowbridge) { + logger.warn(`⚠️ "snowbridge" section not found in ${configFilePath}, creating it.`); + configJson.snowbridge = {}; + } + + configJson.snowbridge.initialValidatorHashes = authorityHashes; + configJson.snowbridge.nextValidatorHashes = authorityHashes; + + await Bun.write(configFilePath, JSON.stringify(configJson, null, 2)); + logger.success(`DataHaven authority hashes updated in: ${configFilePath}`); + } catch (error) { + logger.error(`❌ Failed to read or update ${configFilePath}: ${error}`); + throw new Error(`Failed to update authority hashes in ${configFilePath}.`); + } +}; + +/** + * Checks if any DataHaven containers are currently running. + * + * @returns True if any DataHaven containers are running, false otherwise. + */ +export const checkDataHavenRunning = async (): Promise => { + // Check for any container whose name starts with "datahaven-" + const containerIds = await $`docker ps --format "{{.Names}}" --filter "name=^datahaven-"`.text(); + // Check for any Docker network that starts with "datahaven-" + const networkOutput = + await $`docker network ls --filter "name=^datahaven-" --format "{{.Name}}"`.text(); + + // Check if containerIds has any actual IDs (not just whitespace) + const containersExist = containerIds.trim().length > 0; + if (containersExist) { + logger.info(`ℹ️ DataHaven containers already running: \n${containerIds}`); + } + + // Check if networkOutput has any network names (not just whitespace or empty lines) + const networksExist = + networkOutput + .trim() + .split("\n") + .filter((line) => line.trim().length > 0).length > 0; + if (networksExist) { + logger.info(`ℹ️ DataHaven network already running: ${networkOutput}`); + } + + return containersExist || networksExist; +}; + +/** + * Stops and removes all DataHaven containers and the associated Docker network. + * + * This function: + * - Kills all containers using the specified DataHaven image tag + * - Optionally kills relayer containers if a relayer image tag is provided + * - Removes the DataHaven Docker network + * - Verifies that all containers and networks have been successfully removed + * + * @param datahavenImageTag - The Docker image tag for DataHaven nodes to remove (required) + * @param relayerImageTag - The Docker image tag for relayer nodes to remove (optional) + * + * @throws {Error} If the DataHaven image tag is not provided + * @throws {Error} If containers or networks were not successfully removed + */ +export const cleanDataHavenContainers = async (networkId: string): Promise => { + logger.info("🧹 Stopping and removing existing DataHaven containers..."); + + await killExistingContainers("datahaven-"); + + logger.info( + "🧹 Stopping and removing existing relayer containers (relayers depend on DataHaven nodes)..." + ); + await killExistingContainers("snowbridge-"); + + logger.info("✅ Existing DataHaven containers stopped and removed."); + + logger.debug(await $`docker network rm -f datahaven-${networkId}`.text()); + logger.info("✅ DataHaven Docker network removed."); + + invariant( + (await checkDataHavenRunning()) === false, + "❌ DataHaven containers were not stopped and removed" + ); +}; + +/** + * Builds a local Docker image for DataHaven. + * + * This function: + * - Runs cargo crossbuild with the specified build arguments + * - Builds the Docker image using the 'bun build:docker:operator' command + * - Logs progress at trace level for debugging + * + * @param options - Configuration options for building the image + * @param options.datahavenBuildExtraArgs - Extra arguments to pass to cargo crossbuild (e.g., "--features=fast-runtime") + */ +export const buildLocalImage = async (options: DataHavenOptions) => { + await cargoCrossbuild({ + datahavenBuildExtraArgs: options.datahavenBuildExtraArgs, + networkId: options.networkId + }); + + logger.info("🐳 Building DataHaven node local Docker image..."); + logger.trace(await $`bun build:docker:operator`.text()); + logger.success("DataHaven node local Docker image build completed successfully"); +}; + +/** + * Checks if a Docker image exists locally or on Docker Hub. + * + * @param tag - The tag of the image to check. + * @returns A promise that resolves when the image is found. + * @throws {Error} If the image is not found locally or on Docker Hub. + */ +export const checkTagExists = async (tag: string) => { + const cleaned = tag.trim(); + logger.debug(`Checking if image ${cleaned} is available locally`); + const { exitCode: localExists } = await $`docker image inspect ${cleaned}`.nothrow().quiet(); + + if (localExists !== 0) { + logger.debug(`Checking if image ${cleaned} is available on docker hub`); + const result = await $`docker manifest inspect ${cleaned}`.nothrow().quiet(); + invariant( + result.exitCode === 0, + `❌ Image ${tag} not found.\n Does this image exist?\n Are you logged and have access to the repository?` + ); + } + + logger.success(`Image ${tag} found locally`); +}; + +/** + * Registers the primary DataHaven node (alice) in the launched network. + * + * This function: + * - Checks if the 'datahaven-alice' container is running + * - If running and not already registered, queries its dynamic port + * - Registers it with the dynamically assigned port + * - If not running, logs a warning and returns without error + * + * Note: Only the alice node is registered as it's the primary node exposed on the default port. + * Other nodes can be accessed via the Docker network but aren't directly exposed. + * + * @param launchedNetwork - The launched network instance to register nodes in + */ +export const registerNodes = async (networkId: string, launchedNetwork: LaunchedNetwork) => { + const targetContainerName = `datahaven-alice-${networkId}`; + + logger.debug(`Checking Docker status for container: ${targetContainerName}`); + // Use ^ and $ for an exact name match in the filter. + const dockerPsOutput = await $`docker ps -q --filter "name=^${targetContainerName}"`.text(); + const isContainerRunning = dockerPsOutput.trim().length > 0; + + if (!isContainerRunning) { + // If the target Docker container is not running, we cannot register it. + logger.warn(`⚠️ Docker container ${targetContainerName} is not running. Cannot register node.`); + return; + } + + // Check if already registered + const existingContainer = launchedNetwork.containers.find((c) => c.name === targetContainerName); + if (existingContainer) { + logger.debug( + `Container ${targetContainerName} already registered with port ${existingContainer.publicPorts.ws}` + ); + return; + } + + // Query the dynamic port and register + const dynamicPort = await getPublicPort(targetContainerName, 9944); + logger.debug( + `Docker container ${targetContainerName} is running. Registering with dynamic port ${dynamicPort}.` + ); + launchedNetwork.addContainer(targetContainerName, { ws: dynamicPort }); + logger.info( + `📝 Node ${targetContainerName} successfully registered in ${networkId} as datahaven-alice` + ); +}; diff --git a/test/launcher/index.ts b/test/launcher/index.ts new file mode 100644 index 00000000..e8adee7d --- /dev/null +++ b/test/launcher/index.ts @@ -0,0 +1,7 @@ +// Export the main network launch function +export { launchNetwork } from "./network"; + +// Export types +export * from "./types"; +// Export utilities +export * from "./utils"; diff --git a/test/launcher/kurtosis.ts b/test/launcher/kurtosis.ts new file mode 100644 index 00000000..2c735bda --- /dev/null +++ b/test/launcher/kurtosis.ts @@ -0,0 +1,346 @@ +import { $ } from "bun"; +import invariant from "tiny-invariant"; +import { + getPortFromKurtosis, + type KurtosisEnclaveInfo, + KurtosisEnclaveInfoSchema, + logger, + runShellCommandWithLogger +} from "utils"; +import { parse, stringify } from "yaml"; +import type { LaunchedNetwork } from "./types/launchedNetwork"; + +/** + * Configuration options for Kurtosis-related operations. + */ +export interface KurtosisOptions { + kurtosisEnclaveName: string; + blockscout?: boolean; + slotTime?: number; + kurtosisNetworkArgs?: string; +} + +/** + * Result of launching a Kurtosis network. + */ +export interface KurtosisLaunchResult { + success: boolean; + cleanup?: () => Promise; +} + +/** + * Launches a local Kurtosis Ethereum network for testing. + * + * This function handles the complete setup of a Kurtosis test network including: + * - Checking and handling existing enclaves + * - Pulling required Docker images (macOS-specific handling) + * - Running the Kurtosis enclave with the specified configuration + * - Registering service endpoints in the launched network + * + * @param options - Configuration options for launching the network + * @param options.kurtosisEnclaveName - Name of the Kurtosis enclave to create + * @param options.blockscout - Whether to include Blockscout block explorer + * @param options.slotTime - Seconds per slot for the network + * @param options.kurtosisNetworkArgs - Additional network parameters + * @param launchedNetwork - The launched network instance to track the network's state + * @param configFilePath - Path to the Kurtosis configuration file (default: "configs/kurtosis/minimal.yaml") + * + * @throws {Error} If the Kurtosis network fails to start properly + */ +export const launchKurtosisNetwork = async ( + options: KurtosisOptions, + launchedNetwork: LaunchedNetwork, + configFilePath = "configs/kurtosis/minimal.yaml" +): Promise => { + logger.info("🚀 Launching Kurtosis Ethereum network..."); + + // Handle macOS-specific Docker image requirements + if (process.platform === "darwin") { + await pullMacOSImages(); + } + + await runKurtosisEnclave(options, configFilePath); + await registerServices(launchedNetwork, options.kurtosisEnclaveName); + + logger.success("Kurtosis network launched successfully"); +}; + +/** + * Checks if a Kurtosis enclave with the specified name is currently running. + * + * @param enclaveName - The name of the Kurtosis enclave to check + * @returns True if the enclave is running, false otherwise + */ +export const checkKurtosisEnclaveRunning = async (enclaveName: string): Promise => { + const enclaves = await getRunningKurtosisEnclaves(); + return enclaves.some((enclave) => enclave.name === enclaveName); +}; + +/** + * Gets a list of currently running Kurtosis enclaves. + * + * This function executes the `kurtosis enclave ls` command and parses the output + * to extract information about running enclaves. + * + * @returns Array of running enclave information including UUID, name, status, and creation time + */ +export const getRunningKurtosisEnclaves = async (): Promise => { + logger.debug("🔎 Checking for running Kurtosis enclaves..."); + + try { + const lines = (await Array.fromAsync($`kurtosis enclave ls`.lines())).filter( + (line) => line.length > 0 + ); + logger.trace(lines); + + // Remove header line + lines.shift(); + + const enclaves: KurtosisEnclaveInfo[] = []; + + if (lines.length === 0) { + logger.debug("🤷‍ No Kurtosis enclaves found running."); + return enclaves; + } + + logger.debug(`🔎 Found ${lines.length} Kurtosis enclave(s) running.`); + // Updated regex to match the actual format: "uuid name status creationTime" + const enclaveRegex = /^(\S+)\s+(\S+)\s+(\S+)\s+(.+)$/; + + for (const line of lines) { + const match = line.match(enclaveRegex); + if (match) { + const [, uuid, name, status, creationTime] = match; + const parseResult = KurtosisEnclaveInfoSchema.safeParse({ + uuid: uuid.trim(), + name: name.trim(), + status: status.trim(), + creationTime: creationTime.trim() + }); + + if (parseResult.success) { + enclaves.push(parseResult.data); + } else { + logger.warn( + `⚠️ Could not parse enclave line: "${line}". Error: ${parseResult.error.message}` + ); + } + } else { + logger.warn(`⚠️ Could not parse enclave line (regex mismatch): "${line}"`); + } + } + + if (lines.length > 0 && enclaves.length === 0) { + logger.warn("⚠️ Found enclave lines in output, but failed to parse any of them."); + } + + return enclaves; + } catch (error) { + logger.debug("🤷‍ Kurtosis engine is not running or command failed. Returning empty array."); + logger.trace(`Error: ${error}`); + return []; + } +}; + +/** + * Cleans and removes a Kurtosis enclave and optionally performs system cleanup. + * + * This function: + * - Stops the specified Kurtosis enclave + * - Cleans Kurtosis artifacts + * - Stops the Kurtosis engine + * - Optionally prunes Docker system resources + * + * @param enclaveName - The name of the Kurtosis enclave to clean + * @param pruneDocker - Whether to run docker system prune (default: true) + */ +export const cleanKurtosisEnclave = async ( + enclaveName: string, + pruneDocker = true +): Promise => { + logger.info("🧹 Cleaning up Docker and Kurtosis environments..."); + + logger.debug(await $`kurtosis enclave stop ${enclaveName}`.nothrow().text()); + logger.debug(await $`kurtosis clean`.text()); + logger.debug(await $`kurtosis engine stop`.nothrow().text()); + + if (pruneDocker) { + logger.debug(await $`docker system prune -f`.nothrow().text()); + } + + logger.success("Kurtosis enclave cleaned successfully"); +}; + +/** + * Modifies a Kurtosis configuration file based on deployment options. + * + * This function reads a YAML configuration file, applies modifications based on the provided + * deployment options, and writes the modified configuration to a new file in the tmp/configs directory. + * + * @param options - Configuration options + * @param options.blockscout - If true, adds "blockscout" to the additional_services array + * @param options.slotTime - If provided, sets the network_params.seconds_per_slot value + * @param options.kurtosisNetworkArgs - Space-separated key=value pairs to add to network_params + * @param configFile - Path to the original YAML configuration file to modify + * @returns Path to the modified configuration file in tmp/configs/ + * + * @throws {Error} If the config file is not found + */ +export const modifyConfig = async ( + options: { + blockscout?: boolean; + slotTime?: number; + kurtosisNetworkArgs?: string; + kurtosisEnclaveName?: string; + }, + configFile: string +): Promise => { + const outputDir = "tmp/configs"; + logger.debug(`Ensuring output directory exists: ${outputDir}`); + await $`mkdir -p ${outputDir}`.quiet(); + + const file = Bun.file(configFile); + invariant(file, `❌ Config file ${configFile} not found`); + + const config = await file.text(); + logger.debug(`Parsing config at ${configFile}`); + logger.trace(config); + + const parsedConfig = parse(config); + + if (options.blockscout) { + parsedConfig.additional_services.push("blockscout"); + } + + if (options.slotTime) { + parsedConfig.network_params.seconds_per_slot = options.slotTime; + } + + if (options.kurtosisNetworkArgs) { + logger.debug(`Using custom Kurtosis network args: ${options.kurtosisNetworkArgs}`); + const args = options.kurtosisNetworkArgs.split(" "); + for (const arg of args) { + const [key, value] = arg.split("="); + parsedConfig.network_params[key] = value; + } + } + + logger.trace(parsedConfig); + // Use a unique filename based on the enclave name to avoid conflicts in parallel execution + const configFileName = options.kurtosisEnclaveName + ? `modified-config-${options.kurtosisEnclaveName}.yaml` + : "modified-config.yaml"; + const outputFile = `${outputDir}/${configFileName}`; + logger.debug(`Modified config saving to ${outputFile}`); + + await Bun.write(outputFile, stringify(parsedConfig)); + return outputFile; +}; + +/** + * Registers the Execution Layer (EL) and Consensus Layer (CL) service endpoints with the LaunchedNetwork instance. + * + * This function retrieves the public ports for the Ethereum network services from Kurtosis and configures + * the LaunchedNetwork instance with the appropriate RPC URLs and endpoints for client communication. + * + * Services registered: + * - Execution Layer (EL): Reth RPC endpoint via "el-1-reth-lodestar" service + * - Consensus Layer (CL): Lodestar HTTP endpoint via "cl-1-lodestar-reth" service + * + * @param launchedNetwork - The LaunchedNetwork instance to populate with service endpoints + * @param enclaveName - The name of the Kurtosis enclave containing the services + * + * @throws {Error} If EL RPC port cannot be found + * @throws {Error} If CL endpoint cannot be determined + */ +export const registerServices = async ( + launchedNetwork: LaunchedNetwork, + enclaveName: string +): Promise => { + logger.info("📝 Registering Kurtosis service endpoints..."); + + // Configure EL RPC URL + try { + const rethPublicPort = await getPortFromKurtosis("el-1-reth-lodestar", "rpc", enclaveName); + invariant(rethPublicPort && rethPublicPort > 0, "❌ Could not find EL RPC port"); + const elRpcUrl = `http://127.0.0.1:${rethPublicPort}`; + launchedNetwork.elRpcUrl = elRpcUrl; + logger.info(`📝 Execution Layer RPC URL configured: ${elRpcUrl}`); + + // Configure CL Endpoint + const lodestarPublicPort = await getPortFromKurtosis("cl-1-lodestar-reth", "http", enclaveName); + const clEndpoint = `http://127.0.0.1:${lodestarPublicPort}`; + invariant( + clEndpoint, + "❌ CL Endpoint could not be determined from Kurtosis service cl-1-lodestar-reth" + ); + launchedNetwork.clEndpoint = clEndpoint; + logger.info(`📝 Consensus Layer Endpoint configured: ${clEndpoint}`); + } catch (error) { + logger.warn(`⚠️ Kurtosis service endpoints could not be determined: ${error}`); + throw error; + } +}; + +/** + * Runs a Kurtosis Ethereum network enclave with the specified configuration. + * + * This function handles the complete process of starting a Kurtosis enclave: + * 1. Modifies the configuration file based on the provided options + * 2. Executes the kurtosis run command with the modified configuration + * 3. Handles error cases and logs appropriate debug information + * + * @param options - Configuration options containing kurtosisEnclaveName and other settings + * @param configFilePath - Path to the base YAML configuration file to use + * + * @throws {Error} If the Kurtosis network fails to start properly + */ +export const runKurtosisEnclave = async ( + options: { + kurtosisEnclaveName: string; + blockscout?: boolean; + slotTime?: number; + kurtosisNetworkArgs?: string; + }, + configFilePath: string +): Promise => { + logger.info("🚀 Starting Kurtosis enclave..."); + + const configFile = await modifyConfig(options, configFilePath); + + logger.info(`⚙️ Using Kurtosis config file: ${configFile}`); + + await runShellCommandWithLogger( + `kurtosis run github.com/ethpandaops/ethereum-package --args-file ${configFile} --enclave ${options.kurtosisEnclaveName}`, + { + logLevel: "debug" + } + ); +}; + +/** + * Pulls required Docker images for macOS with the correct platform architecture. + * + * This function is specifically for macOS users who need to pull linux/amd64 images + * to ensure compatibility with Kurtosis. + */ +const pullMacOSImages = async (): Promise => { + logger.debug("Detected macOS, pulling container images with linux/amd64 platform..."); + logger.debug( + await $`docker pull ghcr.io/blockscout/smart-contract-verifier:latest --platform linux/amd64`.text() + ); +}; + +/** + * Gets the Blockscout URL for a given Kurtosis enclave. + * + * @param enclaveName - The name of the Kurtosis enclave + * @returns The Blockscout backend URL + * + * @throws {Error} If the Blockscout service is not found in the enclave + */ +export const getBlockscoutUrl = async (enclaveName: string): Promise => { + const blockscoutPort = await getPortFromKurtosis("blockscout", "http", enclaveName); + invariant(blockscoutPort, "❌ Could not find Blockscout service port"); + return `http://127.0.0.1:${blockscoutPort}`; +}; diff --git a/test/launcher/network/index.ts b/test/launcher/network/index.ts new file mode 100644 index 00000000..085792b3 --- /dev/null +++ b/test/launcher/network/index.ts @@ -0,0 +1,284 @@ +import { $ } from "bun"; +import { getContainersMatchingImage, getPortFromKurtosis, logger } from "utils"; +import { ParameterCollection } from "utils/parameters"; +import { deployContracts } from "../contracts"; +import { launchLocalDataHavenSolochain } from "../datahaven"; +import { getRunningKurtosisEnclaves, launchKurtosisNetwork } from "../kurtosis"; +import { setDataHavenParameters } from "../parameters"; +import { launchRelayers } from "../relayers"; +import type { LaunchNetworkResult, NetworkLaunchOptions } from "../types"; +import { LaunchedNetwork } from "../types/launchedNetwork"; +import { checkBaseDependencies } from "../utils"; +import { COMPONENTS } from "../utils/constants"; +import { fundValidators, setupValidators, updateValidatorSet } from "../validators"; + +// Authority IDs for test networks +const TEST_AUTHORITY_IDS = ["alice", "bob"] as const; + +/** + * Validates that the network ID is unique and no resources with this ID exist. + * @throws {Error} if resources with the network ID already exist + */ +const validateNetworkIdUnique = async (networkId: string): Promise => { + logger.info(`🔍 Validating network ID uniqueness: ${networkId}`); + + // Check for existing DataHaven containers + const datahavenContainers = await getContainersMatchingImage(COMPONENTS.datahaven.imageName); + const conflictingDatahaven = datahavenContainers.filter((c) => + c.Names.some((name) => name.includes(networkId)) + ); + if (conflictingDatahaven.length > 0) { + throw new Error( + `DataHaven containers with network ID '${networkId}' already exist. ` + + `Run 'bun cli stop --all' or remove containers manually.` + ); + } + + // Check for existing relayer containers + const relayerContainers = await getContainersMatchingImage(COMPONENTS.snowbridge.imageName); + const conflictingRelayers = relayerContainers.filter((c) => + c.Names.some((name) => name.includes(networkId)) + ); + if (conflictingRelayers.length > 0) { + throw new Error( + `Relayer containers with network ID '${networkId}' already exist. ` + + `Run 'bun cli stop --all' or remove containers manually.` + ); + } + + // Check for existing Kurtosis enclaves + const enclaves = await getRunningKurtosisEnclaves(); + const enclaveName = `eth-${networkId}`; + const conflictingEnclaves = enclaves.filter((e) => e.name === enclaveName); + if (conflictingEnclaves.length > 0) { + throw new Error( + `Kurtosis enclave '${enclaveName}' already exists. ` + + `Run 'kurtosis enclave rm ${enclaveName}' to remove it.` + ); + } + + // Check for existing Docker network + const dockerNetworkName = `datahaven-${networkId}`; + const networkOutput = + await $`docker network ls --filter "name=^${dockerNetworkName}$" --format "{{.Name}}"`.text(); + if (networkOutput.trim()) { + throw new Error( + `Docker network '${dockerNetworkName}' already exists. ` + + `Run 'docker network rm ${dockerNetworkName}' to remove it.` + ); + } + + logger.success(`Network ID '${networkId}' is available`); +}; + +/** + * Creates a cleanup function for the test network. + */ +const createCleanupFunction = (networkId: string) => { + return async () => { + logger.info(`🧹 Cleaning up test network: ${networkId}`); + + try { + // 1. Stop relayer containers + const relayerContainers = await getContainersMatchingImage(COMPONENTS.snowbridge.imageName); + const networkRelayers = relayerContainers.filter((c) => + c.Names.some((name) => name.includes(networkId)) + ); + if (networkRelayers.length > 0) { + logger.info(`🔨 Stopping ${networkRelayers.length} relayer containers...`); + for (const container of networkRelayers) { + await $`docker stop ${container.Id}`.nothrow(); + await $`docker rm ${container.Id}`.nothrow(); + } + } + + // 2. Stop DataHaven containers + const datahavenContainers = await getContainersMatchingImage(COMPONENTS.datahaven.imageName); + const networkDatahaven = datahavenContainers.filter((c) => + c.Names.some((name) => name.includes(networkId)) + ); + if (networkDatahaven.length > 0) { + logger.info(`🔨 Stopping ${networkDatahaven.length} DataHaven containers...`); + for (const container of networkDatahaven) { + await $`docker stop ${container.Id}`.nothrow(); + await $`docker rm ${container.Id}`.nothrow(); + } + } + + // 3. Remove Docker network + const dockerNetworkName = `datahaven-${networkId}`; + logger.info(`🔨 Removing Docker network: ${dockerNetworkName}`); + await $`docker network rm -f ${dockerNetworkName}`.nothrow(); + + // 4. Remove Kurtosis enclave + const enclaveName = `eth-${networkId}`; + logger.info(`🔨 Removing Kurtosis enclave: ${enclaveName}`); + await $`kurtosis enclave rm ${enclaveName} -f`.nothrow(); + + logger.success(`Cleanup completed for network: ${networkId}`); + } catch (error) { + logger.error(`❌ Cleanup failed for network ${networkId}:`, error); + // Continue cleanup, don't throw + } + }; +}; + +/** + * Launches a complete network stack for E2E testing. + * + * This function orchestrates the launch of all network components: + * 1. DataHaven blockchain nodes + * 2. Kurtosis Ethereum network + * 3. Smart contracts deployment + * 4. Validator setup + * 5. Runtime parameter configuration + * 6. Relayer services + * 7. Validator set update + * + * @param options - Configuration options for the network launch + * @returns NetworkConnectors with cleanup function + * @throws {Error} if network ID is not unique or any component fails to launch + */ +export const launchNetwork = async ( + options: NetworkLaunchOptions +): Promise => { + const networkId = options.networkId; + const launchedNetwork = new LaunchedNetwork(); + launchedNetwork.networkName = networkId; + + let cleanup: (() => Promise) | undefined; + + try { + logger.info(`🚀 Launching complete network stack with ID: ${networkId}`); + const startTime = performance.now(); + + // Check base dependencies + await checkBaseDependencies(); + + // Validate network ID is unique + await validateNetworkIdUnique(networkId); + + // Create cleanup function + cleanup = createCleanupFunction(networkId); + + // Create parameter collection for use throughout the launch + const parameterCollection = new ParameterCollection(); + + // 1. Launch DataHaven network + logger.info("📦 Launching DataHaven network..."); + await launchLocalDataHavenSolochain( + { + networkId, + datahavenImageTag: options.datahavenImageTag || "moonsonglabs/datahaven:local", + relayerImageTag: options.relayerImageTag || "moonsonglabs/snowbridge-relay:latest", + authorityIds: TEST_AUTHORITY_IDS, + buildDatahaven: options.buildDatahaven ?? true, + datahavenBuildExtraArgs: options.datahavenBuildExtraArgs || "--features=fast-runtime" + }, + launchedNetwork + ); + + // 2. Launch Ethereum/Kurtosis network + logger.info("⚡️ Launching Kurtosis Ethereum network..."); + const kurtosisEnclaveName = `eth-${networkId}`; + await launchKurtosisNetwork( + { + kurtosisEnclaveName: kurtosisEnclaveName, + blockscout: options.blockscout ?? false, + slotTime: options.slotTime || 2, + kurtosisNetworkArgs: options.kurtosisNetworkArgs + }, + launchedNetwork + ); + + // 3. Deploy contracts + logger.info("📄 Deploying smart contracts..."); + let blockscoutBackendUrl: string | undefined; + if (options.blockscout) { + const blockscoutPort = await getPortFromKurtosis("blockscout", "http", kurtosisEnclaveName); + blockscoutBackendUrl = `http://127.0.0.1:${blockscoutPort}`; + } + + if (!launchedNetwork.elRpcUrl) { + throw new Error("Ethereum RPC URL not available"); + } + + await deployContracts({ + rpcUrl: launchedNetwork.elRpcUrl, + verified: options.verified ?? false, + blockscoutBackendUrl, + parameterCollection + }); + + // 4. Fund validators + logger.info("💰 Funding validators..."); + await fundValidators({ + rpcUrl: launchedNetwork.elRpcUrl + }); + + // 5. Setup validators + logger.info("🔐 Setting up validators..."); + await setupValidators({ + rpcUrl: launchedNetwork.elRpcUrl + }); + + // 6. Set DataHaven runtime parameters + logger.info("⚙️ Setting DataHaven parameters..."); + await setDataHavenParameters({ + launchedNetwork, + collection: parameterCollection + }); + + // 7. Launch relayers + logger.info("❄️ Launching Snowbridge relayers..."); + if (!options.relayerImageTag) { + throw new Error("Relayer image tag not specified"); + } + + await launchRelayers( + { + networkId, + relayerImageTag: options.relayerImageTag, + kurtosisEnclaveName + }, + launchedNetwork + ); + + // 8. Update validator set (after relayers are running) + logger.info("🔄 Updating validator set..."); + await updateValidatorSet({ + rpcUrl: launchedNetwork.elRpcUrl + }); + + // Log success + const endTime = performance.now(); + const minutes = ((endTime - startTime) / (1000 * 60)).toFixed(1); + logger.success(`Network launched successfully in ${minutes} minutes`); + + // Validate required endpoints + if (!launchedNetwork.clEndpoint) { + throw new Error("Consensus layer endpoint not available"); + } + + // Return connectors + const aliceContainerName = `datahaven-alice-${networkId}`; + const wsPort = launchedNetwork.getContainerPort(aliceContainerName); + return { + launchedNetwork, + dataHavenRpcUrl: `http://127.0.0.1:${wsPort}`, + ethereumRpcUrl: launchedNetwork.elRpcUrl, + ethereumClEndpoint: launchedNetwork.clEndpoint, + cleanup + }; + } catch (error) { + logger.error("❌ Failed to launch network", error); + + // Run cleanup if we created it + if (cleanup) { + logger.info("🧹 Running cleanup due to launch failure..."); + await cleanup(); + } + + throw error; + } +}; diff --git a/test/launcher/parameters.ts b/test/launcher/parameters.ts new file mode 100644 index 00000000..2397ce6f --- /dev/null +++ b/test/launcher/parameters.ts @@ -0,0 +1,54 @@ +import { setDataHavenParameters as setDataHavenParametersScript } from "scripts/set-datahaven-parameters"; +import { logger } from "utils"; +import type { ParameterCollection } from "utils/parameters"; +import type { LaunchedNetwork } from "./types/launchedNetwork"; + +/** + * Configuration options for setting DataHaven runtime parameters. + */ +export interface ParametersOptions { + launchedNetwork: LaunchedNetwork; + collection: ParameterCollection; +} + +/** + * Sets DataHaven runtime parameters from a parameter collection. + * + * This function updates various runtime parameters on the DataHaven chain: + * - Bridge configuration parameters + * - Network timing parameters + * - Validator configuration + * - Fee structures + * - Other protocol-specific settings + * + * The parameters are collected throughout the deployment process and + * applied in a single transaction to minimize gas costs and ensure + * consistency. + * + * @param options - Configuration options for setting parameters + * @param options.launchedNetwork - The launched network instance containing connection details + * @param options.collection - The parameter collection containing all parameters to set + * + * @throws {Error} If the parameter file generation fails + * @throws {Error} If the RPC connection cannot be established + * @throws {Error} If the parameter update transaction fails + */ +export const setDataHavenParameters = async (options: ParametersOptions): Promise => { + logger.info("⚙️ Setting DataHaven runtime parameters..."); + + const { launchedNetwork, collection } = options; + + // Generate the parameters file from the collection + const parametersFilePath = await collection.generateParametersFile(); + + // Get the WebSocket RPC URL from the launched network + const rpcUrl = `ws://127.0.0.1:${launchedNetwork.getPublicWsPort()}`; + + // Execute the parameter update + await setDataHavenParametersScript({ + rpcUrl, + parametersFilePath + }); + + logger.success("DataHaven parameters set successfully"); +}; diff --git a/test/launcher/relayers.ts b/test/launcher/relayers.ts new file mode 100644 index 00000000..2536b140 --- /dev/null +++ b/test/launcher/relayers.ts @@ -0,0 +1,708 @@ +import path from "node:path"; +import { datahaven } from "@polkadot-api/descriptors"; +import { $ } from "bun"; +import { createClient, type PolkadotClient } from "polkadot-api"; +import { withPolkadotSdkCompat } from "polkadot-api/polkadot-sdk-compat"; +import { getWsProvider } from "polkadot-api/ws-provider/web"; +import invariant from "tiny-invariant"; +import { + ANVIL_FUNDED_ACCOUNTS, + getEvmEcdsaSigner, + getPortFromKurtosis, + killExistingContainers, + logger, + parseDeploymentsFile, + parseRelayConfig, + runShellCommandWithLogger, + SUBSTRATE_FUNDED_ACCOUNTS, + waitForContainerToStart +} from "utils"; +import type { BeaconCheckpoint, FinalityCheckpointsResponse } from "utils/types"; +import { parseJsonToBeaconCheckpoint } from "utils/types"; +import { waitFor } from "utils/waits"; +import type { LaunchedNetwork } from "./types/launchedNetwork"; +import { ZERO_HASH } from "./utils/constants"; + +// Type definitions +export type BeaconConfig = { + type: "beacon"; + ethClEndpoint: string; + substrateWsEndpoint: string; +}; + +export type BeefyConfig = { + type: "beefy"; + ethElRpcEndpoint: string; + substrateWsEndpoint: string; + beefyClientAddress: string; + gatewayAddress: string; +}; + +export type ExecutionConfig = { + type: "execution"; + ethElRpcEndpoint: string; + ethClEndpoint: string; + substrateWsEndpoint: string; + gatewayAddress: string; +}; + +export type SolochainConfig = { + type: "solochain"; + ethElRpcEndpoint: string; + substrateWsEndpoint: string; + beefyClientAddress: string; + gatewayAddress: string; + rewardsRegistryAddress: string; + ethClEndpoint: string; +}; + +export type RelayerConfigType = BeaconConfig | BeefyConfig | ExecutionConfig | SolochainConfig; + +export type RelayerSpec = { + name: string; + configFilePath: string; + templateFilePath?: string; + config: RelayerConfigType; + pk: { ethereum?: string; substrate?: string }; +}; + +// Constants +export const INITIAL_CHECKPOINT_DIR = "tmp/beacon-checkpoint"; +export const getInitialCheckpointFile = (networkId: string) => + `dump-initial-checkpoint-${networkId}.json`; +export const getInitialCheckpointPath = (networkId: string) => + path.join(INITIAL_CHECKPOINT_DIR, getInitialCheckpointFile(networkId)); + +/** + * Configuration options for launching Snowbridge relayers. + */ +export interface RelayersOptions { + networkId: string; + relayerImageTag: string; + kurtosisEnclaveName: string; +} + +/** + * Configuration paths for different relayer types. + */ +export const RELAYER_CONFIG_DIR = "tmp/configs"; +export const RELAYER_CONFIG_PATHS = { + BEACON: path.join(RELAYER_CONFIG_DIR, "beacon-relay.json"), + BEEFY: path.join(RELAYER_CONFIG_DIR, "beefy-relay.json"), + EXECUTION: path.join(RELAYER_CONFIG_DIR, "execution-relay.json"), + SOLOCHAIN: path.join(RELAYER_CONFIG_DIR, "solochain-relay.json") +}; + +/** + * Generates configuration files for relayers. + * + * @param relayerSpec - The relayer specification containing name, type, and config path. + * @param environment - The environment to use for template files (e.g., "local", "stagenet", "testnet", "mainnet"). + * @param configDir - The directory where config files should be written. + */ +export const generateRelayerConfig = async ( + relayerSpec: RelayerSpec, + environment: string, + configDir: string +) => { + const { name, configFilePath, templateFilePath: _templateFilePath, config } = relayerSpec; + const { type } = config; + const configFileName = path.basename(configFilePath); + + logger.debug(`Creating config for ${name}`); + const templateFilePath = + _templateFilePath ?? `configs/snowbridge/${environment}/${configFileName}`; + const outputFilePath = path.resolve(configDir, configFileName); + logger.debug(`Reading config file ${templateFilePath}`); + const file = Bun.file(templateFilePath); + + if (!(await file.exists())) { + logger.error(`File ${templateFilePath} does not exist`); + throw new Error("Error reading snowbridge config file"); + } + const json = await file.json(); + + logger.debug(`Generating ${type} relayer configuration for ${name}`); + + switch (type) { + case "beacon": { + const cfg = parseRelayConfig(json, type); + cfg.source.beacon.endpoint = config.ethClEndpoint; + cfg.source.beacon.stateEndpoint = config.ethClEndpoint; + cfg.source.beacon.datastore.location = "/data"; + cfg.sink.parachain.endpoint = config.substrateWsEndpoint; + + await Bun.write(outputFilePath, JSON.stringify(cfg, null, 4)); + logger.success(`Updated beacon config written to ${outputFilePath}`); + break; + } + case "beefy": { + const cfg = parseRelayConfig(json, type); + cfg.source.polkadot.endpoint = config.substrateWsEndpoint; + cfg.sink.ethereum.endpoint = config.ethElRpcEndpoint; + cfg.sink.contracts.BeefyClient = config.beefyClientAddress; + cfg.sink.contracts.Gateway = config.gatewayAddress; + + await Bun.write(outputFilePath, JSON.stringify(cfg, null, 4)); + logger.success(`Updated beefy config written to ${outputFilePath}`); + break; + } + case "execution": { + const cfg = parseRelayConfig(json, type); + cfg.source.ethereum.endpoint = config.ethElRpcEndpoint; + cfg.source.beacon.endpoint = config.ethClEndpoint; + cfg.source.beacon.stateEndpoint = config.ethClEndpoint; + cfg.source.beacon.datastore.location = "/data"; + cfg.sink.parachain.endpoint = config.substrateWsEndpoint; + cfg.source.contracts.Gateway = config.gatewayAddress; + + await Bun.write(outputFilePath, JSON.stringify(cfg, null, 4)); + logger.success(`Updated execution config written to ${outputFilePath}`); + break; + } + case "solochain": { + const cfg = parseRelayConfig(json, type); + cfg.source.ethereum.endpoint = config.ethElRpcEndpoint; + cfg.source.solochain.endpoint = config.substrateWsEndpoint; + cfg.source.contracts.BeefyClient = config.beefyClientAddress; + cfg.source.contracts.Gateway = config.gatewayAddress; + cfg.source.beacon.endpoint = config.ethClEndpoint; + cfg.source.beacon.stateEndpoint = config.ethClEndpoint; + cfg.source.beacon.datastore.location = "/data"; + cfg.sink.ethereum.endpoint = config.ethElRpcEndpoint; + cfg.sink.contracts.Gateway = config.gatewayAddress; + cfg["reward-address"] = config.rewardsRegistryAddress; + + await Bun.write(outputFilePath, JSON.stringify(cfg, null, 4)); + logger.success(`Updated solochain config written to ${outputFilePath}`); + break; + } + default: + throw new Error(`Unsupported relayer type with config: \n${JSON.stringify(config)}`); + } +}; + +/** + * Waits for the beacon chain to be ready by polling its finality checkpoints. + * + * @param launchedNetwork - An instance of LaunchedNetwork to get the CL endpoint. + * @param pollIntervalMs - The interval in milliseconds to poll the beacon chain. + * @param timeoutMs - The total time in milliseconds to wait before timing out. + * @throws Error if the beacon chain is not ready within the timeout. + */ +export const waitBeaconChainReady = async ( + launchedNetwork: LaunchedNetwork, + pollIntervalMs: number, + timeoutMs: number +) => { + const iterations = Math.floor(timeoutMs / pollIntervalMs); + + logger.trace("Waiting for beacon chain to be ready..."); + + await waitFor({ + lambda: async () => { + try { + const response = await fetch( + `${launchedNetwork.clEndpoint}/eth/v1/beacon/states/head/finality_checkpoints` + ); + if (!response.ok) { + throw new Error(`HTTP error! Status: ${response.status}`); + } + + const data = (await response.json()) as FinalityCheckpointsResponse; + logger.debug(`Beacon chain state: ${JSON.stringify(data)}`); + + invariant(data.data, "❌ No data returned from beacon chain"); + invariant(data.data.finalized, "❌ No finalised block returned from beacon chain"); + invariant( + data.data.finalized.root, + "❌ No finalised block root returned from beacon chain" + ); + + const initialBeaconBlock = data.data.finalized.root; + + if (initialBeaconBlock && initialBeaconBlock !== ZERO_HASH) { + logger.info(`⏲️ Beacon chain is ready with finalised block: ${initialBeaconBlock}`); + return true; + } + + logger.info(`⌛️ Retrying beacon chain state fetch in ${pollIntervalMs / 1000}s...`); + return false; + } catch (error) { + logger.error(`Failed to fetch beacon chain state: ${error}`); + return false; + } + }, + iterations, + delay: pollIntervalMs, + errorMessage: "Beacon chain is not ready. Relayers cannot be launched." + }); +}; + +/** + * Initialises the Ethereum Beacon Client pallet on the Substrate chain. + * It waits for the beacon chain to be ready, generates an initial checkpoint, + * and submits this checkpoint to the Substrate runtime via a sudo call. + * + * @param beaconConfigHostPath - The host path to the beacon configuration file. + * @param relayerImageTag - The Docker image tag for the relayer. + * @param datastorePath - The path to the datastore directory. + * @param launchedNetwork - An instance of LaunchedNetwork to interact with the running network. + * @throws If there's an error generating the beacon checkpoint or submitting it to Substrate. + */ +export const initEthClientPallet = async ( + networkId: string, + beaconConfigHostPath: string, + relayerImageTag: string, + datastorePath: string, + launchedNetwork: LaunchedNetwork +) => { + logger.debug("Initialising eth client pallet"); + // Poll the beacon chain until it's ready every 10 seconds for 10 minutes + await waitBeaconChainReady(launchedNetwork, 10000, 600000); + + const beaconConfigContainerPath = "/app/beacon-relay.json"; + const checkpointHostPath = path.resolve(getInitialCheckpointPath(networkId)); + const checkpointContainerPath = "/app/dump-initial-checkpoint.json"; // Hardcoded filename that generate-beacon-checkpoint expects + + logger.debug("Generating beacon checkpoint"); + // Pre-create the checkpoint file so that Docker doesn't interpret it as a directory + await Bun.write(getInitialCheckpointPath(networkId), ""); + + logger.debug(`Removing 'generate-beacon-checkpoint-${networkId}' container if it exists`); + logger.debug(await $`docker rm -f generate-beacon-checkpoint-${networkId}`.text()); + + // When running in Linux, `host.docker.internal` is not pre-defined when running in a container. + // So we need to add the parameter `--add-host host.docker.internal:host-gateway` to the command. + // In Mac this is not needed and could cause issues. + const addHostParam = + process.platform === "linux" ? "--add-host host.docker.internal:host-gateway" : ""; + + // Opportunistic pull - pull the image from Docker Hub only if it's not a local image + const isLocal = relayerImageTag.endsWith(":local"); + + logger.debug("Generating beacon checkpoint"); + const datastoreHostPath = path.resolve(datastorePath); + const command = `docker run \ + -v ${beaconConfigHostPath}:${beaconConfigContainerPath}:ro \ + -v ${checkpointHostPath}:${checkpointContainerPath} \ + -v ${datastoreHostPath}:/data \ + --name generate-beacon-checkpoint-${networkId} \ + --platform linux/amd64 \ + --workdir /app \ + ${addHostParam} \ + ${launchedNetwork.networkName ? `--network ${launchedNetwork.networkName}` : ""} \ + ${isLocal ? "" : "--pull always"} \ + ${relayerImageTag} \ + generate-beacon-checkpoint --config beacon-relay.json --export-json`; + logger.debug(`Running command: ${command}`); + logger.debug(await $`sh -c "${command}"`.text()); + + // Load the checkpoint into a JSON object and clean it up + const initialCheckpointFile = Bun.file(getInitialCheckpointPath(networkId)); + const initialCheckpointRaw = await initialCheckpointFile.text(); + const initialCheckpoint = parseJsonToBeaconCheckpoint(JSON.parse(initialCheckpointRaw)); + await initialCheckpointFile.delete(); + + logger.trace("Initial checkpoint:"); + logger.trace(initialCheckpoint.toJSON()); + + // Send the checkpoint to the Substrate runtime + const substrateRpcUrl = `http://127.0.0.1:${launchedNetwork.getPublicWsPort()}`; + await sendCheckpointToSubstrate(substrateRpcUrl, initialCheckpoint); + logger.success("Ethereum Beacon Client pallet initialised"); +}; + +/** + * Sends the beacon checkpoint to the Substrate runtime, waiting for the transaction to be finalised and successful. + * + * @param networkRpcUrl - The RPC URL of the Substrate network. + * @param checkpoint - The beacon checkpoint to send. + * @throws If the transaction signing fails, it becomes an invalid transaction, or the transaction is included but fails. + */ +const sendCheckpointToSubstrate = async (networkRpcUrl: string, checkpoint: BeaconCheckpoint) => { + logger.trace("Sending checkpoint to Substrate..."); + + const client = createClient(withPolkadotSdkCompat(getWsProvider(networkRpcUrl))); + const dhApi = client.getTypedApi(datahaven); + + logger.trace("Client created"); + + const signer = getEvmEcdsaSigner(SUBSTRATE_FUNDED_ACCOUNTS.ALITH.privateKey); + logger.trace("Signer created"); + + const forceCheckpointCall = dhApi.tx.EthereumBeaconClient.force_checkpoint({ + update: checkpoint + }); + + logger.debug("Force checkpoint call:"); + logger.debug(forceCheckpointCall.decodedCall); + + const tx = dhApi.tx.Sudo.sudo({ + call: forceCheckpointCall.decodedCall + }); + + logger.debug("Sudo call:"); + logger.debug(tx.decodedCall); + + try { + const txFinalisedPayload = await tx.signAndSubmit(signer); + + if (!txFinalisedPayload.ok) { + throw new Error("❌ Beacon checkpoint transaction failed"); + } + + logger.info( + `📪 "force_checkpoint" transaction with hash ${txFinalisedPayload.txHash} submitted successfully and finalised in block ${txFinalisedPayload.block.hash}` + ); + } catch (error) { + logger.error(`Failed to submit checkpoint transaction: ${error}`); + throw new Error(`Failed to submit checkpoint: ${error}`); + } finally { + client.destroy(); + logger.debug("Destroyed client"); + } +}; + +/** + * Launches Snowbridge relayers for cross-chain communication. + * + * This function sets up and launches all required Snowbridge relayers: + * - BEEFY relayer: Handles BEEFY protocol messages + * - Beacon relayer: Syncs Ethereum beacon chain state + * - Execution relayer: Processes execution layer events + * - Solochain relayer: Handles solochain-specific operations + * + * The function performs the following steps: + * 1. Kills any existing relayer containers + * 2. Waits for BEEFY protocol to be ready + * 3. Retrieves contract addresses from deployments + * 4. Creates configuration directories + * 5. Generates relayer configurations + * 6. Initializes the Ethereum client pallet + * 7. Starts all relayer containers + * + * @param options - Configuration options for launching relayers + * @param options.relayerImageTag - Docker image tag for the relayer containers + * @param options.kurtosisEnclaveName - Name of the Kurtosis enclave for Ethereum services + * @param launchedNetwork - The launched network instance containing connection details + * + * @throws {Error} If the relayer image tag is not provided + * @throws {Error} If BEEFY protocol is not ready within timeout + * @throws {Error} If required contract addresses are not found + * @throws {Error} If Docker operations fail + */ +export const launchRelayers = async ( + options: RelayersOptions, + launchedNetwork: LaunchedNetwork +): Promise => { + logger.info("🚀 Launching Snowbridge relayers..."); + + const { relayerImageTag, kurtosisEnclaveName } = options; + + invariant(relayerImageTag, "❌ relayerImageTag is required"); + + await killExistingContainers("snowbridge-"); + + // Get DataHaven node port + const dhNodes = launchedNetwork.containers.filter((container) => + container.name.includes("datahaven") + ); + let substrateWsPort: number; + let substrateNodeId: string; + + if (dhNodes.length === 0) { + logger.warn( + "⚠️ No DataHaven nodes found in launchedNetwork. Assuming DataHaven is running and defaulting to port 9944 for relayers." + ); + substrateWsPort = 9944; + substrateNodeId = "default (assumed)"; + } else { + const firstDhNode = dhNodes[0]; + substrateWsPort = firstDhNode.publicPorts.ws; + substrateNodeId = firstDhNode.name; + logger.info( + `🔌 Using DataHaven node ${substrateNodeId} on port ${substrateWsPort} for relayers and BEEFY check.` + ); + } + + // Check if BEEFY is ready before proceeding + await waitBeefyReady(launchedNetwork, 2000, 60000); + + const anvilDeployments = await parseDeploymentsFile(); + const beefyClientAddress = anvilDeployments.BeefyClient; + const gatewayAddress = anvilDeployments.Gateway; + const rewardsRegistryAddress = anvilDeployments.RewardsRegistry; + invariant(beefyClientAddress, "❌ BeefyClient address not found in anvil.json"); + invariant(gatewayAddress, "❌ Gateway address not found in anvil.json"); + invariant(rewardsRegistryAddress, "❌ RewardsRegistry address not found in anvil.json"); + + logger.debug(`Ensuring output directory exists: ${RELAYER_CONFIG_DIR}`); + await $`mkdir -p ${RELAYER_CONFIG_DIR}`.quiet(); + + const datastorePath = "tmp/datastore"; + logger.debug(`Ensuring datastore directory exists: ${datastorePath}`); + await $`mkdir -p ${datastorePath}`.quiet(); + + const ethWsPort = await getPortFromKurtosis("el-1-reth-lodestar", "ws", kurtosisEnclaveName); + const ethHttpPort = await getPortFromKurtosis("cl-1-lodestar-reth", "http", kurtosisEnclaveName); + + const ethElRpcEndpoint = `ws://host.docker.internal:${ethWsPort}`; + const ethClEndpoint = `http://host.docker.internal:${ethHttpPort}`; + const substrateWsEndpoint = `ws://${substrateNodeId}:${substrateWsPort}`; + + const relayersToStart: RelayerSpec[] = [ + { + name: "relayer-🥩", + configFilePath: RELAYER_CONFIG_PATHS.BEEFY, + config: { + type: "beefy", + ethElRpcEndpoint, + substrateWsEndpoint, + beefyClientAddress, + gatewayAddress + }, + pk: { + ethereum: ANVIL_FUNDED_ACCOUNTS[1].privateKey + } + }, + { + name: "relayer-🥓", + configFilePath: RELAYER_CONFIG_PATHS.BEACON, + config: { + type: "beacon", + ethClEndpoint, + substrateWsEndpoint + }, + pk: { + substrate: SUBSTRATE_FUNDED_ACCOUNTS.BALTATHAR.privateKey + } + }, + { + name: "relayer-⛓️", + configFilePath: RELAYER_CONFIG_PATHS.SOLOCHAIN, + config: { + type: "solochain", + ethElRpcEndpoint, + substrateWsEndpoint, + beefyClientAddress, + gatewayAddress, + rewardsRegistryAddress, + ethClEndpoint + }, + pk: { + ethereum: ANVIL_FUNDED_ACCOUNTS[1].privateKey, + substrate: SUBSTRATE_FUNDED_ACCOUNTS.CHARLETH.privateKey + } + }, + { + name: "relayer-⚙️", + configFilePath: RELAYER_CONFIG_PATHS.EXECUTION, + config: { + type: "execution", + ethElRpcEndpoint, + ethClEndpoint, + substrateWsEndpoint, + gatewayAddress + }, + pk: { + substrate: SUBSTRATE_FUNDED_ACCOUNTS.DOROTHY.privateKey + } + } + ]; + + // Generate configurations for all relayers + for (const relayerSpec of relayersToStart) { + await generateRelayerConfig(relayerSpec, "local", RELAYER_CONFIG_DIR); + } + + invariant( + launchedNetwork.networkName, + "❌ Docker network name not found in LaunchedNetwork instance" + ); + + // Initialize Ethereum client pallet + await initEthClientPallet( + options.networkId, + path.resolve(RELAYER_CONFIG_PATHS.BEACON), + relayerImageTag, + datastorePath, + launchedNetwork + ); + + // Launch all relayers + await launchRelayerContainers( + relayersToStart, + relayerImageTag, + launchedNetwork, + options.networkId + ); + + logger.success("Snowbridge relayers launched successfully"); +}; + +/** + * Waits for the BEEFY protocol to be ready by polling its finalized head. + * + * @param launchedNetwork - An instance of LaunchedNetwork to get the node endpoint + * @param pollIntervalMs - The interval in milliseconds to poll the BEEFY endpoint + * @param timeoutMs - The total time in milliseconds to wait before timing out + * + * @throws {Error} If BEEFY is not ready within the timeout + */ +const waitBeefyReady = async ( + launchedNetwork: LaunchedNetwork, + pollIntervalMs: number, + timeoutMs: number +): Promise => { + const port = launchedNetwork.getPublicWsPort(); + const wsUrl = `ws://127.0.0.1:${port}`; + const iterations = Math.floor(timeoutMs / pollIntervalMs); + + logger.info(`⌛️ Waiting for BEEFY to be ready on port ${port}...`); + + let client: PolkadotClient | undefined; + const clientTimeoutMs = pollIntervalMs / 2; + const delayMs = pollIntervalMs / 2; + try { + client = createClient(withPolkadotSdkCompat(getWsProvider(wsUrl))); + + await waitFor({ + lambda: async () => { + try { + logger.debug("Attempting to to check beefy_getFinalizedHead"); + + // Add timeout to the RPC call to prevent hanging. + const finalisedHeadPromise = client?._request("beefy_getFinalizedHead", []); + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => reject(new Error("RPC call timeout")), clientTimeoutMs); + }); + + const finalisedHeadHex = await Promise.race([finalisedHeadPromise, timeoutPromise]); + + if (finalisedHeadHex && finalisedHeadHex !== ZERO_HASH) { + logger.info(`🥩 BEEFY is ready. Finalised head: ${finalisedHeadHex}.`); + return true; + } + + logger.debug( + `BEEFY not ready or finalised head is zero. Retrying in ${delayMs / 1000}s...` + ); + return false; + } catch (rpcError) { + logger.warn(`RPC error checking BEEFY status: ${rpcError}. Retrying...`); + return false; + } + }, + iterations, + delay: delayMs, + errorMessage: "BEEFY protocol not ready. Relayers cannot be launched." + }); + } catch (error) { + logger.error(`❌ Failed to connect to DataHaven node for BEEFY check: ${error}`); + throw new Error("BEEFY protocol not ready. Relayers cannot be launched."); + } finally { + if (client) { + client.destroy(); + } + } +}; + +/** + * Launches individual relayer containers. + * + * @param relayersToStart - Array of relayer specifications + * @param relayerImageTag - Docker image tag for the relayers + * @param launchedNetwork - The launched network instance + * @param networkId - The network ID to suffix container names + */ +const launchRelayerContainers = async ( + relayersToStart: RelayerSpec[], + relayerImageTag: string, + launchedNetwork: LaunchedNetwork, + networkId: string +): Promise => { + const isLocal = relayerImageTag.endsWith(":local"); + const networkName = launchedNetwork.networkName; + invariant(networkName, "❌ Docker network name not found in LaunchedNetwork instance"); + + for (const { configFilePath, name, config, pk } of relayersToStart) { + try { + const containerName = `snowbridge-${config.type}-relay-${networkId}`; + logger.info(`🚀 Starting relayer ${containerName} ...`); + + const hostConfigFilePath = path.resolve(configFilePath); + const containerConfigFilePath = `/${configFilePath}`; + + const commandBase: string[] = [ + "docker", + "run", + "-d", + "--platform", + "linux/amd64", + "--add-host", + "host.docker.internal:host-gateway", + "--name", + containerName, + "--network", + networkName, + ...(isLocal ? [] : ["--pull", "always"]) + ]; + + const volumeMounts: string[] = ["-v", `${hostConfigFilePath}:${containerConfigFilePath}`]; + + if (config.type === "beacon" || config.type === "execution") { + const hostDatastorePath = path.resolve("tmp/datastore"); + const containerDatastorePath = "/data"; + volumeMounts.push("-v", `${hostDatastorePath}:${containerDatastorePath}`); + } + + const relayerCommandArgs: string[] = ["run", config.type, "--config", configFilePath]; + + switch (config.type) { + case "beacon": + invariant(pk.substrate, "❌ Substrate private key is required for beacon relayer"); + relayerCommandArgs.push("--substrate.private-key", pk.substrate); + break; + case "beefy": + invariant(pk.ethereum, "❌ Ethereum private key is required for beefy relayer"); + relayerCommandArgs.push("--ethereum.private-key", pk.ethereum); + break; + case "solochain": + invariant(pk.ethereum, "❌ Ethereum private key is required for solochain relayer"); + relayerCommandArgs.push("--ethereum.private-key", pk.ethereum); + if (pk.substrate) { + relayerCommandArgs.push("--substrate.private-key", pk.substrate); + } else { + logger.warn( + "⚠️ No substrate private key provided for solochain relayer. This might be an issue depending on the configuration." + ); + } + break; + case "execution": + invariant(pk.substrate, "❌ Substrate private key is required for execution relayer"); + relayerCommandArgs.push("--substrate.private-key", pk.substrate); + break; + } + + const command: string[] = [ + ...commandBase, + ...volumeMounts, + relayerImageTag, + ...relayerCommandArgs + ]; + + logger.debug(`Running command: ${command.join(" ")}`); + await runShellCommandWithLogger(command.join(" "), { logLevel: "debug" }); + + launchedNetwork.addContainer(containerName); + + await waitForContainerToStart(containerName); + + logger.success(`Started relayer ${name} with process ${process.pid}`); + } catch (e) { + logger.error(`Error starting relayer ${name}`); + logger.error(e); + } + } +}; diff --git a/test/launcher/types/index.ts b/test/launcher/types/index.ts new file mode 100644 index 00000000..14e62aa0 --- /dev/null +++ b/test/launcher/types/index.ts @@ -0,0 +1,28 @@ +export { LaunchedNetwork } from "./launchedNetwork"; + +import type { LaunchedNetwork } from "./launchedNetwork"; + +// Network launch options (combines all component options) +export interface NetworkLaunchOptions { + networkId: string; + environment?: "local" | "stagenet" | "testnet" | "mainnet"; + slotTime?: number; + datahavenImageTag?: string; + relayerImageTag?: string; + buildDatahaven?: boolean; + datahavenBuildExtraArgs?: string; + verified?: boolean; + blockscout?: boolean; + kurtosisNetworkArgs?: string; + elRpcUrl?: string; + clEndpoint?: string; +} + +// Network connectors returned by the launcher +export interface LaunchNetworkResult { + launchedNetwork: LaunchedNetwork; + dataHavenRpcUrl: string; + ethereumRpcUrl: string; + ethereumClEndpoint: string; + cleanup: () => Promise; +} diff --git a/test/cli/handlers/common/launchedNetwork.ts b/test/launcher/types/launchedNetwork.ts similarity index 87% rename from test/cli/handlers/common/launchedNetwork.ts rename to test/launcher/types/launchedNetwork.ts index 90d95989..af32ee27 100644 --- a/test/cli/handlers/common/launchedNetwork.ts +++ b/test/launcher/types/launchedNetwork.ts @@ -18,6 +18,8 @@ export class LaunchedNetwork { protected _clEndpoint?: string; /** The Kubernetes namespace for the network. Used only for deploy commands. */ protected _kubeNamespace?: string; + /** The DataHaven authorities for the network. */ + protected _datahavenAuthorities?: string[]; constructor() { this.runId = crypto.randomUUID(); @@ -27,6 +29,7 @@ export class LaunchedNetwork { this._elRpcUrl = undefined; this._clEndpoint = undefined; this._kubeNamespace = undefined; + this._datahavenAuthorities = undefined; } public set networkName(name: string) { @@ -125,4 +128,20 @@ export class LaunchedNetwork { invariant(this._kubeNamespace, "❌ Kubernetes namespace not set in LaunchedNetwork"); return this._kubeNamespace; } + + /** + * Sets the DataHaven authorities for the network. + * @param authorities - Array of authority hashes. + */ + public set datahavenAuthorities(authorities: string[]) { + this._datahavenAuthorities = authorities; + } + + /** + * Gets the DataHaven authorities for the network. + * @returns Array of authority hashes. + */ + public get datahavenAuthorities(): string[] { + return this._datahavenAuthorities || []; + } } diff --git a/test/launcher/utils/checks.ts b/test/launcher/utils/checks.ts new file mode 100644 index 00000000..26e4016f --- /dev/null +++ b/test/launcher/utils/checks.ts @@ -0,0 +1,150 @@ +import { $ } from "bun"; +import { logger } from "utils"; + +// Minimum Bun version required +const MIN_BUN_VERSION = { major: 1, minor: 1 }; + +/** + * Checks if all base dependencies are installed and available. + * These checks are needed for both CLI and test environments. + */ +export const checkBaseDependencies = async (): Promise => { + if (!(await checkKurtosisInstalled())) { + logger.error("Kurtosis CLI is required to be installed: https://docs.kurtosis.com/install"); + throw Error("❌ Kurtosis CLI application not found."); + } + + logger.success("Kurtosis CLI found"); + + if (!(await checkBunVersion())) { + logger.error( + `Bun version must be ${MIN_BUN_VERSION.major}.${MIN_BUN_VERSION.minor} or higher: https://bun.sh/docs/installation#upgrading` + ); + throw Error("❌ Bun version is too old."); + } + + logger.success("Bun is installed and up to date"); + + if (!(await checkDockerRunning())) { + logger.error("Is Docker Running? Unable to make connection to docker daemon"); + throw Error("❌ Error connecting to Docker"); + } + + logger.success("Docker is running"); + + if (!(await checkForgeInstalled())) { + logger.error("Is foundry installed? https://book.getfoundry.sh/getting-started/installation"); + throw Error("❌ Forge binary not found in PATH"); + } + + logger.success("Forge is installed"); +}; + +/** + * Checks if Bun version meets minimum requirements + */ +export const checkBunVersion = async (): Promise => { + const bunVersion = Bun.version; + const [major, minor] = bunVersion.split(".").map(Number); + + // Check if version meets minimum requirements + const isVersionValid = + major > MIN_BUN_VERSION.major || + (major === MIN_BUN_VERSION.major && minor >= MIN_BUN_VERSION.minor); + + if (!isVersionValid) { + logger.debug(`Bun version: ${bunVersion} (too old)`); + return false; + } + + logger.debug(`Bun version: ${bunVersion}`); + return true; +}; + +/** + * Checks if Kurtosis CLI is installed + */ +export const checkKurtosisInstalled = async (): Promise => { + const { exitCode, stderr, stdout } = await $`kurtosis version`.nothrow().quiet(); + if (exitCode !== 0) { + logger.debug(`Kurtosis check failed: ${stderr.toString()}`); + return false; + } + logger.debug(`Kurtosis version: ${stdout.toString().trim()}`); + return true; +}; + +/** + * Checks if Docker daemon is running + */ +export const checkDockerRunning = async (): Promise => { + const { exitCode, stderr } = await $`docker system info`.nothrow().quiet(); + if (exitCode !== 0) { + logger.debug(`Docker check failed: ${stderr.toString()}`); + return false; + } + logger.debug("Docker daemon is running"); + return true; +}; + +/** + * Checks if Forge (Foundry) is installed + */ +export const checkForgeInstalled = async (): Promise => { + const { exitCode, stderr, stdout } = await $`forge --version`.nothrow().quiet(); + if (exitCode !== 0) { + logger.debug(`Forge check failed: ${stderr.toString()}`); + return false; + } + logger.debug(`Forge version: ${stdout.toString().trim()}`); + return true; +}; + +/** + * Checks if the Kurtosis cluster type that is configured is compatible with the expected type + * @param kubernetes - Whether the cluster is expected to be a Kubernetes cluster + * @returns true if the cluster type is compatible, false otherwise + */ +export const checkKurtosisCluster = async (kubernetes?: boolean): Promise => { + // First check if kurtosis cluster get works + const { exitCode, stderr, stdout } = await $`kurtosis cluster get`.nothrow().quiet(); + + if (exitCode !== 0) { + logger.warn(`⚠️ Kurtosis cluster get failed: ${stderr.toString()}`); + logger.info("ℹ️ Assuming local launch mode and continuing."); + return true; + } + + const currentCluster = stdout.toString().trim(); + logger.debug(`Current Kurtosis cluster: ${currentCluster}`); + + // Try to get the cluster type from config, but don't fail if config path is not reachable + const clusterTypeResult = + await $`CURRENT_CLUSTER=${currentCluster} && sed -n "/^ $CURRENT_CLUSTER:$/,/^ [^ ]/p" "$(kurtosis config path)" | grep "type:" | sed 's/.*type: "\(.*\)"/\1/'` + .nothrow() + .quiet(); + + if (clusterTypeResult.exitCode !== 0) { + logger.warn("⚠️ Failed to read Kurtosis cluster type from config"); + logger.debug(clusterTypeResult.stderr.toString()); + logger.info("ℹ️ Assuming local launch mode and continuing gracefully"); + return true; // Continue gracefully for local launch + } + + const clusterType = clusterTypeResult.stdout.toString().trim(); + logger.debug(`Kurtosis cluster type: ${clusterType}`); + + // Validate cluster type against expected type + if (kubernetes && clusterType !== "kubernetes") { + logger.error(`❌ Kurtosis cluster type is "${clusterType}" but kubernetes is required`); + return false; + } + + if (!kubernetes && clusterType !== "docker") { + logger.error(`❌ Kurtosis cluster type is "${clusterType}" but docker is required`); + return false; + } + + logger.success(`Kurtosis cluster type "${clusterType}" is compatible`); + return true; +}; diff --git a/test/cli/handlers/common/consts.ts b/test/launcher/utils/constants.ts similarity index 62% rename from test/cli/handlers/common/consts.ts rename to test/launcher/utils/constants.ts index 9a11b874..be6eaa7e 100644 --- a/test/cli/handlers/common/consts.ts +++ b/test/launcher/utils/constants.ts @@ -4,21 +4,20 @@ export const ZERO_HASH = "0x0000000000000000000000000000000000000000000000000000 * The name of the Docker network that DataHaven nodes and * Snowbridge relayers will be connected to, in a local deployment. */ + export const DOCKER_NETWORK_NAME = "datahaven-net"; /** - * 33-byte compressed public keys for DataHaven next validator set - * These correspond to Alice & Bob - * These are the fallback keys if we can't fetch the next authorities directly from the network + * The base services that are always launched when Kurtosis is used. */ -export const FALLBACK_DATAHAVEN_AUTHORITY_PUBLIC_KEYS: Record = { - alice: "0x020a1091341fe5664bfa1782d5e04779689068c916b04cb365ec3153755684d9a1", - bob: "0x0390084fdbf27d2b79d26a4f13f0ccd982cb755a661969143c37cbc49ef5b91f27" -} as const; +export const BASE_SERVICES = [ + "cl-1-lodestar-reth", + "cl-2-lodestar-reth", + "el-1-reth-lodestar", + "el-2-reth-lodestar", + "dora" +]; -/** - * The components (Docker containers) that can be launched and stopped. - */ export const COMPONENTS = { datahaven: { imageName: "moonsonglabs/datahaven", @@ -32,17 +31,6 @@ export const COMPONENTS = { } } as const; -/** - * The base services that are always launched when Kurtosis is used. - */ -export const BASE_SERVICES = [ - "cl-1-lodestar-reth", - "cl-2-lodestar-reth", - "el-1-reth-lodestar", - "el-2-reth-lodestar", - "dora" -]; - /** * Minimum required Bun version */ diff --git a/test/launcher/utils/crypto.ts b/test/launcher/utils/crypto.ts new file mode 100644 index 00000000..7d5a8fa3 --- /dev/null +++ b/test/launcher/utils/crypto.ts @@ -0,0 +1,42 @@ +import { secp256k1 } from "@noble/curves/secp256k1"; +import { keccak_256 } from "@noble/hashes/sha3"; +import type { Hex } from "viem"; + +/** + * Converts a compressed ECDSA public key to an Ethereum address. + * Used for converting BEEFY authorities public keys to Ethereum addresses. + * + * @param compressedPubKey - The compressed public key (33 bytes) + * @returns The Ethereum address derived from the public key + */ +export const compressedPubKeyToEthereumAddress = (compressedPubKey: Hex): Hex => { + // Remove 0x prefix if present + const pubKeyBytes = compressedPubKey.startsWith("0x") + ? compressedPubKey.slice(2) + : compressedPubKey; + + // Convert hex string to Uint8Array + const matches = pubKeyBytes.match(/.{1,2}/g); + if (!matches) { + throw new Error("Invalid hex string format"); + } + const compressedBytes = new Uint8Array(matches.map((byte) => Number.parseInt(byte, 16))); + + // Get the uncompressed point + const point = secp256k1.ProjectivePoint.fromHex(compressedBytes); + const uncompressedBytes = point.toRawBytes(false); // false = uncompressed + + // Remove the first byte (0x04) which indicates uncompressed format + const publicKeyBytes = uncompressedBytes.slice(1); + + // Keccak256 hash of the public key + const hash = keccak_256(publicKeyBytes); + + // Take the last 20 bytes as the Ethereum address + const address = hash.slice(-20); + + // Convert to hex string with 0x prefix + return `0x${Array.from(address) + .map((b) => b.toString(16).padStart(2, "0")) + .join("")}` as Hex; +}; diff --git a/test/launcher/utils/index.ts b/test/launcher/utils/index.ts new file mode 100644 index 00000000..0b71877d --- /dev/null +++ b/test/launcher/utils/index.ts @@ -0,0 +1,3 @@ +export * from "./checks"; +export * from "./constants"; +export * from "./crypto"; diff --git a/test/launcher/validators.ts b/test/launcher/validators.ts new file mode 100644 index 00000000..4d2c6ce8 --- /dev/null +++ b/test/launcher/validators.ts @@ -0,0 +1,81 @@ +import { fundValidators as fundValidatorsScript } from "scripts/fund-validators"; +import { setupValidators as setupValidatorsScript } from "scripts/setup-validators"; +import { updateValidatorSet as updateValidatorSetScript } from "scripts/update-validator-set"; +import { logger } from "utils"; + +/** + * Configuration options for validator operations. + */ +export interface ValidatorOptions { + rpcUrl: string; +} + +/** + * Funds validators with tokens and ETH. + * + * This function ensures validators have the necessary funds to operate by: + * - Sending ETH for gas fees + * - Sending required tokens for staking + * - Verifying balances after funding + * + * @param options - Configuration options for funding + * @param options.rpcUrl - The RPC URL of the Ethereum network + * + * @throws {Error} If funding transactions fail + * @throws {Error} If the network is unreachable + */ +export const fundValidators = async (options: ValidatorOptions): Promise => { + logger.info("💰 Funding validators with tokens and ETH..."); + + await fundValidatorsScript({ + rpcUrl: options.rpcUrl + }); +}; + +/** + * Registers validators in the EigenLayer protocol. + * + * This function handles the validator registration process: + * - Creates operator registrations in EigenLayer + * - Registers operators with the AVS (Actively Validated Service) + * - Sets up delegation relationships + * - Configures operator metadata + * + * @param options - Configuration options for setup + * @param options.rpcUrl - The RPC URL of the Ethereum network + * + * @throws {Error} If registration transactions fail + * @throws {Error} If validators are already registered + * @throws {Error} If required contracts are not deployed + */ +export const setupValidators = async (options: ValidatorOptions): Promise => { + logger.info("📝 Registering validators in EigenLayer..."); + + await setupValidatorsScript({ + rpcUrl: options.rpcUrl + }); +}; + +/** + * Updates the validator set on the Substrate chain. + * + * This function synchronizes the validator set between Ethereum and Substrate: + * - Fetches the current validator set from EigenLayer + * - Prepares validator set update transaction + * - Submits the update through the bridge + * - Waits for confirmation on the Substrate side + * + * @param options - Configuration options for the update + * @param options.rpcUrl - The RPC URL of the Ethereum network + * + * @throws {Error} If the update transaction fails + * @throws {Error} If the bridge is not initialized + * @throws {Error} If validators are not properly registered + */ +export const updateValidatorSet = async (options: ValidatorOptions): Promise => { + logger.info("🔄 Updating validator set on Substrate chain..."); + + await updateValidatorSetScript({ + rpcUrl: options.rpcUrl + }); +}; diff --git a/test/package.json b/test/package.json index 8a979467..141e434c 100644 --- a/test/package.json +++ b/test/package.json @@ -24,7 +24,8 @@ "stop:sb": "bun cli stop --relayer --no-datahaven --no-enclave", "stop:eth": "bun cli stop --enclave --no-datahaven --no-relayer", "stop:engine": "bun cli stop --kurtosisEngine --no-datahaven --no-relayer --no-enclave", - "test:e2e": "bun test suites/e2e --timeout 60000", + "test:e2e": "bun test ./suites --timeout 900000", + "test:e2e:parallel": "bun scripts/test-parallel.ts", "typecheck": "tsc --noEmit", "tsgo": "tsgo tsc --noEmit --pretty --skipLibCheck", "postinstall": "papi" @@ -73,4 +74,4 @@ "ssh2", "utf-8-validate" ] -} +} \ No newline at end of file diff --git a/test/scripts/cargo-crossbuild.ts b/test/scripts/cargo-crossbuild.ts index bc1ff6d1..336c17bb 100644 --- a/test/scripts/cargo-crossbuild.ts +++ b/test/scripts/cargo-crossbuild.ts @@ -8,7 +8,10 @@ const LOG_LEVEL = Bun.env.LOG_LEVEL || "info"; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); -export const cargoCrossbuild = async (options: { datahavenBuildExtraArgs?: string }) => { +export const cargoCrossbuild = async (options: { + datahavenBuildExtraArgs?: string; + networkId?: string; +}) => { logger.info("🔀 Cross-building DataHaven node for Linux AMD64"); const ARCH = (await $`uname -m`.text()).trim(); @@ -32,7 +35,7 @@ export const cargoCrossbuild = async (options: { datahavenBuildExtraArgs?: strin await addRustupTarget(target); // Build and copy libpq.so before cargo zigbuild - await buildAndCopyLibpq(target); + await buildAndCopyLibpq(target, options.networkId); // Get additional arguments from command line const additionalArgs = options.datahavenBuildExtraArgs ?? ""; @@ -95,7 +98,7 @@ const addRustupTarget = async (target: string): Promise => { }; // Updated function to build and copy libpq.so -const buildAndCopyLibpq = async (target: string): Promise => { +const buildAndCopyLibpq = async (target: string, networkId?: string): Promise => { logger.info("🏗️ Building and copying libpq.so..."); // Set Docker platform @@ -107,8 +110,9 @@ const buildAndCopyLibpq = async (target: string): Promise => { await $`docker build -f ${dockerfilePath} -t crossbuild-libpq ${path.join(__dirname, "..", "..")}`.text() ); - // Create container and copy libpq.so - logger.debug(await $`docker create --name linux-libpq-container crossbuild-libpq`.text()); + // Create container with unique name + const containerName = networkId ? `linux-libpq-container-${networkId}` : "linux-libpq-container"; + logger.debug(await $`docker create --name ${containerName} crossbuild-libpq`.text()); const destPath = path.join( __dirname, @@ -125,11 +129,11 @@ const buildAndCopyLibpq = async (target: string): Promise => { fs.mkdirSync(destPath, { recursive: true }); logger.debug( - await $`docker cp linux-libpq-container:/artifacts/libpq.so ${path.join(destPath, "libpq.so")}`.text() + await $`docker cp ${containerName}:/artifacts/libpq.so ${path.join(destPath, "libpq.so")}`.text() ); // Remove container - logger.debug(await $`docker rm linux-libpq-container`.text()); + logger.debug(await $`docker rm ${containerName}`.text()); // Set RUSTFLAGS with the correct library path process.env.RUSTFLAGS = `-C link-arg=-Wl,-rpath,$ORIGIN/../release/deps -L ${destPath}`; diff --git a/test/scripts/fund-validators.ts b/test/scripts/fund-validators.ts index a3216f62..d27c7f03 100644 --- a/test/scripts/fund-validators.ts +++ b/test/scripts/fund-validators.ts @@ -3,7 +3,7 @@ import path from "node:path"; // Script to fund validators with tokens and ETH for local testing import { $ } from "bun"; import invariant from "tiny-invariant"; -import { logger, printDivider, printHeader } from "../utils/index"; +import { logger } from "../utils/index"; interface FundValidatorsOptions { rpcUrl: string; @@ -52,8 +52,6 @@ interface DeploymentInfo { export const fundValidators = async (options: FundValidatorsOptions): Promise => { const { rpcUrl, validatorsConfig, networkName = "anvil", deploymentPath } = options; - printHeader("Funding DataHaven Validators for Local Testing"); - // Validate RPC URL invariant(rpcUrl, "❌ RPC URL is required"); @@ -127,8 +125,6 @@ export const fundValidators = async (options: FundValidatorsOptions): Promise => { const { rpcUrl, validatorsConfig, networkName = "anvil" } = options; - printHeader("Setting Up DataHaven Validators"); - // Validate RPC URL invariant(rpcUrl, "❌ RPC URL is required"); @@ -121,8 +119,6 @@ export const setupValidators = async (options: SetupValidatorsOptions): Promise< logger.success(`Successfully registered validator ${validator.publicKey}`); } - printDivider(); - return true; }; diff --git a/test/scripts/test-parallel.ts b/test/scripts/test-parallel.ts new file mode 100644 index 00000000..cbf75ae6 --- /dev/null +++ b/test/scripts/test-parallel.ts @@ -0,0 +1,357 @@ +#!/usr/bin/env bun +import { existsSync, mkdirSync } from "node:fs"; +import { basename, join } from "node:path"; +import { $ } from "bun"; +import { logger, printHeader } from "../utils"; + +/** + * Script to run all test suites in parallel with concurrency control + */ + +const TEST_TIMEOUT = 900000; // 15 minutes +const LOG_DIR = "tmp/e2e-test-logs"; +const MAX_CONCURRENT_TESTS = 3; // Limit concurrent tests to prevent resource exhaustion + +// Track all spawned processes for cleanup +const spawnedProcesses: Set> = new Set(); + +async function ensureLogDirectory() { + const logPath = join(process.cwd(), LOG_DIR); + if (!existsSync(logPath)) { + mkdirSync(logPath, { recursive: true }); + } + + // Clear content of existing .log files + try { + const existingLogs = await $`find ${logPath} -name "*.log" -type f`.text().catch(() => ""); + const logFiles = existingLogs + .trim() + .split("\n") + .filter((file) => file.length > 0); + + if (logFiles.length > 0) { + logger.info(`🧹 Clearing content of ${logFiles.length} existing log files...`); + // Truncate files to 0 bytes using Bun.write + for (const logFile of logFiles) { + await Bun.write(logFile, ""); + } + } + } catch (error) { + logger.warn("Failed to clear existing log files:", error); + } + return logPath; +} + +async function killAllProcesses() { + logger.info("🛑 Killing all spawned processes..."); + + // Kill all tracked processes and their children + const killPromises = Array.from(spawnedProcesses).map(async (proc) => { + try { + const pid = proc.pid; + logger.info(`Killing process tree for PID ${pid}...`); + + // First, try to get all child processes + try { + // Get all descendant PIDs using pgrep + const childPids = await $`pgrep -P ${pid}`.text().catch(() => ""); + const allPids = [ + pid, + ...childPids + .trim() + .split("\n") + .filter((p) => p) + ] + .map((p) => Number.parseInt(p.toString())) + .filter((p) => !Number.isNaN(p)); + + logger.info(`Found PIDs to kill: ${allPids.join(", ")}`); + + // Kill all processes in reverse order (children first) + for (const targetPid of allPids.reverse()) { + try { + await $`kill -TERM ${targetPid}`.quiet(); + } catch { + // Process might already be dead + } + } + + // Give processes a moment to clean up + await Bun.sleep(500); + + // Force kill any remaining processes + for (const targetPid of allPids) { + try { + await $`kill -KILL ${targetPid}`.quiet(); + } catch { + // Process already dead + } + } + } catch { + // Fallback: try process group kill + try { + await $`kill -TERM -${pid}`.quiet(); + await Bun.sleep(500); + await $`kill -KILL -${pid}`.quiet(); + } catch { + // Process group might not exist + } + } + + // Also try to kill the process directly + try { + proc.kill("SIGKILL"); + } catch { + // Process already dead + } + } catch (error) { + logger.error("Error killing process:", error); + } + }); + + await Promise.all(killPromises); + spawnedProcesses.clear(); + + // Also kill any lingering kurtosis or docker processes started by tests + try { + logger.info("Cleaning up any lingering test processes..."); + + // Kill kurtosis processes + await $`pkill -f "kurtosis.*e2e-test" || true`.quiet(); + + // Find and kill all containers with e2e-test prefix + const containers = await $`docker ps -q --filter "name=e2e-test"`.text().catch(() => ""); + if (containers.trim()) { + logger.info("Killing e2e-test containers..."); + await $`docker kill ${containers.trim().split("\n").join(" ")}`.quiet().catch(() => {}); + } + + // Also clean up any snowbridge containers + const snowbridgeContainers = await $`docker ps -q --filter "name=snowbridge"` + .text() + .catch(() => ""); + if (snowbridgeContainers.trim()) { + logger.info("Killing snowbridge containers..."); + await $`docker kill ${snowbridgeContainers.trim().split("\n").join(" ")}` + .quiet() + .catch(() => {}); + } + + // Kill any remaining bun test processes + await $`pkill -f "bun.*test.*\\.test\\.ts" || true`.quiet(); + } catch { + // Ignore errors - processes might not exist + } +} + +// Set up signal handlers for graceful shutdown +process.on("SIGINT", async () => { + logger.info("\n⚠️ Received SIGINT, cleaning up..."); + await killAllProcesses(); + process.exit(130); // Standard exit code for SIGINT +}); + +process.on("SIGTERM", async () => { + logger.info("\n⚠️ Received SIGTERM, cleaning up..."); + await killAllProcesses(); + process.exit(143); // Standard exit code for SIGTERM +}); + +// Handle uncaught exceptions +process.on("uncaughtException", async (error) => { + logger.error("💥 Uncaught exception:", error); + await killAllProcesses(); + process.exit(1); +}); + +// Handle unhandled promise rejections +process.on("unhandledRejection", async (reason, _promise) => { + logger.error("💥 Unhandled promise rejection:", reason); + await killAllProcesses(); + process.exit(1); +}); + +async function getTestFiles(): Promise { + const result = await $`find suites -name "*.test.ts" -type f`.text(); + return result + .trim() + .split("\n") + .filter((file) => file.length > 0); +} + +async function runTest( + file: string, + logPath: string +): Promise<{ + file: string; + success: boolean; + duration: string; + logFile: string; + exitCode?: number; + error?: any; +}> { + const startTime = Date.now(); + const testName = basename(file, ".test.ts"); + const logFile = join(logPath, `${testName}.log`); + + logger.info(`📋 Starting ${file}...`); + + try { + // Run each test file in its own process group, capturing all output to log file + const proc = Bun.spawn(["bun", "test", file, "--timeout", TEST_TIMEOUT.toString()], { + stdout: "pipe", + stderr: "pipe", + // Create a new process group so we can kill all child processes + env: { + ...process.env, + // This will help identify processes started by this test run + E2E_TEST_RUN_ID: `e2e-test-${Date.now()}-${Math.random().toString(36).slice(2)}` + } + }); + + // Track the spawned process + spawnedProcesses.add(proc); + + // Create write stream for log file + const logFileHandle = Bun.file(logFile); + const writer = logFileHandle.writer(); + + // Write both stdout and stderr to the same log file + const decoder = new TextDecoder(); + + // Handle stdout + const stdoutReader = proc.stdout.getReader(); + const stdoutPromise = (async () => { + while (true) { + const { done, value } = await stdoutReader.read(); + if (done) break; + const text = decoder.decode(value); + await writer.write(text); + } + })(); + + // Handle stderr + const stderrReader = proc.stderr.getReader(); + const stderrPromise = (async () => { + while (true) { + const { done, value } = await stderrReader.read(); + if (done) break; + const text = decoder.decode(value); + await writer.write(text); + } + })(); + + // Wait for process to complete + await Promise.all([stdoutPromise, stderrPromise]); + const exitCode = await proc.exited; + await writer.end(); + + // Remove from tracked processes + spawnedProcesses.delete(proc); + + const duration = ((Date.now() - startTime) / 1000).toFixed(1); + if (exitCode === 0) { + logger.success(`${file} passed (${duration}s) - Log: ${logFile}`); + return { file, success: true, duration, logFile }; + } + logger.error(`❌ ${file} failed (${duration}s) - Log: ${logFile}`); + return { file, success: false, duration, logFile, exitCode }; + } catch (error) { + const duration = ((Date.now() - startTime) / 1000).toFixed(1); + logger.error(`❌ ${file} crashed (${duration}s) - Log: ${logFile}:`, error); + + // Write error to log file + const errorLog = Bun.file(logFile); + await Bun.write(errorLog, `Test crashed with error:\n${error}\n`); + + return { file, success: false, duration, error, logFile }; + } +} + +async function runTestsWithConcurrencyLimit() { + logger.info(`🚀 Starting test suites with max concurrency of ${MAX_CONCURRENT_TESTS}...`); + + // Ensure log directory exists + const logPath = await ensureLogDirectory(); + logger.info(`📁 Logs will be saved to: ${LOG_DIR}/`); + + // Get all test files dynamically + const testFiles = await getTestFiles(); + logger.info(`📋 Found ${testFiles.length} test files:`); + testFiles.forEach((file) => logger.info(` - ${file}`)); + + // Create a queue of test files + const testQueue = [...testFiles]; + const results: Array>> = []; + const runningTests = new Map>(); + + // Process tests with concurrency limit + while (testQueue.length > 0 || runningTests.size > 0) { + // Start new tests if we have capacity + while (runningTests.size < MAX_CONCURRENT_TESTS && testQueue.length > 0) { + const testFile = testQueue.shift(); + if (!testFile) continue; + const testPromise = runTest(testFile, logPath); + + runningTests.set(testFile, testPromise); + + // Add 1 second delay between starting test suites to prevent resource contention + if (testQueue.length > 0) { + await Bun.sleep(1000); + } + + // When test completes, remove it from running tests and store result + testPromise + .then((result) => { + runningTests.delete(testFile); + results.push(result); + }) + .catch((error) => { + runningTests.delete(testFile); + results.push({ + file: testFile, + success: false, + duration: "0", + logFile: join(logPath, `${basename(testFile, ".test.ts")}.log`), + error + }); + }); + } + + // Wait for at least one test to complete before checking again + if (runningTests.size > 0) { + await Promise.race(runningTests.values()); + } + } + + // Summary + printHeader("📊 Test Summary"); + const passed = results.filter((r) => r.success).length; + const failed = results.filter((r) => !r.success).length; + + results.forEach((result) => { + const icon = result.success ? "✅" : "❌"; + logger.info(`${icon} ${result.file} (${result.duration}s)`); + logger.info(` 📄 Log: ${result.logFile}`); + }); + + logger.info(`Total: ${passed} passed, ${failed} failed`); + logger.info(`📁 All logs saved to: ${LOG_DIR}/`); + + // Exit with error if any tests failed + if (failed > 0) { + logger.error("❌ Some tests failed! Check the logs for details."); + await killAllProcesses(); + process.exit(1); + } else { + logger.success("All tests passed!"); + await killAllProcesses(); + } +} + +// Run the tests +runTestsWithConcurrencyLimit().catch(async (error) => { + logger.error("Failed to run tests:", error); + await killAllProcesses(); + process.exit(1); +}); diff --git a/test/scripts/update-validator-set.ts b/test/scripts/update-validator-set.ts index 53d497da..26b9064f 100644 --- a/test/scripts/update-validator-set.ts +++ b/test/scripts/update-validator-set.ts @@ -3,7 +3,7 @@ import path from "node:path"; // Update validator set on DataHaven substrate chain import { $ } from "bun"; import invariant from "tiny-invariant"; -import { logger, printDivider, printHeader } from "../utils/index"; +import { logger } from "../utils/index"; interface UpdateValidatorSetOptions { rpcUrl: string; @@ -19,8 +19,6 @@ interface UpdateValidatorSetOptions { export const updateValidatorSet = async (options: UpdateValidatorSetOptions): Promise => { const { rpcUrl } = options; - printHeader("Updating DataHaven Validator Set"); - // Validate RPC URL invariant(rpcUrl, "❌ RPC URL is required"); @@ -83,8 +81,6 @@ export const updateValidatorSet = async (options: UpdateValidatorSetOptions): Pr } */ - printDivider(); - return true; }; diff --git a/test/suites/contracts.test.ts b/test/suites/contracts.test.ts new file mode 100644 index 00000000..68324ddd --- /dev/null +++ b/test/suites/contracts.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it } from "bun:test"; +import { logger, parseDeploymentsFile } from "utils"; +import { BaseTestSuite } from "../framework"; + +class ContractsTestSuite extends BaseTestSuite { + constructor() { + super({ + suiteName: "contracts" + }); + + this.setupHooks(); + } +} + +// Create the test suite instance +const suite = new ContractsTestSuite(); + +describe("Smart Contract Interactions", () => { + it("should query contract deployment addresses", async () => { + const _connectors = suite.getTestConnectors(); + const deployments = await parseDeploymentsFile(); + + // Check that we have basic contract addresses + expect(deployments.BeefyClient).toBeDefined(); + expect(deployments.Gateway).toBeDefined(); + expect(deployments.ServiceManager).toBeDefined(); + + logger.info(`BeefyClient deployed at: ${deployments.BeefyClient}`); + logger.info(`Gateway deployed at: ${deployments.Gateway}`); + logger.info(`ServiceManager deployed at: ${deployments.ServiceManager}`); + }); + + it("should check contract code exists", async () => { + const connectors = suite.getTestConnectors(); + const deployments = await parseDeploymentsFile(); + + // Get deployment transaction receipt for BeefyClient + const code = await connectors.publicClient.getCode({ + address: deployments.BeefyClient as `0x${string}` + }); + + expect(code).toBeDefined(); + expect(code?.length).toBeGreaterThan(2); // More than just "0x" + + logger.info(`BeefyClient contract code size: ${code?.length} bytes`); + }); + + it("should check contract balances", async () => { + const connectors = suite.getTestConnectors(); + const deployments = await parseDeploymentsFile(); + + // Check ETH balance of contracts + const beefyBalance = await connectors.publicClient.getBalance({ + address: deployments.BeefyClient as `0x${string}` + }); + + const serviceManagerBalance = await connectors.publicClient.getBalance({ + address: deployments.ServiceManager as `0x${string}` + }); + + logger.info(`BeefyClient ETH balance: ${beefyBalance}`); + logger.info(`ServiceManager ETH balance: ${serviceManagerBalance}`); + + // Contracts typically start with 0 balance + expect(beefyBalance).toBeGreaterThanOrEqual(0n); + expect(serviceManagerBalance).toBeGreaterThanOrEqual(0n); + }); + + it("should verify contract addresses are valid", async () => { + const connectors = suite.getTestConnectors(); + const deployments = await parseDeploymentsFile(); + + // List of expected contracts + const expectedContracts = [ + "BeefyClient", + "ServiceManager", + "RewardsRegistry", + "AVSDirectory", + "DelegationManager", + "StrategyManager" + ]; + + for (const contractName of expectedContracts) { + const address = deployments[contractName as keyof typeof deployments]; + + if (address && typeof address === "string") { + // Verify it's a valid address format + expect(address.startsWith("0x")).toBeTrue(); + expect(address.length).toBe(42); + + // Verify contract exists (has code) + const code = await connectors.publicClient.getCode({ + address: address as `0x${string}` + }); + + expect(code).toBeDefined(); + expect(code?.length).toBeGreaterThan(2); + + logger.info(`✓ ${contractName} deployed at ${address}`); + } else { + logger.warn(`⚠️ ${contractName} not found in deployments`); + } + } + }); +}); diff --git a/test/suites/cross-chain.test.ts b/test/suites/cross-chain.test.ts new file mode 100644 index 00000000..565fed02 --- /dev/null +++ b/test/suites/cross-chain.test.ts @@ -0,0 +1,109 @@ +import { beforeAll, describe, expect, it } from "bun:test"; +import type { PolkadotSigner } from "polkadot-api"; +import { getPapiSigner, logger, SUBSTRATE_FUNDED_ACCOUNTS } from "utils"; +import { BaseTestSuite } from "../framework"; + +class CrossChainTestSuite extends BaseTestSuite { + constructor() { + super({ + suiteName: "cross-chain" + }); + + this.setupHooks(); + } + + override async onSetup(): Promise { + // Wait a bit for relayers to fully initialize + logger.info("Waiting for relayers to initialize..."); + await Bun.sleep(10000); // 10 seconds + } +} + +// Create the test suite instance +const suite = new CrossChainTestSuite(); + +describe("Cross-Chain Communication", () => { + let _signer: PolkadotSigner; + + beforeAll(() => { + _signer = getPapiSigner(); + }); + + it("should query Ethereum client state on DataHaven", async () => { + const connectors = suite.getTestConnectors(); + + // Check basic chain connectivity + const blockNumber = await connectors.papiClient.getBlockHeader(); + + logger.info(`Connected to DataHaven at block: ${blockNumber.number}`); + expect(blockNumber.number).toBeGreaterThan(0); + }); + + it("should check beacon relayer status", async () => { + const connectors = suite.getTestConnectors(); + + // Check if we can access chain state + try { + const blockHash = await connectors.papiClient.getFinalizedBlock(); + logger.info(`Finalized block hash: ${blockHash}`); + expect(blockHash).toBeDefined(); + } catch (_error) { + logger.warn("Unable to get finalized block - relayers may still be syncing"); + } + }); + + it("should verify validator registry connection", async () => { + const connectors = suite.getTestConnectors(); + + // For now, just check that we can connect + // The specific storage items depend on the runtime configuration + const blockNumber = await connectors.papiClient.getBlockHeader(); + + logger.info(`Current block number: ${blockNumber.number}`); + expect(blockNumber.number).toBeGreaterThan(0); + }); + + it("should check system information", async () => { + const connectors = suite.getTestConnectors(); + + // Query basic system information + const blockNumber = await connectors.dhApi.query.System.Number.getValue(); + const parentHash = await connectors.dhApi.query.System.ParentHash.getValue(); + + logger.info(`Current block: ${blockNumber}`); + logger.info(`Parent hash: ${parentHash}`); + + expect(blockNumber).toBeGreaterThan(0); + expect(parentHash).toBeDefined(); + }); + + it("should query ethereum client pallet", async () => { + const connectors = suite.getTestConnectors(); + + // Check if we can access account info + const accountInfo = await connectors.dhApi.query.System.Account.getValue( + SUBSTRATE_FUNDED_ACCOUNTS.ALITH.publicKey + ); + + logger.info(`Account nonce: ${accountInfo.nonce}`); + logger.info(`Account providers: ${accountInfo.providers}`); + + expect(accountInfo.providers).toBeGreaterThan(0); + }); + + it("should check BEEFY consensus status", async () => { + const connectors = suite.getTestConnectors(); + + // Query BEEFY validator set + const validatorSet = await connectors.papiClient.getUnsafeApi().apis.BeefyApi.validator_set(); + + if (validatorSet) { + logger.info(`BEEFY validator set ID: ${validatorSet.id}`); + logger.info(`BEEFY validator count: ${validatorSet.validators.length}`); + + expect(validatorSet.validators.length).toBeGreaterThan(0); + } else { + logger.warn("BEEFY validator set not yet available"); + } + }); +}); diff --git a/test/suites/datahaven-substrate.test.ts b/test/suites/datahaven-substrate.test.ts new file mode 100644 index 00000000..7d510f14 --- /dev/null +++ b/test/suites/datahaven-substrate.test.ts @@ -0,0 +1,71 @@ +import { beforeAll, describe, expect, it } from "bun:test"; +import type { PolkadotSigner } from "polkadot-api"; +import { getPapiSigner, logger, SUBSTRATE_FUNDED_ACCOUNTS } from "utils"; +import { isAddress } from "viem"; +import { BaseTestSuite } from "../framework"; + +class DataHavenSubstrateTestSuite extends BaseTestSuite { + constructor() { + super({ + suiteName: "datahaven-substrate" + }); + + this.setupHooks(); + } +} + +// Create the test suite instance +const suite = new DataHavenSubstrateTestSuite(); + +describe("DataHaven Substrate Operations", () => { + let _signer: PolkadotSigner; + + beforeAll(() => { + _signer = getPapiSigner(); + }); + + it("should query runtime API", async () => { + const connectors = suite.getTestConnectors(); + const address = await connectors.dhApi.apis.EthereumRuntimeRPCApi.author(); + + logger.info(`Author address: ${address.asHex()}`); + expect(isAddress(address.asHex())).toBeTrue(); + }); + + it("should lookup account balance", async () => { + const connectors = suite.getTestConnectors(); + const { + data: { free: freeBalance } + } = await connectors.dhApi.query.System.Account.getValue( + SUBSTRATE_FUNDED_ACCOUNTS.ALITH.publicKey + ); + + logger.info(`Balance of ALITH: ${freeBalance}`); + expect(freeBalance).toBeGreaterThan(0n); + }); + + it("should listen to events", async () => { + const connectors = suite.getTestConnectors(); + + // Pull next ExtrinsicSuccess event + const event = await connectors.dhApi.event.System.ExtrinsicSuccess.pull(); + + expect(event).not.toBeEmpty(); + expect(event[0].payload.dispatch_info.weight.ref_time).toBeGreaterThan(0n); + + logger.info( + `Caught ExtrinsicSuccess event with weight: ${event[0].payload.dispatch_info.weight.ref_time}` + ); + }); + + it("should query block information", async () => { + const connectors = suite.getTestConnectors(); + + // Get current block + const blockHeader = await connectors.papiClient.getBlockHeader(); + + expect(blockHeader.number).toBeGreaterThan(0); + + logger.info(`Current block #${blockHeader.number}`); + }); +}); diff --git a/test/suites/e2e/basic.test.ts b/test/suites/e2e/basic.test.ts deleted file mode 100644 index 26b4b43c..00000000 --- a/test/suites/e2e/basic.test.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { beforeAll, describe, expect, it } from "bun:test"; -import { - ANVIL_FUNDED_ACCOUNTS, - createDefaultClient, - generateRandomAccount, - logger, - type ViemClientInterface -} from "utils"; -import { parseEther } from "viem"; - -describe("E2E: Read-only", () => { - let api: ViemClientInterface; - - beforeAll(async () => { - api = await createDefaultClient(); - }); - - it("should be able to query block number", async () => { - const blockNumber = await api.getBlockNumber(); - expect(blockNumber).toBeGreaterThan(0n); - - const balance = await api.getBalance({ - address: ANVIL_FUNDED_ACCOUNTS[0].publicKey - }); - expect(balance).toBeGreaterThan(parseEther("1")); - }); - - it("funds anvil acc 0", async () => { - const balance = await api.getBalance({ - address: ANVIL_FUNDED_ACCOUNTS[0].publicKey - }); - expect(balance).toBeGreaterThan(parseEther("1")); - }); - - it("can send ETH txs", async () => { - const amount = parseEther("1"); - const randomAddress = generateRandomAccount(); - const balanceBefore = await api.getBalance({ - address: randomAddress.address - }); - logger.debug(`Balance of ${randomAddress.address} before: ${balanceBefore}`); - - const hash = await api.sendTransaction({ - to: randomAddress.address, - value: amount - }); - - const receipt = await api.waitForTransactionReceipt({ hash }); - logger.debug(`Transaction receipt: ${receipt}`); - - const balanceAfter = await api.getBalance({ - address: randomAddress.address - }); - - logger.debug(`Balance of ${randomAddress.address} after: ${balanceAfter}`); - expect(balanceAfter - balanceBefore).toBe(amount); - }); -}); diff --git a/test/suites/e2e/beefy-client.test.ts b/test/suites/e2e/beefy-client.test.ts deleted file mode 100644 index 57af5585..00000000 --- a/test/suites/e2e/beefy-client.test.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { beforeAll, describe, expect, it } from "bun:test"; -import { beefyClientAbi } from "contract-bindings"; -import { - type AnvilDeployments, - type ContractInstance, - createDefaultClient, - getContractInstance, - logger, - parseDeploymentsFile, - type ViemClientInterface -} from "utils"; -import { isAddress } from "viem"; - -describe("BeefyClient contract", async () => { - let api: ViemClientInterface; - let deployments: AnvilDeployments; - let instance: ContractInstance<"BeefyClient">; - - beforeAll(async () => { - api = await createDefaultClient(); - deployments = await parseDeploymentsFile(); - instance = await getContractInstance("BeefyClient"); - }); - - it("BeefyClient contract is deployed", async () => { - const contractAddress = deployments.BeefyClient; - expect(isAddress(contractAddress)).toBeTrue(); - }); - - it("latestBeefyBlock() can be read", async () => { - const value = await api.readContract({ - abi: beefyClientAbi, - functionName: "latestBeefyBlock", - address: deployments.BeefyClient - }); - logger.debug(`latestBeefyBlock() value: ${value}`); - expect(value, "Expected contract read to give positive blocknum").toBeGreaterThan(0n); - }); - - it("latestBeefyBlock() can be read from contract instance", async () => { - const value = await instance.read.latestBeefyBlock(); - - logger.debug(`latestBeefyBlock() value: ${value}`); - expect(value, "Expected contract read to give positive blocknum").toBeGreaterThan(0n); - }); -}); diff --git a/test/suites/e2e/datahaven-basic.test.ts b/test/suites/e2e/datahaven-basic.test.ts deleted file mode 100644 index a9d5b87f..00000000 --- a/test/suites/e2e/datahaven-basic.test.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { beforeAll, describe, expect, it } from "bun:test"; -import type { PolkadotSigner } from "polkadot-api"; -import { - createPapiConnectors, - type DataHavenApi, - generateRandomAccount, - getPapiSigner, - logger, - SUBSTRATE_FUNDED_ACCOUNTS -} from "utils"; -import { isAddress, parseEther } from "viem"; - -describe("DataHaven solochain", () => { - let api: DataHavenApi; - let signer: PolkadotSigner; - - beforeAll(() => { - const { typedApi } = createPapiConnectors(); - api = typedApi; - signer = getPapiSigner(); - }); - - it("Can query runtime API", async () => { - const address = await api.apis.EthereumRuntimeRPCApi.author(); - logger.debug(`Author Address is: ${address.asHex()}`); - expect(isAddress(address.asHex())).toBeTrue(); - }); - - it("Can lookup storages ", async () => { - const { - data: { free: freeBalance } - } = await api.query.System.Account.getValue(SUBSTRATE_FUNDED_ACCOUNTS.ALITH.publicKey); - logger.debug(`Balance of ALITH on DH is ${freeBalance}`); - expect(freeBalance).toBeGreaterThan(0n); - }); - - it("Can submit extrinsics into finalized block", async () => { - const value = parseEther("1"); - const { address: dest } = generateRandomAccount(); - const ext = api.tx.Balances.transfer_allow_death({ - dest, - value - }); - - // This will wait until finalized block - const resp = await ext.signAndSubmit(signer, {}); - logger.debug(`Transaction in finalized block: ${resp.txHash}`); - }); - - // This is way faster and should be how we submit build tests - it("Can submit extrinsics into best block", async () => { - const value = parseEther("1"); - const { address: dest } = generateRandomAccount(); - const ext = api.tx.Balances.transfer_allow_death({ - dest, - value - }); - - const resp = await ext.signAndSubmit(signer, { at: "best" }); - logger.debug(`Transaction submitted: ${resp.txHash}`); - const { - data: { free: freeBalance } - } = await api.query.System.Account.getValue(dest, { at: "best" }); - logger.debug(`Balance of ${dest} on DH is ${freeBalance}`); - expect(freeBalance).toBeGreaterThan(0n); - }); - - it("Can listen to events", async () => { - const event = await api.event.System.ExtrinsicSuccess.pull(); - logger.debug(event[0]); - expect(event).not.toBeEmpty(); - expect(event[0].payload.dispatch_info.weight.ref_time).toBeGreaterThan(0n); - }); -}); diff --git a/test/suites/e2e/service-manager.test.ts b/test/suites/e2e/service-manager.test.ts deleted file mode 100644 index 0b61402f..00000000 --- a/test/suites/e2e/service-manager.test.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { beforeAll, describe, expect, it } from "bun:test"; -import { type ContractInstance, getContractInstance, logger } from "utils"; -import { isAddress } from "viem"; - -describe("BeefyClient contract", async () => { - let instance: ContractInstance<"ServiceManager">; - - beforeAll(async () => { - instance = await getContractInstance("ServiceManager"); - }); - - it("avs() can be read from contract instance", async () => { - const value = await instance.read.avs(); - - logger.debug(`avs() value: ${value}`); - expect(isAddress(value), "AVS getter should return an address").toBeTrue(); - }); -}); diff --git a/test/suites/ethereum-basic.test.ts b/test/suites/ethereum-basic.test.ts new file mode 100644 index 00000000..fcad80a4 --- /dev/null +++ b/test/suites/ethereum-basic.test.ts @@ -0,0 +1,173 @@ +import { describe, expect, it } from "bun:test"; +import { ANVIL_FUNDED_ACCOUNTS, generateRandomAccount, logger } from "utils"; +import { parseEther } from "viem"; +import { BaseTestSuite } from "../framework"; + +class EthereumBasicTestSuite extends BaseTestSuite { + constructor() { + super({ + suiteName: "ethereum-basic" + }); + + // Set up hooks in constructor + this.setupHooks(); + } +} + +// Create the test suite instance +const suite = new EthereumBasicTestSuite(); + +describe("Ethereum Basic Operations", () => { + it("should query block number", async () => { + const connectors = suite.getTestConnectors(); + const blockNumber = await connectors.publicClient.getBlockNumber(); + + expect(blockNumber).toBeGreaterThan(0n); + logger.info(`Current block number: ${blockNumber}`); + }); + + it("should check funded account balance", async () => { + const connectors = suite.getTestConnectors(); + const balance = await connectors.publicClient.getBalance({ + address: ANVIL_FUNDED_ACCOUNTS[0].publicKey + }); + + expect(balance).toBeGreaterThan(parseEther("1")); + logger.info(`Account balance: ${balance} wei`); + }); + + it("should send ETH transaction", async () => { + const connectors = suite.getTestConnectors(); + const amount = parseEther("1"); + const randomAccount = generateRandomAccount(); + + // Check initial balance + const balanceBefore = await connectors.publicClient.getBalance({ + address: randomAccount.address + }); + expect(balanceBefore).toBe(0n); + + // Check balance of the sender + const balance = await connectors.publicClient.getBalance({ + address: connectors.walletClient.account.address + }); + expect(balance).toBeGreaterThan(amount); + + // Send transaction + if (!connectors.walletClient.account) { + throw new Error("Wallet client account not available"); + } + const hash = await connectors.walletClient.sendTransaction({ + account: connectors.walletClient.account, + chain: null, + to: randomAccount.address as `0x${string}`, + value: amount + }); + + // Wait for receipt + const receipt = await connectors.publicClient.waitForTransactionReceipt({ hash }); + expect(receipt.status).toBe("success"); + + // Check final balance + const balanceAfter = await connectors.publicClient.getBalance({ + address: randomAccount.address + }); + expect(balanceAfter).toBe(amount); + + logger.info(`Successfully sent ${amount} wei to ${randomAccount.address}`); + }); + + it("should interact with multiple accounts", async () => { + const connectors = suite.getTestConnectors(); + const factory = suite.getConnectorFactory(); + + // Create wallet clients for multiple accounts + const wallet1 = factory.createWalletClient(ANVIL_FUNDED_ACCOUNTS[1].privateKey); + const wallet2 = factory.createWalletClient(ANVIL_FUNDED_ACCOUNTS[2].privateKey); + + const recipient = generateRandomAccount(); + const amount = parseEther("0.5"); + + // Fund wallet1 and wallet2 with 1ETH to successfully send transaction + const initialAmount = parseEther("1"); + + // Give 1ETH to wallet1 + const hashInit1 = await connectors.walletClient.sendTransaction({ + account: connectors.walletClient.account, + chain: null, + to: wallet1.account.address as `0x${string}`, + value: initialAmount + }); + + // Wait for receipt + const receiptInit1 = await connectors.publicClient.waitForTransactionReceipt({ + hash: hashInit1 + }); + expect(receiptInit1.status).toBe("success"); + + const balance1 = await connectors.publicClient.getBalance({ + address: wallet1.account.address + }); + expect(balance1).toBeGreaterThan(parseEther("1")); + + // Give 1ETH to wallet2 + const hashInit2 = await connectors.walletClient.sendTransaction({ + account: connectors.walletClient.account, + chain: null, + to: wallet2.account.address as `0x${string}`, + value: initialAmount + }); + + // Wait for receipt + const receiptInit2 = await connectors.publicClient.waitForTransactionReceipt({ + hash: hashInit2 + }); + expect(receiptInit2.status).toBe("success"); + + const balance2 = await connectors.publicClient.getBalance({ + address: wallet2.account.address + }); + expect(balance2).toBeGreaterThan(parseEther("1")); + + // Send from account 1 + if (!wallet1.account) { + throw new Error("Wallet1 account not available"); + } + + const hash1 = await wallet1.sendTransaction({ + account: wallet1.account, + chain: null, + to: recipient.address as `0x${string}`, + value: amount + }); + + // Send from account 2 + if (!wallet2.account) { + throw new Error("Wallet2 account not available"); + } + + const hash2 = await wallet2.sendTransaction({ + account: wallet2.account, + chain: null, + to: recipient.address as `0x${string}`, + value: amount + }); + + // Wait for both transactions + const [receipt1, receipt2] = await Promise.all([ + connectors.publicClient.waitForTransactionReceipt({ hash: hash1 }), + connectors.publicClient.waitForTransactionReceipt({ hash: hash2 }) + ]); + + expect(receipt1.status).toBe("success"); + expect(receipt2.status).toBe("success"); + + // Check final balance + const finalBalance = await connectors.publicClient.getBalance({ + address: recipient.address + }); + expect(finalBalance).toBe(amount * 2n); + + logger.info(`Received total of ${finalBalance} wei from multiple accounts`); + }, 20_000); +}); diff --git a/test/tsconfig.json b/test/tsconfig.json index b7b9fe1b..f603137a 100644 --- a/test/tsconfig.json +++ b/test/tsconfig.json @@ -47,6 +47,8 @@ "suites/**/*.ts", "cli/**/*.ts", "wagmi.config.ts", - "contract-bindings/*.ts" + "contract-bindings/*.ts", + "launcher/**/*.ts", + "framework/**/*.ts" ] } \ No newline at end of file diff --git a/test/utils/docker.ts b/test/utils/docker.ts index c8a72e9a..665f541b 100644 --- a/test/utils/docker.ts +++ b/test/utils/docker.ts @@ -71,6 +71,14 @@ export const getContainersMatchingImage = async (imageName: string) => { return matches; }; +export const getContainersByPrefix = async (prefix: string) => { + const containers = await docker.listContainers({ all: true }); + const matches = containers.filter((container) => + container.Names.some((name) => name.startsWith(`/${prefix}`)) + ); + return matches; +}; + export const getPublicPort = async ( containerName: string, internalPort: number @@ -179,20 +187,17 @@ export const waitForContainerToStart = async ( ); }; -export const killExistingContainers = async (imageName: string) => { - logger.debug(`Searching for containers with image ${imageName}...`); - const docker = new Docker(); - const containerInfos = (await docker.listContainers({ all: true })).filter((container) => - container.Image.includes(imageName) - ); +export const killExistingContainers = async (prefix: string) => { + logger.debug(`Searching for containers with image ${prefix}...`); + const containerInfos = await getContainersByPrefix(prefix); if (containerInfos.length === 0) { - logger.debug(`No containers found with image ${imageName}`); + logger.debug(`No containers found with name starting with "${prefix}"`); return; } const promises = containerInfos.map(({ Id }) => docker.getContainer(Id).remove({ force: true })); await Promise.all(promises); - logger.debug(`${containerInfos.length} containers with image ${imageName} killed`); + logger.debug(`${containerInfos.length} containers with name starting with "${prefix}" killed`); }; diff --git a/test/utils/index.ts b/test/utils/index.ts index 5f1e32f2..02ac4ccf 100644 --- a/test/utils/index.ts +++ b/test/utils/index.ts @@ -6,6 +6,7 @@ export * from "./input"; export * from "./kurtosis"; export * from "./logger"; export * from "./papi"; +export * from "./parameters"; export * from "./parser"; export * from "./rpc"; export * from "./shell"; diff --git a/test/utils/parameters.ts b/test/utils/parameters.ts index f3e1b9ce..ecb7b547 100644 --- a/test/utils/parameters.ts +++ b/test/utils/parameters.ts @@ -1,7 +1,7 @@ import path from "node:path"; import { $ } from "bun"; -import { logger } from "utils"; -import type { ParsedDataHavenParameter } from "utils/types"; +import { logger } from "./logger"; +import type { ParsedDataHavenParameter } from "./types"; // Constants for paths export const PARAMETERS_TEMPLATE_PATH = "configs/parameters/datahaven-parameters.json"; diff --git a/test/utils/shell.ts b/test/utils/shell.ts index 20b27a45..4d316a49 100644 --- a/test/utils/shell.ts +++ b/test/utils/shell.ts @@ -84,8 +84,10 @@ export const runShellCommandWithLogger = async ( // Only log stderr if the command failed if (exitCode !== 0) { + logger.error("❌ Command failed with exit code:", exitCode); const trimmedStderr = stderrBuffer.trim(); if (trimmedStderr) { + logger.error("Stderr:"); logger.error( trimmedStderr.includes("\n") ? `>_ \n${trimmedStderr}` : `>_ ${trimmedStderr}` );