mirror of
https://github.com/datahaven-xyz/datahaven
synced 2026-05-24 01:38:32 +00:00
perf(CLI): ⚡ Add option to use local Docker build in CLI for faster iteration (#77)
In this PR: 1. Add new `datahaven-node-local.dockerfile` for building a local image with the locally built DataHaven Node binary. This severely improves iteration speed on running `bun cli` if there are changes in the DH node. Previously, it relied on the published dockerfile, which builds the Cargo project inside of it, taking +20m even with no changes. 2. Building this local dockerfile is integrated to the CLI, which now also asks if the user wants to rebuild the local docker image of the DH node. 3. A new script `cargo-crossbuild` is added, to be able to build the DataHaven node Cargo project both from Mac and Linux, with the target being `x86_64-unknown-linux-gnu`. For building from Mac, it uses `cargo zigbuild`, so `zig` is now a dependency. Building for this target is needed because the docker image is an Ubuntu image, so it will need to run a linux binary. 4. Added `zig` as dependency in docs. 5. CI still uses the docker image built by the CI itself, which builds the Cargo project inside of it. The CI can take advantage of caching for this.
This commit is contained in:
Facundo Farall
GitHub
parent
ae1798a2a4
commit
a86791ec1c
14 changed files with 10544 additions and 9635 deletions
|
|
@ -7,7 +7,8 @@
|
|||
"./target/*",
|
||||
"**/tmp/*",
|
||||
"*.spec.json",
|
||||
"**/.papi/descriptors/**/*"
|
||||
"**/.papi/descriptors/**/*",
|
||||
"**/contract-bindings/**/*"
|
||||
]
|
||||
},
|
||||
"organizeImports": {
|
||||
|
|
|
|||
11
operator/Cargo.lock
generated
11
operator/Cargo.lock
generated
|
|
@ -2434,6 +2434,7 @@ dependencies = [
|
|||
"jsonrpsee 0.24.9",
|
||||
"log",
|
||||
"mmr-rpc",
|
||||
"openssl-sys",
|
||||
"pallet-beefy-mmr",
|
||||
"pallet-ethereum",
|
||||
"pallet-im-online",
|
||||
|
|
@ -7305,6 +7306,15 @@ version = "0.1.6"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e"
|
||||
|
||||
[[package]]
|
||||
name = "openssl-src"
|
||||
version = "300.4.2+3.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "168ce4e058f975fe43e89d9ccf78ca668601887ae736090aacc23ae353c298e2"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openssl-sys"
|
||||
version = "0.9.108"
|
||||
|
|
@ -7313,6 +7323,7 @@ checksum = "e145e1651e858e820e4860f7b9c5e169bc1d8ce1c86043be79fa7b7634821847"
|
|||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
"openssl-src",
|
||||
"pkg-config",
|
||||
"vcpkg",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -50,6 +50,9 @@ jsonrpsee = { version = "0.24.3" }
|
|||
libsecp256k1 = { version = "0.7", default-features = false }
|
||||
log = { version = "0.4.25" }
|
||||
milagro-bls = { version = "1.5.4", default-features = false, package = "snowbridge-milagro-bls" }
|
||||
openssl-sys = { version = "0.9", features = [
|
||||
"vendored",
|
||||
] } # This is just to set the "vendored" feature required for the crossbuild, so that OpenSSL builds from source
|
||||
parity-bytes = { version = "0.1.2", default-features = false }
|
||||
parity-scale-codec = { version = "3.0.0", default-features = false, features = [
|
||||
"derive",
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ futures = { features = ["thread-pool"], workspace = true }
|
|||
hex-literal = { workspace = true }
|
||||
jsonrpsee = { features = ["server"], workspace = true }
|
||||
log = { workspace = true }
|
||||
openssl-sys = { workspace = true }
|
||||
serde_json = { workspace = true, default-features = true }
|
||||
url = { workspace = true }
|
||||
|
||||
|
|
|
|||
|
|
@ -15,6 +15,11 @@
|
|||
- [Bun](https://bun.sh/): TypeScript runtime and package manager
|
||||
- [Docker](https://www.docker.com/): For container management
|
||||
|
||||
##### MacOS
|
||||
|
||||
> [!IMPORTANT]
|
||||
> If you are running this on a Mac, `zig` is a pre-requisite for crossbuilding the node. Instructions for installation can be found [here](https://ziglang.org/learn/getting-started/).
|
||||
|
||||
## QuickStart
|
||||
|
||||
Run:
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import { $ } from "bun";
|
|||
import { type PolkadotClient, createClient } from "polkadot-api";
|
||||
import { withPolkadotSdkCompat } from "polkadot-api/polkadot-sdk-compat";
|
||||
import { getWsProvider } from "polkadot-api/ws-provider/web";
|
||||
import { cargoCrossbuild } from "scripts/cargo-crossbuild";
|
||||
import invariant from "tiny-invariant";
|
||||
import { waitForContainerToStart } from "utils";
|
||||
import { confirmWithTimeout, logger, printDivider, printHeader } from "utils";
|
||||
|
|
@ -14,6 +15,8 @@ import { publicKeyToAddress } from "viem/accounts";
|
|||
import type { LaunchOptions } from ".";
|
||||
import type { LaunchedNetwork } from "./launchedNetwork";
|
||||
|
||||
const LOG_LEVEL = Bun.env.LOG_LEVEL || "info";
|
||||
|
||||
const COMMON_LAUNCH_ARGS = [
|
||||
"--unsafe-force-node-key-generation",
|
||||
"--tmp",
|
||||
|
|
@ -45,6 +48,286 @@ const FALLBACK_DATAHAVEN_AUTHORITY_PUBLIC_KEYS: Record<string, string> = {
|
|||
ferdie: "0x03bc9d0ca094bd5b8b3225d7651eac5d18c1c04bf8ae8f8b263eebca4e1410ed0c"
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* Launches a DataHaven solochain network for testing.
|
||||
*
|
||||
* @param options - Configuration options for launching the network.
|
||||
* @param launchedNetwork - An instance of LaunchedNetwork to track the network's state.
|
||||
*/
|
||||
export const launchDataHavenSolochain = async (
|
||||
options: LaunchOptions,
|
||||
launchedNetwork: LaunchedNetwork
|
||||
) => {
|
||||
printHeader("Starting DataHaven Network");
|
||||
|
||||
let shouldLaunchDataHaven = options.datahaven;
|
||||
|
||||
if ((await checkDataHavenRunning()) && !options.alwaysClean) {
|
||||
logger.info("ℹ️ DataHaven network (Docker containers) is already running.");
|
||||
|
||||
logger.trace("Checking if datahaven option was set via flags");
|
||||
if (options.datahaven === false) {
|
||||
logger.info("Keeping existing DataHaven containers.");
|
||||
|
||||
await registerNodes(launchedNetwork);
|
||||
printDivider();
|
||||
return;
|
||||
}
|
||||
|
||||
if (options.datahaven === true) {
|
||||
logger.info("Proceeding to clean and relaunch DataHaven containers...");
|
||||
await cleanDataHavenContainers();
|
||||
} else {
|
||||
const shouldRelaunch = await confirmWithTimeout(
|
||||
"Do you want to clean and relaunch the DataHaven containers?",
|
||||
true,
|
||||
10
|
||||
);
|
||||
|
||||
if (!shouldRelaunch) {
|
||||
logger.info("Keeping existing DataHaven containers.");
|
||||
|
||||
await registerNodes(launchedNetwork);
|
||||
printDivider();
|
||||
return;
|
||||
}
|
||||
logger.info("Proceeding to clean and relaunch DataHaven containers...");
|
||||
await cleanDataHavenContainers();
|
||||
}
|
||||
}
|
||||
|
||||
if (shouldLaunchDataHaven === undefined) {
|
||||
shouldLaunchDataHaven = await confirmWithTimeout(
|
||||
"Do you want to launch the DataHaven network?",
|
||||
true,
|
||||
10
|
||||
);
|
||||
} else {
|
||||
logger.info(
|
||||
`Using flag option: ${shouldLaunchDataHaven ? "will launch" : "will not launch"} DataHaven network`
|
||||
);
|
||||
}
|
||||
|
||||
if (!shouldLaunchDataHaven) {
|
||||
logger.info("Skipping DataHaven network launch. Done!");
|
||||
printDivider();
|
||||
return;
|
||||
}
|
||||
|
||||
invariant(options.datahavenImageTag, "❌ DataHaven image tag not defined");
|
||||
|
||||
await buildLocalImage(options);
|
||||
await checkTagExists(options.datahavenImageTag);
|
||||
|
||||
for (const id of CLI_AUTHORITY_IDS) {
|
||||
logger.info(`Starting ${id}...`);
|
||||
const containerName = `datahaven-${id}`;
|
||||
|
||||
const command: string[] = [
|
||||
"docker",
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
containerName,
|
||||
...(id === "alice" ? ["-p", `${DEFAULT_PUBLIC_WS_PORT}:9944`] : []),
|
||||
options.datahavenImageTag,
|
||||
`--${id}`,
|
||||
...COMMON_LAUNCH_ARGS
|
||||
];
|
||||
|
||||
logger.debug($`sh -c "${command.join(" ")}"`.text());
|
||||
|
||||
await waitForContainerToStart(containerName);
|
||||
|
||||
// TODO: Un-comment this when it doesn't stop process from hanging
|
||||
// This is working on SH, but not here so probably a Bun defect
|
||||
//
|
||||
// const listeningLine = await waitForLog({
|
||||
// search: "Running JSON-RPC server: addr=0.0.0.0:",
|
||||
// containerName,
|
||||
// timeoutSeconds: 30
|
||||
// });
|
||||
// logger.debug(listeningLine);
|
||||
}
|
||||
|
||||
for (let i = 0; i < 30; i++) {
|
||||
logger.info("Waiting for datahaven to start...");
|
||||
if (await isNetworkReady(DEFAULT_PUBLIC_WS_PORT)) {
|
||||
logger.success(
|
||||
`DataHaven network started, primary node accessible on port ${DEFAULT_PUBLIC_WS_PORT}`
|
||||
);
|
||||
|
||||
await registerNodes(launchedNetwork);
|
||||
|
||||
// Call setupDataHavenValidatorConfig now that nodes are up
|
||||
logger.info("Proceeding with DataHaven validator configuration setup...");
|
||||
await setupDataHavenValidatorConfig(launchedNetwork);
|
||||
|
||||
printDivider();
|
||||
return;
|
||||
}
|
||||
logger.debug("Node not ready, waiting 1 second...");
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
}
|
||||
|
||||
throw new Error("DataHaven network failed to start after 30 seconds");
|
||||
};
|
||||
|
||||
/**
|
||||
* Checks if any DataHaven containers are currently running.
|
||||
*
|
||||
* @returns True if any DataHaven containers are running, false otherwise.
|
||||
*/
|
||||
const checkDataHavenRunning = async (): Promise<boolean> => {
|
||||
// Check for any container whose name starts with "datahaven-"
|
||||
const PIDS = await $`docker ps -q --filter "name=^datahaven-"`.text();
|
||||
return PIDS.trim().length > 0;
|
||||
};
|
||||
|
||||
/**
|
||||
* Stops and removes all DataHaven containers.
|
||||
*/
|
||||
const cleanDataHavenContainers = async (): Promise<void> => {
|
||||
logger.info("🧹 Stopping and removing existing DataHaven containers...");
|
||||
const containerIds = (await $`docker ps -a -q --filter "name=^datahaven-"`.text()).trim();
|
||||
logger.debug(`Container IDs: ${containerIds}`);
|
||||
if (containerIds.length > 0) {
|
||||
const idsArray = containerIds
|
||||
.split("\n")
|
||||
.map((id) => id.trim())
|
||||
.filter((id) => id.length > 0);
|
||||
for (const id of idsArray) {
|
||||
logger.debug(`Stopping container ${id}`);
|
||||
logger.debug(await $`docker stop ${id}`.nothrow().text());
|
||||
logger.debug(await $`docker rm ${id}`.nothrow().text());
|
||||
}
|
||||
}
|
||||
logger.info("✅ Existing DataHaven containers stopped and removed.");
|
||||
};
|
||||
|
||||
/**
|
||||
* Checks if the DataHaven network is ready by sending a POST request to the system_chain method.
|
||||
*
|
||||
* @param port - The port number to check.
|
||||
* @returns True if the network is ready, false otherwise.
|
||||
*/
|
||||
export const isNetworkReady = async (port: number): Promise<boolean> => {
|
||||
const wsUrl = `ws://127.0.0.1:${port}`;
|
||||
let client: PolkadotClient | undefined;
|
||||
try {
|
||||
// Use withPolkadotSdkCompat for consistency, though _request might not strictly need it.
|
||||
client = createClient(withPolkadotSdkCompat(getWsProvider(wsUrl)));
|
||||
const chainName = await client._request<string>("system_chain", []);
|
||||
logger.debug(`isNetworkReady PAPI check successful for port ${port}, chain: ${chainName}`);
|
||||
client.destroy();
|
||||
return !!chainName; // Ensure it's a boolean and chainName is truthy
|
||||
} catch (error) {
|
||||
logger.debug(`isNetworkReady PAPI check failed for port ${port}: ${error}`);
|
||||
if (client) {
|
||||
client.destroy();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
const buildLocalImage = async (options: LaunchOptions) => {
|
||||
let shouldBuildDataHaven = options.buildDatahaven;
|
||||
|
||||
if (shouldBuildDataHaven === undefined) {
|
||||
shouldBuildDataHaven = await confirmWithTimeout(
|
||||
"Do you want to build the DataHaven node local Docker image?",
|
||||
true,
|
||||
10
|
||||
);
|
||||
} else {
|
||||
logger.info(
|
||||
`Using flag option: ${shouldBuildDataHaven ? "will build" : "will not build"} DataHaven node local Docker image`
|
||||
);
|
||||
}
|
||||
|
||||
if (!shouldBuildDataHaven) {
|
||||
logger.info("Skipping DataHaven node local Docker image build. Done!");
|
||||
return;
|
||||
}
|
||||
|
||||
await cargoCrossbuild({ datahavenBuildExtraArgs: options.datahavenBuildExtraArgs });
|
||||
|
||||
logger.info("🐳 Building DataHaven node local Docker image...");
|
||||
if (LOG_LEVEL === "trace") {
|
||||
await $`bun build:docker:operator`;
|
||||
} else {
|
||||
await $`bun build:docker:operator`.quiet();
|
||||
}
|
||||
logger.success("DataHaven node local Docker image build completed successfully");
|
||||
};
|
||||
|
||||
/**
|
||||
* Checks if an image exists locally or on Docker Hub.
|
||||
*
|
||||
* @param tag - The tag of the image to check.
|
||||
* @returns A promise that resolves when the image is found.
|
||||
*/
|
||||
const checkTagExists = async (tag: string) => {
|
||||
const cleaned = tag.trim();
|
||||
logger.debug(`Checking if image ${cleaned} is available locally`);
|
||||
const { exitCode: localExists } = await $`docker image inspect ${cleaned}`.nothrow().quiet();
|
||||
|
||||
if (localExists !== 0) {
|
||||
logger.debug(`Checking if image ${cleaned} is available on docker hub`);
|
||||
const result = await $`docker manifest inspect ${cleaned}`.nothrow().quiet();
|
||||
invariant(
|
||||
result.exitCode === 0,
|
||||
`❌ Image ${tag} not found.\n Does this image exist?\n Are you logged and have access to the repository?`
|
||||
);
|
||||
}
|
||||
|
||||
logger.success(`Image ${tag} found locally`);
|
||||
};
|
||||
|
||||
const registerNodes = async (launchedNetwork: LaunchedNetwork) => {
|
||||
const targetContainerName = "datahaven-alice";
|
||||
const aliceHostWsPort = 9944; // Standard host port for Alice's WS, as set during launch.
|
||||
|
||||
logger.debug(`Checking Docker status for container: ${targetContainerName}`);
|
||||
// Use ^ and $ for an exact name match in the filter.
|
||||
const dockerPsOutput = await $`docker ps -q --filter "name=^${targetContainerName}$"`.text();
|
||||
const isContainerRunning = dockerPsOutput.trim().length > 0;
|
||||
|
||||
if (!isContainerRunning) {
|
||||
// If the target Docker container is not running, we cannot register it.
|
||||
throw new Error(
|
||||
`❌ Docker container ${targetContainerName} is not running. Cannot register node.`
|
||||
);
|
||||
}
|
||||
|
||||
// If the Docker container is running, proceed to register it in launchedNetwork.
|
||||
// We use the standard host WS port that "datahaven-alice" is expected to use.
|
||||
logger.info(
|
||||
`✅ Docker container ${targetContainerName} is running. Registering with WS port ${aliceHostWsPort}.`
|
||||
);
|
||||
launchedNetwork.addContainer(targetContainerName, { ws: aliceHostWsPort });
|
||||
logger.success(`👍 Node ${targetContainerName} successfully registered in launchedNetwork.`);
|
||||
};
|
||||
|
||||
// Function to convert compressed public key to Ethereum address
|
||||
export const compressedPubKeyToEthereumAddress = (compressedPubKey: string): string => {
|
||||
// Ensure the input is a hex string and remove "0x" prefix
|
||||
const compressedKeyHex = compressedPubKey.startsWith("0x")
|
||||
? compressedPubKey.substring(2)
|
||||
: compressedPubKey;
|
||||
|
||||
// Decompress the public key
|
||||
const point = secp256k1.ProjectivePoint.fromHex(compressedKeyHex);
|
||||
// toRawBytes(false) returns the uncompressed key (64 bytes, x and y coordinates)
|
||||
const uncompressedPubKeyBytes = point.toRawBytes(false);
|
||||
const uncompressedPubKeyHex = toHex(uncompressedPubKeyBytes); // Prefixes with "0x"
|
||||
|
||||
// Compute the Ethereum address from the uncompressed public key
|
||||
// publicKeyToAddress expects a 0x-prefixed hex string representing the 64-byte uncompressed public key
|
||||
const address = publicKeyToAddress(uncompressedPubKeyHex);
|
||||
return address;
|
||||
};
|
||||
|
||||
/**
|
||||
* Prepares the configuration for DataHaven authorities by converting their
|
||||
* compressed public keys to Ethereum addresses and saving them to a JSON file.
|
||||
|
|
@ -155,257 +438,3 @@ export async function setupDataHavenValidatorConfig(
|
|||
throw new Error(`Failed to update authority hashes in ${configFilePath}.`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Launches a DataHaven solochain network for testing.
|
||||
*
|
||||
* @param options - Configuration options for launching the network.
|
||||
* @param launchedNetwork - An instance of LaunchedNetwork to track the network's state.
|
||||
*/
|
||||
export const launchDataHavenSolochain = async (
|
||||
options: LaunchOptions,
|
||||
launchedNetwork: LaunchedNetwork
|
||||
) => {
|
||||
printHeader("Starting DataHaven Network");
|
||||
|
||||
let shouldLaunchDataHaven = options.datahaven;
|
||||
|
||||
if ((await checkDataHavenRunning()) && !options.alwaysClean) {
|
||||
logger.info("ℹ️ DataHaven network (Docker containers) is already running.");
|
||||
|
||||
logger.trace("Checking if datahaven option was set via flags");
|
||||
if (options.datahaven === false) {
|
||||
logger.info("Keeping existing DataHaven containers.");
|
||||
|
||||
await registerNodes(launchedNetwork);
|
||||
printDivider();
|
||||
return;
|
||||
}
|
||||
|
||||
if (options.datahaven === true) {
|
||||
logger.info("Proceeding to clean and relaunch DataHaven containers...");
|
||||
await cleanDataHavenContainers();
|
||||
} else {
|
||||
const shouldRelaunch = await confirmWithTimeout(
|
||||
"Do you want to clean and relaunch the DataHaven containers?",
|
||||
true,
|
||||
10
|
||||
);
|
||||
|
||||
if (!shouldRelaunch) {
|
||||
logger.info("Keeping existing DataHaven containers.");
|
||||
|
||||
await registerNodes(launchedNetwork);
|
||||
printDivider();
|
||||
return;
|
||||
}
|
||||
logger.info("Proceeding to clean and relaunch DataHaven containers...");
|
||||
await cleanDataHavenContainers();
|
||||
}
|
||||
}
|
||||
|
||||
if (shouldLaunchDataHaven === undefined) {
|
||||
shouldLaunchDataHaven = await confirmWithTimeout(
|
||||
"Do you want to launch the DataHaven network?",
|
||||
true,
|
||||
10
|
||||
);
|
||||
} else {
|
||||
logger.info(
|
||||
`Using flag option: ${shouldLaunchDataHaven ? "will launch" : "will not launch"} DataHaven network`
|
||||
);
|
||||
}
|
||||
|
||||
if (!shouldLaunchDataHaven) {
|
||||
logger.info("Skipping DataHaven network launch. Done!");
|
||||
printDivider();
|
||||
return;
|
||||
}
|
||||
|
||||
invariant(options.datahavenImageTag, "❌ Datahaven image tag not defined");
|
||||
|
||||
await checkTagExists(options.datahavenImageTag);
|
||||
|
||||
const logsPath = `tmp/logs/${launchedNetwork.getRunId()}/`;
|
||||
logger.debug(`Ensuring logs directory exists: ${logsPath}`);
|
||||
await $`mkdir -p ${logsPath}`.quiet();
|
||||
|
||||
for (const id of CLI_AUTHORITY_IDS) {
|
||||
logger.info(`Starting ${id}...`);
|
||||
const containerName = `datahaven-${id}`;
|
||||
|
||||
const command: string[] = [
|
||||
"docker",
|
||||
"run",
|
||||
"-d",
|
||||
"--platform",
|
||||
"linux/amd64",
|
||||
"--name",
|
||||
containerName,
|
||||
...(id === "alice" ? ["-p", `${DEFAULT_PUBLIC_WS_PORT}:9944`] : []),
|
||||
options.datahavenImageTag,
|
||||
`--${id}`,
|
||||
...COMMON_LAUNCH_ARGS
|
||||
];
|
||||
|
||||
logger.debug($`sh -c "${command.join(" ")}"`.text());
|
||||
|
||||
await waitForContainerToStart(containerName);
|
||||
|
||||
// TODO: Un-comment this when it doesn't stop process from hanging
|
||||
// This is working on SH, but not here so probably a Bun defect
|
||||
//
|
||||
// const listeningLine = await waitForLog({
|
||||
// search: "Running JSON-RPC server: addr=0.0.0.0:",
|
||||
// containerName,
|
||||
// timeoutSeconds: 30
|
||||
// });
|
||||
// logger.debug(listeningLine);
|
||||
}
|
||||
|
||||
for (let i = 0; i < 30; i++) {
|
||||
logger.info("Waiting for datahaven to start...");
|
||||
if (await isNetworkReady(DEFAULT_PUBLIC_WS_PORT)) {
|
||||
logger.success(
|
||||
`DataHaven network started, primary node accessible on port ${DEFAULT_PUBLIC_WS_PORT}`
|
||||
);
|
||||
|
||||
await registerNodes(launchedNetwork);
|
||||
|
||||
// Call setupDataHavenValidatorConfig now that nodes are up
|
||||
logger.info("Proceeding with DataHaven validator configuration setup...");
|
||||
await setupDataHavenValidatorConfig(launchedNetwork);
|
||||
|
||||
printDivider();
|
||||
return;
|
||||
}
|
||||
logger.debug("Node not ready, waiting 1 second...");
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
}
|
||||
|
||||
throw new Error("Datahaven network failed to start after 30 seconds");
|
||||
};
|
||||
|
||||
/**
|
||||
* Checks if any DataHaven containers are currently running.
|
||||
*
|
||||
* @returns True if any DataHaven containers are running, false otherwise.
|
||||
*/
|
||||
const checkDataHavenRunning = async (): Promise<boolean> => {
|
||||
// Check for any container whose name starts with "datahaven-"
|
||||
const PIDS = await $`docker ps -q --filter "name=^datahaven-"`.text();
|
||||
return PIDS.trim().length > 0;
|
||||
};
|
||||
|
||||
/**
|
||||
* Stops and removes all DataHaven containers.
|
||||
*/
|
||||
const cleanDataHavenContainers = async (): Promise<void> => {
|
||||
logger.info("🧹 Stopping and removing existing DataHaven containers...");
|
||||
const containerIds = (await $`docker ps -a -q --filter "name=^datahaven-"`.text()).trim();
|
||||
logger.debug(`Container IDs: ${containerIds}`);
|
||||
if (containerIds.length > 0) {
|
||||
const idsArray = containerIds
|
||||
.split("\n")
|
||||
.map((id) => id.trim())
|
||||
.filter((id) => id.length > 0);
|
||||
for (const id of idsArray) {
|
||||
logger.debug(`Stopping container ${id}`);
|
||||
logger.debug(await $`docker stop ${id}`.nothrow().text());
|
||||
logger.debug(await $`docker rm ${id}`.nothrow().text());
|
||||
}
|
||||
}
|
||||
logger.info("✅ Existing DataHaven containers stopped and removed.");
|
||||
};
|
||||
|
||||
/**
|
||||
* Checks if the DataHaven network is ready by sending a POST request to the system_chain method.
|
||||
*
|
||||
* @param port - The port number to check.
|
||||
* @returns True if the network is ready, false otherwise.
|
||||
*/
|
||||
export const isNetworkReady = async (port: number): Promise<boolean> => {
|
||||
const wsUrl = `ws://127.0.0.1:${port}`;
|
||||
let client: PolkadotClient | undefined;
|
||||
try {
|
||||
// Use withPolkadotSdkCompat for consistency, though _request might not strictly need it.
|
||||
client = createClient(withPolkadotSdkCompat(getWsProvider(wsUrl)));
|
||||
const chainName = await client._request<string>("system_chain", []);
|
||||
logger.debug(`isNetworkReady PAPI check successful for port ${port}, chain: ${chainName}`);
|
||||
client.destroy();
|
||||
return !!chainName; // Ensure it's a boolean and chainName is truthy
|
||||
} catch (error) {
|
||||
logger.debug(`isNetworkReady PAPI check failed for port ${port}: ${error}`);
|
||||
if (client) {
|
||||
client.destroy();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Checks if an image exists locally or on Docker Hub.
|
||||
*
|
||||
* @param tag - The tag of the image to check.
|
||||
* @returns A promise that resolves when the image is found.
|
||||
*/
|
||||
const checkTagExists = async (tag: string) => {
|
||||
const cleaned = tag.trim();
|
||||
logger.debug(`Checking if image ${cleaned} is available locally`);
|
||||
const { exitCode: localExists } = await $`docker image inspect ${cleaned}`.nothrow().quiet();
|
||||
|
||||
if (localExists !== 0) {
|
||||
logger.debug(`Checking if image ${cleaned} is available on docker hub`);
|
||||
const result = await $`docker manifest inspect ${cleaned}`.nothrow().quiet();
|
||||
invariant(
|
||||
result.exitCode === 0,
|
||||
`❌ Image ${tag} not found.\n Does this image exist?\n Are you logged and have access to the repository?`
|
||||
);
|
||||
}
|
||||
|
||||
logger.success(`Image ${tag} found locally`);
|
||||
};
|
||||
|
||||
const registerNodes = async (launchedNetwork: LaunchedNetwork) => {
|
||||
const targetContainerName = "datahaven-alice";
|
||||
const aliceHostWsPort = 9944; // Standard host port for Alice's WS, as set during launch.
|
||||
|
||||
logger.debug(`Checking Docker status for container: ${targetContainerName}`);
|
||||
// Use ^ and $ for an exact name match in the filter.
|
||||
const dockerPsOutput = await $`docker ps -q --filter "name=^${targetContainerName}$"`.text();
|
||||
const isContainerRunning = dockerPsOutput.trim().length > 0;
|
||||
|
||||
if (!isContainerRunning) {
|
||||
// If the target Docker container is not running, we cannot register it.
|
||||
throw new Error(
|
||||
`❌ Docker container ${targetContainerName} is not running. Cannot register node.`
|
||||
);
|
||||
}
|
||||
|
||||
// If the Docker container is running, proceed to register it in launchedNetwork.
|
||||
// We use the standard host WS port that "datahaven-alice" is expected to use.
|
||||
logger.info(
|
||||
`✅ Docker container ${targetContainerName} is running. Registering with WS port ${aliceHostWsPort}.`
|
||||
);
|
||||
launchedNetwork.addContainer(targetContainerName, { ws: aliceHostWsPort });
|
||||
logger.success(`👍 Node ${targetContainerName} successfully registered in launchedNetwork.`);
|
||||
};
|
||||
|
||||
// Function to convert compressed public key to Ethereum address
|
||||
export const compressedPubKeyToEthereumAddress = (compressedPubKey: string): string => {
|
||||
// Ensure the input is a hex string and remove "0x" prefix
|
||||
const compressedKeyHex = compressedPubKey.startsWith("0x")
|
||||
? compressedPubKey.substring(2)
|
||||
: compressedPubKey;
|
||||
|
||||
// Decompress the public key
|
||||
const point = secp256k1.ProjectivePoint.fromHex(compressedKeyHex);
|
||||
// toRawBytes(false) returns the uncompressed key (64 bytes, x and y coordinates)
|
||||
const uncompressedPubKeyBytes = point.toRawBytes(false);
|
||||
const uncompressedPubKeyHex = toHex(uncompressedPubKeyBytes); // Prefixes with "0x"
|
||||
|
||||
// Compute the Ethereum address from the uncompressed public key
|
||||
// publicKeyToAddress expects a 0x-prefixed hex string representing the 64-byte uncompressed public key
|
||||
const address = publicKeyToAddress(uncompressedPubKeyHex);
|
||||
return address;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -22,7 +22,9 @@ export interface LaunchOptions {
|
|||
skipCleaning?: boolean;
|
||||
alwaysClean?: boolean;
|
||||
datahaven?: boolean;
|
||||
buildDatahaven?: boolean;
|
||||
datahavenImageTag?: string;
|
||||
datahavenBuildExtraArgs?: string;
|
||||
kurtosisNetworkArgs?: string;
|
||||
slotTime?: number;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,8 +14,10 @@ function parseIntValue(value: string): number {
|
|||
// So far we only have the launch command
|
||||
// we can expand this to more commands in the future
|
||||
const program = new Command()
|
||||
.option("--d, --datahaven", "(Re)Launch Datahaven network")
|
||||
.option("--nd, --no-datahaven", "Skip launching Datahaven network")
|
||||
.option("--d, --datahaven", "(Re)Launch DataHaven network")
|
||||
.option("--nd, --no-datahaven", "Skip launching DataHaven network")
|
||||
.option("--bd, --build-datahaven", "Build DataHaven node local Docker image")
|
||||
.option("--nbd, --no-build-datahaven", "Skip building DataHaven node local Docker image")
|
||||
.option("--lk, --launch-kurtosis", "Launch Kurtosis Ethereum network with EL and CL clients")
|
||||
.option("--nlk, --no-launch-kurtosis", "Skip launching Kurtosis Ethereum network")
|
||||
.option("--dc, --deploy-contracts", "Deploy smart contracts")
|
||||
|
|
@ -30,6 +32,11 @@ const program = new Command()
|
|||
.option("--nr, --no-relayer", "Skip Snowbridge Relayers")
|
||||
.option("--b, --blockscout", "Enable Blockscout")
|
||||
.option("--slot-time <number>", "Set slot time in seconds", parseIntValue)
|
||||
.option(
|
||||
"--datahaven-build-extra-args <value>",
|
||||
"Extra args for DataHaven node Cargo build (the plain command is `cargo build --release` for linux, `cargo zigbuild --target x86_64-unknown-linux-gnu --release` for mac)",
|
||||
"--features=fast-runtime"
|
||||
)
|
||||
.option("--kurtosis-network-args <value>", "CustomKurtosis network args")
|
||||
.option("--verified", "Verify smart contracts with Blockscout")
|
||||
.option("--always-clean", "Always clean Kurtosis", false)
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
12
test/docker/crossbuild-mac-libpq.dockerfile
Normal file
12
test/docker/crossbuild-mac-libpq.dockerfile
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
# Stage 1: Build libpq
|
||||
FROM ubuntu:noble AS crossbuild-libpq-builder
|
||||
|
||||
RUN apt-get update && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
||||
ca-certificates libpq-dev
|
||||
|
||||
# Stage 2: Copy the compiled libpq.so to a more accessible directory
|
||||
FROM ubuntu:noble AS crossbuild-libpq-artifacts
|
||||
|
||||
# Copy libpq.so from the crossbuild-libpq stage to the /artifacts directory
|
||||
COPY --from=crossbuild-libpq-builder /usr/lib/x86_64-linux-gnu/libpq.so /artifacts/libpq.so
|
||||
40
test/docker/datahaven-node-local.dockerfile
Normal file
40
test/docker/datahaven-node-local.dockerfile
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
# DATAHAVEN_NODE DOCKERFILE
|
||||
#
|
||||
# This Dockerfile expects to have the binary already built.
|
||||
# So it just copies the binary into the image and runs it.
|
||||
#
|
||||
# This is done to speed up iterating while running the E2E CLI.
|
||||
#
|
||||
# Requires to run from /test folder and to copy the binary in the build folder
|
||||
|
||||
FROM ubuntu:noble
|
||||
|
||||
LABEL version="0.1.0"
|
||||
LABEL description="DataHaven Node Local Build"
|
||||
|
||||
ENV RUST_BACKTRACE=1
|
||||
|
||||
RUN apt-get update && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
||||
ca-certificates curl sudo librocksdb-dev libpq-dev && \
|
||||
apt-get autoremove -y && \
|
||||
apt-get clean && \
|
||||
find /var/lib/apt/lists/ -type f -not -name lock -delete && \
|
||||
useradd -m -u 1337 -U -s /bin/sh -d /datahaven datahaven && \
|
||||
mkdir -p /data /datahaven/.local/share /specs /storage && \
|
||||
chown -R datahaven:datahaven /data && \
|
||||
ln -s /data /datahaven/.local/share/datahaven-node && \
|
||||
echo "datahaven ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers && \
|
||||
chmod -R 777 /storage /data
|
||||
|
||||
USER datahaven
|
||||
|
||||
COPY --chown=datahaven:datahaven ./operator/target/x86_64-unknown-linux-gnu/release/datahaven-node /usr/local/bin/datahaven-node
|
||||
RUN chmod uog+x /usr/local/bin/datahaven-node
|
||||
|
||||
EXPOSE 9333 9944 30333 30334 9615
|
||||
|
||||
VOLUME ["/data"]
|
||||
|
||||
ENTRYPOINT ["datahaven-node"]
|
||||
CMD ["--tmp"]
|
||||
|
|
@ -7,14 +7,14 @@
|
|||
"cli": "bun run cli/index.ts",
|
||||
"fmt": "biome check .",
|
||||
"fmt:fix": "biome check --write .",
|
||||
"build:docker:operator": "docker buildx build --platform=linux/amd64 -t moonsonglabs/datahaven:local -f ../operator/Dockerfile ../.",
|
||||
"build:docker:operator": "docker build -t moonsonglabs/datahaven:local -f ./docker/datahaven-node-local.dockerfile ../.",
|
||||
"build:docker:relayer": "bun -e \"import build from './scripts/snowbridge-relayer.ts'; build()\"",
|
||||
"generate:wagmi": "wagmi generate",
|
||||
"generate:snowbridge-cfgs": "bun -e \"import {generateSnowbridgeConfigs} from './scripts/gen-snowbridge-cfgs.ts'; await generateSnowbridgeConfigs()\"",
|
||||
"generate:types": "(cd ../operator && cargo build --release) && bun x papi add --wasm \"../operator/target/release/wbuild/datahaven-stagenet-runtime/datahaven_stagenet_runtime.wasm\" datahaven",
|
||||
"start:e2e:verified": "bun cli --verified --blockscout --deploy-contracts --setup-validators --update-validator-set --fund-validators",
|
||||
"start:e2e:verified:relayers": "bun cli --verified --blockscout --deploy-contracts --setup-validators --update-validator-set --fund-validators --slot-time 1 --relayer --datahaven",
|
||||
"start:e2e:ci": "bun cli --datahaven --launch-kurtosis --deploy-contracts --fund-validators --setup-validators --update-validator-set --relayer --always-clean",
|
||||
"start:e2e:ci": "bun cli --datahaven --no-build-datahaven --launch-kurtosis --deploy-contracts --fund-validators --setup-validators --update-validator-set --relayer --always-clean",
|
||||
"start:e2e:minrelayer": "bun cli --relayer --deploy-contracts --no-setup-validators --no-update-validator-set --no-fund-validators --datahaven",
|
||||
"stop:docker": "docker ps -a --filter 'ancestor=moonsonglabs/datahaven:local' -q | xargs -r docker rm -f",
|
||||
"stop:e2e": "bun stop:docker ;pkill datahaven ; pkill snowbridge-relay ; kurtosis enclave stop datahaven-ethereum && kurtosis clean && kurtosis engine stop && docker container prune -f",
|
||||
|
|
@ -69,4 +69,4 @@
|
|||
"ssh2",
|
||||
"utf-8-validate"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
136
test/scripts/cargo-crossbuild.ts
Normal file
136
test/scripts/cargo-crossbuild.ts
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { $ } from "bun";
|
||||
import { logger } from "utils";
|
||||
|
||||
const LOG_LEVEL = Bun.env.LOG_LEVEL || "info";
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
export const cargoCrossbuild = async (options: {
|
||||
datahavenBuildExtraArgs?: string;
|
||||
}) => {
|
||||
logger.info("🔀 Cross-building DataHaven node for Linux AMD64");
|
||||
|
||||
const ARCH = (await $`uname -m`.text()).trim();
|
||||
const OS = (await $`uname -s`.text()).trim();
|
||||
|
||||
// Case: Apple Silicon
|
||||
if (ARCH === "arm64" && OS === "Darwin") {
|
||||
logger.info("🍎 Apple Silicon detected. Proceeding with cross-building...");
|
||||
|
||||
if (!isCommandAvailable("zig")) {
|
||||
logger.error("Zig is not installed. Please install Zig to proceed.");
|
||||
logger.info(
|
||||
"Instructions to install can be found here: https://ziglang.org/learn/getting-started/"
|
||||
);
|
||||
throw new Error("Zig is not installed");
|
||||
}
|
||||
|
||||
installCargoZigbuild();
|
||||
|
||||
const target = "x86_64-unknown-linux-gnu";
|
||||
addRustupTarget(target);
|
||||
|
||||
// Build and copy libpq.so before cargo zigbuild
|
||||
await buildAndCopyLibpq(target);
|
||||
|
||||
// Get additional arguments from command line
|
||||
const additionalArgs = options.datahavenBuildExtraArgs ?? "";
|
||||
|
||||
const command = `cargo zigbuild --target ${target} --release ${additionalArgs}`;
|
||||
logger.debug(`Running build command: ${command}`);
|
||||
|
||||
if (LOG_LEVEL === "debug") {
|
||||
await $`sh -c "${command}"`.cwd(`${process.cwd()}/../operator`);
|
||||
} else {
|
||||
await $`sh -c "${command}"`.cwd(`${process.cwd()}/../operator`).quiet();
|
||||
}
|
||||
|
||||
// Case: Linux x86
|
||||
} else if (ARCH === "x86_64" && OS === "Linux") {
|
||||
logger.info("🖥️ Linux AMD64 detected. Proceeding with cross-building...");
|
||||
|
||||
const command = "cargo build --release";
|
||||
logger.debug(`Running build command: ${command}`);
|
||||
|
||||
if (LOG_LEVEL === "debug") {
|
||||
await $`sh -c "${command}"`.cwd(`${process.cwd()}/../operator`);
|
||||
} else {
|
||||
await $`sh -c "${command}"`.cwd(`${process.cwd()}/../operator`).quiet();
|
||||
}
|
||||
|
||||
// Case: Unsupported architecture or OS
|
||||
} else {
|
||||
logger.error("🚨 Unsupported architecture or OS. Please use Apple Silicon or Linux AMD64.");
|
||||
logger.info(`Architecture: ${ARCH}; OS: ${OS}`);
|
||||
throw new Error("Unsupported architecture or OS");
|
||||
}
|
||||
};
|
||||
|
||||
const isCommandAvailable = async (command: string): Promise<boolean> => {
|
||||
try {
|
||||
await $`command -v ${command}`.text();
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
const installCargoZigbuild = async (): Promise<void> => {
|
||||
if (!(await $`cargo install --list`.text()).includes("cargo-zigbuild")) {
|
||||
await $`cargo install cargo-zigbuild --locked`.text();
|
||||
}
|
||||
};
|
||||
|
||||
const addRustupTarget = async (target: string): Promise<void> => {
|
||||
if (!(await $`rustup target list --installed`.text()).includes(target)) {
|
||||
await $`rustup target add ${target}`.text();
|
||||
}
|
||||
};
|
||||
|
||||
// Updated function to build and copy libpq.so
|
||||
const buildAndCopyLibpq = async (target: string): Promise<void> => {
|
||||
logger.info("🏗️ Building and copying libpq.so...");
|
||||
|
||||
// Set Docker platform
|
||||
process.env.DOCKER_DEFAULT_PLATFORM = "linux/amd64";
|
||||
|
||||
// Build Docker image
|
||||
const dockerfilePath = path.join(__dirname, "../docker/crossbuild-mac-libpq.dockerfile");
|
||||
logger.debug(
|
||||
await $`docker build -f ${dockerfilePath} -t crossbuild-libpq ${path.join(__dirname, "..", "..")}`.text()
|
||||
);
|
||||
|
||||
// Create container and copy libpq.so
|
||||
logger.debug(await $`docker create --name linux-libpq-container crossbuild-libpq`.text());
|
||||
|
||||
const destPath = path.join(
|
||||
__dirname,
|
||||
"..",
|
||||
"..",
|
||||
"operator",
|
||||
"target",
|
||||
target,
|
||||
"release",
|
||||
"deps"
|
||||
);
|
||||
|
||||
// Ensure the destination directory exists
|
||||
fs.mkdirSync(destPath, { recursive: true });
|
||||
|
||||
logger.debug(
|
||||
await $`docker cp linux-libpq-container:/artifacts/libpq.so ${path.join(destPath, "libpq.so")}`.text()
|
||||
);
|
||||
|
||||
// Remove container
|
||||
logger.debug(await $`docker rm linux-libpq-container`.text());
|
||||
|
||||
// Set RUSTFLAGS with the correct library path
|
||||
process.env.RUSTFLAGS = `-C link-arg=-Wl,-rpath,$ORIGIN/../release/deps -L ${destPath}`;
|
||||
logger.trace(`RUSTFLAGS set to: ${process.env.RUSTFLAGS}`);
|
||||
|
||||
logger.success(`libpq.so has been copied to ${destPath}`);
|
||||
};
|
||||
|
|
@ -10,7 +10,7 @@ import {
|
|||
} from "utils";
|
||||
import { isAddress, parseEther } from "viem";
|
||||
|
||||
describe("Datahaven solochain", () => {
|
||||
describe("DataHaven solochain", () => {
|
||||
let api: DataHavenApi;
|
||||
let signer: PolkadotSigner;
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue