test: storagehub e2e (#394)

## Summary

Add Storage Hub basic end to end test. This PR also include some fixes
to allow Storage Hub node and datahaven node to run on the same network
(local chain). Before that one was running on dev and the other one on
the local chain.

## What changed

* Added `storagehub.test.ts` e2e test. In this file we explicitly start
the storagehub node using the launch function already used in the CI
* Added Storage Hub backend the flow so it can be used in the e2e test
* Fix the `--chain local` vs `--chain dev` issue. The storagehub nodes
were started on the dev network and therefore they were never syncing
with the datahaven node
* Fix  the folder permission issue in the CI by fixing the folder name
* Added StorageHub javascript lib

---------

Co-authored-by: Gonza Montiel <gonzamontiel@users.noreply.github.com>
Co-authored-by: Ahmad Kaouk <56095276+ahmadkaouk@users.noreply.github.com>
Co-authored-by: Gonza Montiel <gon.montiel@gmail.com>
This commit is contained in:
undercover-cactus 2026-03-11 12:39:47 +01:00 committed by GitHub
parent b4e22035a3
commit 406a0dc59e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 285 additions and 11 deletions

View file

@ -17,7 +17,8 @@
"!**/html/**/*",
"!**/moonwall/contracts/out/**/*",
"!**/contracts/out/**/*",
"!**/contracts/deployments/state-diff.checksum"
"!**/contracts/deployments/state-diff.checksum",
"!**/bun.lock"
],
"maxSize": 3000000
},

View file

@ -50,4 +50,3 @@ examples/
Cargo.lock.old
*.toml.old
*.lock.old
**/target/

View file

@ -55,7 +55,7 @@ COPY --from=builder \
RUN useradd -m -u 1001 -U -s /bin/sh -d /datahaven datahaven && \
mkdir -p /datahaven/.local/share /data && \
chown -R datahaven:datahaven /data && \
ln -s /data /datahaven/.local/share/datahaven
ln -s /data /datahaven/.local/share/datahaven-node
USER datahaven

View file

@ -14,6 +14,9 @@
"@noble/curves": "^1.9.2",
"@noble/hashes": "^1.8.0",
"@polkadot-api/descriptors": "file:.papi/descriptors",
"@storagehub-sdk/core": "^0.4.4",
"@storagehub-sdk/msp-client": "^0.4.4",
"@storagehub/api-augment": "^0.4.0",
"@types/dockerode": "^3.3.41",
"@types/node": "^22.15.32",
"@wagmi/cli": "^2.3.1",
@ -568,6 +571,14 @@
"@sqltools/formatter": ["@sqltools/formatter@1.2.5", "", {}, "sha512-Uy0+khmZqUrUGm5dmMqVlnvufZRSK0FbYzVgp0UMstm+F5+W2/jnEEQyc9vo1ZR/E5ZI/B1WjjoTqBqwJL6Krw=="],
"@storagehub-sdk/core": ["@storagehub-sdk/core@0.4.4", "", { "dependencies": { "@polkadot/types": "^16.4.7", "abitype": "^1.0.0", "ethers": "^6.15.0" }, "peerDependencies": { "viem": ">=2.38.3" } }, "sha512-3tvsp5ILx4r1JWzqef02EKKL+u9nZIrl+/PMpj4Ode17v+mDmYI2ME3On9fZ8/+dEIAXWgqGh8/EjkYdP9PAEQ=="],
"@storagehub-sdk/msp-client": ["@storagehub-sdk/msp-client@0.4.4", "", { "peerDependencies": { "@storagehub-sdk/core": ">=0.0.5", "viem": ">=2.38.3" } }, "sha512-7TLSQAhwJ+RFxU5SbknRw37Qkhts3u2DycdZyA7aUe6e+QyD917QNnlYcM/JJLZFFiqGwy+Nrk07xhKv1zKAZg=="],
"@storagehub/api-augment": ["@storagehub/api-augment@0.4.2", "", { "dependencies": { "@polkadot/api": "^16.4.7", "@polkadot/api-base": "^16.4.7", "@polkadot/rpc-core": "^16.4.7", "@polkadot/typegen": "^16.4.7", "@polkadot/types": "^16.4.7", "@polkadot/types-codec": "^16.4.7", "@storagehub/types-bundle": "0.4.2", "tsx": "4.20.5", "typescript": "^5.9.2" } }, "sha512-L3q5ZsZD+iLPEdBs2ZTKeH5fDaihiUJQpyxSC3pj0geOdE97m+FqxgOALEvAZT7Eqi0m38B0xneREzwPpIGtnA=="],
"@storagehub/types-bundle": ["@storagehub/types-bundle@0.4.2", "", { "dependencies": { "@polkadot/api": "^16.4.7", "@polkadot/api-base": "^16.4.7", "@polkadot/rpc-core": "^16.4.6", "@polkadot/typegen": "^16.4.6", "@polkadot/types": "^16.4.7", "@polkadot/types-codec": "^16.4.7", "typescript": "^5.9.2" } }, "sha512-kkWYP1WwiVP0NGQqIWLfcOsIkb1BJXk7Qw+pkNIzf7QW6HpJaPySJybRksK6ClwKdqzNXXyZ4Sw0vBO1//8h0w=="],
"@substrate/connect": ["@substrate/connect@0.8.11", "", { "dependencies": { "@substrate/connect-extension-protocol": "^2.0.0", "@substrate/connect-known-chains": "^1.1.5", "@substrate/light-client-extension-helpers": "^1.0.0", "smoldot": "2.0.26" } }, "sha512-ofLs1PAO9AtDdPbdyTYj217Pe+lBfTLltdHDs3ds8no0BseoLeAGxpz1mHfi7zB4IxI3YyAiLjH6U8cw4pj4Nw=="],
"@substrate/connect-extension-protocol": ["@substrate/connect-extension-protocol@2.2.2", "", {}, "sha512-t66jwrXA0s5Goq82ZtjagLNd7DPGCNjHeehRlE/gcJmJ+G56C0W+2plqOMRicJ8XGR1/YFnUSEqUFiSNbjGrAA=="],
@ -2180,6 +2191,10 @@
"@safe-global/safe-apps-sdk/viem": ["viem@2.29.2", "", { "dependencies": { "@noble/curves": "1.8.2", "@noble/hashes": "1.7.2", "@scure/bip32": "1.6.2", "@scure/bip39": "1.5.4", "abitype": "1.0.8", "isows": "1.0.6", "ox": "0.6.9", "ws": "8.18.1" }, "peerDependencies": { "typescript": ">=5.0.4" }, "optionalPeers": ["typescript"] }, "sha512-cukRxab90jvQ+TDD84sU3qB3UmejYqgCw4cX8SfWzvh7JPfZXI3kAMUaT5OSR2As1Mgvx1EJawccwPjGqkSSwA=="],
"@storagehub/api-augment/typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
"@storagehub/types-bundle/typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
"@substrate/connect/smoldot": ["smoldot@2.0.26", "", { "dependencies": { "ws": "^8.8.1" } }, "sha512-F+qYmH4z2s2FK+CxGj8moYcd1ekSIKH8ywkdqlOz88Dat35iB1DIYL11aILN46YSGMzQW/lbJNS307zBSDN5Ig=="],
"@substrate/light-client-extension-helpers/@polkadot-api/json-rpc-provider": ["@polkadot-api/json-rpc-provider@0.0.1", "", {}, "sha512-/SMC/l7foRjpykLTUTacIH05H3mr9ip8b5xxfwXlVezXrNVLp3Cv0GX6uItkKd+ZjzVPf3PFrDF2B2/HLSNESA=="],

View file

@ -54,6 +54,8 @@ export const launchDatahavenValidator = async (
const COMMON_LAUNCH_ARGS = [
"--unsafe-force-node-key-generation",
"--tmp",
"--chain",
"local",
"--validator",
"--discover-local",
"--no-prometheus",

View file

@ -0,0 +1,228 @@
/**
* StorageHub E2E Tests
*
* Tests the uploading a file to storage through Datahaven
*
* Prerequisites:
* - DataHaven network with StorageHub service running
* - Storage hub MSP and BSP
*/
import "@storagehub/api-augment";
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
import { TypeRegistry } from "@polkadot/types";
import {
FileManager,
initWasm,
ReplicationLevel,
SH_FILE_SYSTEM_PRECOMPILE_ADDRESS,
StorageHubClient
} from "@storagehub-sdk/core";
import { MspClient } from "@storagehub-sdk/msp-client";
import { $ } from "bun";
import { Binary } from "polkadot-api";
import { createPapiConnectors, logger } from "utils";
import { CHAIN_ID, SUBSTRATE_FUNDED_ACCOUNTS } from "utils/constants";
import { getEvmEcdsaSigner } from "utils/papi";
import { createPublicClient, createWalletClient, defineChain, http } from "viem";
import { privateKeyToAccount } from "viem/accounts";
import { launchLocalDataHavenSolochain } from "../../launcher/datahaven";
import {
launchBackend,
launchBspNode,
launchIndexerNode,
launchMspNode,
launchStorageHubPostgres
} from "../../launcher/storagehub-docker";
import { LaunchedNetwork } from "../../launcher/types/launchedNetwork";
import { registerProviders } from "../../scripts/register-providers";
const TEST_AUTHORITY_IDS = ["alice", "bob"] as const;
const networkId = `storagehub-${Date.now()}`.toLowerCase().replace(/[^a-z0-9-]/g, "-");
describe("test uploading file to storage hub", () => {
let aliceUrl: string;
let _mspUrl: string;
let backendUrl: string;
beforeAll(async () => {
await initWasm();
const datahavenImageTag = "datahavenxyz/datahaven:local";
const relayerImageTag = "datahavenxyz/snowbridge-relay:latest";
const authorityIds = TEST_AUTHORITY_IDS;
const buildDatahaven = false;
const datahavenBuildExtraArgs = "";
const options = {
networkId,
datahavenImageTag,
relayerImageTag,
authorityIds,
buildDatahaven,
datahavenBuildExtraArgs
};
const run = new LaunchedNetwork();
// 1. Launch DataHaven validator nodes
logger.info("📦 Launching DataHaven validator nodes...");
aliceUrl = await launchLocalDataHavenSolochain(options, run);
// 2. Launch PostgreSQL database
logger.info("🗄️ Launching StorageHub PostgreSQL...");
await launchStorageHubPostgres(options, run);
// 3. Launch MSP node
logger.info("📦 Launching MSP node...");
_mspUrl = await launchMspNode(options, run);
// 4. Launch BSP node
logger.info("📦 Launching BSP node...");
await launchBspNode(options, run);
// 6. Launch Indexer node
logger.info("📦 Launching Indexer node...");
await launchIndexerNode(options, run);
// // 7. Launch Fisherman node
// logger.info("📦 Launching Fisherman node...");
// await launchFishermanNode(options, run);
// Register providers
logger.info("📝 Registering providers...");
await registerProviders({ launchedNetwork: run });
// Launch Storage Hub Backend
logger.info("📦 Launching Storage hub backend...");
backendUrl = await launchBackend(options, run);
});
it("Create a bucket", async () => {
const { typedApi: dhApi } = createPapiConnectors(aliceUrl);
const mspCount = await dhApi.query.Providers.MspCount.getValue();
const bspCount = await dhApi.query.Providers.BspCount.getValue();
expect(mspCount).toBe(1);
expect(bspCount).toBe(1);
const msp_id = await dhApi.query.Providers.AccountIdToMainStorageProviderId.getValue(
SUBSTRATE_FUNDED_ACCOUNTS.CHARLETH.publicKey
);
expect(msp_id).toBeDefined();
if (!msp_id) {
throw new Error("mspId for Charleth not found");
}
const value_prop_id =
await dhApi.apis.StorageProvidersApi.query_value_propositions_for_msp(msp_id);
const call = await dhApi.tx.FileSystem.create_bucket({
msp_id,
name: Binary.fromText("bucket"),
private: false,
value_prop_id: value_prop_id[0].id
});
const aliceSigner = getEvmEcdsaSigner(SUBSTRATE_FUNDED_ACCOUNTS.ALITH.privateKey);
const mspResult = await call.signAndSubmit(aliceSigner);
expect(mspResult.ok).toBeTrue();
}, 30000);
it("Send a request", async () => {
const { typedApi: dhApi } = createPapiConnectors(aliceUrl);
const msp_id = await dhApi.query.Providers.AccountIdToMainStorageProviderId.getValue(
SUBSTRATE_FUNDED_ACCOUNTS.CHARLETH.publicKey
);
expect(msp_id).toBeDefined();
if (!msp_id) {
throw new Error("mspId for Charleth not found");
}
const buckets = await dhApi.apis.StorageProvidersApi.query_buckets_for_msp(msp_id);
if (!buckets.success) {
throw new Error("Bucket not found for the registered msp");
}
expect(buckets.value.length).toBe(1);
const bucketId = buckets.value[0].asHex();
const fileContent = "foo bar";
const location = "foo/bar.txt";
// Build FileManager from in-memory file content
const fileBytes = new TextEncoder().encode(fileContent);
const fileManager = new FileManager({
size: fileBytes.length,
stream: () =>
new ReadableStream({
start(controller) {
controller.enqueue(fileBytes);
controller.close();
}
}) as ReadableStream<Uint8Array>
});
// Compute fingerprint and file key from the file metadata
const registry = new TypeRegistry();
const account = privateKeyToAccount(SUBSTRATE_FUNDED_ACCOUNTS.ALITH.privateKey);
const owner = registry.createType("AccountId20", account.address);
const bucketIdH256 = registry.createType("H256", bucketId);
const fingerprint = await fileManager.getFingerprint();
const _fileKey = await fileManager.computeFileKey(owner, bucketIdH256, location);
// Set up EVM clients
const httpUrl = aliceUrl.replace("ws://", "http://");
const chain = defineChain({
id: CHAIN_ID,
name: "DataHaven",
nativeCurrency: { decimals: 18, name: "Ether", symbol: "ETH" },
rpcUrls: { default: { http: [httpUrl] } }
});
const walletClient = createWalletClient({ account, chain, transport: http(httpUrl) });
const publicClient = createPublicClient({ chain, transport: http(httpUrl) });
const storageHubClient = new StorageHubClient({
rpcUrl: httpUrl,
chain,
walletClient,
filesystemContractAddress: SH_FILE_SYSTEM_PRECOMPILE_ADDRESS
});
// Issue storage request
const txHash = await storageHubClient.issueStorageRequest(
bucketId as `0x${string}`,
location,
fingerprint.toHex() as `0x${string}`,
BigInt(fileBytes.length),
msp_id.asHex() as `0x${string}`,
[],
ReplicationLevel.Basic,
1
);
// Wait for storage request transaction
// Don't proceed until receipt is confirmed on chain
if (!txHash) {
throw new Error("Storage request transaction was not submitted");
}
const receipt = await publicClient.waitForTransactionReceipt({ hash: txHash });
if (receipt.status !== "success") {
throw new Error(`Storage request failed: ${txHash}`);
}
console.log("issueStorageRequest() txReceipt:", receipt);
// Authenticate with the backend via SIWE and upload the file
let sessionRef: { token: string; user: { address: string } } | undefined;
const sessionProvider = async () => sessionRef;
const mspClient = await MspClient.connect({ baseUrl: backendUrl }, sessionProvider);
const domain = new URL(backendUrl).host;
const siweSession = await mspClient.auth.SIWE(walletClient, domain, backendUrl);
const sessionToken = (siweSession as { token: string }).token;
expect(sessionToken).toBeDefined();
}, 60000);
afterAll(async () => {
// Delete all the containers started by this test suite
await $`docker container rm -f $(docker container ls -q --filter name=${networkId})`;
});
});

View file

@ -84,7 +84,7 @@ export const getPortMappingForNode = (nodeId: string, networkId: string): string
export const launchLocalDataHavenSolochain = async (
options: DataHavenOptions,
launchedNetwork: LaunchedNetwork
): Promise<void> => {
): Promise<string> => {
logger.info("🚀 Launching DataHaven network...");
invariant(options.datahavenImageTag, "❌ DataHaven image tag not defined");
@ -165,6 +165,8 @@ export const launchLocalDataHavenSolochain = async (
await setupDataHavenValidatorConfig(launchedNetwork, "datahaven-");
logger.success(`DataHaven network started, primary node accessible on port ${alicePort}`);
return `ws://127.0.0.1:${alicePort}`;
};
/**

View file

@ -124,7 +124,7 @@ export const injectStorageHubKey = async (
// Use Bun's $ directly with docker exec (no sh -c wrapper needed)
// This properly handles the spaces in the seed phrase
try {
await $`docker exec ${containerName} datahaven-node key insert --base-path /data --key-type bcsv --scheme ecdsa --suri ${secretKey}`;
await $`docker exec ${containerName} datahaven-node key insert --chain local --key-type bcsv --scheme ecdsa --suri ${secretKey}`;
logger.success("Key injected successfully");
} catch (error) {
logger.error(`Failed to inject key : ${error}`);
@ -141,7 +141,7 @@ export const injectStorageHubKey = async (
export const launchMspNode = async (
options: DataHavenOptions,
launchedNetwork: LaunchedNetwork
): Promise<void> => {
): Promise<string> => {
logger.info("🚀 Launching StorageHub MSP node...");
const containerName = `storagehub-msp-${options.networkId}`;
@ -182,7 +182,10 @@ export const launchMspNode = async (
"--max-storage-capacity",
"10737418240", // 10 GiB
"--jump-capacity",
"1073741824" // 1 GiB
"1073741824", // 1 GiB
"--trusted-file-transfer-server",
"--trusted-file-transfer-server-host",
"0.0.0.0" // Listen on all interfaces so the backend container can reach it
];
logger.debug(`Executing: ${command.join(" ")}`);
@ -217,6 +220,8 @@ export const launchMspNode = async (
launchedNetwork.addContainer(containerName, { ws: wsPort }, { ws: DEFAULT_SUBSTRATE_WS_PORT });
logger.success(`MSP node started on port ${wsPort}`);
return `ws://127.0.0.1:${wsPort}`;
};
/**
@ -457,11 +462,12 @@ export const launchFishermanNode = async (
*
* @param options - Configuration options for launching the network
* @param launchedNetwork - The launched network instance to track the node
* @returns The HTTP URL of the backend API (e.g. "http://127.0.0.1:8080")
*/
export const launchBackend = async (
options: DataHavenOptions,
launchedNetwork: LaunchedNetwork
): Promise<void> => {
): Promise<string> => {
logger.info("🚀 Launching StorageHub Backend...");
const backendImage = "moonsonglabs/storage-hub-msp-backend:latest";
@ -484,8 +490,10 @@ export const launchBackend = async (
"-e",
"RUST_LOG=info",
backendImage,
"--chain",
"local",
"--host",
"0.0.0.0",
"--port",
"8080",
"--log-format",
"text",
"--database-url",
@ -507,6 +515,8 @@ export const launchBackend = async (
launchedNetwork.addContainer(containerName, { http: apiPort }, { http: apiPort });
logger.success(`StorageHub Backend container started on port ${apiPort}`);
return `http://127.0.0.1:${apiPort}`;
};
/**

View file

@ -54,6 +54,9 @@
"@noble/curves": "^1.9.2",
"@noble/hashes": "^1.8.0",
"@polkadot-api/descriptors": "file:.papi/descriptors",
"@storagehub-sdk/core": "^0.4.4",
"@storagehub-sdk/msp-client": "^0.4.4",
"@storagehub/api-augment": "^0.4.0",
"@types/dockerode": "^3.3.41",
"@types/node": "^22.15.32",
"@wagmi/cli": "^2.3.1",

View file

@ -212,7 +212,7 @@ export async function verifyProvidersRegistered(
): Promise<boolean> {
logger.info("🔍 Verifying provider registration...");
const aliceContainerName = `datahaven - alice - ${options.launchedNetwork.networkId} `;
const aliceContainerName = `datahaven-alice-${options.launchedNetwork.networkId} `;
const alicePort = options.launchedNetwork.getContainerPort(aliceContainerName);
const { client, typedApi } = createPapiConnectors(`ws://127.0.0.1:${alicePort}`);

View file

@ -1,5 +1,6 @@
import { existsSync } from "node:fs";
import { type Duplex, PassThrough, Transform } from "node:stream";
import { $ } from "bun";
import Docker from "dockerode";
import invariant from "tiny-invariant";
import { logger } from "./logger";
@ -238,6 +239,9 @@ export const waitForContainerToStart = async (
logger.debug(`Waiting for container ${containerName} to start...`);
const seconds = options?.timeoutSeconds ?? 30;
// sleep 2 seconds to see if the started container didn't exit right away
await Bun.sleep(2000);
for (let i = 0; i < seconds; i++) {
const containers = await docker.listContainers();
const container = containers.find((container) =>
@ -245,10 +249,17 @@ export const waitForContainerToStart = async (
);
if (container) {
logger.debug(`Container ${containerName} started after ${i} seconds`);
const result = await $`docker logs ${containerName}`.nothrow().quiet().text();
console.log(result);
return;
}
await Bun.sleep(1000);
}
const result = await $`docker logs ${containerName}`;
console.log(result);
invariant(
false,
`❌ container ${containerName} cannot be found in running container list after ${seconds} seconds`

View file

@ -43,6 +43,7 @@ export const createPapiConnectors = (
): { client: PolkadotClient; typedApi: DataHavenApi } => {
const url = wsUrl ?? "ws://127.0.0.1:9944";
const client = createClient(withPolkadotSdkCompat(getWsProvider(url)));
return { client, typedApi: client.getTypedApi(datahaven) };
};

View file

@ -8,6 +8,8 @@
export const COMMON_LAUNCH_ARGS = [
"--unsafe-force-node-key-generation",
"--tmp",
"--chain",
"local",
"--validator",
"--discover-local",
"--no-prometheus",