From 9344e243cffbc4e2feac9fe143abab39c32d2909 Mon Sep 17 00:00:00 2001 From: Ahmad Kaouk <56095276+ahmadkaouk@users.noreply.github.com> Date: Mon, 22 Dec 2025 15:57:32 +0100 Subject: [PATCH] perf: Batch runtime parameter updates to speed up E2E setup (#368) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Optimizes the DataHaven parameter configuration during E2E test infrastructure setup by batching multiple extrinsics into a single transaction, reducing setup time. ## Problem Setting runtime parameters required 5 separate `Parameters.set_parameter` calls, each waiting for block finality. This created unnecessary delays during infrastructure setup since each call blocked sequentially. ## Solution - **Batch parameter updates:** Combine all `Parameters.set_parameter `calls into a single `Utility.batch_all` transaction wrapped in `Sudo.sudo`. - **~5× faster parameter setup:** Only wait for finality once instead of 5 separate times - **Code simplification:** Refactored parameter handling code, removing ~190 lines of unnecessary abstractions and complexity --- test/cli/handlers/deploy/parameters.ts | 5 +- test/launcher/parameters.ts | 9 +- test/scripts/set-datahaven-parameters.ts | 164 +++++++------------- test/utils/parameters.ts | 14 +- test/utils/types.ts | 181 +++-------------------- 5 files changed, 91 insertions(+), 282 deletions(-) diff --git a/test/cli/handlers/deploy/parameters.ts b/test/cli/handlers/deploy/parameters.ts index 8cf7fa05..d43a54c2 100644 --- a/test/cli/handlers/deploy/parameters.ts +++ b/test/cli/handlers/deploy/parameters.ts @@ -30,10 +30,7 @@ export const setParametersFromCollection = async ({ const rpcUrl = `ws://127.0.0.1:${DEFAULT_SUBSTRATE_WS_PORT}`; - const parametersSet = await setDataHavenParameters({ - rpcUrl, - parametersFilePath - }); + const parametersSet = await setDataHavenParameters(rpcUrl, parametersFilePath); printDivider(); return parametersSet; diff --git a/test/launcher/parameters.ts b/test/launcher/parameters.ts index 2397ce6f..9f6ffaec 100644 --- a/test/launcher/parameters.ts +++ b/test/launcher/parameters.ts @@ -45,10 +45,11 @@ export const setDataHavenParameters = async (options: ParametersOptions): Promis const rpcUrl = `ws://127.0.0.1:${launchedNetwork.getPublicWsPort()}`; // Execute the parameter update - await setDataHavenParametersScript({ - rpcUrl, - parametersFilePath - }); + const success = await setDataHavenParametersScript(rpcUrl, parametersFilePath); + + if (!success) { + throw new Error("Failed to set DataHaven parameters"); + } logger.success("DataHaven parameters set successfully"); }; diff --git a/test/scripts/set-datahaven-parameters.ts b/test/scripts/set-datahaven-parameters.ts index 5d1479fa..716962b8 100644 --- a/test/scripts/set-datahaven-parameters.ts +++ b/test/scripts/set-datahaven-parameters.ts @@ -4,145 +4,87 @@ import { createClient } from "polkadot-api"; import { withPolkadotSdkCompat } from "polkadot-api/polkadot-sdk-compat"; import { getWsProvider } from "polkadot-api/ws-provider/web"; import { getEvmEcdsaSigner, logger, SUBSTRATE_FUNDED_ACCOUNTS } from "utils"; -import { type ParsedDataHavenParameter, parseJsonToParameters } from "utils/types"; - -// Interface for the options object of setDataHavenParameters -interface SetDataHavenParametersOptions { - rpcUrl: string; - parametersFilePath: string; -} +import { parseJsonToParameters } from "utils/types"; /** * Sets DataHaven runtime parameters on the specified RPC URL from a JSON file. - * - * @param options - Configuration options for setting parameters - * @param options.rpcUrl - The RPC URL of the DataHaven node - * @param options.parametersFilePath - Path to the JSON file containing an array of parameters to set - * @returns Promise resolving to true if parameters were set successfully, false if skipped */ export const setDataHavenParameters = async ( - options: SetDataHavenParametersOptions + rpcUrl: string, + parametersFilePath: string ): Promise => { - const { rpcUrl, parametersFilePath } = options; + const parametersJson = await Bun.file(parametersFilePath).json(); + const parameters = parseJsonToParameters(parametersJson).filter((p) => p.value !== undefined); - // Load parameters from the JSON file - let parameters: ParsedDataHavenParameter[]; - try { - const parametersFile = Bun.file(parametersFilePath); - const parametersJson = await parametersFile.text(); - // Parse and convert the parameters using our utility - parameters = parseJsonToParameters(JSON.parse(parametersJson)); - - if (parameters.length === 0) { - logger.warn("⚠️ The parameters file is empty. No parameters to set."); - return false; - } - } catch (error: any) { - logger.error( - `❌ Error reading or parsing parameters file at '${parametersFilePath}': ${error.message}` - ); - throw error; + if (parameters.length === 0) { + logger.warn("⚠️ No parameters to set."); + return false; } const client = createClient(withPolkadotSdkCompat(getWsProvider(rpcUrl))); - const dhApi = client.getTypedApi(datahaven); - logger.trace("Substrate client created"); - - const signer = getEvmEcdsaSigner(SUBSTRATE_FUNDED_ACCOUNTS.ALITH.privateKey); - logger.trace("Signer created for SUDO (ALITH)"); - - let allSuccessful = true; try { - for (const param of parameters) { - // TODO: Add a graceful way to print the value of the parameter, since it won't always be representable as a hex string - logger.info( - `🔧 Attempting to set parameter: ${param.name.toString()} = ${param.value.asHex()}` - ); + const dhApi = client.getTypedApi(datahaven); + const signer = getEvmEcdsaSigner(SUBSTRATE_FUNDED_ACCOUNTS.ALITH.privateKey); - const setParameterArgs: any = { - key_value: { - type: "RuntimeConfig" as const, - value: { - type: param.name, - value: [param.value] - } - } - }; - - try { - const setParameterCall = dhApi.tx.Parameters.set_parameter(setParameterArgs); - - logger.debug("Parameter set call:"); - logger.debug(setParameterCall.decodedCall); - - const sudoCall = dhApi.tx.Sudo.sudo({ - call: setParameterCall.decodedCall - }); - - logger.debug(`Submitting transaction to set ${String(param.name)}...`); - const txFinalisedPayload = await sudoCall.signAndSubmit(signer); - - if (!txFinalisedPayload.ok) { - logger.error( - `❌ Transaction to set parameter ${String(param.name)} failed. Block: ${txFinalisedPayload.block.hash}, Tx Hash: ${txFinalisedPayload.txHash}` - ); - logger.error(`Events: ${JSON.stringify(txFinalisedPayload.events)}`); - allSuccessful = false; - } - } catch (txError: any) { - logger.error( - `❌ Error submitting transaction for parameter ${String(param.name)}: ${txError.message || txError}` - ); - allSuccessful = false; - } + // Log parameters being set + for (const p of parameters) { + logger.debug(`🔧 Setting ${p.name} = ${p.value!.asHex()}`); } + + // Build parameter calls + const calls = parameters.map( + (p) => + dhApi.tx.Parameters.set_parameter({ + key_value: { + type: "RuntimeConfig", + value: { type: p.name, value: [p.value] } + } + }).decodedCall + ); + + // Batch all calls and wrap in sudo + const tx = dhApi.tx.Sudo.sudo({ + call: dhApi.tx.Utility.batch_all({ calls }).decodedCall + }); + + const result = await tx.signAndSubmit(signer); + + if (!result.ok) { + logger.error(`❌ Transaction failed: ${result.block.hash}`); + return false; + } + + logger.success("Runtime parameters set successfully"); + return true; + } catch (error) { + logger.error(`❌ ${error instanceof Error ? error.message : error}`); + return false; } finally { client.destroy(); - logger.trace("Substrate client destroyed"); } - - if (allSuccessful) { - logger.success("All specified DataHaven parameters processed successfully."); - } else { - logger.warn("Some DataHaven parameters could not be set. Please check logs."); - } - - return allSuccessful; }; -// Allow script to be run directly with CLI arguments +// CLI entry point if (import.meta.main) { const { values } = parseArgs({ args: process.argv, options: { - rpcUrl: { - type: "string", - short: "r" - }, - parametersFile: { - type: "string", - short: "f" - } + rpcUrl: { type: "string", short: "r" }, + parametersFile: { type: "string", short: "f" } }, strict: true }); - if (!values.rpcUrl) { - console.error("Error: --rpc-url parameter is required"); + if (!values.rpcUrl || !values.parametersFile) { + console.error("Usage: --rpc-url --parameters-file "); process.exit(1); } - if (!values.parametersFile) { - console.error("Error: --parameters-file parameter is required."); - process.exit(1); - } - - setDataHavenParameters({ - rpcUrl: values.rpcUrl, - parametersFilePath: values.parametersFile - }).catch((error: Error) => { - console.error("Setting DataHaven parameters failed:", error.message || error); - process.exit(1); - }); + setDataHavenParameters(values.rpcUrl, values.parametersFile) + .then((ok) => process.exit(ok ? 0 : 1)) + .catch((e) => { + console.error(e); + process.exit(1); + }); } diff --git a/test/utils/parameters.ts b/test/utils/parameters.ts index ecb7b547..ca402e49 100644 --- a/test/utils/parameters.ts +++ b/test/utils/parameters.ts @@ -1,7 +1,13 @@ import path from "node:path"; import { $ } from "bun"; import { logger } from "./logger"; -import type { ParsedDataHavenParameter } from "./types"; +import type { DataHavenRuntimeParameterKey } from "./types"; + +/** Raw parameter for JSON storage (hex strings, not FixedSizeBinary) */ +interface RawParameter { + name: DataHavenRuntimeParameterKey; + value: string | null | undefined; +} // Constants for paths export const PARAMETERS_TEMPLATE_PATH = "configs/parameters/datahaven-parameters.json"; @@ -15,13 +21,13 @@ export const PARAMETERS_OUTPUT_PATH = path.join(PARAMETERS_OUTPUT_DIR, PARAMETER * and then generate a JSON file to be used by the setDataHavenParameters script. */ export class ParameterCollection { - private parameters: ParsedDataHavenParameter[] = []; + private parameters: RawParameter[] = []; /** * Adds a parameter to the collection * @param param The parameter to add */ - public addParameter(param: ParsedDataHavenParameter): void { + public addParameter(param: RawParameter): void { // Check if parameter with same name already exists const existingIndex = this.parameters.findIndex((p) => p.name === param.name); if (existingIndex !== -1) { @@ -38,7 +44,7 @@ export class ParameterCollection { /** * Returns the current parameters */ - public getParameters(): ParsedDataHavenParameter[] { + public getParameters(): RawParameter[] { return [...this.parameters]; } diff --git a/test/utils/types.ts b/test/utils/types.ts index c0911c37..b974ede3 100644 --- a/test/utils/types.ts +++ b/test/utils/types.ts @@ -180,174 +180,37 @@ export const parseJsonToBeaconCheckpoint = (jsonInput: any): BeaconCheckpoint => } }; -/** - * The key of the DataHaven runtime parameter. - * This is an union type with all the possible parameter keys. - */ -export type DataHavenRuntimeParameterKey = - | "EthereumGatewayAddress" - | "RewardsRegistryAddress" - | "RewardsUpdateSelector" - | "RewardsAgentOrigin" - | "DatahavenServiceManagerAddress"; +/** Valid parameter names for DataHaven runtime configuration */ +const DATAHAVEN_PARAM_NAMES = [ + "EthereumGatewayAddress", + "RewardsRegistryAddress", + "RewardsUpdateSelector", + "RewardsAgentOrigin", + "DatahavenServiceManagerAddress" +] as const; -/** - * Interface for raw JSON parameters before conversion - */ -export interface RawJsonParameter { - name: DataHavenRuntimeParameterKey; - value: string | null | undefined; -} +export type DataHavenRuntimeParameterKey = (typeof DATAHAVEN_PARAM_NAMES)[number]; -/** - * Schema for raw EthereumGatewayAddress parameter - */ -const rawEthereumGatewayAddressSchema = z.object({ - name: z.literal("EthereumGatewayAddress"), +/** Schema for a single parameter: { name, value } */ +const parameterSchema = z.object({ + name: z.enum(DATAHAVEN_PARAM_NAMES), value: hexStringSchema.nullable().optional() }); -/** - * Schema for raw RewardsRegistryAddress parameter - */ -const rawRewardsRegistryAddressSchema = z.object({ - name: z.literal("RewardsRegistryAddress"), - value: hexStringSchema.nullable().optional() -}); - -/** - * Schema for raw RewardsUpdateSelector parameter - */ -const rawRewardsUpdateSelectorSchema = z.object({ - name: z.literal("RewardsUpdateSelector"), - value: hexStringSchema.nullable().optional() -}); - -/** - * Schema for raw RewardsOrigin parameter - */ -const rawRewardsAgentOriginSchema = z.object({ - name: z.literal("RewardsAgentOrigin"), - value: hexStringSchema.nullable().optional() -}); - -/** - * Schema for raw DatahavenServiceManagerAddress parameter - */ -const rawDatahavenServiceManagerAddressSchema = z.object({ - name: z.literal("DatahavenServiceManagerAddress"), - value: hexStringSchema.nullable().optional() -}); - -/** - * Union schema for raw DataHaven parameters (for parsing JSON) - */ -export const rawDataHavenParameterSchema = z.discriminatedUnion("name", [ - rawEthereumGatewayAddressSchema, - rawRewardsRegistryAddressSchema, - rawRewardsUpdateSelectorSchema, - rawRewardsAgentOriginSchema, - rawDatahavenServiceManagerAddressSchema -]); - -/** - * Schema for an array of raw DataHaven parameters - */ -export const rawDataHavenParametersArraySchema = z.array(rawDataHavenParameterSchema); - -/** - * The parsed type of a DataHaven runtime parameter. - */ export interface ParsedDataHavenParameter { name: DataHavenRuntimeParameterKey; - value: any; + value: FixedSizeBinary | undefined; } -/** - * Converts a parsed raw parameter to its typed version - */ -function convertParameter(rawParam: any): ParsedDataHavenParameter { - if (rawParam.name === "EthereumGatewayAddress" && rawParam.value) { - return { - name: rawParam.name, - value: new FixedSizeBinary<20>(hexToUint8Array(rawParam.value)) - }; - } - if (rawParam.name === "RewardsRegistryAddress" && rawParam.value) { - return { - name: rawParam.name, - value: new FixedSizeBinary<20>(hexToUint8Array(rawParam.value)) - }; - } - if (rawParam.name === "RewardsUpdateSelector" && rawParam.value) { - return { - name: rawParam.name, - value: new FixedSizeBinary<4>(hexToUint8Array(rawParam.value)) - }; - } - if (rawParam.name === "RewardsAgentOrigin" && rawParam.value) { - return { - name: rawParam.name, - value: new FixedSizeBinary<32>(hexToUint8Array(rawParam.value)) - }; - } - if (rawParam.name === "DatahavenServiceManagerAddress" && rawParam.value) { - return { - name: rawParam.name, - value: new FixedSizeBinary<20>(hexToUint8Array(rawParam.value)) - }; - } - - // For other parameter types, add conversion logic here - return rawParam; -} - -/** - * Parses and converts a JSON object into a typed DataHaven parameter. - * - * @param jsonInput - The JSON parameter object to parse. - * @returns The parsed and converted parameter. - */ -export const parseJsonToParameter = (jsonInput: any): ParsedDataHavenParameter => { - try { - // First validate the raw structure - const rawParam = rawDataHavenParameterSchema.parse(jsonInput); - // Then convert to typed version - return convertParameter(rawParam); - } catch (error) { - if (error instanceof z.ZodError) { - throw new Error( - `Invalid JSON structure for DataHaven parameter: ${error.errors - .map((e) => `${e.path.join(".")} - ${e.message}`) - .join(", ")}` - ); - } - throw error; - } -}; - -/** - * Parses and converts an array of JSON parameters into typed DataHaven parameters. - * - * @param jsonInput - Array of JSON parameter objects to parse. - * @returns Array of parsed and converted parameters. - */ -export const parseJsonToParameters = (jsonInput: any[]): ParsedDataHavenParameter[] => { - try { - // First validate the raw structure of all parameters - const rawParams = rawDataHavenParametersArraySchema.parse(jsonInput); - // Then convert each parameter to its typed version - return rawParams.map(convertParameter); - } catch (error) { - if (error instanceof z.ZodError) { - throw new Error( - `Invalid JSON structure for DataHaven parameters array: ${error.errors - .map((e) => `${e.path.join(".")} - ${e.message}`) - .join(", ")}` - ); - } - throw error; - } +/** Parses JSON array into typed parameters with FixedSizeBinary values */ +export const parseJsonToParameters = (jsonInput: unknown[]): ParsedDataHavenParameter[] => { + return z + .array(parameterSchema) + .parse(jsonInput) + .map((p) => ({ + name: p.name, + value: p.value ? new FixedSizeBinary(hexToUint8Array(p.value)) : undefined + })); }; /**