From 406a0dc59e2cdc8139cb48579b5dd6d5c6d39f29 Mon Sep 17 00:00:00 2001 From: undercover-cactus Date: Wed, 11 Mar 2026 12:39:47 +0100 Subject: [PATCH 1/2] test: storagehub e2e (#394) ## Summary Add Storage Hub basic end to end test. This PR also include some fixes to allow Storage Hub node and datahaven node to run on the same network (local chain). Before that one was running on dev and the other one on the local chain. ## What changed * Added `storagehub.test.ts` e2e test. In this file we explicitly start the storagehub node using the launch function already used in the CI * Added Storage Hub backend the flow so it can be used in the e2e test * Fix the `--chain local` vs `--chain dev` issue. The storagehub nodes were started on the dev network and therefore they were never syncing with the datahaven node * Fix the folder permission issue in the CI by fixing the folder name * Added StorageHub javascript lib --------- Co-authored-by: Gonza Montiel Co-authored-by: Ahmad Kaouk <56095276+ahmadkaouk@users.noreply.github.com> Co-authored-by: Gonza Montiel --- biome.json | 3 +- operator/.dockerignore | 1 - operator/Dockerfile | 2 +- test/bun.lock | 15 ++ test/e2e/framework/validators.ts | 2 + test/e2e/suites/storagehub.test.ts | 228 +++++++++++++++++++++++++++++ test/launcher/datahaven.ts | 4 +- test/launcher/storagehub-docker.ts | 22 ++- test/package.json | 3 + test/scripts/register-providers.ts | 2 +- test/utils/docker.ts | 11 ++ test/utils/papi.ts | 1 + test/utils/validators.ts | 2 + 13 files changed, 285 insertions(+), 11 deletions(-) create mode 100644 test/e2e/suites/storagehub.test.ts diff --git a/biome.json b/biome.json index aed14936..2b197a48 100644 --- a/biome.json +++ b/biome.json @@ -17,7 +17,8 @@ "!**/html/**/*", "!**/moonwall/contracts/out/**/*", "!**/contracts/out/**/*", - "!**/contracts/deployments/state-diff.checksum" + "!**/contracts/deployments/state-diff.checksum", + "!**/bun.lock" ], "maxSize": 3000000 }, diff --git a/operator/.dockerignore b/operator/.dockerignore index 8f7393e4..13c345c5 100644 --- a/operator/.dockerignore +++ b/operator/.dockerignore @@ -50,4 +50,3 @@ examples/ Cargo.lock.old *.toml.old *.lock.old -**/target/ \ No newline at end of file diff --git a/operator/Dockerfile b/operator/Dockerfile index 7e52508d..c21849c3 100644 --- a/operator/Dockerfile +++ b/operator/Dockerfile @@ -55,7 +55,7 @@ COPY --from=builder \ RUN useradd -m -u 1001 -U -s /bin/sh -d /datahaven datahaven && \ mkdir -p /datahaven/.local/share /data && \ chown -R datahaven:datahaven /data && \ - ln -s /data /datahaven/.local/share/datahaven + ln -s /data /datahaven/.local/share/datahaven-node USER datahaven diff --git a/test/bun.lock b/test/bun.lock index d95f380f..a80af401 100644 --- a/test/bun.lock +++ b/test/bun.lock @@ -14,6 +14,9 @@ "@noble/curves": "^1.9.2", "@noble/hashes": "^1.8.0", "@polkadot-api/descriptors": "file:.papi/descriptors", + "@storagehub-sdk/core": "^0.4.4", + "@storagehub-sdk/msp-client": "^0.4.4", + "@storagehub/api-augment": "^0.4.0", "@types/dockerode": "^3.3.41", "@types/node": "^22.15.32", "@wagmi/cli": "^2.3.1", @@ -568,6 +571,14 @@ "@sqltools/formatter": ["@sqltools/formatter@1.2.5", "", {}, "sha512-Uy0+khmZqUrUGm5dmMqVlnvufZRSK0FbYzVgp0UMstm+F5+W2/jnEEQyc9vo1ZR/E5ZI/B1WjjoTqBqwJL6Krw=="], + "@storagehub-sdk/core": ["@storagehub-sdk/core@0.4.4", "", { "dependencies": { "@polkadot/types": "^16.4.7", "abitype": "^1.0.0", "ethers": "^6.15.0" }, "peerDependencies": { "viem": ">=2.38.3" } }, "sha512-3tvsp5ILx4r1JWzqef02EKKL+u9nZIrl+/PMpj4Ode17v+mDmYI2ME3On9fZ8/+dEIAXWgqGh8/EjkYdP9PAEQ=="], + + "@storagehub-sdk/msp-client": ["@storagehub-sdk/msp-client@0.4.4", "", { "peerDependencies": { "@storagehub-sdk/core": ">=0.0.5", "viem": ">=2.38.3" } }, "sha512-7TLSQAhwJ+RFxU5SbknRw37Qkhts3u2DycdZyA7aUe6e+QyD917QNnlYcM/JJLZFFiqGwy+Nrk07xhKv1zKAZg=="], + + "@storagehub/api-augment": ["@storagehub/api-augment@0.4.2", "", { "dependencies": { "@polkadot/api": "^16.4.7", "@polkadot/api-base": "^16.4.7", "@polkadot/rpc-core": "^16.4.7", "@polkadot/typegen": "^16.4.7", "@polkadot/types": "^16.4.7", "@polkadot/types-codec": "^16.4.7", "@storagehub/types-bundle": "0.4.2", "tsx": "4.20.5", "typescript": "^5.9.2" } }, "sha512-L3q5ZsZD+iLPEdBs2ZTKeH5fDaihiUJQpyxSC3pj0geOdE97m+FqxgOALEvAZT7Eqi0m38B0xneREzwPpIGtnA=="], + + "@storagehub/types-bundle": ["@storagehub/types-bundle@0.4.2", "", { "dependencies": { "@polkadot/api": "^16.4.7", "@polkadot/api-base": "^16.4.7", "@polkadot/rpc-core": "^16.4.6", "@polkadot/typegen": "^16.4.6", "@polkadot/types": "^16.4.7", "@polkadot/types-codec": "^16.4.7", "typescript": "^5.9.2" } }, "sha512-kkWYP1WwiVP0NGQqIWLfcOsIkb1BJXk7Qw+pkNIzf7QW6HpJaPySJybRksK6ClwKdqzNXXyZ4Sw0vBO1//8h0w=="], + "@substrate/connect": ["@substrate/connect@0.8.11", "", { "dependencies": { "@substrate/connect-extension-protocol": "^2.0.0", "@substrate/connect-known-chains": "^1.1.5", "@substrate/light-client-extension-helpers": "^1.0.0", "smoldot": "2.0.26" } }, "sha512-ofLs1PAO9AtDdPbdyTYj217Pe+lBfTLltdHDs3ds8no0BseoLeAGxpz1mHfi7zB4IxI3YyAiLjH6U8cw4pj4Nw=="], "@substrate/connect-extension-protocol": ["@substrate/connect-extension-protocol@2.2.2", "", {}, "sha512-t66jwrXA0s5Goq82ZtjagLNd7DPGCNjHeehRlE/gcJmJ+G56C0W+2plqOMRicJ8XGR1/YFnUSEqUFiSNbjGrAA=="], @@ -2180,6 +2191,10 @@ "@safe-global/safe-apps-sdk/viem": ["viem@2.29.2", "", { "dependencies": { "@noble/curves": "1.8.2", "@noble/hashes": "1.7.2", "@scure/bip32": "1.6.2", "@scure/bip39": "1.5.4", "abitype": "1.0.8", "isows": "1.0.6", "ox": "0.6.9", "ws": "8.18.1" }, "peerDependencies": { "typescript": ">=5.0.4" }, "optionalPeers": ["typescript"] }, "sha512-cukRxab90jvQ+TDD84sU3qB3UmejYqgCw4cX8SfWzvh7JPfZXI3kAMUaT5OSR2As1Mgvx1EJawccwPjGqkSSwA=="], + "@storagehub/api-augment/typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + + "@storagehub/types-bundle/typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + "@substrate/connect/smoldot": ["smoldot@2.0.26", "", { "dependencies": { "ws": "^8.8.1" } }, "sha512-F+qYmH4z2s2FK+CxGj8moYcd1ekSIKH8ywkdqlOz88Dat35iB1DIYL11aILN46YSGMzQW/lbJNS307zBSDN5Ig=="], "@substrate/light-client-extension-helpers/@polkadot-api/json-rpc-provider": ["@polkadot-api/json-rpc-provider@0.0.1", "", {}, "sha512-/SMC/l7foRjpykLTUTacIH05H3mr9ip8b5xxfwXlVezXrNVLp3Cv0GX6uItkKd+ZjzVPf3PFrDF2B2/HLSNESA=="], diff --git a/test/e2e/framework/validators.ts b/test/e2e/framework/validators.ts index 2ecee8a9..a6c0ff38 100644 --- a/test/e2e/framework/validators.ts +++ b/test/e2e/framework/validators.ts @@ -54,6 +54,8 @@ export const launchDatahavenValidator = async ( const COMMON_LAUNCH_ARGS = [ "--unsafe-force-node-key-generation", "--tmp", + "--chain", + "local", "--validator", "--discover-local", "--no-prometheus", diff --git a/test/e2e/suites/storagehub.test.ts b/test/e2e/suites/storagehub.test.ts new file mode 100644 index 00000000..229b47ca --- /dev/null +++ b/test/e2e/suites/storagehub.test.ts @@ -0,0 +1,228 @@ +/** + * StorageHub E2E Tests + * + * Tests the uploading a file to storage through Datahaven + * + * Prerequisites: + * - DataHaven network with StorageHub service running + * - Storage hub MSP and BSP + */ +import "@storagehub/api-augment"; +import { afterAll, beforeAll, describe, expect, it } from "bun:test"; +import { TypeRegistry } from "@polkadot/types"; +import { + FileManager, + initWasm, + ReplicationLevel, + SH_FILE_SYSTEM_PRECOMPILE_ADDRESS, + StorageHubClient +} from "@storagehub-sdk/core"; +import { MspClient } from "@storagehub-sdk/msp-client"; +import { $ } from "bun"; +import { Binary } from "polkadot-api"; +import { createPapiConnectors, logger } from "utils"; +import { CHAIN_ID, SUBSTRATE_FUNDED_ACCOUNTS } from "utils/constants"; +import { getEvmEcdsaSigner } from "utils/papi"; +import { createPublicClient, createWalletClient, defineChain, http } from "viem"; +import { privateKeyToAccount } from "viem/accounts"; +import { launchLocalDataHavenSolochain } from "../../launcher/datahaven"; +import { + launchBackend, + launchBspNode, + launchIndexerNode, + launchMspNode, + launchStorageHubPostgres +} from "../../launcher/storagehub-docker"; +import { LaunchedNetwork } from "../../launcher/types/launchedNetwork"; +import { registerProviders } from "../../scripts/register-providers"; + +const TEST_AUTHORITY_IDS = ["alice", "bob"] as const; +const networkId = `storagehub-${Date.now()}`.toLowerCase().replace(/[^a-z0-9-]/g, "-"); + +describe("test uploading file to storage hub", () => { + let aliceUrl: string; + let _mspUrl: string; + let backendUrl: string; + + beforeAll(async () => { + await initWasm(); + + const datahavenImageTag = "datahavenxyz/datahaven:local"; + const relayerImageTag = "datahavenxyz/snowbridge-relay:latest"; + const authorityIds = TEST_AUTHORITY_IDS; + const buildDatahaven = false; + const datahavenBuildExtraArgs = ""; + + const options = { + networkId, + datahavenImageTag, + relayerImageTag, + authorityIds, + buildDatahaven, + datahavenBuildExtraArgs + }; + + const run = new LaunchedNetwork(); + + // 1. Launch DataHaven validator nodes + logger.info("📦 Launching DataHaven validator nodes..."); + aliceUrl = await launchLocalDataHavenSolochain(options, run); + + // 2. Launch PostgreSQL database + logger.info("🗄️ Launching StorageHub PostgreSQL..."); + await launchStorageHubPostgres(options, run); + + // 3. Launch MSP node + logger.info("📦 Launching MSP node..."); + _mspUrl = await launchMspNode(options, run); + + // 4. Launch BSP node + logger.info("📦 Launching BSP node..."); + await launchBspNode(options, run); + + // 6. Launch Indexer node + logger.info("📦 Launching Indexer node..."); + await launchIndexerNode(options, run); + + // // 7. Launch Fisherman node + // logger.info("📦 Launching Fisherman node..."); + // await launchFishermanNode(options, run); + + // Register providers + logger.info("📝 Registering providers..."); + await registerProviders({ launchedNetwork: run }); + + // Launch Storage Hub Backend + logger.info("📦 Launching Storage hub backend..."); + backendUrl = await launchBackend(options, run); + }); + + it("Create a bucket", async () => { + const { typedApi: dhApi } = createPapiConnectors(aliceUrl); + + const mspCount = await dhApi.query.Providers.MspCount.getValue(); + const bspCount = await dhApi.query.Providers.BspCount.getValue(); + + expect(mspCount).toBe(1); + expect(bspCount).toBe(1); + + const msp_id = await dhApi.query.Providers.AccountIdToMainStorageProviderId.getValue( + SUBSTRATE_FUNDED_ACCOUNTS.CHARLETH.publicKey + ); + expect(msp_id).toBeDefined(); + if (!msp_id) { + throw new Error("mspId for Charleth not found"); + } + + const value_prop_id = + await dhApi.apis.StorageProvidersApi.query_value_propositions_for_msp(msp_id); + + const call = await dhApi.tx.FileSystem.create_bucket({ + msp_id, + name: Binary.fromText("bucket"), + private: false, + value_prop_id: value_prop_id[0].id + }); + const aliceSigner = getEvmEcdsaSigner(SUBSTRATE_FUNDED_ACCOUNTS.ALITH.privateKey); + const mspResult = await call.signAndSubmit(aliceSigner); + expect(mspResult.ok).toBeTrue(); + }, 30000); + + it("Send a request", async () => { + const { typedApi: dhApi } = createPapiConnectors(aliceUrl); + + const msp_id = await dhApi.query.Providers.AccountIdToMainStorageProviderId.getValue( + SUBSTRATE_FUNDED_ACCOUNTS.CHARLETH.publicKey + ); + expect(msp_id).toBeDefined(); + if (!msp_id) { + throw new Error("mspId for Charleth not found"); + } + + const buckets = await dhApi.apis.StorageProvidersApi.query_buckets_for_msp(msp_id); + if (!buckets.success) { + throw new Error("Bucket not found for the registered msp"); + } + expect(buckets.value.length).toBe(1); + + const bucketId = buckets.value[0].asHex(); + const fileContent = "foo bar"; + const location = "foo/bar.txt"; + + // Build FileManager from in-memory file content + const fileBytes = new TextEncoder().encode(fileContent); + const fileManager = new FileManager({ + size: fileBytes.length, + stream: () => + new ReadableStream({ + start(controller) { + controller.enqueue(fileBytes); + controller.close(); + } + }) as ReadableStream + }); + + // Compute fingerprint and file key from the file metadata + const registry = new TypeRegistry(); + const account = privateKeyToAccount(SUBSTRATE_FUNDED_ACCOUNTS.ALITH.privateKey); + const owner = registry.createType("AccountId20", account.address); + const bucketIdH256 = registry.createType("H256", bucketId); + const fingerprint = await fileManager.getFingerprint(); + const _fileKey = await fileManager.computeFileKey(owner, bucketIdH256, location); + + // Set up EVM clients + const httpUrl = aliceUrl.replace("ws://", "http://"); + const chain = defineChain({ + id: CHAIN_ID, + name: "DataHaven", + nativeCurrency: { decimals: 18, name: "Ether", symbol: "ETH" }, + rpcUrls: { default: { http: [httpUrl] } } + }); + const walletClient = createWalletClient({ account, chain, transport: http(httpUrl) }); + const publicClient = createPublicClient({ chain, transport: http(httpUrl) }); + const storageHubClient = new StorageHubClient({ + rpcUrl: httpUrl, + chain, + walletClient, + filesystemContractAddress: SH_FILE_SYSTEM_PRECOMPILE_ADDRESS + }); + + // Issue storage request + const txHash = await storageHubClient.issueStorageRequest( + bucketId as `0x${string}`, + location, + fingerprint.toHex() as `0x${string}`, + BigInt(fileBytes.length), + msp_id.asHex() as `0x${string}`, + [], + ReplicationLevel.Basic, + 1 + ); + + // Wait for storage request transaction + // Don't proceed until receipt is confirmed on chain + if (!txHash) { + throw new Error("Storage request transaction was not submitted"); + } + const receipt = await publicClient.waitForTransactionReceipt({ hash: txHash }); + if (receipt.status !== "success") { + throw new Error(`Storage request failed: ${txHash}`); + } + console.log("issueStorageRequest() txReceipt:", receipt); + + // Authenticate with the backend via SIWE and upload the file + let sessionRef: { token: string; user: { address: string } } | undefined; + const sessionProvider = async () => sessionRef; + const mspClient = await MspClient.connect({ baseUrl: backendUrl }, sessionProvider); + + const domain = new URL(backendUrl).host; + const siweSession = await mspClient.auth.SIWE(walletClient, domain, backendUrl); + const sessionToken = (siweSession as { token: string }).token; + expect(sessionToken).toBeDefined(); + }, 60000); + + afterAll(async () => { + // Delete all the containers started by this test suite + await $`docker container rm -f $(docker container ls -q --filter name=${networkId})`; + }); +}); diff --git a/test/launcher/datahaven.ts b/test/launcher/datahaven.ts index fcc004cc..429a326a 100644 --- a/test/launcher/datahaven.ts +++ b/test/launcher/datahaven.ts @@ -84,7 +84,7 @@ export const getPortMappingForNode = (nodeId: string, networkId: string): string export const launchLocalDataHavenSolochain = async ( options: DataHavenOptions, launchedNetwork: LaunchedNetwork -): Promise => { +): Promise => { logger.info("🚀 Launching DataHaven network..."); invariant(options.datahavenImageTag, "❌ DataHaven image tag not defined"); @@ -165,6 +165,8 @@ export const launchLocalDataHavenSolochain = async ( await setupDataHavenValidatorConfig(launchedNetwork, "datahaven-"); logger.success(`DataHaven network started, primary node accessible on port ${alicePort}`); + + return `ws://127.0.0.1:${alicePort}`; }; /** diff --git a/test/launcher/storagehub-docker.ts b/test/launcher/storagehub-docker.ts index a9f64c0e..9cb9c7df 100644 --- a/test/launcher/storagehub-docker.ts +++ b/test/launcher/storagehub-docker.ts @@ -124,7 +124,7 @@ export const injectStorageHubKey = async ( // Use Bun's $ directly with docker exec (no sh -c wrapper needed) // This properly handles the spaces in the seed phrase try { - await $`docker exec ${containerName} datahaven-node key insert --base-path /data --key-type bcsv --scheme ecdsa --suri ${secretKey}`; + await $`docker exec ${containerName} datahaven-node key insert --chain local --key-type bcsv --scheme ecdsa --suri ${secretKey}`; logger.success("Key injected successfully"); } catch (error) { logger.error(`Failed to inject key : ${error}`); @@ -141,7 +141,7 @@ export const injectStorageHubKey = async ( export const launchMspNode = async ( options: DataHavenOptions, launchedNetwork: LaunchedNetwork -): Promise => { +): Promise => { logger.info("🚀 Launching StorageHub MSP node..."); const containerName = `storagehub-msp-${options.networkId}`; @@ -182,7 +182,10 @@ export const launchMspNode = async ( "--max-storage-capacity", "10737418240", // 10 GiB "--jump-capacity", - "1073741824" // 1 GiB + "1073741824", // 1 GiB + "--trusted-file-transfer-server", + "--trusted-file-transfer-server-host", + "0.0.0.0" // Listen on all interfaces so the backend container can reach it ]; logger.debug(`Executing: ${command.join(" ")}`); @@ -217,6 +220,8 @@ export const launchMspNode = async ( launchedNetwork.addContainer(containerName, { ws: wsPort }, { ws: DEFAULT_SUBSTRATE_WS_PORT }); logger.success(`MSP node started on port ${wsPort}`); + + return `ws://127.0.0.1:${wsPort}`; }; /** @@ -457,11 +462,12 @@ export const launchFishermanNode = async ( * * @param options - Configuration options for launching the network * @param launchedNetwork - The launched network instance to track the node + * @returns The HTTP URL of the backend API (e.g. "http://127.0.0.1:8080") */ export const launchBackend = async ( options: DataHavenOptions, launchedNetwork: LaunchedNetwork -): Promise => { +): Promise => { logger.info("🚀 Launching StorageHub Backend..."); const backendImage = "moonsonglabs/storage-hub-msp-backend:latest"; @@ -484,8 +490,10 @@ export const launchBackend = async ( "-e", "RUST_LOG=info", backendImage, - "--chain", - "local", + "--host", + "0.0.0.0", + "--port", + "8080", "--log-format", "text", "--database-url", @@ -507,6 +515,8 @@ export const launchBackend = async ( launchedNetwork.addContainer(containerName, { http: apiPort }, { http: apiPort }); logger.success(`StorageHub Backend container started on port ${apiPort}`); + + return `http://127.0.0.1:${apiPort}`; }; /** diff --git a/test/package.json b/test/package.json index ad1f1156..4d521359 100644 --- a/test/package.json +++ b/test/package.json @@ -54,6 +54,9 @@ "@noble/curves": "^1.9.2", "@noble/hashes": "^1.8.0", "@polkadot-api/descriptors": "file:.papi/descriptors", + "@storagehub-sdk/core": "^0.4.4", + "@storagehub-sdk/msp-client": "^0.4.4", + "@storagehub/api-augment": "^0.4.0", "@types/dockerode": "^3.3.41", "@types/node": "^22.15.32", "@wagmi/cli": "^2.3.1", diff --git a/test/scripts/register-providers.ts b/test/scripts/register-providers.ts index 216d509b..5bc850cc 100644 --- a/test/scripts/register-providers.ts +++ b/test/scripts/register-providers.ts @@ -212,7 +212,7 @@ export async function verifyProvidersRegistered( ): Promise { logger.info("🔍 Verifying provider registration..."); - const aliceContainerName = `datahaven - alice - ${options.launchedNetwork.networkId} `; + const aliceContainerName = `datahaven-alice-${options.launchedNetwork.networkId} `; const alicePort = options.launchedNetwork.getContainerPort(aliceContainerName); const { client, typedApi } = createPapiConnectors(`ws://127.0.0.1:${alicePort}`); diff --git a/test/utils/docker.ts b/test/utils/docker.ts index b1c00b62..9d5b438e 100644 --- a/test/utils/docker.ts +++ b/test/utils/docker.ts @@ -1,5 +1,6 @@ import { existsSync } from "node:fs"; import { type Duplex, PassThrough, Transform } from "node:stream"; +import { $ } from "bun"; import Docker from "dockerode"; import invariant from "tiny-invariant"; import { logger } from "./logger"; @@ -238,6 +239,9 @@ export const waitForContainerToStart = async ( logger.debug(`Waiting for container ${containerName} to start...`); const seconds = options?.timeoutSeconds ?? 30; + // sleep 2 seconds to see if the started container didn't exit right away + await Bun.sleep(2000); + for (let i = 0; i < seconds; i++) { const containers = await docker.listContainers(); const container = containers.find((container) => @@ -245,10 +249,17 @@ export const waitForContainerToStart = async ( ); if (container) { logger.debug(`Container ${containerName} started after ${i} seconds`); + const result = await $`docker logs ${containerName}`.nothrow().quiet().text(); + console.log(result); + return; } await Bun.sleep(1000); } + + const result = await $`docker logs ${containerName}`; + console.log(result); + invariant( false, `❌ container ${containerName} cannot be found in running container list after ${seconds} seconds` diff --git a/test/utils/papi.ts b/test/utils/papi.ts index 42a4dc63..1ec97545 100644 --- a/test/utils/papi.ts +++ b/test/utils/papi.ts @@ -43,6 +43,7 @@ export const createPapiConnectors = ( ): { client: PolkadotClient; typedApi: DataHavenApi } => { const url = wsUrl ?? "ws://127.0.0.1:9944"; const client = createClient(withPolkadotSdkCompat(getWsProvider(url))); + return { client, typedApi: client.getTypedApi(datahaven) }; }; diff --git a/test/utils/validators.ts b/test/utils/validators.ts index 637c9ae6..5e635200 100644 --- a/test/utils/validators.ts +++ b/test/utils/validators.ts @@ -8,6 +8,8 @@ export const COMMON_LAUNCH_ARGS = [ "--unsafe-force-node-key-generation", "--tmp", + "--chain", + "local", "--validator", "--discover-local", "--no-prometheus", From 8d82f63efa048ae965572f38819dee847c2de273 Mon Sep 17 00:00:00 2001 From: Steve Degosserie <723552+stiiifff@users.noreply.github.com> Date: Wed, 11 Mar 2026 15:09:02 +0100 Subject: [PATCH 2/2] feat: add retry mechanism for failed Snowbridge rewards messages (#462) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary When `send_rewards_message` fails at era end (bridge paused, queue full, etc.), tokens have already been minted to the Ethereum sovereign account but the `submitRewards` message to EigenLayer is silently lost. This creates an inconsistency: wHAVE tokens exist on DataHaven but validators never receive their rewards for that era. This PR adds: - **Automatic retry via `on_initialize`** — processes one failed era per block - **Head-of-line blocking avoidance** — failed entries are moved to the back of the queue so a single stuck era doesn't block retries for subsequent ones - **Governance escape hatch** — `retry_unsent_reward_era` extrinsic gated by configurable `GovernanceOrigin` - **Automatic cleanup** — expired entries (reward points pruned past `HistoryDepth`) are discarded ### Context: HistoryDepth window Reward points are kept in storage for `HistoryDepth` eras (64 on mainnet, ~16 days with 6-hour eras). Retries are only possible while the data exists. After that window, the entry is automatically expired and dropped from the queue. This means governance has up to ~16 days to intervene if automatic retries keep failing. ### Storage design A ring buffer (`StorageMap` + head/tail pointers) with capacity 64, matching `HistoryDepth`. Each entry stores: - `era_index` — which era's rewards message failed - `era_start_timestamp` — preserved from the original era (seconds since epoch) - `scaled_inflation` — the exact minted amount (stored because recomputing later could yield a different value if `EraInflationProvider` has changed) ### Retry behavior | Scenario | Action | |----------|--------| | Queue empty | Return minimal weight (2 reads) | | Entry reward points pruned | Remove entry, emit `UnsentEraExpired` | | Retry succeeds | Remove entry, emit `RewardsMessageRetried` | | Retry fails | Move entry to back of queue, try next entry next block | ### Changes - **`lib.rs`** — Ring buffer storage, events (`RewardsMessageSendFailed`, `RewardsMessageRetried`, `UnsentEraExpired`, `UnsentQueueFull`), errors, `on_initialize` hook, `retry_unsent_reward_era` extrinsic with configurable `GovernanceOrigin`, modified `on_era_end` (queue on failure) and `on_era_start` (prune expired entries where `idx <= era_index_to_delete`) - **`mock.rs`** — Configurable `send_message_fails` flag on `MockOkOutboundQueue` - **`tests.rs`** — 14 new test cases covering all retry/expiry/governance paths including head-of-line blocking avoidance - **`benchmarking.rs`** — 5 new benchmarks (empty queue, expired entry, success, failure, governance extrinsic) - **`weights.rs`** — New weight functions in trait and both impls - **Runtime configs** — `GovernanceOrigin = EnsureRoot` + placeholder weight functions for mainnet/stagenet/testnet ## Test plan - [x] `cargo test -p pallet-external-validators-rewards` — 90 tests pass - [x] `cargo clippy -p pallet-external-validators-rewards` — no new warnings - [x] `cargo check -p datahaven-mainnet-runtime` — compiles - [ ] Run benchmarks to generate production weight values --------- Co-authored-by: Claude Opus 4.6 Co-authored-by: Gonza Montiel Co-authored-by: undercover-cactus Co-authored-by: Tobi Demeco <50408393+TDemeco@users.noreply.github.com> Co-authored-by: Ahmad Kaouk <56095276+ahmadkaouk@users.noreply.github.com> --- .../src/benchmarking.rs | 109 +++- .../external-validators-rewards/src/lib.rs | 316 +++++++++++- .../external-validators-rewards/src/mock.rs | 6 + .../external-validators-rewards/src/tests.rs | 471 +++++++++++++++++- .../src/weights.rs | 60 +++ operator/runtime/mainnet/src/configs/mod.rs | 2 + .../pallet_external_validators_rewards.rs | 25 + operator/runtime/stagenet/src/configs/mod.rs | 2 + .../pallet_external_validators_rewards.rs | 25 + operator/runtime/testnet/src/configs/mod.rs | 2 + .../pallet_external_validators_rewards.rs | 25 + test/.papi/descriptors/package.json | 2 +- test/.papi/metadata/datahaven.scale | Bin 632664 -> 634460 bytes 13 files changed, 1018 insertions(+), 27 deletions(-) diff --git a/operator/pallets/external-validators-rewards/src/benchmarking.rs b/operator/pallets/external-validators-rewards/src/benchmarking.rs index 335557f4..4b84bc3f 100644 --- a/operator/pallets/external-validators-rewards/src/benchmarking.rs +++ b/operator/pallets/external-validators-rewards/src/benchmarking.rs @@ -21,9 +21,9 @@ use super::*; #[allow(unused)] use crate::Pallet as ExternalValidatorsRewards; use { - crate::{types::BenchmarkHelper, OnEraEnd}, + crate::types::BenchmarkHelper, frame_benchmarking::{account, v2::*, BenchmarkError}, - frame_support::traits::Currency, + frame_support::traits::{Currency, EnsureOrigin}, sp_std::prelude::*, }; @@ -43,6 +43,11 @@ fn create_funded_user( user } +/// Helper: insert a single entry into the ring buffer at slot 0. +fn push_unsent_entry(era_index: u32, timestamp: u32, inflation: u128) { + ExternalValidatorsRewards::::unsent_queue_push((era_index, timestamp, inflation)); +} + #[allow(clippy::multiple_bound_locations)] #[benchmarks(where T: pallet_balances::Config)] mod benchmarks { @@ -72,6 +77,106 @@ mod benchmarks { Ok(()) } + /// Helper to populate reward points for an era with 1000 validators. + fn setup_era_reward_points(era_index: u32) { + let mut era_reward_points = EraRewardPoints::default(); + era_reward_points.total = 20 * 1000; + + for i in 0..1000 { + let account_id = create_funded_user::("candidate", i, 100); + era_reward_points.individual.insert(account_id, 20); + } + + >::insert(era_index, era_reward_points); + } + + // on_initialize: unsent queue is empty (2 reads for head+tail) + #[benchmark] + fn process_unsent_reward_eras_empty() -> Result<(), BenchmarkError> { + // Ensure queue is empty (default state: head == tail == 0) + assert!(ExternalValidatorsRewards::::unsent_queue_is_empty()); + + #[block] + { + ExternalValidatorsRewards::::process_unsent_reward_eras(); + } + + Ok(()) + } + + // on_initialize: oldest entry has pruned reward points + #[benchmark] + fn process_unsent_reward_eras_expired() -> Result<(), BenchmarkError> { + // Push an entry whose reward points do NOT exist in storage + push_unsent_entry::(999, 0, 42); + + #[block] + { + ExternalValidatorsRewards::::process_unsent_reward_eras(); + } + + // Entry should have been removed + assert!(ExternalValidatorsRewards::::unsent_queue_is_empty()); + + Ok(()) + } + + // on_initialize: oldest entry retried successfully + #[benchmark] + fn process_unsent_reward_eras_success() -> Result<(), BenchmarkError> { + frame_system::Pallet::::set_block_number(0u32.into()); + T::BenchmarkHelper::setup(); + setup_era_reward_points::(1); + + push_unsent_entry::(1, 0, 42); + + #[block] + { + ExternalValidatorsRewards::::process_unsent_reward_eras(); + } + + assert!(ExternalValidatorsRewards::::unsent_queue_is_empty()); + + Ok(()) + } + + // Use success weight as upper bound for the failed path + #[benchmark] + fn process_unsent_reward_eras_failed() -> Result<(), BenchmarkError> { + frame_system::Pallet::::set_block_number(0u32.into()); + T::BenchmarkHelper::setup(); + setup_era_reward_points::(1); + + push_unsent_entry::(1, 0, 42); + + #[block] + { + ExternalValidatorsRewards::::process_unsent_reward_eras(); + } + + Ok(()) + } + + // Governance extrinsic: retry a specific unsent era + #[benchmark] + fn retry_unsent_reward_era() -> Result<(), BenchmarkError> { + frame_system::Pallet::::set_block_number(0u32.into()); + T::BenchmarkHelper::setup(); + setup_era_reward_points::(1); + + push_unsent_entry::(1, 0, 42); + + let origin = + T::GovernanceOrigin::try_successful_origin().map_err(|_| BenchmarkError::Weightless)?; + + #[extrinsic_call] + _(origin as T::RuntimeOrigin, 1u32); + + assert!(ExternalValidatorsRewards::::unsent_queue_is_empty()); + + Ok(()) + } + impl_benchmark_test_suite!( ExternalValidatorsRewards, crate::mock::new_test_ext(), diff --git a/operator/pallets/external-validators-rewards/src/lib.rs b/operator/pallets/external-validators-rewards/src/lib.rs index 376d3a55..8aeaa123 100644 --- a/operator/pallets/external-validators-rewards/src/lib.rs +++ b/operator/pallets/external-validators-rewards/src/lib.rs @@ -66,13 +66,13 @@ pub mod pallet { pub use crate::weights::WeightInfo; use { - super::*, frame_support::pallet_prelude::*, + super::*, frame_support::pallet_prelude::*, frame_system::pallet_prelude::OriginFor, pallet_external_validators::traits::EraIndexProvider, sp_runtime::Saturating, sp_std::collections::btree_map::BTreeMap, }; /// The current storage version. - const STORAGE_VERSION: StorageVersion = StorageVersion::new(0); + const STORAGE_VERSION: StorageVersion = StorageVersion::new(1); pub type RewardPoints = u32; pub type EraIndex = u32; @@ -168,6 +168,9 @@ pub mod pallet { /// Hook for minting inflation tokens. type HandleInflation: HandleInflation; + /// Origin for governance calls (e.g., retrying unsent reward messages). + type GovernanceOrigin: EnsureOrigin; + #[cfg(feature = "runtime-benchmarks")] type BenchmarkHelper: types::BenchmarkHelper; } @@ -175,6 +178,62 @@ pub mod pallet { #[pallet::storage_version(STORAGE_VERSION)] pub struct Pallet(_); + #[pallet::hooks] + impl Hooks> for Pallet { + fn on_initialize(_n: frame_system::pallet_prelude::BlockNumberFor) -> Weight { + Self::process_unsent_reward_eras() + } + } + + #[pallet::call] + impl Pallet { + /// Governance escape hatch: manually retry sending a rewards message for + /// an era that is stuck in the unsent queue. + #[pallet::call_index(0)] + #[pallet::weight(T::WeightInfo::retry_unsent_reward_era())] + pub fn retry_unsent_reward_era( + origin: OriginFor, + era_index: EraIndex, + ) -> DispatchResult { + T::GovernanceOrigin::ensure_origin(origin)?; + + // Scan the ring buffer for the requested era + let head = UnsentRewardHead::::get(); + let tail = UnsentRewardTail::::get(); + let mut found = None; + let mut slot = head; + while slot != tail { + if let Some(entry @ (idx, _, _)) = UnsentRewardEra::::get(slot) { + if idx == era_index { + found = Some((slot, entry)); + break; + } + } + slot = (slot + 1) % UNSENT_QUEUE_CAPACITY; + } + let (slot, (_, timestamp, inflation)) = found.ok_or(Error::::EraNotInUnsentQueue)?; + + let reward_points = RewardPointsForEra::::get(era_index); + let info = reward_points + .generate_era_rewards_info(era_index, inflation, timestamp) + .ok_or(Error::::RewardPointsPruned)?; + + let message_id = + Self::send_rewards_message(&info).ok_or(Error::::MessageSendFailed)?; + + Self::unsent_queue_remove_slot(slot); + + Self::deposit_event(Event::RewardsMessageRetried { + message_id, + era_index, + total_points: info.total_points, + inflation_amount: inflation, + }); + + Ok(()) + } + } + #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { @@ -185,6 +244,29 @@ pub mod pallet { total_points: u128, inflation_amount: u128, }, + /// The rewards message failed to send; era queued for retry. + RewardsMessageSendFailed { era_index: EraIndex }, + /// A previously failed rewards message was retried and sent successfully. + RewardsMessageRetried { + message_id: H256, + era_index: EraIndex, + total_points: u128, + inflation_amount: u128, + }, + /// An unsent era was dropped because its reward points have been pruned. + UnsentEraExpired { era_index: EraIndex }, + /// The unsent queue is full; this era could not be enqueued for retry. + UnsentQueueFull { era_index: EraIndex }, + } + + #[pallet::error] + pub enum Error { + /// The specified era is not in the unsent queue. + EraNotInUnsentQueue, + /// Reward points for the era have been pruned from storage. + RewardPointsPruned, + /// The message delivery still failed on retry. + MessageSendFailed, } /// Keep tracks of distributed points per validator and total. @@ -200,7 +282,7 @@ pub mod pallet { /// - individual_points: (address, points) tuples for each validator. /// - inflation_amount: total inflation tokens to distribute. /// - era_start_timestamp: timestamp when the era started (seconds since Unix epoch). - pub fn generate_era_rewards_utils( + pub fn generate_era_rewards_info( &self, era_index: EraIndex, inflation_amount: u128, @@ -260,6 +342,33 @@ pub mod pallet { pub type BlocksProducedInEra = StorageMap<_, Twox64Concat, EraIndex, u32, ValueQuery>; + /// Maximum number of unsent reward entries in the ring buffer. + pub const UNSENT_QUEUE_CAPACITY: u32 = 64; + + /// Ring buffer of eras whose rewards messages failed to send. + /// Each slot stores (era_index, era_start_timestamp, scaled_inflation). + /// Keyed by slot index [0, UNSENT_QUEUE_CAPACITY). + #[pallet::storage] + pub type UnsentRewardEra = StorageMap< + _, + Twox64Concat, + u32, + ( + EraIndex, + /* era_start_timestamp */ u32, + /* scaled_inflation */ u128, + ), + >; + + /// Ring buffer head: next slot to be processed by `on_initialize`. + #[pallet::storage] + pub type UnsentRewardHead = StorageValue<_, u32, ValueQuery>; + + /// Ring buffer tail: next slot to write a new entry into. + /// When head == tail the buffer is empty. + #[pallet::storage] + pub type UnsentRewardTail = StorageValue<_, u32, ValueQuery>; + impl Pallet { /// Reward validators. Does not check if the validators are valid, caller needs to make sure of that. pub fn reward_by_ids(points: impl IntoIterator) { @@ -276,8 +385,8 @@ pub mod pallet { /// Helper to build, validate and deliver an outbound message. /// Logs any error and returns None on failure. - fn send_rewards_message(utils: &EraRewardsUtils) -> Option { - let outbound = T::SendMessage::build(utils).or_else(|| { + fn send_rewards_message(info: &EraRewardsUtils) -> Option { + let outbound = T::SendMessage::build(info).or_else(|| { log::error!(target: "ext_validators_rewards", "Failed to build outbound message"); None })?; @@ -303,6 +412,147 @@ pub mod pallet { .ok() } + // ── Ring-buffer helpers ────────────────────────────────────────── + + /// Returns true when the ring buffer is empty (head == tail). + #[allow(dead_code)] + pub(crate) fn unsent_queue_is_empty() -> bool { + UnsentRewardHead::::get() == UnsentRewardTail::::get() + } + + /// Number of entries currently in the ring buffer. + #[allow(dead_code)] + pub(crate) fn unsent_queue_len() -> u32 { + let head = UnsentRewardHead::::get(); + let tail = UnsentRewardTail::::get(); + tail.wrapping_sub(head) % UNSENT_QUEUE_CAPACITY + } + + /// Push a new entry into the ring buffer. + /// Returns `true` on success, `false` if the buffer is full. + pub(crate) fn unsent_queue_push(entry: (EraIndex, u32, u128)) -> bool { + let head = UnsentRewardHead::::get(); + let tail = UnsentRewardTail::::get(); + let next_tail = (tail + 1) % UNSENT_QUEUE_CAPACITY; + if next_tail == head { + // Buffer full + return false; + } + UnsentRewardEra::::insert(tail, entry); + UnsentRewardTail::::put(next_tail); + true + } + + /// Remove the entry at a given slot and compact the buffer by shifting + /// subsequent entries back. Used by the extrinsic and `on_era_start`. + fn unsent_queue_remove_slot(slot: u32) { + let tail = UnsentRewardTail::::get(); + // Shift entries after `slot` backward to fill the gap + let mut cur = slot; + loop { + let next = (cur + 1) % UNSENT_QUEUE_CAPACITY; + if next == tail { + break; + } + // Move next → cur + if let Some(entry) = UnsentRewardEra::::get(next) { + UnsentRewardEra::::insert(cur, entry); + } + cur = next; + } + // Remove the now-duplicate last entry and shrink tail + UnsentRewardEra::::remove(cur); + let new_tail = if tail == 0 { + UNSENT_QUEUE_CAPACITY - 1 + } else { + tail - 1 + }; + UnsentRewardTail::::put(new_tail); + + // If head was after the removed slot, adjust it too + let head = UnsentRewardHead::::get(); + // We also need to handle head potentially pointing past the buffer + // after a removal. Since we shifted everything between slot..tail back, + // the head only needs adjustment if it was == tail (now new_tail) — but + // that means the buffer just became empty, which is fine (head == new_tail). + // However, if head was pointing *at* a slot beyond the removed one, the + // entry it pointed to slid back by one, so head should also slide back. + // In practice, removal only happens when we know the slot, so we can + // simply recalculate emptiness. + if head == tail { + // Was already at tail, buffer must be empty now + UnsentRewardHead::::put(new_tail); + } + } + + // ── Core retry logic ────────────────────────────────────────────── + + /// Process at most one unsent reward era per block. + /// On failure the head pointer advances to the next entry so a single + /// stuck era does not block retries for subsequent eras. + pub(crate) fn process_unsent_reward_eras() -> Weight { + let head = UnsentRewardHead::::get(); + let tail = UnsentRewardTail::::get(); + + if head == tail { + return T::WeightInfo::process_unsent_reward_eras_empty(); + } + + let Some((era_index, timestamp, inflation)) = UnsentRewardEra::::get(head) else { + // Slot unexpectedly empty — advance head past it + UnsentRewardHead::::put((head + 1) % UNSENT_QUEUE_CAPACITY); + return T::WeightInfo::process_unsent_reward_eras_empty(); + }; + + // Check if reward points are still available + let reward_points = RewardPointsForEra::::get(era_index); + let info = + match reward_points.generate_era_rewards_info(era_index, inflation, timestamp) { + Some(info) => info, + None => { + // Reward points have been pruned — discard this entry + log::warn!( + target: "ext_validators_rewards", + "Unsent era {era_index} expired: reward points pruned", + ); + UnsentRewardEra::::remove(head); + UnsentRewardHead::::put((head + 1) % UNSENT_QUEUE_CAPACITY); + Self::deposit_event(Event::UnsentEraExpired { era_index }); + return T::WeightInfo::process_unsent_reward_eras_expired(); + } + }; + + // Attempt to resend + match Self::send_rewards_message(&info) { + Some(message_id) => { + UnsentRewardEra::::remove(head); + UnsentRewardHead::::put((head + 1) % UNSENT_QUEUE_CAPACITY); + Self::deposit_event(Event::RewardsMessageRetried { + message_id, + era_index, + total_points: info.total_points, + inflation_amount: inflation, + }); + T::WeightInfo::process_unsent_reward_eras_success() + } + None => { + // Move the failed entry to the back of the queue so the + // next block tries a different era (avoids head-of-line + // blocking). The entry is not lost — it will be retried + // after all other pending entries. + UnsentRewardEra::::remove(head); + UnsentRewardHead::::put((head + 1) % UNSENT_QUEUE_CAPACITY); + UnsentRewardEra::::insert(tail, (era_index, timestamp, inflation)); + UnsentRewardTail::::put((tail + 1) % UNSENT_QUEUE_CAPACITY); + log::warn!( + target: "ext_validators_rewards", + "Retry for unsent era {era_index} still failing, moved to back of queue", + ); + T::WeightInfo::process_unsent_reward_eras_failed() + } + } + } + /// Track a block authored by a validator pub fn note_block_author(author: T::AccountId) { // Track per-session authorship for performance points @@ -619,6 +869,24 @@ pub mod pallet { RewardPointsForEra::::remove(era_index_to_delete); BlocksProducedInEra::::remove(era_index_to_delete); + + // Proactively clean up any unsent entries whose reward points + // have been pruned (this era and any older ones still lingering). + let head = UnsentRewardHead::::get(); + let mut tail = UnsentRewardTail::::get(); + let mut slot = head; + while slot != tail { + if let Some((idx, _, _)) = UnsentRewardEra::::get(slot) { + if idx <= era_index_to_delete { + Self::unsent_queue_remove_slot(slot); + tail = UnsentRewardTail::::get(); + Self::deposit_event(Event::UnsentEraExpired { era_index: idx }); + // Don't advance slot — next entry slid into this position + continue; + } + } + slot = (slot + 1) % UNSENT_QUEUE_CAPACITY; + } } } @@ -671,17 +939,17 @@ pub mod pallet { // Generate era rewards utils with the actual rewards amount (post-treasury split). // This ensures the message to EigenLayer matches the actual minted rewards. - let utils = match era_reward_points.generate_era_rewards_utils( + let info = match RewardPointsForEra::::get(&era_index).generate_era_rewards_info( era_index, mint_result.rewards_amount, era_start_timestamp, ) { - Some(utils) => utils, + Some(info) => info, None => { // Returns None when total_points is zero or no validators have rewards log::error!( target: "ext_validators_rewards", - "Failed to generate era rewards utils (no rewards to distribute)" + "Failed to generate era rewards info (no rewards to distribute)" ); return; } @@ -692,13 +960,31 @@ pub mod pallet { DispatchClass::Mandatory, ); - if let Some(message_id) = Self::send_rewards_message(&utils) { - Self::deposit_event(Event::RewardsMessageSent { - message_id, - era_index, - total_points: utils.total_points, - inflation_amount: mint_result.rewards_amount, - }); + match Self::send_rewards_message(&info) { + Some(message_id) => { + Self::deposit_event(Event::RewardsMessageSent { + message_id, + era_index, + total_points: info.total_points, + inflation_amount: mint_result.rewards_amount, + }); + } + None => { + // Message failed — queue for automatic retry via on_initialize + if Self::unsent_queue_push(( + era_index, + era_start_timestamp, + mint_result.rewards_amount, + )) { + Self::deposit_event(Event::RewardsMessageSendFailed { era_index }); + } else { + log::error!( + target: "ext_validators_rewards", + "Unsent reward queue full, cannot enqueue era {era_index}", + ); + Self::deposit_event(Event::UnsentQueueFull { era_index }); + } + } } } } diff --git a/operator/pallets/external-validators-rewards/src/mock.rs b/operator/pallets/external-validators-rewards/src/mock.rs index 3c892f35..6b99b7c3 100644 --- a/operator/pallets/external-validators-rewards/src/mock.rs +++ b/operator/pallets/external-validators-rewards/src/mock.rs @@ -131,6 +131,9 @@ impl crate::types::SendMessage for MockOkOutboundQueue { } fn validate(ticket: Self::Ticket) -> Result { + if Mock::mock().send_message_fails { + return Err(SendError::MessageTooLarge); + } Ok(ticket) } @@ -223,6 +226,7 @@ impl pallet_external_validators_rewards::Config for Test { type HandleInflation = InflationMinter; type Currency = Balances; type RewardsEthereumSovereignAccount = RewardsEthereumSovereignAccount; + type GovernanceOrigin = frame_system::EnsureRoot; type WeightInfo = (); #[cfg(feature = "runtime-benchmarks")] type BenchmarkHelper = (); @@ -292,6 +296,8 @@ pub mod mock_data { pub offline_validators: sp_std::vec::Vec, /// Set of (era_index, validator_id) pairs that are slashed pub slashed_validators: sp_std::vec::Vec<(u32, sp_core::H160)>, + /// When true, MockOkOutboundQueue::validate will return Err(SendError::MessageTooLarge) + pub send_message_fails: bool, } #[pallet::config] diff --git a/operator/pallets/external-validators-rewards/src/tests.rs b/operator/pallets/external-validators-rewards/src/tests.rs index 1a66daa0..752a55c0 100644 --- a/operator/pallets/external-validators-rewards/src/tests.rs +++ b/operator/pallets/external-validators-rewards/src/tests.rs @@ -16,7 +16,7 @@ use { crate::{self as pallet_external_validators_rewards, mock::*}, - frame_support::traits::fungible::Mutate, + frame_support::{assert_noop, assert_ok, traits::fungible::Mutate}, pallet_external_validators::traits::{ActiveEraInfo, OnEraEnd, OnEraStart}, sp_core::H160, sp_std::collections::btree_map::BTreeMap, @@ -165,8 +165,8 @@ fn test_on_era_end() { let treasury_amount = InflationTreasuryProportion::get().mul_floor(inflation); let rewards_amount = inflation - treasury_amount; // Use 0 for era_start_timestamp in tests - let rewards_utils = era_rewards.generate_era_rewards_utils(1, rewards_amount, 0); - assert!(rewards_utils.is_some()); + let rewards_info = era_rewards.generate_era_rewards_info(1, inflation, 0); + assert!(rewards_info.is_some()); System::assert_last_event(RuntimeEvent::ExternalValidatorsRewards( crate::Event::RewardsMessageSent { message_id: Default::default(), @@ -207,8 +207,8 @@ fn test_on_era_end_with_zero_inflation() { let era_rewards = pallet_external_validators_rewards::RewardPointsForEra::::get(1); let inflation = ::EraInflationProvider::get(); - let rewards_utils = era_rewards.generate_era_rewards_utils(1, inflation, 0); - assert!(rewards_utils.is_some()); + let rewards_info = era_rewards.generate_era_rewards_info(1, inflation, 0); + assert!(rewards_info.is_some()); // With zero inflation, no RewardsMessageSent event should be emitted let events = System::events(); assert!( @@ -246,15 +246,15 @@ fn test_on_era_end_with_zero_points() { ExternalValidatorsRewards::reward_by_ids(accounts_points); ExternalValidatorsRewards::on_era_end(1); - // When all validators have zero points, generate_era_rewards_utils should return None + // When all validators have zero points, generate_era_rewards_info should return None // to prevent inflation from being minted with no way to distribute it let era_rewards = pallet_external_validators_rewards::RewardPointsForEra::::get(1); let inflation = ::EraInflationProvider::get(); - let rewards_utils = era_rewards.generate_era_rewards_utils(1, inflation, 0); + let rewards_info = era_rewards.generate_era_rewards_info(1, inflation, 0); assert!( - rewards_utils.is_none(), - "generate_era_rewards_utils should return None when total_points is zero" + rewards_info.is_none(), + "generate_era_rewards_info should return None when total_points is zero" ); // Verify no RewardsMessageSent event was emitted @@ -3722,3 +3722,456 @@ fn test_era_end_uses_correct_era_blocks_not_session() { ); }) } + +// ═══════════════════════════════════════════════════════════════════════════ +// Retry mechanism tests (ring-buffer storage) +// ═══════════════════════════════════════════════════════════════════════════ + +/// Helper: push an entry into the unsent ring buffer via the pallet API. +fn push_unsent(era_index: u32, timestamp: u32, inflation: u128) { + assert!( + ExternalValidatorsRewards::unsent_queue_push((era_index, timestamp, inflation)), + "unsent_queue_push should succeed" + ); +} + +/// Helper: return the number of entries in the unsent ring buffer. +fn unsent_len() -> u32 { + ExternalValidatorsRewards::unsent_queue_len() +} + +/// Helper: check if unsent queue is empty. +fn unsent_is_empty() -> bool { + ExternalValidatorsRewards::unsent_queue_is_empty() +} + +#[test] +fn send_failure_queues_era() { + new_test_ext().execute_with(|| { + run_to_block(1); + + Mock::mutate(|mock| { + mock.active_era = Some(ActiveEraInfo { + index: 1, + start: Some(30_000), + }); + mock.send_message_fails = true; + }); + + // Give validators some points + ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]); + // Author expected blocks for 100% inflation + for _ in 0..600 { + ExternalValidatorsRewards::note_block_author(H160::from_low_u64_be(1)); + } + + ExternalValidatorsRewards::on_era_end(1); + + // Verify era is queued + assert_eq!(unsent_len(), 1); + + // Verify event + System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards( + crate::Event::RewardsMessageSendFailed { era_index: 1 }, + )); + }) +} + +#[test] +fn on_initialize_retries_and_succeeds() { + new_test_ext().execute_with(|| { + run_to_block(1); + + Mock::mutate(|mock| { + mock.active_era = Some(ActiveEraInfo { + index: 1, + start: Some(30_000), + }); + }); + + // Set up reward points for era 1 + ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]); + + // Manually populate the unsent queue + push_unsent(1, 30, 42); + + // Sending should succeed (send_message_fails is false by default) + System::reset_events(); + ExternalValidatorsRewards::process_unsent_reward_eras(); + + // Queue should be empty + assert!(unsent_is_empty()); + + // Verify retry event + System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards( + crate::Event::RewardsMessageRetried { + message_id: Default::default(), + era_index: 1, + total_points: 100, + inflation_amount: 42, + }, + )); + }) +} + +#[test] +fn on_initialize_moves_failed_entry_to_back() { + new_test_ext().execute_with(|| { + run_to_block(1); + + Mock::mutate(|mock| { + mock.active_era = Some(ActiveEraInfo { + index: 2, + start: Some(30_000), + }); + mock.send_message_fails = true; + }); + + // Set up reward points for eras 1 and 2 + ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]); + Mock::mutate(|mock| { + mock.active_era = Some(ActiveEraInfo { + index: 1, + start: Some(30_000), + }); + }); + ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 200)]); + + // Push two entries: era 1 then era 2 + push_unsent(1, 30, 42); + push_unsent(2, 30, 84); + + // First call: tries era 1, fails, moves era 1 to back of queue + ExternalValidatorsRewards::process_unsent_reward_eras(); + // Queue length stays the same (entry moved, not removed) + assert_eq!(unsent_len(), 2); + + // Second call: tries era 2 (NOT era 1 again), fails, moves era 2 to back + ExternalValidatorsRewards::process_unsent_reward_eras(); + assert_eq!(unsent_len(), 2); + + // Re-enable sending + Mock::mutate(|mock| mock.send_message_fails = false); + + // Third call: era 1 (now at front again), succeeds + System::reset_events(); + ExternalValidatorsRewards::process_unsent_reward_eras(); + assert_eq!(unsent_len(), 1); + System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards( + crate::Event::RewardsMessageRetried { + message_id: Default::default(), + era_index: 1, + total_points: 200, + inflation_amount: 42, + }, + )); + + // Fourth call: era 2, succeeds + ExternalValidatorsRewards::process_unsent_reward_eras(); + assert!(unsent_is_empty()); + }) +} + +#[test] +fn on_initialize_removes_expired_era() { + new_test_ext().execute_with(|| { + run_to_block(1); + + // Populate unsent queue with era 999 but do NOT add RewardPointsForEra for it + push_unsent(999, 0, 42); + + System::reset_events(); + ExternalValidatorsRewards::process_unsent_reward_eras(); + + // Entry should be removed + assert!(unsent_is_empty()); + + // Verify expired event + System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards( + crate::Event::UnsentEraExpired { era_index: 999 }, + )); + }) +} + +#[test] +fn on_initialize_noop_when_queue_empty() { + new_test_ext().execute_with(|| { + run_to_block(1); + System::reset_events(); + + ExternalValidatorsRewards::process_unsent_reward_eras(); + + // No events should be emitted + let events = System::events(); + assert!( + events.is_empty(), + "No events should be emitted when unsent queue is empty" + ); + }) +} + +#[test] +fn on_initialize_processes_only_head() { + new_test_ext().execute_with(|| { + run_to_block(1); + + Mock::mutate(|mock| { + mock.active_era = Some(ActiveEraInfo { + index: 3, + start: Some(30_000), + }); + }); + + // Set up reward points for both eras + ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]); + Mock::mutate(|mock| { + mock.active_era = Some(ActiveEraInfo { + index: 2, + start: Some(30_000), + }); + }); + ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(2), 200)]); + + // Push two entries + push_unsent(3, 30, 42); + push_unsent(2, 20, 84); + + System::reset_events(); + ExternalValidatorsRewards::process_unsent_reward_eras(); + + // Only the head entry (era 3) should be processed (and removed on success) + assert_eq!(unsent_len(), 1); + }) +} + +#[test] +fn retry_extrinsic_success() { + new_test_ext().execute_with(|| { + run_to_block(1); + + Mock::mutate(|mock| { + mock.active_era = Some(ActiveEraInfo { + index: 1, + start: Some(30_000), + }); + }); + + // Set up reward points + ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]); + + // Populate unsent queue + push_unsent(1, 30, 42); + + System::reset_events(); + assert_ok!(ExternalValidatorsRewards::retry_unsent_reward_era( + RuntimeOrigin::root(), + 1 + )); + + // Queue should be empty + assert!(unsent_is_empty()); + + // Verify retry event + System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards( + crate::Event::RewardsMessageRetried { + message_id: Default::default(), + era_index: 1, + total_points: 100, + inflation_amount: 42, + }, + )); + }) +} + +#[test] +fn retry_extrinsic_era_not_in_queue() { + new_test_ext().execute_with(|| { + run_to_block(1); + + assert_noop!( + ExternalValidatorsRewards::retry_unsent_reward_era(RuntimeOrigin::root(), 1), + crate::Error::::EraNotInUnsentQueue + ); + }) +} + +#[test] +fn retry_extrinsic_pruned_data() { + new_test_ext().execute_with(|| { + run_to_block(1); + + // Queue an era but don't create reward points for it + push_unsent(999, 0, 42); + + assert_noop!( + ExternalValidatorsRewards::retry_unsent_reward_era(RuntimeOrigin::root(), 999), + crate::Error::::RewardPointsPruned + ); + }) +} + +#[test] +fn retry_extrinsic_requires_root() { + new_test_ext().execute_with(|| { + run_to_block(1); + + assert_noop!( + ExternalValidatorsRewards::retry_unsent_reward_era( + RuntimeOrigin::signed(H160::from_low_u64_be(1)), + 1 + ), + sp_runtime::DispatchError::BadOrigin + ); + }) +} + +#[test] +fn unsent_queue_full() { + new_test_ext().execute_with(|| { + run_to_block(1); + + Mock::mutate(|mock| { + mock.active_era = Some(ActiveEraInfo { + index: 65, + start: Some(30_000), + }); + mock.send_message_fails = true; + }); + + // Fill the ring buffer to capacity (63 entries, since capacity=64 + // means 63 usable slots in a ring buffer with head==tail==empty). + for i in 0..63u32 { + push_unsent(i, 0, 42); + } + assert_eq!(unsent_len(), 63); + + // Give validators some points so on_era_end doesn't bail early + ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]); + for _ in 0..600 { + ExternalValidatorsRewards::note_block_author(H160::from_low_u64_be(1)); + } + + System::reset_events(); + ExternalValidatorsRewards::on_era_end(65); + + // Verify UnsentQueueFull event + System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards( + crate::Event::UnsentQueueFull { era_index: 65 }, + )); + + // Queue should still be at 63 + assert_eq!(unsent_len(), 63); + }) +} + +#[test] +fn on_era_start_prunes_unsent_entry() { + new_test_ext().execute_with(|| { + run_to_block(1); + + // Set up: era 1 has an unsent entry + push_unsent(1, 0, 42); + + // HistoryDepth is 10, so era 11 should prune era 1 + System::reset_events(); + ExternalValidatorsRewards::on_era_start(11, 0, 11); + + // Unsent entry should be removed + assert!(unsent_is_empty()); + + // Verify expired event + System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards( + crate::Event::UnsentEraExpired { era_index: 1 }, + )); + }) +} + +#[test] +fn retry_extrinsic_send_still_fails() { + new_test_ext().execute_with(|| { + run_to_block(1); + + Mock::mutate(|mock| { + mock.active_era = Some(ActiveEraInfo { + index: 1, + start: Some(30_000), + }); + mock.send_message_fails = true; + }); + + // Set up reward points + ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]); + + // Populate unsent queue + push_unsent(1, 30, 42); + + assert_noop!( + ExternalValidatorsRewards::retry_unsent_reward_era(RuntimeOrigin::root(), 1), + crate::Error::::MessageSendFailed + ); + + // Queue should still have the entry + assert_eq!(unsent_len(), 1); + }) +} + +#[test] +fn head_of_line_blocking_avoided() { + new_test_ext().execute_with(|| { + run_to_block(1); + + // Set up reward points for eras 1, 2, 3 + for era in 1..=3u32 { + Mock::mutate(|mock| { + mock.active_era = Some(ActiveEraInfo { + index: era, + start: Some(30_000), + }); + }); + ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]); + } + + // Push eras 1, 2, 3 into the queue + push_unsent(1, 30, 10); + push_unsent(2, 30, 20); + push_unsent(3, 30, 30); + + // Make sending fail + Mock::mutate(|mock| mock.send_message_fails = true); + + // Block 1: tries era 1, fails, advances head → era 2 + ExternalValidatorsRewards::process_unsent_reward_eras(); + // Block 2: tries era 2, fails, advances head → era 3 + ExternalValidatorsRewards::process_unsent_reward_eras(); + + // Now re-enable sending + Mock::mutate(|mock| mock.send_message_fails = false); + + // Block 3: tries era 3, succeeds + System::reset_events(); + ExternalValidatorsRewards::process_unsent_reward_eras(); + System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards( + crate::Event::RewardsMessageRetried { + message_id: Default::default(), + era_index: 3, + total_points: 100, + inflation_amount: 30, + }, + )); + + // Block 4: wraps around to era 1, succeeds + ExternalValidatorsRewards::process_unsent_reward_eras(); + System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards( + crate::Event::RewardsMessageRetried { + message_id: Default::default(), + era_index: 1, + total_points: 100, + inflation_amount: 10, + }, + )); + + // Block 5: era 2, succeeds + ExternalValidatorsRewards::process_unsent_reward_eras(); + assert!(unsent_is_empty()); + }) +} diff --git a/operator/pallets/external-validators-rewards/src/weights.rs b/operator/pallets/external-validators-rewards/src/weights.rs index 766adfcf..a7585778 100644 --- a/operator/pallets/external-validators-rewards/src/weights.rs +++ b/operator/pallets/external-validators-rewards/src/weights.rs @@ -54,6 +54,11 @@ use sp_std::marker::PhantomData; /// Weight functions needed for pallet_external_validators_rewards. pub trait WeightInfo { fn on_era_end() -> Weight; + fn process_unsent_reward_eras_empty() -> Weight; + fn process_unsent_reward_eras_expired() -> Weight; + fn process_unsent_reward_eras_success() -> Weight; + fn process_unsent_reward_eras_failed() -> Weight; + fn retry_unsent_reward_era() -> Weight; } /// Weights for pallet_external_validators_rewards using the Substrate node and recommended hardware. @@ -84,6 +89,36 @@ impl WeightInfo for SubstrateWeight { .saturating_add(T::DbWeight::get().reads(5_u64)) .saturating_add(T::DbWeight::get().writes(5_u64)) } + + fn process_unsent_reward_eras_empty() -> Weight { + // 1 read for UnsentRewardEras + Weight::from_parts(5_000_000, 0) + .saturating_add(T::DbWeight::get().reads(1_u64)) + } + + fn process_unsent_reward_eras_expired() -> Weight { + // 1 read UnsentRewardEras + 1 read RewardPointsForEra + 1 write UnsentRewardEras + Weight::from_parts(10_000_000, 0) + .saturating_add(T::DbWeight::get().reads(2_u64)) + .saturating_add(T::DbWeight::get().writes(1_u64)) + } + + fn process_unsent_reward_eras_success() -> Weight { + // Same as on_era_end + queue read/write + Weight::from_parts(1_136_401_000, 39987) + .saturating_add(T::DbWeight::get().reads(7_u64)) + .saturating_add(T::DbWeight::get().writes(6_u64)) + } + + fn process_unsent_reward_eras_failed() -> Weight { + // Use success weight as upper bound + Self::process_unsent_reward_eras_success() + } + + fn retry_unsent_reward_era() -> Weight { + // Same as success path + Self::process_unsent_reward_eras_success() + } } // For backwards compatibility and tests @@ -113,4 +148,29 @@ impl WeightInfo for () { .saturating_add(RocksDbWeight::get().reads(5_u64)) .saturating_add(RocksDbWeight::get().writes(5_u64)) } + + fn process_unsent_reward_eras_empty() -> Weight { + Weight::from_parts(5_000_000, 0) + .saturating_add(RocksDbWeight::get().reads(1_u64)) + } + + fn process_unsent_reward_eras_expired() -> Weight { + Weight::from_parts(10_000_000, 0) + .saturating_add(RocksDbWeight::get().reads(2_u64)) + .saturating_add(RocksDbWeight::get().writes(1_u64)) + } + + fn process_unsent_reward_eras_success() -> Weight { + Weight::from_parts(1_136_401_000, 39987) + .saturating_add(RocksDbWeight::get().reads(7_u64)) + .saturating_add(RocksDbWeight::get().writes(6_u64)) + } + + fn process_unsent_reward_eras_failed() -> Weight { + Self::process_unsent_reward_eras_success() + } + + fn retry_unsent_reward_era() -> Weight { + Self::process_unsent_reward_eras_success() + } } diff --git a/operator/runtime/mainnet/src/configs/mod.rs b/operator/runtime/mainnet/src/configs/mod.rs index 0deeff2a..48f1aad9 100644 --- a/operator/runtime/mainnet/src/configs/mod.rs +++ b/operator/runtime/mainnet/src/configs/mod.rs @@ -1598,6 +1598,8 @@ impl pallet_external_validators_rewards::Config for Runtime { type RewardsEthereumSovereignAccount = ExternalValidatorRewardsAccount; type SendMessage = RewardsSendAdapter; type HandleInflation = ExternalRewardsInflationHandler; + type GovernanceOrigin = + EitherOfDiverse, governance::custom_origins::GeneralAdmin>; type WeightInfo = mainnet_weights::pallet_external_validators_rewards::WeightInfo; #[cfg(feature = "runtime-benchmarks")] type BenchmarkHelper = (); diff --git a/operator/runtime/mainnet/src/weights/pallet_external_validators_rewards.rs b/operator/runtime/mainnet/src/weights/pallet_external_validators_rewards.rs index b8be1393..10854100 100644 --- a/operator/runtime/mainnet/src/weights/pallet_external_validators_rewards.rs +++ b/operator/runtime/mainnet/src/weights/pallet_external_validators_rewards.rs @@ -74,4 +74,29 @@ impl pallet_external_validators_rewards::WeightInfo for .saturating_add(T::DbWeight::get().reads(9_u64)) .saturating_add(T::DbWeight::get().writes(2_u64)) } + + fn process_unsent_reward_eras_empty() -> Weight { + Weight::from_parts(5_000_000, 0) + .saturating_add(T::DbWeight::get().reads(1_u64)) + } + + fn process_unsent_reward_eras_expired() -> Weight { + Weight::from_parts(10_000_000, 0) + .saturating_add(T::DbWeight::get().reads(2_u64)) + .saturating_add(T::DbWeight::get().writes(1_u64)) + } + + fn process_unsent_reward_eras_success() -> Weight { + Weight::from_parts(1_905_623_000, 29162) + .saturating_add(T::DbWeight::get().reads(11_u64)) + .saturating_add(T::DbWeight::get().writes(3_u64)) + } + + fn process_unsent_reward_eras_failed() -> Weight { + Self::process_unsent_reward_eras_success() + } + + fn retry_unsent_reward_era() -> Weight { + Self::process_unsent_reward_eras_success() + } } diff --git a/operator/runtime/stagenet/src/configs/mod.rs b/operator/runtime/stagenet/src/configs/mod.rs index 60fee86c..12b4a960 100644 --- a/operator/runtime/stagenet/src/configs/mod.rs +++ b/operator/runtime/stagenet/src/configs/mod.rs @@ -1594,6 +1594,8 @@ impl pallet_external_validators_rewards::Config for Runtime { type RewardsEthereumSovereignAccount = ExternalValidatorRewardsAccount; type SendMessage = RewardsSendAdapter; type HandleInflation = ExternalRewardsInflationHandler; + type GovernanceOrigin = + EitherOfDiverse, governance::custom_origins::GeneralAdmin>; type WeightInfo = stagenet_weights::pallet_external_validators_rewards::WeightInfo; #[cfg(feature = "runtime-benchmarks")] type BenchmarkHelper = (); diff --git a/operator/runtime/stagenet/src/weights/pallet_external_validators_rewards.rs b/operator/runtime/stagenet/src/weights/pallet_external_validators_rewards.rs index 4d223163..34d31953 100644 --- a/operator/runtime/stagenet/src/weights/pallet_external_validators_rewards.rs +++ b/operator/runtime/stagenet/src/weights/pallet_external_validators_rewards.rs @@ -74,4 +74,29 @@ impl pallet_external_validators_rewards::WeightInfo for .saturating_add(T::DbWeight::get().reads(9_u64)) .saturating_add(T::DbWeight::get().writes(2_u64)) } + + fn process_unsent_reward_eras_empty() -> Weight { + Weight::from_parts(5_000_000, 0) + .saturating_add(T::DbWeight::get().reads(1_u64)) + } + + fn process_unsent_reward_eras_expired() -> Weight { + Weight::from_parts(10_000_000, 0) + .saturating_add(T::DbWeight::get().reads(2_u64)) + .saturating_add(T::DbWeight::get().writes(1_u64)) + } + + fn process_unsent_reward_eras_success() -> Weight { + Weight::from_parts(1_894_953_000, 29162) + .saturating_add(T::DbWeight::get().reads(11_u64)) + .saturating_add(T::DbWeight::get().writes(3_u64)) + } + + fn process_unsent_reward_eras_failed() -> Weight { + Self::process_unsent_reward_eras_success() + } + + fn retry_unsent_reward_era() -> Weight { + Self::process_unsent_reward_eras_success() + } } diff --git a/operator/runtime/testnet/src/configs/mod.rs b/operator/runtime/testnet/src/configs/mod.rs index caca5de5..27dbc538 100644 --- a/operator/runtime/testnet/src/configs/mod.rs +++ b/operator/runtime/testnet/src/configs/mod.rs @@ -1598,6 +1598,8 @@ impl pallet_external_validators_rewards::Config for Runtime { type RewardsEthereumSovereignAccount = ExternalValidatorRewardsAccount; type SendMessage = RewardsSendAdapter; type HandleInflation = ExternalRewardsInflationHandler; + type GovernanceOrigin = + EitherOfDiverse, governance::custom_origins::GeneralAdmin>; type WeightInfo = testnet_weights::pallet_external_validators_rewards::WeightInfo; #[cfg(feature = "runtime-benchmarks")] type BenchmarkHelper = (); diff --git a/operator/runtime/testnet/src/weights/pallet_external_validators_rewards.rs b/operator/runtime/testnet/src/weights/pallet_external_validators_rewards.rs index b2403bcf..9b7e752d 100644 --- a/operator/runtime/testnet/src/weights/pallet_external_validators_rewards.rs +++ b/operator/runtime/testnet/src/weights/pallet_external_validators_rewards.rs @@ -74,4 +74,29 @@ impl pallet_external_validators_rewards::WeightInfo for .saturating_add(T::DbWeight::get().reads(9_u64)) .saturating_add(T::DbWeight::get().writes(2_u64)) } + + fn process_unsent_reward_eras_empty() -> Weight { + Weight::from_parts(5_000_000, 0) + .saturating_add(T::DbWeight::get().reads(1_u64)) + } + + fn process_unsent_reward_eras_expired() -> Weight { + Weight::from_parts(10_000_000, 0) + .saturating_add(T::DbWeight::get().reads(2_u64)) + .saturating_add(T::DbWeight::get().writes(1_u64)) + } + + fn process_unsent_reward_eras_success() -> Weight { + Weight::from_parts(1_893_280_000, 29162) + .saturating_add(T::DbWeight::get().reads(11_u64)) + .saturating_add(T::DbWeight::get().writes(3_u64)) + } + + fn process_unsent_reward_eras_failed() -> Weight { + Self::process_unsent_reward_eras_success() + } + + fn retry_unsent_reward_era() -> Weight { + Self::process_unsent_reward_eras_success() + } } diff --git a/test/.papi/descriptors/package.json b/test/.papi/descriptors/package.json index 5ad366a9..185bc678 100644 --- a/test/.papi/descriptors/package.json +++ b/test/.papi/descriptors/package.json @@ -1,5 +1,5 @@ { - "version": "0.1.0-autogenerated.15484599658830368838", + "version": "0.1.0-autogenerated.18139584469151706411", "name": "@polkadot-api/descriptors", "files": [ "dist" diff --git a/test/.papi/metadata/datahaven.scale b/test/.papi/metadata/datahaven.scale index 1ad174a797dc11e0174c4289ff51aa16bb594ea0..3f4882b211f71d8afcb306069413c129280f4b53 100644 GIT binary patch delta 14621 zcmZ{L4_H-Iw)oj+@6A2u-g6Zc5ES%+fS`a-pi-csVv?eu;-sc{g`?htd*O0XDb1L2 z8k+or_u(2#>tR`0S;I>^&o^25zMMBs&&p3sBC`L4|{8$a|y0Ae0+DG zz0dly_u6}}wf5TUy#H3(haaXz)ae2D@Yc?VuP8b9(lz=eBUgf-D7mCBn4l(;q~Hv7 zD$B5uH{e&FVC_*glVk;duP!FJ!7=s)ti(qC71qScal!R0D)`V-qgbhpJTF?t1pi=9 zCS}1M`xI6wpt_oATyTcNMLfYZj;U;$jqHJj=UH^{MQrN`zT=q24%^7DVa8P&8%)+* z@(~++k;z8|S8Ah_qc&1o@2;&4R%mg-y;=e}9(++-%ud+I@1gB7OAbzOX2~aQuriCM z2Jdl>A*X`i%pqrjzjqePXKfIf&C_1W9rn?Er2D1l-Gu~;ialjm1S|Fw1mAD-hn^nT zdvEBeFsNHkFa0@qpG+C1hKqv5M8J`%3quG%7)bTWcm3zEm~ z46fWi!fcZ>UBSuDRCX$Y^qMbTd2X_NIwDy7#OmPu=f|-#5v0>>|HJb+?A##iy~Awx zHy(Yk@hCAGkIL@tu;((HxIJ<<4xX{45@C1x^?<**s>$1^djnN|eY4wNQ>FXejHKXi zmB(A7Z;2$X62H4tJW7%hLvHhJ(*0hyw_11Ujn(dY-L=6TsNQgktIq9hLdwl9lc)$H8kl> zI`l3f^Js*2AFRsaF5_w@Js{)YVcQ2Z-gtg89W_io69H|PXdGNDqKV`zs*jdWPz_{7; z0O^5OX4B%Fo(LAVSZ{2q4Jd3W>5X6|xB(22nXVNu%CX&<(S#xM1 zxe9m9p{ZEx5ydrlc@E7Uh2RQFsrNMoaB9~1dLu|=DVEUTrSVkiivZVU5(kq@=rWnu z;K34lHzBZK9*Sf1TpC3 zko8dI^LwbDcntSex^*O}GcN6>|7Ii2MjOyGOs=y;lHL#!Nz)*aG@vM$l6Q?uO+qMG zZ8o82$=ht;d!EI@nirTF?3t(Qar1zonFtqej=O)G>;5LeF`6S?;t!*LXS-KKdVe2RRP6zDJwo6E@g= znN3168uYwJ_itab*wS!irh?O#pyr$F_SEV%|4fe(_VH5GbK!~+^`eR73iwRrd8?@) zO(j?(RHmg@vzLZ5TdW5dilAoS`}EH3GK(k0u6n<|$>VEkL{%K3cq60D?naXu;7d!FKgvXeZ27py*ep>ZO7h%jt^iDJ~TSUh#=g`r?*T~5H zD>g|ZabBOl&Ru)OMuZj5^wr~V#mfJ&DFtnDjXU7;H=8g(KBn=o<*&4`{AvW5wy54z z9Y51O>o)|ZO9Y{}R0Vvf?SJYmPFthy-=x?4#7n6M*vF`-dtvg&bhO+%&{5d(G48*s z@chRprfYCTKt(}xH+5l=-%YDXAGCMV>FIp~mV+p$knsoYApXc)k?jmRawI-NYuR zGm68o_Y_TbwoJgrej2dgD z#%n$F7$Ie*L}MyKILim&RKTlW;==jx3p#&zr9u)G>+VLMw^5QRm7tG|g0e4ZvbtQ* zt5Bp>109HQhluHeh`9x^fT{~{w<384f!Bq=+XjKxK}#>q9oe8H%yQQXZp$nWW*5vLTA8QT6$Q}WQ~IRy>hql0-`ydG;D|DLwUiDv}UvxfLnSTc+k9Sk*n7Nk4bVwS6-SNFOT<-ga- z#^WY_*~#YKl#4hh%{VDPj28mmsBZ)$@+yGrVJv@i5dvgZdi^>Yu*QI1gJMQsd7~aE zhHb;xbW#GZ3}f@A9B>(Ca%M#GF4wwScQ-WXOD zw?id!&{y8%snPv~AuWPSeg~+rY`xYh`ld;dJ5*_>u{9RM5B4^*yV5k=!fz(Br_s%9 z9Lv(>{Z`+}LF4jRwwjR+m@%H!k;8CwJT9>#)`(z33L8m|!qyb_IXaIUQ&|EQzf47w zavUzEvdQEGIMdif`J@_rXya&jzl3QsrCuWSsd(-k2eJ8Vp-k+sE}zXI@$g(e zy9ul2tLyn}CZ0EEOkwlT-tU^m#*r9!VH!)3x!v5)>8jn@g--jxepc;bi$toj(s+xBeHy#}2I`3+>PaE$se`B|nbdpHj-(3eSj{l02OoP- zH5L`H`zdZ<_Z$`vpB1u&D5G18*d#Q^>xJ4oQ$RH3S`40=!OATLD8-p?ZU=B}o_k`)G#8PF!<6TlAXJDR~lcZK?9DTi2)uZfI`)sb*P1 z@8jnsEF-GUPBahVOLf=(TqcjqZs14p&z*dYK;7=k&ufjTtdabC8O_;tPXJwx5MztuBoPgi$ZFEc_`|Z*ZQKH>a<%ys%4AV`>fG zQs6%pv+45bP-W~AhEZ5pv;>XbS=hCN#X8R*CKaPUYnRSK=Mr|)uyb~jaGT%PRDWBp zB%QDg*aOm1^Z!mQ0TIYe~CM!?oekxUdrOPU$K+Oxjz3! ziNv{U18$Fhpdn?gUR&!9_&r+&+7d)G6Uf<@meoxJM*jkB-xyLwF$sVPo*I2b21t|BLJsb;1SFla&sy!4%D_zOb7#`H> ztJqTcnjJ=egW~_Bih1pQI53$wB;o*H1jwswT}mFEnAIyeB)Yn!QvVNE2`Olq8Ul7VYkVN4lDHbU=4c) zBXBu7+d-2Y3e;E7RJf?4-AQ$rkvOPW$F3tp`Fi#~9?W)o*eIB}f!&3w{J9ORG$YF~ zSUcuAhG@rJ>>2L2TnjbN!^$1`*u<0$9*Hg(5)7ga9)B0h#G?Ezh9Orl_W@LV7cgYEo@+EqyK5-uhN7yUTk1@5VG7t%xGlaJ64%P$jg)5 z)(j=OLoqjf&@pTu9RrhgqyOGyJ^vNW zmE3I6joB8O`IbR+w^?v^?_rnJ7Qthi1D!5+8}9g4AbXM0P7C71UiLO>w$>o-`8JCn z2)~20i}x`Gp10EC?EB5JMCPH2@jlPm<&Hs+^5IZFek!JT1m1avrNU!>WTS4*vli%4 z2g#hZ)URWVE?~w^k2;vEzd;=T^DqR_g$hOL3~|YRf!U&u4;tNZF(Q|ALUd-7f#J`~ z!yG5gE}|AZmvStIP#?ud?641FxZfYQ7l#8oNtAYl4n54!_3E zILJA}_a=*=@_C09Z+z%2c0A^SAd4<)q;%0_AVo@D=6hFcw*xz>kZH4hwL&GJ@?Pw# z)#07Das79^&2r@{Awqw9n z${A+Gg?i$PKe1RjEA(#kc~(qvjXTdX)D`)#>oW7AH`aHVrR5Z8p{0TYSJ-?av1KF@ zXQ2ptVgATCOU0r~T4ZFLR3eZww?~`TEar8awgAJa3w?o7@1RI0f*%vn%*J}X+Otjs zWJT-}Lzp6P`4jhH+4HRUZ@C#oEjL4_XxT1Xt^L7O5iS-tY5E!@c4MT~wa)LW!@Cxr z-@RVXo--tBEy?V8(IxA?PIuMlwH~}7YDN@KZEb%{+2=Jw$k33*Qdp_he?f^;)K!te;L#M~O*}o@NqJ-q-nuPzRq;jNHqQ$LAZY^~A_T2xnLnF>d zV5nlcu1Ci+9 zAho%uu}q?`_c0f$t*$<{2sKvjbvE76u93(xUk$pX?V`a2kKm8I9~V+D#_!(0&NTU; zNf5rfjvPB6@duV1b3`N4gg;fX1&zPA(Nm4vrvsM$z}7|Jz|e7m*MDGh<-;asdjelL z!fYFwr}OPlg{@4;u*`1YlH1ur#8gCbs`elw-X3Aa0~ekUP7KkSo0F{ z9V8vPgFI;gpM-NVPlgSIr;$^zoABf@r!_LK(z_8|3-8hahfz>IC4k4n7le0V7zv*n z&`Fz3c`7<-9?B=?p1on;lv3O{g*eo6$j$tb&ubF?;~p=ih3Id+LHXj5XEc&i>fNCG z@ovI&axpk=c56Q4Q*ye(#?!409#FTS8$P0L=w$ref-7Ot>KQ7n9!mhCkb5-=|FKhr z=M~sxiT z^EgzR?kJuxJp4@A3~vnQg&EtzI@dCE;4Rj`TOcEnPe8R&5y_L1Zj48hX?}fORlrlH zS2ba-h;D)ATb&$tm*+7ULQ*1;oc9WtU;F``6>jN(yZSmUF3O8Bs*h7W6J*m&B9 zhCbTyT!g}IgAb$lgzfEN;75moivuu%@5hcYZKt!%+3u7Mh9Mmv8t^=hAvZ+o5J<{S zfpR!(%r2b7BPeQRl*lG=4b0*ZOV~%CVFcfleKZUy5|KIulEA^>eE-aMS~#5oCkdvE zmW~m?sdtAN7=lDw^bzPU$4%NAq;}G6sKkM5Asz4W3v&1NMv$Q0$A3q$Ubu~+B&kxkz{r2EfX$mgvA{Uer_XI}_I73%V0 zKjt8HdC@|=Xd&(!#cy8I$ z4Ymzp+WD}d)6Ow~foqJzmt)Y3FCN3aIcgYUs|_)o{g@&*qrdY`3)Lx54`xS&VVS0= zA8C+mV=SB)i`}IJUX>jmhA8@(*pE19c_vzji5B9nL|&SmGz2mAClFID#8eA0eJsCw zY(^MjnFCp7^<$Pf;GS{30IrSYg@w5YIADJ;UFSxXi^pdnz5FnOog4`kK*Km5trS?? z3&5MiC&00Bd{ItO81My(z{UOCFW?Pvzj%u+)M5*Dg^L$wmxQ4T(_Pw+nKOV`Y9W?d zh+n#J;g=0TtoRAU3JbA9AZ|&h48sxgvb-NB5?)G18Qq%9mrhxQNSLW4xx>KY5M1vE zj~nQlZn4m&G0tpX)5tItz9Ccs@6$VJK?TPoXwhs7(Sj3zAd# z^z7y^RIyyQ^zLPsS~2 z#w)x!CaJJ2m1kgd=)P2*DQ=Qd`7P|ULY@&BsH1_W@kC5X9G}J~<0Z|KG@gU5<^yR6 zszRqIqTrHf%z)t&crH3E#S^f$6a1n$2v1BvkPdi96epo?0?)#Xl1b^jK<-w9yKS+s zH67ja9{4Vu&opn%uZqVzScYBX*5F<@aCQ=(!aM@d zyrv(QiSBee%*;gECmf-4Uf7+7j50cqg~F+Tv!bRO!?SrgC+CedH}lmp-e`sa zQ`K%%EydMWDho5kkkycK7GuHau_%{;r!h0Tmf^-{gcX13KUV%?G z;;QQbrIr_9w0Kr6znhY)#z`NKlyUjJ)xc{=pK-IFzemOBwN1PhOXKcMypUsQ+{5oB zc>DevXvpT7P_vB}57hp&jemfP_QiWqxH0hQy?h-ecP`$}*UPwu|Dy%ZnOV@)!o|&6 z($9Ip@J^*vWvE%>DY-16m^?p6pRvMd`#HZzh#Qg~!L*m3KY+dWK;aJFMVr-a;Q9q$ zhWUq^f5Go3q}BNTLH?Z$_r$lmc^Yj~+l_I1_+A-z$G`36J2542W)NjV533AngS->$ z?ffBnq`x+AAIkbDyt|M87fSaLK*}eKUO-8>PO7O=G+rxjX(m>oB#4?vZzP0qJPu>$ z-auK4J{s2^;s3)(m+{WyyoREv^Pk`^pr{SkGdzg^e+m!FJ+S#Ho{RUchW82H6neQ= zcmXdR;ZyiEwNHIth;$01Jc|n~(Rlq?-Y&$p>?j8m+vi95Y*J?2^aB5ik>y6}OZ*8L zx9*wahR~agU8D! z?9llie7s5;7)i28xTI$NJ6}Y|DP!*mT;1gKgV)E~jO*i-4>rK>#@nV2G<15RGHgnY zxb|3v*GJ-3sT*G3sk|)r*n_V>8E>@hQpSr^iQ~JKSLCbq;B|F0?A)WI;@Wy?kMh4V zE|W)sN)iU)UI{8Q2rifZ|1U}ni8tKsN(-SSHU(z>S(yMDC{0O;jv3OtCH)s3x}{#1 z-6WBAPD-)gsc;CCn`nk3AVio5-+*#I&36>Q-M>~Qq12xGwX#9RMLzsdr5r_WtT~|I zRw#xO2bHXd5(ksgd!X|l=RzWGcq2)E@Sz2i;hw>Qp1>?ilmEYKL`R;sE$-yJq zt8Xd^h~H#fd{c=Pl)pKv%w$S4c0s$(RSqqGR1%Hl?<%WggxX=8_>+RkwyiMyJ>?_J zul?#hWj@6O-|2r=k}-O2xIb4=nY9>y_(B;^$$sPWuarA+=^OK}Dlf^n_J8c&`d0>+F}3vgvrj#FR6x7?*3Ac%b^NiCG= zNyjOpYrHy=;?rs4A8G2%l%8>v3pqy{#S_)$NM!KaThysM{;XrEls*d+{Pn=Xb%92S zR7Q{^I6LRSF;g9fDfp9Ss_kg+PR&&Jpjuj7tRA4Wew3Mp3ELU40!P}*3 z9K2PED8~)?c6BVlRQaU&Y8+;Z-!fkXvOSc4Y6Vh7JP@~YWU+=5@R@6|_>y656NV%G zF5fy4qHT0--r&Q;Rx7Es|KkQ7(@?{bc(Y^h^9gtL23KP(#&bo?6+zMw-}<;BQ;@2{ z^ml(?yLs%z$GUnODapMW*3(i&n{iMJflB@*BXwvy(Quz0sj~wVdL3=?{0B6d1{S?>6&1MzrWwKxNnIrM*3}1^vk875bHe!30(Cy4q(lQ(g}O$3 z8gmjJs8IimrghC?H9};jtzV+P#!g3|&YHMX&1PstmfxX@sO0uL)I9Og%o#B5P#+f2 z=`XW*0=So}`Iu(`%hmgFr;k~oItW_k2`kk_SZrRYR^SRfvrOdW(#S3a8hoY1=(o%DmYd^y-b6Lb^it`$R6d?yu91%`{t=V3a?k78|t4X=dBf zEuoesEi%V!xeaf98$FH9TAl?_=JR86t|!pEP0LH~r(Uh^(2ixAH7k?1RXer@e&}Oy zlXhyy=9n+a@PjPO19s1IH*RRtKHRUZiJXlqP)opYzEI85H8|>43(+Wk;#Q}jIT*iI zeTj8y>|q#Jt;WKaYt>1(T@tF*ql9*7C*Z$p)VJvw?Fc-kt9RKCYLSBa8SMg$Sf}B` z^J>*Sa-RmfHmN1XxH|O{nGq*<8q)*nXxW(HS23MY4K-lSk<-|@N&O2$Q&D%1It6vW zllQ0_Q3phBRe$G5bw=ab^{vZt<{MpG)nAgZmcYI0YXqaNdH1PHv1q+dJ%diik{0zB z5rToM1ZD=*NrwIBsN>>_orN29%puwA)%{f_Q4^-ff@ys~Js?|HtE`q~52{6>-aSH$ z>|d#O6XtX(C9r3c8g0D(u$n|Bb!b|x?u{&QN^>xy55K-zr|VTUI8mP3Mn!Ad5(oj5 zIwhFATOA7%cd5%U@#leENUhZPVHbW%hZn=id(;omO!Vzhd(k5@zSyf)QFLn;?NbjD zv;;k%enM!SvjINXuWpK|bMA0TdDTsR5lt7FD%BgMouEIWPR84e_D9qrbPWFZh&nX^ z<5M$nw^~;dRqH&p0o~8EErG@(S_h;(s;P++_J`lwj81uNAg^IrPapZ9U9(-IKFWl>=brvTcJvDi5ICNMoL5uzO!>A`3VC+*!xEW?VrA~=% d(F%31du^>=B|d=^d&1QMtxu`{O?M2F{tvBDH3t9y delta 13172 zcmZ{L3tUxI*7(_H@6A2?-g6P;A(w}Of`E#Gf=Pmk3W|z?ijNBM3Rk`Hex;a9G3A(* z72Uy=C5=>;WLRX*89$|RW^7{0CZ^9(XR@LhDr+d2qA#3m{_EUx32w~)_q#v#>9H^_ge0JQFz#QM+k~YrZ9tI9!`8H_7!TxI2m4TrXQ9NQ>)T z%N)7Y1ZR?Xm@7t&AV*w_)EP>fiIi5_OG{mOYKZGmHJo&~UQ}1JlP2;eG=0Hr@JIp` zu2^e=e98pcOdjL9(;7xPU0_Wl=Ui`EGgz02{9Xr*Ix*e1_ioa2BJklff<@jVIas*z z9!YgwYO40s?tS!Uo?4pAVbo6i&2_g-F1v0!5KpeSesRD_UN~{>!0JGlzd_h!snhkJ zV-Lw^eOwt&Z*uj$FoB)(!7-92Z*!HlCzA87!edpB`kL+uI=(mPIMITR%l5lq^(QP& zdu=*xmJtymKc+UVB8iUkmCyM=>*p*44$r3%qzhi1PY)VJ2vjbhuljcRND@*!EYAPRB{>qwTMQ` zSA3vnue?z1@o`n|3xzut(^Yb>52SxcZ1B-y`XspucV*Iy#H&6mWVNHFuC!KRYe}CE z%OnFnL{V0iFsTnmBuQ#!bv2U)uudeY53Xg>G|~@;Kc#_VEA90)gmm71Iw1s zU3%%ajE;jVOQ=5~@YND(Tg*))I&-J97ROOhjf24P-r=aOWm1rd#OWW5wj@h|S@v3c zDU)m-40ltaIC5CAlrAP5nwHWtM8L{ydJC~acP|ZsSF`CD?0P4gMn_mpOkGs5gP|(? zO%jvvkL3QtWhn^8EThv~`90&_;qd;=BOSjSY8O<^ILGA!* z>1{i`i{wG;oit3_wuO$7NxrtDm_F^p3QQyy>^o_&mRv&LC&aG#m(gYm*{D6YlMW)B zj#OmZXd)}t*-Psjxz!bD63$v@MY(%)aQIF-5$5lvZKOa4jRJE6-9zj~y`zCv%MKHC zKPU%+-ERwVEF10+2s0SibebTlQdr@4O(g|{ZJ{INQx|vW?Ed*3~weNm{fS&(oV_w;Mf#lHUIU{nAf9VuJKPu?W`} zkwQD%Nhu|5+B2_H%Q(^wFI}V`VbOGnmXZ$m;S#NvPnsaAmnG=nguzRf=|NcYK8@1- zYu+bps@D2GeU6Y$nD8+Tg`y8|I?rkgKBOPWu=E2O2B{y@N%A?9?#Du5=f^b4a9}n# z@iEOFa%Mq)r)z1K$?#y>Lw~3D6LKDGy)<9x)&;*LU4T1!ht^*(I;_un>22f+bbpHL z>ef$a7U_YfKB1H49uxFu(oolHBw$XjNm5Bjc|~=Zy|mXv*jkB9tEhBT+jTqou`9aF zUS4dkt*EXyV5DAyYUZajZ9=~fnX{_WZFlP(&TZRk=Sc(+b{5uFpx1D_wD6$+0wi74 ziA*7V@cpMWRPGxd6=Z&f6Vwm&pP^42fM0)xP*&*qj7AzoFoax1#SFrht8`x6;IMPK zp*t7+Bf0;`M3L}^wvbKzB(A-Al|E0=C1rd~Z;rJp#)9@&h8MKe4V*3~O8!`ru^FTT z2gByQa@e9d27+w_f=vgTBSk1I#L$55;i43x_^qrcci=Gm)>M=^m=uL@=;foJ`zsm_ z#b45KBnEbWNn>e(LZI&}YJ=ClM9-WA-Cxph{N{!;e?@N#h*wDBy6UZKoZHInwRP1< zG#=`|qU%Y54pas2enlgo@EWxTj)qK8pw+?y;L~d~holaRGCAENe8vb_(%r%*`binO zP}s`U9_**D5R&aySWu1!Cw~M^4m|QTEwug3C{BZznrV73optDIG-0>|xw>68Ir)X8mz^VcCDqmyYtaO8Ns*p0gNT-h`t%D-zOt%K$ zqYyS$&K9^#Lftc@y&b@6Ws(CK!K?s7o?ivC3M_POCFOgxwQ+>D^4*O6I;gY}Yce(q z+&v+7A=!1bMNm18%}jTCaH~e(V&4c&myI53nK9HdeJ~*~H-wdlDxHf{NOi-*C-ug6 zi;b;;e}(9qvyEk;V|dNRg0XNTe`#Z@tb6pKsghr#koF1J_lcoeaVVQ&GVgZ}S(<}u z`mreX9L63CCbBpz^zoB}TKhz{iODU(wZIt+lVaIA(h9p`**;vRx?&(Djs=i5$c|%Q zV#qNko`qwPACGp{0nf*?8MsLQ8qcQ4r-bYNl29#j3cHC~I^9gqiV*3nn@Z3*kBYiR zsOTIVO<+v{UAolggmm7G(~UUY2F?W!&gBs}7vRz9>}TS#j&ngsS2WuUmMvpgQIy0U zmU{(M`H%oG&18?0tJ=FM>@yjcqFatLDJVz3(Wef$c}e|58WebnngD;F!&b^Sa1E!j z#Uu!Jrm~sv+c_)>-b!T)@VKJh%$DNLe`^|>fa~kdG!`urvpX)2<7Q)>3G?uJu1*10ST`fq08P%)P+fi-iH#l^Yoo}bXRd1yy?Xh)5p9pR?kkK0X@ zP8*vsW{7+ZPa>h|Y!AiNd3XWahAL4q*i>Ak$r)j)g21_(EK9-d*Jj zna!(fzE>+2G8Unk{%QeQV9g?yYmi61En*L|Br_?6yoD?bniivyJxtzS%*?pDFD_>B z2u(8CL|om`nQSt83>={iiZj`gycF}ub%K$sCQ7Y2fGu&ekO<+faLIjz{fzBh;Q_0EI|-Sdp=nua_Im68;dopA?v-0z<%)`J7(B<7R&1*&+)`Rm zQPzk8EYvHK)PQ($lu2rY?o}*k;vO?uaBJP1xseN3tXh>*xF-9S%mPW;gFsZDF-6)3 zO}Q)p!g5*gr2S^HI5RtUcFLU9jv|M%vUV{Z=rKGE$Ih9lvl5xKAJ*owiD3x16#J4= zQc_1kqPY8Ya<@zv9uV=A6oIC|IdtJwkbK~(QjUA+h4)zxfjREw^KOeVGJWwTj2 zqL--1HodGjOYLA@!+wL_@po%js^uj1GUXgemQHD=wd@vI?lePgKZ%CD>)1T7uVa(p zi**dsUJ$;X&5Q0rJ>}MIDRmYtchu)rJ1Xtfj$%nVhh!+`L8LCT)CE6V&u03bHOOhn%JiNM|#fEmHjc4Ubgw_98+hx??^m=qUVH=nlcmV}k=-eg|b*H1|if%l! zX2O5;&+9iZ-?+o;?RDiPb0b%-$XtnYdRgc0i?embtp6l#k$OB*bZlU@yL!#U zFRP-uL?R*f(ptN-dblHci=(vEUR&+lIouasWUthFhkF8x9p#QK!|iZKsiO#|ZMes8 zYh7vGk4>r9tX9^PSM1nQ?JV8~xAtT5X3Iu&cUQHCHnJxu>C>hZpi<>N_b|u9u1)L? z^#1*u*m8#Fx%48oRvs|JnQw3+v?5k+9z?O^8)oT-8D|mSVWe0j$%=(-_USqehi@2qHzKt!BZ5Ei9Pba}g zSE+!)?W~1Fz>V$bwIeJFBo?z6c;1N{QIthb-@y3W*$)U1QNk{<7>i;gajd26cJ%Q( zOIh}`c*{uVnP3^^JQHwmxD*qHkVzTK^-aP~rnGWDDP^d?ri|@IUwy5NO~XPL*&RHQ zbj@DQ+Nhji(N7UkkWkHHNhai1v)OW{yDt*jt64J3wvf#*ze5Rv>(wlriHrPcZZ%uDy+mb(*HN7l;IgZQkgXD>)h6ur9&)0#DY)K4Q}B`fL@LB!?RjW2BvIUx@6q=;xv^FgLM& zQU>2Qv5(|3cV8%c_b59gS9uyQxiDg_bCVCQ_t-?k2=etFxFx@2ADA0-UiB8K(bJjT z%!_9wC*)&aW(3mp7nxyk)0P{YL77YeVVuizo zmoN}-af4#Kri*JHPEvuT1H+2jF0#qCWW#RTEM(fkwbc&C8b_@=HP;3m9T)?&yHUd< z4XBP0l62^UkCaZj`58$w*!2l+A@g5m@Az6yxd((!vD2cTB10hYEbEZZ4mW&&{Fc1| zJ!e@YeEM5hoeaF7)5HkYPrB@;F8N7U z^v_Y_dMub!QOKM{6{WggKtH~^qSPVbHphaY{5_ULdg1VUY_|VZOLRsh+7B914e7H; zDx8hr!Eoa}CMNb-$nZyrgcm#9$;zu16291|Ld$z>JY@YZ#=w2LVuE!Yj)C9)FPoIv zZy|Hs8qBd*rY)ZqxeoVQnICC?R4m#i0)1_}zcxL5qYxTy7G6xM2u8?*iK{ z6BVM~!}W3c0vjiD6>htL)4`$d0tTMXUSL6x{1;XpCRD`r^XsKeKFvOoRd3-vg1;*H zd(hwg3oB8B-2EXvM6#)ROd6z*J7H>s`jbhIP(9;CE6~$KNKHC~BaKp_RiA}7FS2fU z`64nueu-Ia@hW~#B+0pWk>jY%attMZJgh&x#4<>N7Jr!~SumuF?8UVA-hZ%#82|n8 z9~b~(eEAPH&NmeYC+UA=oYizK_Y*cpmNQiM{Ry1<94+ejXSnLns~rBEy%(0PL*}Rw z{?Hiw{1T-+y(eFl^5LN`*c*Y}K6vKey48Uhm*viK+!{Bko+}khND9sMPu5OD3v}#g zyXsJ#YMEN4)~gL_qq;}MU@M^?!^7HZxH&kC`fJy4-*6gr-jDvROs}WGGuJSny|8lYX~H`7MdIMH*;XOcb8NO>e|pnSY# zpGy34DvB|2womU}K=&i$*^F@bh;obE-3oZurvZUA!~uKv?&k(Yr|8b7j%Dtt=P*&!3fWu`BwY>I0t4MjeO zEZu+PE>*%GE`)5FhH=kZ3SS-2sglI(a=f8&lzRfpZk@%3#0y^XTvkU4bXh0qEzxCT zV1E_-rMXwURMsfDy1kPQp%TO65{6-C<@xx4C&v^V+|FYW`QxeplJ2Mk`vR30Y}s&GN&(ee!y z{*CRq#5!`)Z;YmK!=S+tZg|p@QAYyo;XFVQR<}_IsJHSncUir_5_K@E$2Jm&HR-|e z_W};Gj)V?EvO|jZfa_i8+N@Dl$%a)q!YV~rmBlrcg=J;c7!f$jobZY-kH>qq4}AG- zjI)Bq@Qi{O9X3&l_hRRHl6yYbS@+2gR28%Auj|ahCb1zXL#|;9L+D&;FqbR z?v2m(0?!!@o@0RL!0TiAFE-_k65ZeAnUoQt>$jaKQoa{$9q=cjd*C+eYQ=g1JQKig z#tXX-19D<@nJ;t^xhq(Cywfj7B@)$Z&(-U1f|@=54p zR|WGB%!?lm126O*c%#>dn?~mwn(&Z;)3C=2?#O63 z{YZ&$``~mik1gMiV`6Hfb&qwQRci9W?HHX`?7-9(;x_9@N~4Z)&`aD%oVFHJrrW1S z4o_Q)q0JUZjNnmla2(GMX!Sz$L&PIMLeMW9F<_4vu<0TE4|CeQVB2+_o*d%Rj^IPq zaneB5yW|clru@@vJPvmcXkh@UlxpEan3 zhW$t_@SGQg9vO$%1!u!}pweaFbQ$b^2;;c{=e>Y+f7osK!)^l>_ZEFyT}kGo)d-;g@Gsx#NH@g z7-026)CcGf{S(p5{0&%t1Gaf0&khJ01#2^4Z3e7OhiyrW@B+)WAkCfxkqDB z238cj8^dD{MDw()7{naj9q;_%CLH8GNJS)&K8rW$7I}Ko`0X!+5FP>iJ1+1^qsv${7*jtr> zh_}=@!*C{^rzO^TA*LbXk1?(uKzm1o^}g}uvp^3_MGy|Z(F-f|>YXtVJdG#e<@VBP z*xCp6dT|h*pN1eUa8WN#0ZZTs_<~|)0#B8DgzI5bFx-=X@$yw5)A<7TRWe>LVts=g zV|Fzt5wLSQ565k)c{-j#s^HD($f6ytPsi}(6ilAMZ$i{fGk7jyy*PufX8sn>^FWA8 z#5mpts}lJl7G)XnA}~fyx46DE+u+MY9?l#V&znG)oWvtBQQ@*%#zA2cZn!7mt|UGe zJ?rTto{CRH29i((2@o?Ay`}N)E**Y7lV@OP^xaGpHWT7!@eH<6^}O}d9-hUganh}w znZq~9&+VM^ymP>S|+{t`;DXz1KO+e_zoF0JUfFg zCqnyq20w-`Ewt&G{0V~Tw?Ag_jU+;gUcy5e#&PqP^Icf{&vJf?^ucF0_d0nnA?LIwZs+gl!rfkqvvEN? zU8)Ok8D>@RQ=~`htKb{RgkD8RnB~Q{FO_yZ-RrJ|2d=5L*E$FuG>WTG(S6#JReTr4 z{HCpr`^oZv;(8b(*9ANHN;0TDyn|n)@(l&H-^oj{)c$fOPs4WyT0kTJ8KFYF1l_21 zc>U)*W4M)g5C1C~s^wl(ygw|t7tb(J+VOk&Hd(jZy=W=%kiM7ew?qwldFq&UB~vi; zj5bO>l2C=`_F_zyqxJ3OR|vlTdTSql8&&z({k(_PiF(-m0B&&&+ItW1dkER1t>4eD zn{WxNZswEee$k}e*~}l6aWVKl#`og`sY8$HDr<%Q$N34^`Z&Lz9MQ0jFGAa3`XT-Y zRQ9)rkm5-#rv){DmcHp%d^`q#4Zq^Q$7IvO!~7~ESF~wIcriugHXY^13F*}qJkKB1 zciQ)k@g%uVXn#D)>pcBY1X_;sDm1V$FY?)ZKn#jYx}s*o-7lg^*|hMNc(bmImtW?9 z%2;-SFCy96LnrvZ7`~uu{0)Cv#$_0D8j*W_BwUJ{Ph-4L2fsYc+ej14JAYbBKyJ&H3+NW<9E0@{choiF)dO9bSa! z_ai+J?oxg&Up2d4f5xWuxs*xzeZPORat1Z=PP4KAEo#gG;7NN|PEKcqZ~?&O0*N*K??(MG`!hm@J9u%s5{DVk(SfxomUQ&DB| zlM3d;`n8h7N-n|jOslei;7KazX(fSVXp5d!vN*}p?mw#R(cdQe9aHX?@eSe=?Mf;2 z!M$CIj(~@cDnZ)B4rQ*HZnSLFHlJ3;5n5ofYrD=U*C;8_N?up07#iZ0-z(44JX0>b z5GsyqbKX#1GNWByeOF1u98hSt5{~#)T57ivtn4J|iKm+E+g)&oDIYUs4g_L)x{Mh1kGIy=L(hCBf)j zcSMU+J!cy&Uc|5V7;%Uo_79UqnoLhwI<@q85kQqwmb1E2_CagA5)Q9jR}^jcG-0On zoF!Mc;6SZqhN$;Le&ZI1o4BpZvQ~;)SXWfys9m|WwnifPKI8~a*m>BNDJEd-yEjub zf$eoMtHJTWg`DyCAQTA^BR z2f?#Taq##=d8xR7w(#q0aX-Fj%vdJS^TxoHl_CjuiZQFiO46ZiS|!F4%#b$ZiV%Dn zdmvW;!8^s1+F#a)<;>M{EDYvt5Lv9#2Uq%i8^qu6qM~@C@X=rHl-wd-V`qJEQB1#8 zOlN3_1%)C8_oRCYMY8_P`y8Au6c5r&b-OF$SRlUg2FGTRf>+c}Y!>&RIZm((3&Ce0 zQ@4m!SlqcqH|1}t!~yiA4^)f0Sf9#%54kmB z5+v1#P)vTUs6o@|*RIrvM9K!#K~?kLArct7;i-noa9e{&CPZs$5Faw!#;blVW}}Na z`g2i&t|jmu@rJL>8i=X=img%Bcx&J{uzQaP)#Q8eyJf@|ZRZ~G8o>nCoV{W#7W?;# zE;Ow5_laK^+S)_O#9(YzvLTi8VSuquhzdeiedm!LBaRY-4 z|Dz(B$OBfmsZAv5G&H~Y3VjcZDG-N{hFYBo1?JC;V5Q)wa%gvKMwD<;o`%s zqD{PwKHzuHi|IVVSGsK*WW?|QbD^`CsXMk;NHGv_Oiag}X5ldrhqnnO$3!A(@X#@I z3mNc^-pYpm91{=XihS?|VPpBe(hA7fk2ff1UO<%;z&~FQDH9#O(&DN*M|FMS7Pmhu u#F?#S(Vm90IJpcKwTsz-b!wWU+`gsMQKQC_