diff --git a/operator/pallets/external-validators-rewards/src/benchmarking.rs b/operator/pallets/external-validators-rewards/src/benchmarking.rs index 335557f4..4b84bc3f 100644 --- a/operator/pallets/external-validators-rewards/src/benchmarking.rs +++ b/operator/pallets/external-validators-rewards/src/benchmarking.rs @@ -21,9 +21,9 @@ use super::*; #[allow(unused)] use crate::Pallet as ExternalValidatorsRewards; use { - crate::{types::BenchmarkHelper, OnEraEnd}, + crate::types::BenchmarkHelper, frame_benchmarking::{account, v2::*, BenchmarkError}, - frame_support::traits::Currency, + frame_support::traits::{Currency, EnsureOrigin}, sp_std::prelude::*, }; @@ -43,6 +43,11 @@ fn create_funded_user( user } +/// Helper: insert a single entry into the ring buffer at slot 0. +fn push_unsent_entry(era_index: u32, timestamp: u32, inflation: u128) { + ExternalValidatorsRewards::::unsent_queue_push((era_index, timestamp, inflation)); +} + #[allow(clippy::multiple_bound_locations)] #[benchmarks(where T: pallet_balances::Config)] mod benchmarks { @@ -72,6 +77,106 @@ mod benchmarks { Ok(()) } + /// Helper to populate reward points for an era with 1000 validators. + fn setup_era_reward_points(era_index: u32) { + let mut era_reward_points = EraRewardPoints::default(); + era_reward_points.total = 20 * 1000; + + for i in 0..1000 { + let account_id = create_funded_user::("candidate", i, 100); + era_reward_points.individual.insert(account_id, 20); + } + + >::insert(era_index, era_reward_points); + } + + // on_initialize: unsent queue is empty (2 reads for head+tail) + #[benchmark] + fn process_unsent_reward_eras_empty() -> Result<(), BenchmarkError> { + // Ensure queue is empty (default state: head == tail == 0) + assert!(ExternalValidatorsRewards::::unsent_queue_is_empty()); + + #[block] + { + ExternalValidatorsRewards::::process_unsent_reward_eras(); + } + + Ok(()) + } + + // on_initialize: oldest entry has pruned reward points + #[benchmark] + fn process_unsent_reward_eras_expired() -> Result<(), BenchmarkError> { + // Push an entry whose reward points do NOT exist in storage + push_unsent_entry::(999, 0, 42); + + #[block] + { + ExternalValidatorsRewards::::process_unsent_reward_eras(); + } + + // Entry should have been removed + assert!(ExternalValidatorsRewards::::unsent_queue_is_empty()); + + Ok(()) + } + + // on_initialize: oldest entry retried successfully + #[benchmark] + fn process_unsent_reward_eras_success() -> Result<(), BenchmarkError> { + frame_system::Pallet::::set_block_number(0u32.into()); + T::BenchmarkHelper::setup(); + setup_era_reward_points::(1); + + push_unsent_entry::(1, 0, 42); + + #[block] + { + ExternalValidatorsRewards::::process_unsent_reward_eras(); + } + + assert!(ExternalValidatorsRewards::::unsent_queue_is_empty()); + + Ok(()) + } + + // Use success weight as upper bound for the failed path + #[benchmark] + fn process_unsent_reward_eras_failed() -> Result<(), BenchmarkError> { + frame_system::Pallet::::set_block_number(0u32.into()); + T::BenchmarkHelper::setup(); + setup_era_reward_points::(1); + + push_unsent_entry::(1, 0, 42); + + #[block] + { + ExternalValidatorsRewards::::process_unsent_reward_eras(); + } + + Ok(()) + } + + // Governance extrinsic: retry a specific unsent era + #[benchmark] + fn retry_unsent_reward_era() -> Result<(), BenchmarkError> { + frame_system::Pallet::::set_block_number(0u32.into()); + T::BenchmarkHelper::setup(); + setup_era_reward_points::(1); + + push_unsent_entry::(1, 0, 42); + + let origin = + T::GovernanceOrigin::try_successful_origin().map_err(|_| BenchmarkError::Weightless)?; + + #[extrinsic_call] + _(origin as T::RuntimeOrigin, 1u32); + + assert!(ExternalValidatorsRewards::::unsent_queue_is_empty()); + + Ok(()) + } + impl_benchmark_test_suite!( ExternalValidatorsRewards, crate::mock::new_test_ext(), diff --git a/operator/pallets/external-validators-rewards/src/lib.rs b/operator/pallets/external-validators-rewards/src/lib.rs index 376d3a55..8aeaa123 100644 --- a/operator/pallets/external-validators-rewards/src/lib.rs +++ b/operator/pallets/external-validators-rewards/src/lib.rs @@ -66,13 +66,13 @@ pub mod pallet { pub use crate::weights::WeightInfo; use { - super::*, frame_support::pallet_prelude::*, + super::*, frame_support::pallet_prelude::*, frame_system::pallet_prelude::OriginFor, pallet_external_validators::traits::EraIndexProvider, sp_runtime::Saturating, sp_std::collections::btree_map::BTreeMap, }; /// The current storage version. - const STORAGE_VERSION: StorageVersion = StorageVersion::new(0); + const STORAGE_VERSION: StorageVersion = StorageVersion::new(1); pub type RewardPoints = u32; pub type EraIndex = u32; @@ -168,6 +168,9 @@ pub mod pallet { /// Hook for minting inflation tokens. type HandleInflation: HandleInflation; + /// Origin for governance calls (e.g., retrying unsent reward messages). + type GovernanceOrigin: EnsureOrigin; + #[cfg(feature = "runtime-benchmarks")] type BenchmarkHelper: types::BenchmarkHelper; } @@ -175,6 +178,62 @@ pub mod pallet { #[pallet::storage_version(STORAGE_VERSION)] pub struct Pallet(_); + #[pallet::hooks] + impl Hooks> for Pallet { + fn on_initialize(_n: frame_system::pallet_prelude::BlockNumberFor) -> Weight { + Self::process_unsent_reward_eras() + } + } + + #[pallet::call] + impl Pallet { + /// Governance escape hatch: manually retry sending a rewards message for + /// an era that is stuck in the unsent queue. + #[pallet::call_index(0)] + #[pallet::weight(T::WeightInfo::retry_unsent_reward_era())] + pub fn retry_unsent_reward_era( + origin: OriginFor, + era_index: EraIndex, + ) -> DispatchResult { + T::GovernanceOrigin::ensure_origin(origin)?; + + // Scan the ring buffer for the requested era + let head = UnsentRewardHead::::get(); + let tail = UnsentRewardTail::::get(); + let mut found = None; + let mut slot = head; + while slot != tail { + if let Some(entry @ (idx, _, _)) = UnsentRewardEra::::get(slot) { + if idx == era_index { + found = Some((slot, entry)); + break; + } + } + slot = (slot + 1) % UNSENT_QUEUE_CAPACITY; + } + let (slot, (_, timestamp, inflation)) = found.ok_or(Error::::EraNotInUnsentQueue)?; + + let reward_points = RewardPointsForEra::::get(era_index); + let info = reward_points + .generate_era_rewards_info(era_index, inflation, timestamp) + .ok_or(Error::::RewardPointsPruned)?; + + let message_id = + Self::send_rewards_message(&info).ok_or(Error::::MessageSendFailed)?; + + Self::unsent_queue_remove_slot(slot); + + Self::deposit_event(Event::RewardsMessageRetried { + message_id, + era_index, + total_points: info.total_points, + inflation_amount: inflation, + }); + + Ok(()) + } + } + #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { @@ -185,6 +244,29 @@ pub mod pallet { total_points: u128, inflation_amount: u128, }, + /// The rewards message failed to send; era queued for retry. + RewardsMessageSendFailed { era_index: EraIndex }, + /// A previously failed rewards message was retried and sent successfully. + RewardsMessageRetried { + message_id: H256, + era_index: EraIndex, + total_points: u128, + inflation_amount: u128, + }, + /// An unsent era was dropped because its reward points have been pruned. + UnsentEraExpired { era_index: EraIndex }, + /// The unsent queue is full; this era could not be enqueued for retry. + UnsentQueueFull { era_index: EraIndex }, + } + + #[pallet::error] + pub enum Error { + /// The specified era is not in the unsent queue. + EraNotInUnsentQueue, + /// Reward points for the era have been pruned from storage. + RewardPointsPruned, + /// The message delivery still failed on retry. + MessageSendFailed, } /// Keep tracks of distributed points per validator and total. @@ -200,7 +282,7 @@ pub mod pallet { /// - individual_points: (address, points) tuples for each validator. /// - inflation_amount: total inflation tokens to distribute. /// - era_start_timestamp: timestamp when the era started (seconds since Unix epoch). - pub fn generate_era_rewards_utils( + pub fn generate_era_rewards_info( &self, era_index: EraIndex, inflation_amount: u128, @@ -260,6 +342,33 @@ pub mod pallet { pub type BlocksProducedInEra = StorageMap<_, Twox64Concat, EraIndex, u32, ValueQuery>; + /// Maximum number of unsent reward entries in the ring buffer. + pub const UNSENT_QUEUE_CAPACITY: u32 = 64; + + /// Ring buffer of eras whose rewards messages failed to send. + /// Each slot stores (era_index, era_start_timestamp, scaled_inflation). + /// Keyed by slot index [0, UNSENT_QUEUE_CAPACITY). + #[pallet::storage] + pub type UnsentRewardEra = StorageMap< + _, + Twox64Concat, + u32, + ( + EraIndex, + /* era_start_timestamp */ u32, + /* scaled_inflation */ u128, + ), + >; + + /// Ring buffer head: next slot to be processed by `on_initialize`. + #[pallet::storage] + pub type UnsentRewardHead = StorageValue<_, u32, ValueQuery>; + + /// Ring buffer tail: next slot to write a new entry into. + /// When head == tail the buffer is empty. + #[pallet::storage] + pub type UnsentRewardTail = StorageValue<_, u32, ValueQuery>; + impl Pallet { /// Reward validators. Does not check if the validators are valid, caller needs to make sure of that. pub fn reward_by_ids(points: impl IntoIterator) { @@ -276,8 +385,8 @@ pub mod pallet { /// Helper to build, validate and deliver an outbound message. /// Logs any error and returns None on failure. - fn send_rewards_message(utils: &EraRewardsUtils) -> Option { - let outbound = T::SendMessage::build(utils).or_else(|| { + fn send_rewards_message(info: &EraRewardsUtils) -> Option { + let outbound = T::SendMessage::build(info).or_else(|| { log::error!(target: "ext_validators_rewards", "Failed to build outbound message"); None })?; @@ -303,6 +412,147 @@ pub mod pallet { .ok() } + // ── Ring-buffer helpers ────────────────────────────────────────── + + /// Returns true when the ring buffer is empty (head == tail). + #[allow(dead_code)] + pub(crate) fn unsent_queue_is_empty() -> bool { + UnsentRewardHead::::get() == UnsentRewardTail::::get() + } + + /// Number of entries currently in the ring buffer. + #[allow(dead_code)] + pub(crate) fn unsent_queue_len() -> u32 { + let head = UnsentRewardHead::::get(); + let tail = UnsentRewardTail::::get(); + tail.wrapping_sub(head) % UNSENT_QUEUE_CAPACITY + } + + /// Push a new entry into the ring buffer. + /// Returns `true` on success, `false` if the buffer is full. + pub(crate) fn unsent_queue_push(entry: (EraIndex, u32, u128)) -> bool { + let head = UnsentRewardHead::::get(); + let tail = UnsentRewardTail::::get(); + let next_tail = (tail + 1) % UNSENT_QUEUE_CAPACITY; + if next_tail == head { + // Buffer full + return false; + } + UnsentRewardEra::::insert(tail, entry); + UnsentRewardTail::::put(next_tail); + true + } + + /// Remove the entry at a given slot and compact the buffer by shifting + /// subsequent entries back. Used by the extrinsic and `on_era_start`. + fn unsent_queue_remove_slot(slot: u32) { + let tail = UnsentRewardTail::::get(); + // Shift entries after `slot` backward to fill the gap + let mut cur = slot; + loop { + let next = (cur + 1) % UNSENT_QUEUE_CAPACITY; + if next == tail { + break; + } + // Move next → cur + if let Some(entry) = UnsentRewardEra::::get(next) { + UnsentRewardEra::::insert(cur, entry); + } + cur = next; + } + // Remove the now-duplicate last entry and shrink tail + UnsentRewardEra::::remove(cur); + let new_tail = if tail == 0 { + UNSENT_QUEUE_CAPACITY - 1 + } else { + tail - 1 + }; + UnsentRewardTail::::put(new_tail); + + // If head was after the removed slot, adjust it too + let head = UnsentRewardHead::::get(); + // We also need to handle head potentially pointing past the buffer + // after a removal. Since we shifted everything between slot..tail back, + // the head only needs adjustment if it was == tail (now new_tail) — but + // that means the buffer just became empty, which is fine (head == new_tail). + // However, if head was pointing *at* a slot beyond the removed one, the + // entry it pointed to slid back by one, so head should also slide back. + // In practice, removal only happens when we know the slot, so we can + // simply recalculate emptiness. + if head == tail { + // Was already at tail, buffer must be empty now + UnsentRewardHead::::put(new_tail); + } + } + + // ── Core retry logic ────────────────────────────────────────────── + + /// Process at most one unsent reward era per block. + /// On failure the head pointer advances to the next entry so a single + /// stuck era does not block retries for subsequent eras. + pub(crate) fn process_unsent_reward_eras() -> Weight { + let head = UnsentRewardHead::::get(); + let tail = UnsentRewardTail::::get(); + + if head == tail { + return T::WeightInfo::process_unsent_reward_eras_empty(); + } + + let Some((era_index, timestamp, inflation)) = UnsentRewardEra::::get(head) else { + // Slot unexpectedly empty — advance head past it + UnsentRewardHead::::put((head + 1) % UNSENT_QUEUE_CAPACITY); + return T::WeightInfo::process_unsent_reward_eras_empty(); + }; + + // Check if reward points are still available + let reward_points = RewardPointsForEra::::get(era_index); + let info = + match reward_points.generate_era_rewards_info(era_index, inflation, timestamp) { + Some(info) => info, + None => { + // Reward points have been pruned — discard this entry + log::warn!( + target: "ext_validators_rewards", + "Unsent era {era_index} expired: reward points pruned", + ); + UnsentRewardEra::::remove(head); + UnsentRewardHead::::put((head + 1) % UNSENT_QUEUE_CAPACITY); + Self::deposit_event(Event::UnsentEraExpired { era_index }); + return T::WeightInfo::process_unsent_reward_eras_expired(); + } + }; + + // Attempt to resend + match Self::send_rewards_message(&info) { + Some(message_id) => { + UnsentRewardEra::::remove(head); + UnsentRewardHead::::put((head + 1) % UNSENT_QUEUE_CAPACITY); + Self::deposit_event(Event::RewardsMessageRetried { + message_id, + era_index, + total_points: info.total_points, + inflation_amount: inflation, + }); + T::WeightInfo::process_unsent_reward_eras_success() + } + None => { + // Move the failed entry to the back of the queue so the + // next block tries a different era (avoids head-of-line + // blocking). The entry is not lost — it will be retried + // after all other pending entries. + UnsentRewardEra::::remove(head); + UnsentRewardHead::::put((head + 1) % UNSENT_QUEUE_CAPACITY); + UnsentRewardEra::::insert(tail, (era_index, timestamp, inflation)); + UnsentRewardTail::::put((tail + 1) % UNSENT_QUEUE_CAPACITY); + log::warn!( + target: "ext_validators_rewards", + "Retry for unsent era {era_index} still failing, moved to back of queue", + ); + T::WeightInfo::process_unsent_reward_eras_failed() + } + } + } + /// Track a block authored by a validator pub fn note_block_author(author: T::AccountId) { // Track per-session authorship for performance points @@ -619,6 +869,24 @@ pub mod pallet { RewardPointsForEra::::remove(era_index_to_delete); BlocksProducedInEra::::remove(era_index_to_delete); + + // Proactively clean up any unsent entries whose reward points + // have been pruned (this era and any older ones still lingering). + let head = UnsentRewardHead::::get(); + let mut tail = UnsentRewardTail::::get(); + let mut slot = head; + while slot != tail { + if let Some((idx, _, _)) = UnsentRewardEra::::get(slot) { + if idx <= era_index_to_delete { + Self::unsent_queue_remove_slot(slot); + tail = UnsentRewardTail::::get(); + Self::deposit_event(Event::UnsentEraExpired { era_index: idx }); + // Don't advance slot — next entry slid into this position + continue; + } + } + slot = (slot + 1) % UNSENT_QUEUE_CAPACITY; + } } } @@ -671,17 +939,17 @@ pub mod pallet { // Generate era rewards utils with the actual rewards amount (post-treasury split). // This ensures the message to EigenLayer matches the actual minted rewards. - let utils = match era_reward_points.generate_era_rewards_utils( + let info = match RewardPointsForEra::::get(&era_index).generate_era_rewards_info( era_index, mint_result.rewards_amount, era_start_timestamp, ) { - Some(utils) => utils, + Some(info) => info, None => { // Returns None when total_points is zero or no validators have rewards log::error!( target: "ext_validators_rewards", - "Failed to generate era rewards utils (no rewards to distribute)" + "Failed to generate era rewards info (no rewards to distribute)" ); return; } @@ -692,13 +960,31 @@ pub mod pallet { DispatchClass::Mandatory, ); - if let Some(message_id) = Self::send_rewards_message(&utils) { - Self::deposit_event(Event::RewardsMessageSent { - message_id, - era_index, - total_points: utils.total_points, - inflation_amount: mint_result.rewards_amount, - }); + match Self::send_rewards_message(&info) { + Some(message_id) => { + Self::deposit_event(Event::RewardsMessageSent { + message_id, + era_index, + total_points: info.total_points, + inflation_amount: mint_result.rewards_amount, + }); + } + None => { + // Message failed — queue for automatic retry via on_initialize + if Self::unsent_queue_push(( + era_index, + era_start_timestamp, + mint_result.rewards_amount, + )) { + Self::deposit_event(Event::RewardsMessageSendFailed { era_index }); + } else { + log::error!( + target: "ext_validators_rewards", + "Unsent reward queue full, cannot enqueue era {era_index}", + ); + Self::deposit_event(Event::UnsentQueueFull { era_index }); + } + } } } } diff --git a/operator/pallets/external-validators-rewards/src/mock.rs b/operator/pallets/external-validators-rewards/src/mock.rs index 3c892f35..6b99b7c3 100644 --- a/operator/pallets/external-validators-rewards/src/mock.rs +++ b/operator/pallets/external-validators-rewards/src/mock.rs @@ -131,6 +131,9 @@ impl crate::types::SendMessage for MockOkOutboundQueue { } fn validate(ticket: Self::Ticket) -> Result { + if Mock::mock().send_message_fails { + return Err(SendError::MessageTooLarge); + } Ok(ticket) } @@ -223,6 +226,7 @@ impl pallet_external_validators_rewards::Config for Test { type HandleInflation = InflationMinter; type Currency = Balances; type RewardsEthereumSovereignAccount = RewardsEthereumSovereignAccount; + type GovernanceOrigin = frame_system::EnsureRoot; type WeightInfo = (); #[cfg(feature = "runtime-benchmarks")] type BenchmarkHelper = (); @@ -292,6 +296,8 @@ pub mod mock_data { pub offline_validators: sp_std::vec::Vec, /// Set of (era_index, validator_id) pairs that are slashed pub slashed_validators: sp_std::vec::Vec<(u32, sp_core::H160)>, + /// When true, MockOkOutboundQueue::validate will return Err(SendError::MessageTooLarge) + pub send_message_fails: bool, } #[pallet::config] diff --git a/operator/pallets/external-validators-rewards/src/tests.rs b/operator/pallets/external-validators-rewards/src/tests.rs index 1a66daa0..752a55c0 100644 --- a/operator/pallets/external-validators-rewards/src/tests.rs +++ b/operator/pallets/external-validators-rewards/src/tests.rs @@ -16,7 +16,7 @@ use { crate::{self as pallet_external_validators_rewards, mock::*}, - frame_support::traits::fungible::Mutate, + frame_support::{assert_noop, assert_ok, traits::fungible::Mutate}, pallet_external_validators::traits::{ActiveEraInfo, OnEraEnd, OnEraStart}, sp_core::H160, sp_std::collections::btree_map::BTreeMap, @@ -165,8 +165,8 @@ fn test_on_era_end() { let treasury_amount = InflationTreasuryProportion::get().mul_floor(inflation); let rewards_amount = inflation - treasury_amount; // Use 0 for era_start_timestamp in tests - let rewards_utils = era_rewards.generate_era_rewards_utils(1, rewards_amount, 0); - assert!(rewards_utils.is_some()); + let rewards_info = era_rewards.generate_era_rewards_info(1, inflation, 0); + assert!(rewards_info.is_some()); System::assert_last_event(RuntimeEvent::ExternalValidatorsRewards( crate::Event::RewardsMessageSent { message_id: Default::default(), @@ -207,8 +207,8 @@ fn test_on_era_end_with_zero_inflation() { let era_rewards = pallet_external_validators_rewards::RewardPointsForEra::::get(1); let inflation = ::EraInflationProvider::get(); - let rewards_utils = era_rewards.generate_era_rewards_utils(1, inflation, 0); - assert!(rewards_utils.is_some()); + let rewards_info = era_rewards.generate_era_rewards_info(1, inflation, 0); + assert!(rewards_info.is_some()); // With zero inflation, no RewardsMessageSent event should be emitted let events = System::events(); assert!( @@ -246,15 +246,15 @@ fn test_on_era_end_with_zero_points() { ExternalValidatorsRewards::reward_by_ids(accounts_points); ExternalValidatorsRewards::on_era_end(1); - // When all validators have zero points, generate_era_rewards_utils should return None + // When all validators have zero points, generate_era_rewards_info should return None // to prevent inflation from being minted with no way to distribute it let era_rewards = pallet_external_validators_rewards::RewardPointsForEra::::get(1); let inflation = ::EraInflationProvider::get(); - let rewards_utils = era_rewards.generate_era_rewards_utils(1, inflation, 0); + let rewards_info = era_rewards.generate_era_rewards_info(1, inflation, 0); assert!( - rewards_utils.is_none(), - "generate_era_rewards_utils should return None when total_points is zero" + rewards_info.is_none(), + "generate_era_rewards_info should return None when total_points is zero" ); // Verify no RewardsMessageSent event was emitted @@ -3722,3 +3722,456 @@ fn test_era_end_uses_correct_era_blocks_not_session() { ); }) } + +// ═══════════════════════════════════════════════════════════════════════════ +// Retry mechanism tests (ring-buffer storage) +// ═══════════════════════════════════════════════════════════════════════════ + +/// Helper: push an entry into the unsent ring buffer via the pallet API. +fn push_unsent(era_index: u32, timestamp: u32, inflation: u128) { + assert!( + ExternalValidatorsRewards::unsent_queue_push((era_index, timestamp, inflation)), + "unsent_queue_push should succeed" + ); +} + +/// Helper: return the number of entries in the unsent ring buffer. +fn unsent_len() -> u32 { + ExternalValidatorsRewards::unsent_queue_len() +} + +/// Helper: check if unsent queue is empty. +fn unsent_is_empty() -> bool { + ExternalValidatorsRewards::unsent_queue_is_empty() +} + +#[test] +fn send_failure_queues_era() { + new_test_ext().execute_with(|| { + run_to_block(1); + + Mock::mutate(|mock| { + mock.active_era = Some(ActiveEraInfo { + index: 1, + start: Some(30_000), + }); + mock.send_message_fails = true; + }); + + // Give validators some points + ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]); + // Author expected blocks for 100% inflation + for _ in 0..600 { + ExternalValidatorsRewards::note_block_author(H160::from_low_u64_be(1)); + } + + ExternalValidatorsRewards::on_era_end(1); + + // Verify era is queued + assert_eq!(unsent_len(), 1); + + // Verify event + System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards( + crate::Event::RewardsMessageSendFailed { era_index: 1 }, + )); + }) +} + +#[test] +fn on_initialize_retries_and_succeeds() { + new_test_ext().execute_with(|| { + run_to_block(1); + + Mock::mutate(|mock| { + mock.active_era = Some(ActiveEraInfo { + index: 1, + start: Some(30_000), + }); + }); + + // Set up reward points for era 1 + ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]); + + // Manually populate the unsent queue + push_unsent(1, 30, 42); + + // Sending should succeed (send_message_fails is false by default) + System::reset_events(); + ExternalValidatorsRewards::process_unsent_reward_eras(); + + // Queue should be empty + assert!(unsent_is_empty()); + + // Verify retry event + System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards( + crate::Event::RewardsMessageRetried { + message_id: Default::default(), + era_index: 1, + total_points: 100, + inflation_amount: 42, + }, + )); + }) +} + +#[test] +fn on_initialize_moves_failed_entry_to_back() { + new_test_ext().execute_with(|| { + run_to_block(1); + + Mock::mutate(|mock| { + mock.active_era = Some(ActiveEraInfo { + index: 2, + start: Some(30_000), + }); + mock.send_message_fails = true; + }); + + // Set up reward points for eras 1 and 2 + ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]); + Mock::mutate(|mock| { + mock.active_era = Some(ActiveEraInfo { + index: 1, + start: Some(30_000), + }); + }); + ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 200)]); + + // Push two entries: era 1 then era 2 + push_unsent(1, 30, 42); + push_unsent(2, 30, 84); + + // First call: tries era 1, fails, moves era 1 to back of queue + ExternalValidatorsRewards::process_unsent_reward_eras(); + // Queue length stays the same (entry moved, not removed) + assert_eq!(unsent_len(), 2); + + // Second call: tries era 2 (NOT era 1 again), fails, moves era 2 to back + ExternalValidatorsRewards::process_unsent_reward_eras(); + assert_eq!(unsent_len(), 2); + + // Re-enable sending + Mock::mutate(|mock| mock.send_message_fails = false); + + // Third call: era 1 (now at front again), succeeds + System::reset_events(); + ExternalValidatorsRewards::process_unsent_reward_eras(); + assert_eq!(unsent_len(), 1); + System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards( + crate::Event::RewardsMessageRetried { + message_id: Default::default(), + era_index: 1, + total_points: 200, + inflation_amount: 42, + }, + )); + + // Fourth call: era 2, succeeds + ExternalValidatorsRewards::process_unsent_reward_eras(); + assert!(unsent_is_empty()); + }) +} + +#[test] +fn on_initialize_removes_expired_era() { + new_test_ext().execute_with(|| { + run_to_block(1); + + // Populate unsent queue with era 999 but do NOT add RewardPointsForEra for it + push_unsent(999, 0, 42); + + System::reset_events(); + ExternalValidatorsRewards::process_unsent_reward_eras(); + + // Entry should be removed + assert!(unsent_is_empty()); + + // Verify expired event + System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards( + crate::Event::UnsentEraExpired { era_index: 999 }, + )); + }) +} + +#[test] +fn on_initialize_noop_when_queue_empty() { + new_test_ext().execute_with(|| { + run_to_block(1); + System::reset_events(); + + ExternalValidatorsRewards::process_unsent_reward_eras(); + + // No events should be emitted + let events = System::events(); + assert!( + events.is_empty(), + "No events should be emitted when unsent queue is empty" + ); + }) +} + +#[test] +fn on_initialize_processes_only_head() { + new_test_ext().execute_with(|| { + run_to_block(1); + + Mock::mutate(|mock| { + mock.active_era = Some(ActiveEraInfo { + index: 3, + start: Some(30_000), + }); + }); + + // Set up reward points for both eras + ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]); + Mock::mutate(|mock| { + mock.active_era = Some(ActiveEraInfo { + index: 2, + start: Some(30_000), + }); + }); + ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(2), 200)]); + + // Push two entries + push_unsent(3, 30, 42); + push_unsent(2, 20, 84); + + System::reset_events(); + ExternalValidatorsRewards::process_unsent_reward_eras(); + + // Only the head entry (era 3) should be processed (and removed on success) + assert_eq!(unsent_len(), 1); + }) +} + +#[test] +fn retry_extrinsic_success() { + new_test_ext().execute_with(|| { + run_to_block(1); + + Mock::mutate(|mock| { + mock.active_era = Some(ActiveEraInfo { + index: 1, + start: Some(30_000), + }); + }); + + // Set up reward points + ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]); + + // Populate unsent queue + push_unsent(1, 30, 42); + + System::reset_events(); + assert_ok!(ExternalValidatorsRewards::retry_unsent_reward_era( + RuntimeOrigin::root(), + 1 + )); + + // Queue should be empty + assert!(unsent_is_empty()); + + // Verify retry event + System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards( + crate::Event::RewardsMessageRetried { + message_id: Default::default(), + era_index: 1, + total_points: 100, + inflation_amount: 42, + }, + )); + }) +} + +#[test] +fn retry_extrinsic_era_not_in_queue() { + new_test_ext().execute_with(|| { + run_to_block(1); + + assert_noop!( + ExternalValidatorsRewards::retry_unsent_reward_era(RuntimeOrigin::root(), 1), + crate::Error::::EraNotInUnsentQueue + ); + }) +} + +#[test] +fn retry_extrinsic_pruned_data() { + new_test_ext().execute_with(|| { + run_to_block(1); + + // Queue an era but don't create reward points for it + push_unsent(999, 0, 42); + + assert_noop!( + ExternalValidatorsRewards::retry_unsent_reward_era(RuntimeOrigin::root(), 999), + crate::Error::::RewardPointsPruned + ); + }) +} + +#[test] +fn retry_extrinsic_requires_root() { + new_test_ext().execute_with(|| { + run_to_block(1); + + assert_noop!( + ExternalValidatorsRewards::retry_unsent_reward_era( + RuntimeOrigin::signed(H160::from_low_u64_be(1)), + 1 + ), + sp_runtime::DispatchError::BadOrigin + ); + }) +} + +#[test] +fn unsent_queue_full() { + new_test_ext().execute_with(|| { + run_to_block(1); + + Mock::mutate(|mock| { + mock.active_era = Some(ActiveEraInfo { + index: 65, + start: Some(30_000), + }); + mock.send_message_fails = true; + }); + + // Fill the ring buffer to capacity (63 entries, since capacity=64 + // means 63 usable slots in a ring buffer with head==tail==empty). + for i in 0..63u32 { + push_unsent(i, 0, 42); + } + assert_eq!(unsent_len(), 63); + + // Give validators some points so on_era_end doesn't bail early + ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]); + for _ in 0..600 { + ExternalValidatorsRewards::note_block_author(H160::from_low_u64_be(1)); + } + + System::reset_events(); + ExternalValidatorsRewards::on_era_end(65); + + // Verify UnsentQueueFull event + System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards( + crate::Event::UnsentQueueFull { era_index: 65 }, + )); + + // Queue should still be at 63 + assert_eq!(unsent_len(), 63); + }) +} + +#[test] +fn on_era_start_prunes_unsent_entry() { + new_test_ext().execute_with(|| { + run_to_block(1); + + // Set up: era 1 has an unsent entry + push_unsent(1, 0, 42); + + // HistoryDepth is 10, so era 11 should prune era 1 + System::reset_events(); + ExternalValidatorsRewards::on_era_start(11, 0, 11); + + // Unsent entry should be removed + assert!(unsent_is_empty()); + + // Verify expired event + System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards( + crate::Event::UnsentEraExpired { era_index: 1 }, + )); + }) +} + +#[test] +fn retry_extrinsic_send_still_fails() { + new_test_ext().execute_with(|| { + run_to_block(1); + + Mock::mutate(|mock| { + mock.active_era = Some(ActiveEraInfo { + index: 1, + start: Some(30_000), + }); + mock.send_message_fails = true; + }); + + // Set up reward points + ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]); + + // Populate unsent queue + push_unsent(1, 30, 42); + + assert_noop!( + ExternalValidatorsRewards::retry_unsent_reward_era(RuntimeOrigin::root(), 1), + crate::Error::::MessageSendFailed + ); + + // Queue should still have the entry + assert_eq!(unsent_len(), 1); + }) +} + +#[test] +fn head_of_line_blocking_avoided() { + new_test_ext().execute_with(|| { + run_to_block(1); + + // Set up reward points for eras 1, 2, 3 + for era in 1..=3u32 { + Mock::mutate(|mock| { + mock.active_era = Some(ActiveEraInfo { + index: era, + start: Some(30_000), + }); + }); + ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]); + } + + // Push eras 1, 2, 3 into the queue + push_unsent(1, 30, 10); + push_unsent(2, 30, 20); + push_unsent(3, 30, 30); + + // Make sending fail + Mock::mutate(|mock| mock.send_message_fails = true); + + // Block 1: tries era 1, fails, advances head → era 2 + ExternalValidatorsRewards::process_unsent_reward_eras(); + // Block 2: tries era 2, fails, advances head → era 3 + ExternalValidatorsRewards::process_unsent_reward_eras(); + + // Now re-enable sending + Mock::mutate(|mock| mock.send_message_fails = false); + + // Block 3: tries era 3, succeeds + System::reset_events(); + ExternalValidatorsRewards::process_unsent_reward_eras(); + System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards( + crate::Event::RewardsMessageRetried { + message_id: Default::default(), + era_index: 3, + total_points: 100, + inflation_amount: 30, + }, + )); + + // Block 4: wraps around to era 1, succeeds + ExternalValidatorsRewards::process_unsent_reward_eras(); + System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards( + crate::Event::RewardsMessageRetried { + message_id: Default::default(), + era_index: 1, + total_points: 100, + inflation_amount: 10, + }, + )); + + // Block 5: era 2, succeeds + ExternalValidatorsRewards::process_unsent_reward_eras(); + assert!(unsent_is_empty()); + }) +} diff --git a/operator/pallets/external-validators-rewards/src/weights.rs b/operator/pallets/external-validators-rewards/src/weights.rs index 766adfcf..a7585778 100644 --- a/operator/pallets/external-validators-rewards/src/weights.rs +++ b/operator/pallets/external-validators-rewards/src/weights.rs @@ -54,6 +54,11 @@ use sp_std::marker::PhantomData; /// Weight functions needed for pallet_external_validators_rewards. pub trait WeightInfo { fn on_era_end() -> Weight; + fn process_unsent_reward_eras_empty() -> Weight; + fn process_unsent_reward_eras_expired() -> Weight; + fn process_unsent_reward_eras_success() -> Weight; + fn process_unsent_reward_eras_failed() -> Weight; + fn retry_unsent_reward_era() -> Weight; } /// Weights for pallet_external_validators_rewards using the Substrate node and recommended hardware. @@ -84,6 +89,36 @@ impl WeightInfo for SubstrateWeight { .saturating_add(T::DbWeight::get().reads(5_u64)) .saturating_add(T::DbWeight::get().writes(5_u64)) } + + fn process_unsent_reward_eras_empty() -> Weight { + // 1 read for UnsentRewardEras + Weight::from_parts(5_000_000, 0) + .saturating_add(T::DbWeight::get().reads(1_u64)) + } + + fn process_unsent_reward_eras_expired() -> Weight { + // 1 read UnsentRewardEras + 1 read RewardPointsForEra + 1 write UnsentRewardEras + Weight::from_parts(10_000_000, 0) + .saturating_add(T::DbWeight::get().reads(2_u64)) + .saturating_add(T::DbWeight::get().writes(1_u64)) + } + + fn process_unsent_reward_eras_success() -> Weight { + // Same as on_era_end + queue read/write + Weight::from_parts(1_136_401_000, 39987) + .saturating_add(T::DbWeight::get().reads(7_u64)) + .saturating_add(T::DbWeight::get().writes(6_u64)) + } + + fn process_unsent_reward_eras_failed() -> Weight { + // Use success weight as upper bound + Self::process_unsent_reward_eras_success() + } + + fn retry_unsent_reward_era() -> Weight { + // Same as success path + Self::process_unsent_reward_eras_success() + } } // For backwards compatibility and tests @@ -113,4 +148,29 @@ impl WeightInfo for () { .saturating_add(RocksDbWeight::get().reads(5_u64)) .saturating_add(RocksDbWeight::get().writes(5_u64)) } + + fn process_unsent_reward_eras_empty() -> Weight { + Weight::from_parts(5_000_000, 0) + .saturating_add(RocksDbWeight::get().reads(1_u64)) + } + + fn process_unsent_reward_eras_expired() -> Weight { + Weight::from_parts(10_000_000, 0) + .saturating_add(RocksDbWeight::get().reads(2_u64)) + .saturating_add(RocksDbWeight::get().writes(1_u64)) + } + + fn process_unsent_reward_eras_success() -> Weight { + Weight::from_parts(1_136_401_000, 39987) + .saturating_add(RocksDbWeight::get().reads(7_u64)) + .saturating_add(RocksDbWeight::get().writes(6_u64)) + } + + fn process_unsent_reward_eras_failed() -> Weight { + Self::process_unsent_reward_eras_success() + } + + fn retry_unsent_reward_era() -> Weight { + Self::process_unsent_reward_eras_success() + } } diff --git a/operator/runtime/mainnet/src/configs/mod.rs b/operator/runtime/mainnet/src/configs/mod.rs index 0deeff2a..48f1aad9 100644 --- a/operator/runtime/mainnet/src/configs/mod.rs +++ b/operator/runtime/mainnet/src/configs/mod.rs @@ -1598,6 +1598,8 @@ impl pallet_external_validators_rewards::Config for Runtime { type RewardsEthereumSovereignAccount = ExternalValidatorRewardsAccount; type SendMessage = RewardsSendAdapter; type HandleInflation = ExternalRewardsInflationHandler; + type GovernanceOrigin = + EitherOfDiverse, governance::custom_origins::GeneralAdmin>; type WeightInfo = mainnet_weights::pallet_external_validators_rewards::WeightInfo; #[cfg(feature = "runtime-benchmarks")] type BenchmarkHelper = (); diff --git a/operator/runtime/mainnet/src/weights/pallet_external_validators_rewards.rs b/operator/runtime/mainnet/src/weights/pallet_external_validators_rewards.rs index b8be1393..10854100 100644 --- a/operator/runtime/mainnet/src/weights/pallet_external_validators_rewards.rs +++ b/operator/runtime/mainnet/src/weights/pallet_external_validators_rewards.rs @@ -74,4 +74,29 @@ impl pallet_external_validators_rewards::WeightInfo for .saturating_add(T::DbWeight::get().reads(9_u64)) .saturating_add(T::DbWeight::get().writes(2_u64)) } + + fn process_unsent_reward_eras_empty() -> Weight { + Weight::from_parts(5_000_000, 0) + .saturating_add(T::DbWeight::get().reads(1_u64)) + } + + fn process_unsent_reward_eras_expired() -> Weight { + Weight::from_parts(10_000_000, 0) + .saturating_add(T::DbWeight::get().reads(2_u64)) + .saturating_add(T::DbWeight::get().writes(1_u64)) + } + + fn process_unsent_reward_eras_success() -> Weight { + Weight::from_parts(1_905_623_000, 29162) + .saturating_add(T::DbWeight::get().reads(11_u64)) + .saturating_add(T::DbWeight::get().writes(3_u64)) + } + + fn process_unsent_reward_eras_failed() -> Weight { + Self::process_unsent_reward_eras_success() + } + + fn retry_unsent_reward_era() -> Weight { + Self::process_unsent_reward_eras_success() + } } diff --git a/operator/runtime/stagenet/src/configs/mod.rs b/operator/runtime/stagenet/src/configs/mod.rs index 60fee86c..12b4a960 100644 --- a/operator/runtime/stagenet/src/configs/mod.rs +++ b/operator/runtime/stagenet/src/configs/mod.rs @@ -1594,6 +1594,8 @@ impl pallet_external_validators_rewards::Config for Runtime { type RewardsEthereumSovereignAccount = ExternalValidatorRewardsAccount; type SendMessage = RewardsSendAdapter; type HandleInflation = ExternalRewardsInflationHandler; + type GovernanceOrigin = + EitherOfDiverse, governance::custom_origins::GeneralAdmin>; type WeightInfo = stagenet_weights::pallet_external_validators_rewards::WeightInfo; #[cfg(feature = "runtime-benchmarks")] type BenchmarkHelper = (); diff --git a/operator/runtime/stagenet/src/weights/pallet_external_validators_rewards.rs b/operator/runtime/stagenet/src/weights/pallet_external_validators_rewards.rs index 4d223163..34d31953 100644 --- a/operator/runtime/stagenet/src/weights/pallet_external_validators_rewards.rs +++ b/operator/runtime/stagenet/src/weights/pallet_external_validators_rewards.rs @@ -74,4 +74,29 @@ impl pallet_external_validators_rewards::WeightInfo for .saturating_add(T::DbWeight::get().reads(9_u64)) .saturating_add(T::DbWeight::get().writes(2_u64)) } + + fn process_unsent_reward_eras_empty() -> Weight { + Weight::from_parts(5_000_000, 0) + .saturating_add(T::DbWeight::get().reads(1_u64)) + } + + fn process_unsent_reward_eras_expired() -> Weight { + Weight::from_parts(10_000_000, 0) + .saturating_add(T::DbWeight::get().reads(2_u64)) + .saturating_add(T::DbWeight::get().writes(1_u64)) + } + + fn process_unsent_reward_eras_success() -> Weight { + Weight::from_parts(1_894_953_000, 29162) + .saturating_add(T::DbWeight::get().reads(11_u64)) + .saturating_add(T::DbWeight::get().writes(3_u64)) + } + + fn process_unsent_reward_eras_failed() -> Weight { + Self::process_unsent_reward_eras_success() + } + + fn retry_unsent_reward_era() -> Weight { + Self::process_unsent_reward_eras_success() + } } diff --git a/operator/runtime/testnet/src/configs/mod.rs b/operator/runtime/testnet/src/configs/mod.rs index caca5de5..27dbc538 100644 --- a/operator/runtime/testnet/src/configs/mod.rs +++ b/operator/runtime/testnet/src/configs/mod.rs @@ -1598,6 +1598,8 @@ impl pallet_external_validators_rewards::Config for Runtime { type RewardsEthereumSovereignAccount = ExternalValidatorRewardsAccount; type SendMessage = RewardsSendAdapter; type HandleInflation = ExternalRewardsInflationHandler; + type GovernanceOrigin = + EitherOfDiverse, governance::custom_origins::GeneralAdmin>; type WeightInfo = testnet_weights::pallet_external_validators_rewards::WeightInfo; #[cfg(feature = "runtime-benchmarks")] type BenchmarkHelper = (); diff --git a/operator/runtime/testnet/src/weights/pallet_external_validators_rewards.rs b/operator/runtime/testnet/src/weights/pallet_external_validators_rewards.rs index b2403bcf..9b7e752d 100644 --- a/operator/runtime/testnet/src/weights/pallet_external_validators_rewards.rs +++ b/operator/runtime/testnet/src/weights/pallet_external_validators_rewards.rs @@ -74,4 +74,29 @@ impl pallet_external_validators_rewards::WeightInfo for .saturating_add(T::DbWeight::get().reads(9_u64)) .saturating_add(T::DbWeight::get().writes(2_u64)) } + + fn process_unsent_reward_eras_empty() -> Weight { + Weight::from_parts(5_000_000, 0) + .saturating_add(T::DbWeight::get().reads(1_u64)) + } + + fn process_unsent_reward_eras_expired() -> Weight { + Weight::from_parts(10_000_000, 0) + .saturating_add(T::DbWeight::get().reads(2_u64)) + .saturating_add(T::DbWeight::get().writes(1_u64)) + } + + fn process_unsent_reward_eras_success() -> Weight { + Weight::from_parts(1_893_280_000, 29162) + .saturating_add(T::DbWeight::get().reads(11_u64)) + .saturating_add(T::DbWeight::get().writes(3_u64)) + } + + fn process_unsent_reward_eras_failed() -> Weight { + Self::process_unsent_reward_eras_success() + } + + fn retry_unsent_reward_era() -> Weight { + Self::process_unsent_reward_eras_success() + } } diff --git a/test/.papi/descriptors/package.json b/test/.papi/descriptors/package.json index 5ad366a9..185bc678 100644 --- a/test/.papi/descriptors/package.json +++ b/test/.papi/descriptors/package.json @@ -1,5 +1,5 @@ { - "version": "0.1.0-autogenerated.15484599658830368838", + "version": "0.1.0-autogenerated.18139584469151706411", "name": "@polkadot-api/descriptors", "files": [ "dist" diff --git a/test/.papi/metadata/datahaven.scale b/test/.papi/metadata/datahaven.scale index 1ad174a7..3f4882b2 100644 Binary files a/test/.papi/metadata/datahaven.scale and b/test/.papi/metadata/datahaven.scale differ