diff --git a/operator/pallets/external-validators-rewards/src/benchmarking.rs b/operator/pallets/external-validators-rewards/src/benchmarking.rs index 4b84bc3f..2b06aca1 100644 --- a/operator/pallets/external-validators-rewards/src/benchmarking.rs +++ b/operator/pallets/external-validators-rewards/src/benchmarking.rs @@ -43,9 +43,13 @@ fn create_funded_user( user } -/// Helper: insert a single entry into the ring buffer at slot 0. -fn push_unsent_entry(era_index: u32, timestamp: u32, inflation: u128) { - ExternalValidatorsRewards::::unsent_queue_push((era_index, timestamp, inflation)); +/// Helper: insert a single queued window into the ring buffer at slot 0. +fn push_unsent_entry(window_start: u32, window_index: u32, duration: u32) { + ExternalValidatorsRewards::::unsent_queue_push(QueuedRewardsWindow { + window_start, + window_index, + duration, + }); } #[allow(clippy::multiple_bound_locations)] @@ -77,17 +81,20 @@ mod benchmarks { Ok(()) } - /// Helper to populate reward points for an era with 1000 validators. - fn setup_era_reward_points(era_index: u32) { - let mut era_reward_points = EraRewardPoints::default(); - era_reward_points.total = 20 * 1000; + /// Helper to populate persisted state for a closed window with 1000 operators. + fn setup_window_reward_state( + window_start: u32, + inflation_amount: u128, + ) { + let mut operator_points = sp_std::collections::btree_map::BTreeMap::new(); for i in 0..1000 { - let account_id = create_funded_user::("candidate", i, 100); - era_reward_points.individual.insert(account_id, 20); + let _ = create_funded_user::("candidate", i, 100); + operator_points.insert(sp_core::H160::from_low_u64_be(i as u64 + 1), 20); } - >::insert(era_index, era_reward_points); + >::insert(window_start, operator_points); + >::insert(window_start, inflation_amount); } // on_initialize: unsent queue is empty (2 reads for head+tail) @@ -104,11 +111,10 @@ mod benchmarks { Ok(()) } - // on_initialize: oldest entry has pruned reward points + // on_initialize: oldest queued window no longer has persisted state #[benchmark] fn process_unsent_reward_eras_expired() -> Result<(), BenchmarkError> { - // Push an entry whose reward points do NOT exist in storage - push_unsent_entry::(999, 0, 42); + push_unsent_entry::(999, 99, 10); #[block] { @@ -126,9 +132,9 @@ mod benchmarks { fn process_unsent_reward_eras_success() -> Result<(), BenchmarkError> { frame_system::Pallet::::set_block_number(0u32.into()); T::BenchmarkHelper::setup(); - setup_era_reward_points::(1); + setup_window_reward_state::(0, 42); - push_unsent_entry::(1, 0, 42); + push_unsent_entry::(0, 0, 10); #[block] { @@ -145,9 +151,9 @@ mod benchmarks { fn process_unsent_reward_eras_failed() -> Result<(), BenchmarkError> { frame_system::Pallet::::set_block_number(0u32.into()); T::BenchmarkHelper::setup(); - setup_era_reward_points::(1); + setup_window_reward_state::(0, 42); - push_unsent_entry::(1, 0, 42); + push_unsent_entry::(0, 0, 10); #[block] { @@ -157,20 +163,20 @@ mod benchmarks { Ok(()) } - // Governance extrinsic: retry a specific unsent era + // Governance extrinsic: retry a specific unsent window #[benchmark] - fn retry_unsent_reward_era() -> Result<(), BenchmarkError> { + fn retry_unsent_reward_window() -> Result<(), BenchmarkError> { frame_system::Pallet::::set_block_number(0u32.into()); T::BenchmarkHelper::setup(); - setup_era_reward_points::(1); + setup_window_reward_state::(0, 42); - push_unsent_entry::(1, 0, 42); + push_unsent_entry::(0, 0, 10); let origin = T::GovernanceOrigin::try_successful_origin().map_err(|_| BenchmarkError::Weightless)?; #[extrinsic_call] - _(origin as T::RuntimeOrigin, 1u32); + _(origin as T::RuntimeOrigin, 0u32); assert!(ExternalValidatorsRewards::::unsent_queue_is_empty()); diff --git a/operator/pallets/external-validators-rewards/src/lib.rs b/operator/pallets/external-validators-rewards/src/lib.rs index 323dba4f..457998f7 100644 --- a/operator/pallets/external-validators-rewards/src/lib.rs +++ b/operator/pallets/external-validators-rewards/src/lib.rs @@ -37,7 +37,7 @@ use { crate::types::{HandleInflation, RewardsPeriodUtils, SendMessage}, frame_support::traits::{Get, UnixTime, ValidatorSet}, pallet_external_validators::traits::{ExternalIndexProvider, OnEraEnd, OnEraStart}, - parity_scale_codec::{Decode, Encode}, + parity_scale_codec::{Decode, Encode, MaxEncodedLen}, sp_core::{H160, H256}, sp_runtime::{ traits::{Hash, SaturatedConversion}, @@ -199,46 +199,50 @@ pub mod pallet { #[pallet::call] impl Pallet { /// Governance escape hatch: manually retry sending a rewards message for - /// an era that is stuck in the unsent queue. + /// a closed window that is stuck in the unsent queue. #[pallet::call_index(0)] #[pallet::weight(T::WeightInfo::retry_unsent_reward_era())] - pub fn retry_unsent_reward_era( + pub fn retry_unsent_reward_window( origin: OriginFor, - era_index: EraIndex, + window_start: u32, ) -> DispatchResult { T::GovernanceOrigin::ensure_origin(origin)?; - // Scan the ring buffer for the requested era + // Scan the ring buffer for the requested window let head = UnsentRewardHead::::get(); let tail = UnsentRewardTail::::get(); let mut found = None; let mut slot = head; while slot != tail { - if let Some(entry @ (idx, _, _)) = UnsentRewardEra::::get(slot) { - if idx == era_index { + if let Some(entry) = UnsentRewardWindow::::get(slot) { + if entry.window_start == window_start { found = Some((slot, entry)); break; } } slot = (slot + 1) % UNSENT_QUEUE_CAPACITY; } - let (slot, (_, timestamp, inflation)) = found.ok_or(Error::::EraNotInUnsentQueue)?; + let (slot, window) = found.ok_or(Error::::WindowNotInUnsentQueue)?; - let reward_points = RewardPointsForEra::::get(era_index); - let info = reward_points - .generate_era_rewards_info(era_index, inflation, timestamp) - .ok_or(Error::::RewardPointsPruned)?; + let info = Self::window_rewards_info( + window.window_start, + window.window_index, + window.duration, + ) + .ok_or(Error::::WindowRewardsMissing)?; let message_id = Self::send_rewards_message(&info).ok_or(Error::::MessageSendFailed)?; + Self::clear_window(window.window_start); Self::unsent_queue_remove_slot(slot); - Self::deposit_event(Event::RewardsMessageRetried { + Self::deposit_event(Event::RewardsWindowRetried { message_id, - era_index, + window_start: window.window_start, + window_index: window.window_index, total_points: info.total_points, - inflation_amount: inflation, + inflation_amount: info.inflation_amount, }); Ok(()) @@ -256,7 +260,7 @@ pub mod pallet { total_points: u128, inflation_amount: u128, }, - /// Window submission failed. Window data has been cleared. + /// Window submission failed on the initial attempt. RewardsWindowSubmissionFailed { window_start: u32, window_index: u32, @@ -266,27 +270,32 @@ pub mod pallet { window_start: u32, window_index: u32, }, - /// The rewards message failed to send; era queued for retry. - RewardsMessageSendFailed { era_index: EraIndex }, - /// A previously failed rewards message was retried and sent successfully. - RewardsMessageRetried { + /// A previously failed rewards window was retried and sent successfully. + RewardsWindowRetried { message_id: H256, - era_index: EraIndex, + window_start: u32, + window_index: u32, total_points: u128, inflation_amount: u128, }, - /// An unsent era was dropped because its reward points have been pruned. - UnsentEraExpired { era_index: EraIndex }, - /// The unsent queue is full; this era could not be enqueued for retry. - UnsentQueueFull { era_index: EraIndex }, + /// A queued window was dropped because its stored rewards data is no longer available. + UnsentWindowExpired { + window_start: u32, + window_index: u32, + }, + /// The unsent queue is full; this failed window could not be enqueued for retry. + UnsentWindowQueueFull { + window_start: u32, + window_index: u32, + }, } #[pallet::error] pub enum Error { - /// The specified era is not in the unsent queue. - EraNotInUnsentQueue, - /// Reward points for the era have been pruned from storage. - RewardPointsPruned, + /// The specified window is not in the unsent queue. + WindowNotInUnsentQueue, + /// Rewards data for the window is no longer available. + WindowRewardsMissing, /// The message delivery still failed on retry. MessageSendFailed, } @@ -298,42 +307,6 @@ pub mod pallet { pub individual: BTreeMap, } - impl EraRewardPoints { - /// Generate utils needed for EigenLayer rewards submission from era data. - /// Used by the unsent-era retry mechanism to reconstruct a `RewardsPeriodUtils` - /// from stored era reward points. - pub fn generate_era_rewards_info( - &self, - era_index: EraIndex, - inflation_amount: u128, - era_start_timestamp: u32, - ) -> Option { - let mut individual_points = Vec::with_capacity(self.individual.len()); - - for (account_id, reward_points) in self.individual.iter() { - // Convert AccountId to H160 for EigenLayer rewards submission. - // In DataHaven, AccountId is H160, so encode() produces exactly 20 bytes. - individual_points - .push((H160::from_slice(&account_id.encode()[..20]), *reward_points)); - } - - let total_points: u128 = individual_points.iter().map(|(_, pts)| *pts as u128).sum(); - - if total_points.is_zero() { - return None; - } - - Some(RewardsPeriodUtils { - period_index: era_index, - period_start: era_start_timestamp, - duration: 0, - total_points, - individual_points, - inflation_amount, - }) - } - } - impl Default for EraRewardPoints { fn default() -> Self { EraRewardPoints { @@ -380,20 +353,18 @@ pub mod pallet { /// Maximum number of unsent reward entries in the ring buffer. pub const UNSENT_QUEUE_CAPACITY: u32 = 64; - /// Ring buffer of eras whose rewards messages failed to send. - /// Each slot stores (era_index, era_start_timestamp, scaled_inflation). + /// Metadata for a failed rewards window kept in the retry ring buffer. + #[derive(RuntimeDebug, Encode, Decode, MaxEncodedLen, PartialEq, Eq, TypeInfo, Clone, Copy)] + pub struct QueuedRewardsWindow { + pub window_start: u32, + pub window_index: u32, + pub duration: u32, + } + + /// Ring buffer of windows whose rewards messages failed to send. /// Keyed by slot index [0, UNSENT_QUEUE_CAPACITY). #[pallet::storage] - pub type UnsentRewardEra = StorageMap< - _, - Twox64Concat, - u32, - ( - EraIndex, - /* era_start_timestamp */ u32, - /* scaled_inflation */ u128, - ), - >; + pub type UnsentRewardWindow = StorageMap<_, Twox64Concat, u32, QueuedRewardsWindow>; /// Ring buffer head: next slot to be processed by `on_initialize`. #[pallet::storage] @@ -420,6 +391,10 @@ pub mod pallet { genesis + (timestamp.saturating_sub(genesis) / interval) * interval } + fn window_index_for(window_start: u32, genesis: u32, interval: u32) -> u32 { + window_start.saturating_sub(genesis) / interval + } + fn account_to_h160(account_id: &T::AccountId) -> H160 { H160::from_slice(&account_id.encode()[..20]) } @@ -442,6 +417,29 @@ pub mod pallet { earliest } + fn window_rewards_info( + window_start: u32, + window_index: u32, + duration: u32, + ) -> Option { + let inflation_amount = WindowInflationAmount::::get(window_start); + let operator_points = WindowOperatorPoints::::get(window_start); + let total_points: u128 = operator_points.values().map(|p| *p as u128).sum(); + + if total_points == 0 || inflation_amount == 0 { + return None; + } + + Some(RewardsPeriodUtils { + period_index: window_index, + period_start: window_start, + duration, + total_points, + individual_points: operator_points.into_iter().collect(), + inflation_amount, + }) + } + fn allocate_era_inflation_to_windows( era_start: u32, era_end: u32, @@ -503,7 +501,7 @@ pub mod pallet { let inflation_amount = WindowInflationAmount::::get(next_window); let operator_points = WindowOperatorPoints::::get(next_window); let total_points: u128 = operator_points.values().map(|p| *p as u128).sum(); - let window_index = next_window.saturating_sub(genesis) / interval; + let window_index = Self::window_index_for(next_window, genesis, interval); if total_points == 0 || inflation_amount == 0 { Self::clear_window(next_window); @@ -539,6 +537,28 @@ pub mod pallet { window_start: next_window, window_index, }); + + let queued_window = QueuedRewardsWindow { + window_start: next_window, + window_index, + duration: interval, + }; + + if Self::unsent_queue_push(queued_window) { + next_window = next_window.saturating_add(interval); + continue; + } + + log::error!( + target: "ext_validators_rewards", + "Unsent reward queue full, cannot enqueue window {}", + next_window, + ); + Self::deposit_event(Event::UnsentWindowQueueFull { + window_start: next_window, + window_index, + }); + break; } } @@ -618,9 +638,9 @@ pub mod pallet { tail.wrapping_sub(head) % UNSENT_QUEUE_CAPACITY } - /// Push a new entry into the ring buffer. + /// Push a new window into the ring buffer. /// Returns `true` on success, `false` if the buffer is full. - pub(crate) fn unsent_queue_push(entry: (EraIndex, u32, u128)) -> bool { + pub(crate) fn unsent_queue_push(entry: QueuedRewardsWindow) -> bool { let head = UnsentRewardHead::::get(); let tail = UnsentRewardTail::::get(); let next_tail = (tail + 1) % UNSENT_QUEUE_CAPACITY; @@ -628,7 +648,7 @@ pub mod pallet { // Buffer full return false; } - UnsentRewardEra::::insert(tail, entry); + UnsentRewardWindow::::insert(tail, entry); UnsentRewardTail::::put(next_tail); true } @@ -645,13 +665,13 @@ pub mod pallet { break; } // Move next → cur - if let Some(entry) = UnsentRewardEra::::get(next) { - UnsentRewardEra::::insert(cur, entry); + if let Some(entry) = UnsentRewardWindow::::get(next) { + UnsentRewardWindow::::insert(cur, entry); } cur = next; } // Remove the now-duplicate last entry and shrink tail - UnsentRewardEra::::remove(cur); + UnsentRewardWindow::::remove(cur); let new_tail = if tail == 0 { UNSENT_QUEUE_CAPACITY - 1 } else { @@ -677,9 +697,9 @@ pub mod pallet { // ── Core retry logic ────────────────────────────────────────────── - /// Process at most one unsent reward era per block. + /// Process at most one unsent reward window per block. /// On failure the head pointer advances to the next entry so a single - /// stuck era does not block retries for subsequent eras. + /// stuck window does not block retries for subsequent windows. pub(crate) fn process_unsent_reward_eras() -> Weight { let head = UnsentRewardHead::::get(); let tail = UnsentRewardTail::::get(); @@ -688,55 +708,63 @@ pub mod pallet { return T::WeightInfo::process_unsent_reward_eras_empty(); } - let Some((era_index, timestamp, inflation)) = UnsentRewardEra::::get(head) else { + let Some(window) = UnsentRewardWindow::::get(head) else { // Slot unexpectedly empty — advance head past it UnsentRewardHead::::put((head + 1) % UNSENT_QUEUE_CAPACITY); return T::WeightInfo::process_unsent_reward_eras_empty(); }; - // Check if reward points are still available - let reward_points = RewardPointsForEra::::get(era_index); - let info = - match reward_points.generate_era_rewards_info(era_index, inflation, timestamp) { - Some(info) => info, - None => { - // Reward points have been pruned — discard this entry - log::warn!( - target: "ext_validators_rewards", - "Unsent era {era_index} expired: reward points pruned", - ); - UnsentRewardEra::::remove(head); - UnsentRewardHead::::put((head + 1) % UNSENT_QUEUE_CAPACITY); - Self::deposit_event(Event::UnsentEraExpired { era_index }); - return T::WeightInfo::process_unsent_reward_eras_expired(); - } - }; + let info = match Self::window_rewards_info( + window.window_start, + window.window_index, + window.duration, + ) { + Some(info) => info, + None => { + log::warn!( + target: "ext_validators_rewards", + "Unsent window {} expired: rewards state missing", + window.window_start, + ); + Self::clear_window(window.window_start); + UnsentRewardWindow::::remove(head); + UnsentRewardHead::::put((head + 1) % UNSENT_QUEUE_CAPACITY); + Self::deposit_event(Event::UnsentWindowExpired { + window_start: window.window_start, + window_index: window.window_index, + }); + return T::WeightInfo::process_unsent_reward_eras_expired(); + } + }; // Attempt to resend match Self::send_rewards_message(&info) { Some(message_id) => { - UnsentRewardEra::::remove(head); + Self::clear_window(window.window_start); + UnsentRewardWindow::::remove(head); UnsentRewardHead::::put((head + 1) % UNSENT_QUEUE_CAPACITY); - Self::deposit_event(Event::RewardsMessageRetried { + Self::deposit_event(Event::RewardsWindowRetried { message_id, - era_index, + window_start: window.window_start, + window_index: window.window_index, total_points: info.total_points, - inflation_amount: inflation, + inflation_amount: info.inflation_amount, }); T::WeightInfo::process_unsent_reward_eras_success() } None => { // Move the failed entry to the back of the queue so the - // next block tries a different era (avoids head-of-line + // next block tries a different window (avoids head-of-line // blocking). The entry is not lost — it will be retried // after all other pending entries. - UnsentRewardEra::::remove(head); + UnsentRewardWindow::::remove(head); UnsentRewardHead::::put((head + 1) % UNSENT_QUEUE_CAPACITY); - UnsentRewardEra::::insert(tail, (era_index, timestamp, inflation)); + UnsentRewardWindow::::insert(tail, window); UnsentRewardTail::::put((tail + 1) % UNSENT_QUEUE_CAPACITY); log::warn!( target: "ext_validators_rewards", - "Retry for unsent era {era_index} still failing, moved to back of queue", + "Retry for unsent window {} still failing, moved to back of queue", + window.window_start, ); T::WeightInfo::process_unsent_reward_eras_failed() } @@ -1060,17 +1088,27 @@ pub mod pallet { RewardPointsForEra::::remove(era_index_to_delete); BlocksProducedInEra::::remove(era_index_to_delete); - // Proactively clean up any unsent entries whose reward points - // have been pruned (this era and any older ones still lingering). + // Proactively clean up any unsent entries whose window state has + // been removed while they were waiting for retry. let head = UnsentRewardHead::::get(); let mut tail = UnsentRewardTail::::get(); let mut slot = head; while slot != tail { - if let Some((idx, _, _)) = UnsentRewardEra::::get(slot) { - if idx <= era_index_to_delete { + if let Some(window) = UnsentRewardWindow::::get(slot) { + if Self::window_rewards_info( + window.window_start, + window.window_index, + window.duration, + ) + .is_none() + { + Self::clear_window(window.window_start); Self::unsent_queue_remove_slot(slot); tail = UnsentRewardTail::::get(); - Self::deposit_event(Event::UnsentEraExpired { era_index: idx }); + Self::deposit_event(Event::UnsentWindowExpired { + window_start: window.window_start, + window_index: window.window_index, + }); // Don't advance slot — next entry slid into this position continue; } diff --git a/operator/pallets/external-validators-rewards/src/tests.rs b/operator/pallets/external-validators-rewards/src/tests.rs index e9c17cf1..5a71ceaa 100644 --- a/operator/pallets/external-validators-rewards/src/tests.rs +++ b/operator/pallets/external-validators-rewards/src/tests.rs @@ -424,15 +424,14 @@ fn window_mode_delivery_failure_emits_submission_failed_event() { "should not have emitted RewardsWindowSubmitted on delivery failure" ); - // Storage should still be cleared even on failure + // Failed window state should be retained for retry assert!( - pallet_external_validators_rewards::WindowOperatorPoints::::get(30).is_empty(), - "window points should be cleared after failed submission" + !pallet_external_validators_rewards::WindowOperatorPoints::::get(30).is_empty(), + "window points should be retained after failed submission" ); - assert_eq!( - pallet_external_validators_rewards::WindowInflationAmount::::get(30), - 0, - "window inflation should be cleared after failed submission" + assert!( + pallet_external_validators_rewards::WindowInflationAmount::::get(30) > 0, + "window inflation should be retained after failed submission" ); // NextWindowToSubmit should still advance @@ -440,6 +439,7 @@ fn window_mode_delivery_failure_emits_submission_failed_event() { pallet_external_validators_rewards::NextWindowToSubmit::::get(), 40 ); + assert_eq!(unsent_len(), 1, "failed window should be queued for retry"); }) } @@ -4096,14 +4096,30 @@ fn test_era_end_uses_correct_era_blocks_not_session() { // Retry mechanism tests (ring-buffer storage) // ═══════════════════════════════════════════════════════════════════════════ -/// Helper: push an entry into the unsent ring buffer via the pallet API. -fn push_unsent(era_index: u32, timestamp: u32, inflation: u128) { +/// Helper: push a window entry into the unsent ring buffer via the pallet API. +fn push_unsent(window_start: u32, window_index: u32, duration: u32) { assert!( - ExternalValidatorsRewards::unsent_queue_push((era_index, timestamp, inflation)), + ExternalValidatorsRewards::unsent_queue_push(crate::QueuedRewardsWindow { + window_start, + window_index, + duration, + }), "unsent_queue_push should succeed" ); } +/// Helper: seed persisted rewards state for a closed window. +fn seed_window(window_start: u32, inflation_amount: u128, operator_points: &[(u64, u32)]) { + crate::WindowInflationAmount::::insert(window_start, inflation_amount); + crate::WindowOperatorPoints::::insert( + window_start, + operator_points + .iter() + .map(|(operator, points)| (H160::from_low_u64_be(*operator), *points)) + .collect::>(), + ); +} + /// Helper: return the number of entries in the unsent ring buffer. fn unsent_len() -> u32 { ExternalValidatorsRewards::unsent_queue_len() @@ -4115,7 +4131,7 @@ fn unsent_is_empty() -> bool { } #[test] -fn send_failure_emits_window_submission_failed() { +fn send_failure_emits_window_submission_failed_and_queues_window() { new_test_ext().execute_with(|| { run_to_block(1); @@ -4144,9 +4160,6 @@ fn send_failure_emits_window_submission_failed() { System::reset_events(); ExternalValidatorsRewards::on_era_end(1); - // Unsent queue should NOT be populated (window approach doesn't use it) - assert!(unsent_is_empty()); - // Verify window submission failure event let window_index = window_start.saturating_sub(genesis) / window_duration; System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards( @@ -4155,11 +4168,21 @@ fn send_failure_emits_window_submission_failed() { window_index, }, )); + + assert_eq!(unsent_len(), 1, "failed window should be queued for retry"); + assert!( + !crate::WindowOperatorPoints::::get(window_start).is_empty(), + "failed window points must be retained for retry" + ); + assert!( + crate::WindowInflationAmount::::get(window_start) > 0, + "failed window inflation must be retained for retry" + ); }) } #[test] -fn on_initialize_retries_and_succeeds() { +fn failed_window_is_retried_on_initialize() { new_test_ext().execute_with(|| { run_to_block(1); @@ -4168,13 +4191,26 @@ fn on_initialize_retries_and_succeeds() { index: 1, start: Some(30_000), }); + mock.send_message_fails = true; }); - // Set up reward points for era 1 ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]); + for _ in 0..600 { + ExternalValidatorsRewards::note_block_author(H160::from_low_u64_be(1)); + } - // Manually populate the unsent queue - push_unsent(1, 30, 42); + let duration = RewardsWindowDuration::get(); + let genesis = RewardsWindowGenesisTimestamp::get(); + let now = (Timestamp::get() / 1000) as u32; + let window_start = now - (now - genesis) % duration; + let window_index = window_start.saturating_sub(genesis) / duration; + + Timestamp::set_timestamp(((window_start + duration + 1) as u64) * 1000); + ExternalValidatorsRewards::on_era_end(1); + + assert_eq!(unsent_len(), 1); + + Mock::mutate(|mock| mock.send_message_fails = false); // Sending should succeed (send_message_fails is false by default) System::reset_events(); @@ -4185,9 +4221,49 @@ fn on_initialize_retries_and_succeeds() { // Verify retry event System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards( - crate::Event::RewardsMessageRetried { + crate::Event::RewardsWindowRetried { message_id: Default::default(), - era_index: 1, + window_start, + window_index, + total_points: 100, + inflation_amount: 30, + }, + )); + + assert!( + crate::WindowOperatorPoints::::get(window_start).is_empty(), + "window points should be cleared after successful retry" + ); + assert_eq!( + crate::WindowInflationAmount::::get(window_start), + 0, + "window inflation should be cleared after successful retry" + ); + }) +} + +#[test] +fn on_initialize_retries_and_succeeds() { + new_test_ext().execute_with(|| { + run_to_block(1); + + let window_start = 30; + let window_index = 3; + let duration = RewardsWindowDuration::get(); + + seed_window(window_start, 42, &[(1, 100)]); + push_unsent(window_start, window_index, duration); + + System::reset_events(); + ExternalValidatorsRewards::process_unsent_reward_eras(); + + assert!(unsent_is_empty()); + + System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards( + crate::Event::RewardsWindowRetried { + message_id: Default::default(), + window_start, + window_index, total_points: 100, inflation_amount: 42, }, @@ -4201,75 +4277,64 @@ fn on_initialize_moves_failed_entry_to_back() { run_to_block(1); Mock::mutate(|mock| { - mock.active_era = Some(ActiveEraInfo { - index: 2, - start: Some(30_000), - }); mock.send_message_fails = true; }); - // Set up reward points for eras 1 and 2 - ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]); - Mock::mutate(|mock| { - mock.active_era = Some(ActiveEraInfo { - index: 1, - start: Some(30_000), - }); - }); - ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 200)]); + let duration = RewardsWindowDuration::get(); + seed_window(30, 42, &[(1, 100)]); + seed_window(40, 84, &[(1, 200)]); - // Push two entries: era 1 then era 2 - push_unsent(1, 30, 42); - push_unsent(2, 30, 84); + push_unsent(30, 3, duration); + push_unsent(40, 4, duration); - // First call: tries era 1, fails, moves era 1 to back of queue - ExternalValidatorsRewards::process_unsent_reward_eras(); - // Queue length stays the same (entry moved, not removed) - assert_eq!(unsent_len(), 2); - - // Second call: tries era 2 (NOT era 1 again), fails, moves era 2 to back + // First call: tries window 30, fails, moves it to the back of the queue + ExternalValidatorsRewards::process_unsent_reward_eras(); + assert_eq!(unsent_len(), 2); + + // Second call: tries window 40, fails, moves it to the back ExternalValidatorsRewards::process_unsent_reward_eras(); assert_eq!(unsent_len(), 2); - // Re-enable sending Mock::mutate(|mock| mock.send_message_fails = false); - // Third call: era 1 (now at front again), succeeds + // Third call: window 30 (now at front again), succeeds System::reset_events(); ExternalValidatorsRewards::process_unsent_reward_eras(); assert_eq!(unsent_len(), 1); System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards( - crate::Event::RewardsMessageRetried { + crate::Event::RewardsWindowRetried { message_id: Default::default(), - era_index: 1, - total_points: 200, + window_start: 30, + window_index: 3, + total_points: 100, inflation_amount: 42, }, )); - // Fourth call: era 2, succeeds + // Fourth call: window 40 succeeds ExternalValidatorsRewards::process_unsent_reward_eras(); assert!(unsent_is_empty()); }) } #[test] -fn on_initialize_removes_expired_era() { +fn on_initialize_removes_expired_window() { new_test_ext().execute_with(|| { run_to_block(1); - // Populate unsent queue with era 999 but do NOT add RewardPointsForEra for it - push_unsent(999, 0, 42); + let duration = RewardsWindowDuration::get(); + push_unsent(999, 99, duration); System::reset_events(); ExternalValidatorsRewards::process_unsent_reward_eras(); - // Entry should be removed assert!(unsent_is_empty()); - // Verify expired event System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards( - crate::Event::UnsentEraExpired { era_index: 999 }, + crate::Event::UnsentWindowExpired { + window_start: 999, + window_index: 99, + }, )); }) } @@ -4296,31 +4361,16 @@ fn on_initialize_processes_only_head() { new_test_ext().execute_with(|| { run_to_block(1); - Mock::mutate(|mock| { - mock.active_era = Some(ActiveEraInfo { - index: 3, - start: Some(30_000), - }); - }); + let duration = RewardsWindowDuration::get(); + seed_window(30, 42, &[(1, 100)]); + seed_window(40, 84, &[(2, 200)]); - // Set up reward points for both eras - ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]); - Mock::mutate(|mock| { - mock.active_era = Some(ActiveEraInfo { - index: 2, - start: Some(30_000), - }); - }); - ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(2), 200)]); - - // Push two entries - push_unsent(3, 30, 42); - push_unsent(2, 20, 84); + push_unsent(30, 3, duration); + push_unsent(40, 4, duration); System::reset_events(); ExternalValidatorsRewards::process_unsent_reward_eras(); - // Only the head entry (era 3) should be processed (and removed on success) assert_eq!(unsent_len(), 1); }) } @@ -4330,33 +4380,25 @@ fn retry_extrinsic_success() { new_test_ext().execute_with(|| { run_to_block(1); - Mock::mutate(|mock| { - mock.active_era = Some(ActiveEraInfo { - index: 1, - start: Some(30_000), - }); - }); - - // Set up reward points - ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]); - - // Populate unsent queue - push_unsent(1, 30, 42); + let window_start = 30; + let window_index = 3; + let duration = RewardsWindowDuration::get(); + seed_window(window_start, 42, &[(1, 100)]); + push_unsent(window_start, window_index, duration); System::reset_events(); - assert_ok!(ExternalValidatorsRewards::retry_unsent_reward_era( + assert_ok!(ExternalValidatorsRewards::retry_unsent_reward_window( RuntimeOrigin::root(), - 1 + window_start )); - // Queue should be empty assert!(unsent_is_empty()); - // Verify retry event System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards( - crate::Event::RewardsMessageRetried { + crate::Event::RewardsWindowRetried { message_id: Default::default(), - era_index: 1, + window_start, + window_index, total_points: 100, inflation_amount: 42, }, @@ -4365,28 +4407,27 @@ fn retry_extrinsic_success() { } #[test] -fn retry_extrinsic_era_not_in_queue() { +fn retry_extrinsic_window_not_in_queue() { new_test_ext().execute_with(|| { run_to_block(1); assert_noop!( - ExternalValidatorsRewards::retry_unsent_reward_era(RuntimeOrigin::root(), 1), - crate::Error::::EraNotInUnsentQueue + ExternalValidatorsRewards::retry_unsent_reward_window(RuntimeOrigin::root(), 1), + crate::Error::::WindowNotInUnsentQueue ); }) } #[test] -fn retry_extrinsic_pruned_data() { +fn retry_extrinsic_missing_window_state() { new_test_ext().execute_with(|| { run_to_block(1); - // Queue an era but don't create reward points for it - push_unsent(999, 0, 42); + push_unsent(999, 99, RewardsWindowDuration::get()); assert_noop!( - ExternalValidatorsRewards::retry_unsent_reward_era(RuntimeOrigin::root(), 999), - crate::Error::::RewardPointsPruned + ExternalValidatorsRewards::retry_unsent_reward_window(RuntimeOrigin::root(), 999), + crate::Error::::WindowRewardsMissing ); }) } @@ -4397,7 +4438,7 @@ fn retry_extrinsic_requires_root() { run_to_block(1); assert_noop!( - ExternalValidatorsRewards::retry_unsent_reward_era( + ExternalValidatorsRewards::retry_unsent_reward_window( RuntimeOrigin::signed(H160::from_low_u64_be(1)), 1 ), @@ -4407,7 +4448,7 @@ fn retry_extrinsic_requires_root() { } #[test] -fn send_failure_clears_window_and_advances_pointer() { +fn send_failure_retains_window_and_advances_pointer() { new_test_ext().execute_with(|| { run_to_block(1); @@ -4436,45 +4477,45 @@ fn send_failure_clears_window_and_advances_pointer() { System::reset_events(); ExternalValidatorsRewards::on_era_end(1); - // Even though send failed, the window data should be cleared - // and the next_window pointer should have advanced let next_window = crate::NextWindowToSubmit::::get(); assert!( next_window > window_start, "NextWindowToSubmit should advance past the failed window" ); - // Window storage should be cleared assert!( - crate::WindowOperatorPoints::::get(window_start).is_empty(), - "Window operator points should be cleared after failed submission" + !crate::WindowOperatorPoints::::get(window_start).is_empty(), + "Window operator points should be retained after failed submission" + ); + assert!( + crate::WindowInflationAmount::::get(window_start) > 0, + "Window inflation should be retained after failed submission" ); assert_eq!( - crate::WindowInflationAmount::::get(window_start), - 0, - "Window inflation should be cleared after failed submission" + unsent_len(), + 1, + "failed window should be present in retry queue" ); }) } #[test] -fn on_era_start_prunes_unsent_entry() { +fn on_era_start_prunes_unsent_window_without_state() { new_test_ext().execute_with(|| { run_to_block(1); - // Set up: era 1 has an unsent entry - push_unsent(1, 0, 42); + push_unsent(30, 3, RewardsWindowDuration::get()); - // HistoryDepth is 10, so era 11 should prune era 1 System::reset_events(); ExternalValidatorsRewards::on_era_start(11, 0, 11); - // Unsent entry should be removed assert!(unsent_is_empty()); - // Verify expired event System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards( - crate::Event::UnsentEraExpired { era_index: 1 }, + crate::Event::UnsentWindowExpired { + window_start: 30, + window_index: 3, + }, )); }) } @@ -4484,26 +4525,15 @@ fn retry_extrinsic_send_still_fails() { new_test_ext().execute_with(|| { run_to_block(1); - Mock::mutate(|mock| { - mock.active_era = Some(ActiveEraInfo { - index: 1, - start: Some(30_000), - }); - mock.send_message_fails = true; - }); - - // Set up reward points - ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]); - - // Populate unsent queue - push_unsent(1, 30, 42); + Mock::mutate(|mock| mock.send_message_fails = true); + seed_window(30, 42, &[(1, 100)]); + push_unsent(30, 3, RewardsWindowDuration::get()); assert_noop!( - ExternalValidatorsRewards::retry_unsent_reward_era(RuntimeOrigin::root(), 1), + ExternalValidatorsRewards::retry_unsent_reward_window(RuntimeOrigin::root(), 30), crate::Error::::MessageSendFailed ); - // Queue should still have the entry assert_eq!(unsent_len(), 1); }) } @@ -4513,57 +4543,50 @@ fn head_of_line_blocking_avoided() { new_test_ext().execute_with(|| { run_to_block(1); - // Set up reward points for eras 1, 2, 3 - for era in 1..=3u32 { - Mock::mutate(|mock| { - mock.active_era = Some(ActiveEraInfo { - index: era, - start: Some(30_000), - }); - }); - ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]); - } + let duration = RewardsWindowDuration::get(); + seed_window(30, 10, &[(1, 100)]); + seed_window(40, 20, &[(1, 100)]); + seed_window(50, 30, &[(1, 100)]); - // Push eras 1, 2, 3 into the queue - push_unsent(1, 30, 10); - push_unsent(2, 30, 20); - push_unsent(3, 30, 30); + push_unsent(30, 3, duration); + push_unsent(40, 4, duration); + push_unsent(50, 5, duration); - // Make sending fail Mock::mutate(|mock| mock.send_message_fails = true); - // Block 1: tries era 1, fails, advances head → era 2 + // Block 1: tries window 30, fails, advances head → window 40 ExternalValidatorsRewards::process_unsent_reward_eras(); - // Block 2: tries era 2, fails, advances head → era 3 + // Block 2: tries window 40, fails, advances head → window 50 ExternalValidatorsRewards::process_unsent_reward_eras(); - // Now re-enable sending Mock::mutate(|mock| mock.send_message_fails = false); - // Block 3: tries era 3, succeeds + // Block 3: tries window 50, succeeds System::reset_events(); ExternalValidatorsRewards::process_unsent_reward_eras(); System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards( - crate::Event::RewardsMessageRetried { + crate::Event::RewardsWindowRetried { message_id: Default::default(), - era_index: 3, + window_start: 50, + window_index: 5, total_points: 100, inflation_amount: 30, }, )); - // Block 4: wraps around to era 1, succeeds + // Block 4: wraps around to window 30, succeeds ExternalValidatorsRewards::process_unsent_reward_eras(); System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards( - crate::Event::RewardsMessageRetried { + crate::Event::RewardsWindowRetried { message_id: Default::default(), - era_index: 1, + window_start: 30, + window_index: 3, total_points: 100, inflation_amount: 10, }, )); - // Block 5: era 2, succeeds + // Block 5: window 40 succeeds ExternalValidatorsRewards::process_unsent_reward_eras(); assert!(unsent_is_empty()); })