mirror of
https://github.com/datahaven-xyz/datahaven
synced 2026-05-23 17:28:23 +00:00
Fix window-based rewards retry flow
This commit is contained in:
Ahmad Kaouk
parent
21ca8072f8
commit
03bd305e37
3 changed files with 363 additions and 296 deletions
|
|
@ -43,9 +43,13 @@ fn create_funded_user<T: Config + pallet_balances::Config>(
|
|||
user
|
||||
}
|
||||
|
||||
/// Helper: insert a single entry into the ring buffer at slot 0.
|
||||
fn push_unsent_entry<T: Config>(era_index: u32, timestamp: u32, inflation: u128) {
|
||||
ExternalValidatorsRewards::<T>::unsent_queue_push((era_index, timestamp, inflation));
|
||||
/// Helper: insert a single queued window into the ring buffer at slot 0.
|
||||
fn push_unsent_entry<T: Config>(window_start: u32, window_index: u32, duration: u32) {
|
||||
ExternalValidatorsRewards::<T>::unsent_queue_push(QueuedRewardsWindow {
|
||||
window_start,
|
||||
window_index,
|
||||
duration,
|
||||
});
|
||||
}
|
||||
|
||||
#[allow(clippy::multiple_bound_locations)]
|
||||
|
|
@ -77,17 +81,20 @@ mod benchmarks {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Helper to populate reward points for an era with 1000 validators.
|
||||
fn setup_era_reward_points<T: Config + pallet_balances::Config>(era_index: u32) {
|
||||
let mut era_reward_points = EraRewardPoints::default();
|
||||
era_reward_points.total = 20 * 1000;
|
||||
/// Helper to populate persisted state for a closed window with 1000 operators.
|
||||
fn setup_window_reward_state<T: Config + pallet_balances::Config>(
|
||||
window_start: u32,
|
||||
inflation_amount: u128,
|
||||
) {
|
||||
let mut operator_points = sp_std::collections::btree_map::BTreeMap::new();
|
||||
|
||||
for i in 0..1000 {
|
||||
let account_id = create_funded_user::<T>("candidate", i, 100);
|
||||
era_reward_points.individual.insert(account_id, 20);
|
||||
let _ = create_funded_user::<T>("candidate", i, 100);
|
||||
operator_points.insert(sp_core::H160::from_low_u64_be(i as u64 + 1), 20);
|
||||
}
|
||||
|
||||
<RewardPointsForEra<T>>::insert(era_index, era_reward_points);
|
||||
<WindowOperatorPoints<T>>::insert(window_start, operator_points);
|
||||
<WindowInflationAmount<T>>::insert(window_start, inflation_amount);
|
||||
}
|
||||
|
||||
// on_initialize: unsent queue is empty (2 reads for head+tail)
|
||||
|
|
@ -104,11 +111,10 @@ mod benchmarks {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
// on_initialize: oldest entry has pruned reward points
|
||||
// on_initialize: oldest queued window no longer has persisted state
|
||||
#[benchmark]
|
||||
fn process_unsent_reward_eras_expired() -> Result<(), BenchmarkError> {
|
||||
// Push an entry whose reward points do NOT exist in storage
|
||||
push_unsent_entry::<T>(999, 0, 42);
|
||||
push_unsent_entry::<T>(999, 99, 10);
|
||||
|
||||
#[block]
|
||||
{
|
||||
|
|
@ -126,9 +132,9 @@ mod benchmarks {
|
|||
fn process_unsent_reward_eras_success() -> Result<(), BenchmarkError> {
|
||||
frame_system::Pallet::<T>::set_block_number(0u32.into());
|
||||
T::BenchmarkHelper::setup();
|
||||
setup_era_reward_points::<T>(1);
|
||||
setup_window_reward_state::<T>(0, 42);
|
||||
|
||||
push_unsent_entry::<T>(1, 0, 42);
|
||||
push_unsent_entry::<T>(0, 0, 10);
|
||||
|
||||
#[block]
|
||||
{
|
||||
|
|
@ -145,9 +151,9 @@ mod benchmarks {
|
|||
fn process_unsent_reward_eras_failed() -> Result<(), BenchmarkError> {
|
||||
frame_system::Pallet::<T>::set_block_number(0u32.into());
|
||||
T::BenchmarkHelper::setup();
|
||||
setup_era_reward_points::<T>(1);
|
||||
setup_window_reward_state::<T>(0, 42);
|
||||
|
||||
push_unsent_entry::<T>(1, 0, 42);
|
||||
push_unsent_entry::<T>(0, 0, 10);
|
||||
|
||||
#[block]
|
||||
{
|
||||
|
|
@ -157,20 +163,20 @@ mod benchmarks {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
// Governance extrinsic: retry a specific unsent era
|
||||
// Governance extrinsic: retry a specific unsent window
|
||||
#[benchmark]
|
||||
fn retry_unsent_reward_era() -> Result<(), BenchmarkError> {
|
||||
fn retry_unsent_reward_window() -> Result<(), BenchmarkError> {
|
||||
frame_system::Pallet::<T>::set_block_number(0u32.into());
|
||||
T::BenchmarkHelper::setup();
|
||||
setup_era_reward_points::<T>(1);
|
||||
setup_window_reward_state::<T>(0, 42);
|
||||
|
||||
push_unsent_entry::<T>(1, 0, 42);
|
||||
push_unsent_entry::<T>(0, 0, 10);
|
||||
|
||||
let origin =
|
||||
T::GovernanceOrigin::try_successful_origin().map_err(|_| BenchmarkError::Weightless)?;
|
||||
|
||||
#[extrinsic_call]
|
||||
_(origin as T::RuntimeOrigin, 1u32);
|
||||
_(origin as T::RuntimeOrigin, 0u32);
|
||||
|
||||
assert!(ExternalValidatorsRewards::<T>::unsent_queue_is_empty());
|
||||
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ use {
|
|||
crate::types::{HandleInflation, RewardsPeriodUtils, SendMessage},
|
||||
frame_support::traits::{Get, UnixTime, ValidatorSet},
|
||||
pallet_external_validators::traits::{ExternalIndexProvider, OnEraEnd, OnEraStart},
|
||||
parity_scale_codec::{Decode, Encode},
|
||||
parity_scale_codec::{Decode, Encode, MaxEncodedLen},
|
||||
sp_core::{H160, H256},
|
||||
sp_runtime::{
|
||||
traits::{Hash, SaturatedConversion},
|
||||
|
|
@ -199,46 +199,50 @@ pub mod pallet {
|
|||
#[pallet::call]
|
||||
impl<T: Config> Pallet<T> {
|
||||
/// Governance escape hatch: manually retry sending a rewards message for
|
||||
/// an era that is stuck in the unsent queue.
|
||||
/// a closed window that is stuck in the unsent queue.
|
||||
#[pallet::call_index(0)]
|
||||
#[pallet::weight(T::WeightInfo::retry_unsent_reward_era())]
|
||||
pub fn retry_unsent_reward_era(
|
||||
pub fn retry_unsent_reward_window(
|
||||
origin: OriginFor<T>,
|
||||
era_index: EraIndex,
|
||||
window_start: u32,
|
||||
) -> DispatchResult {
|
||||
T::GovernanceOrigin::ensure_origin(origin)?;
|
||||
|
||||
// Scan the ring buffer for the requested era
|
||||
// Scan the ring buffer for the requested window
|
||||
let head = UnsentRewardHead::<T>::get();
|
||||
let tail = UnsentRewardTail::<T>::get();
|
||||
let mut found = None;
|
||||
let mut slot = head;
|
||||
while slot != tail {
|
||||
if let Some(entry @ (idx, _, _)) = UnsentRewardEra::<T>::get(slot) {
|
||||
if idx == era_index {
|
||||
if let Some(entry) = UnsentRewardWindow::<T>::get(slot) {
|
||||
if entry.window_start == window_start {
|
||||
found = Some((slot, entry));
|
||||
break;
|
||||
}
|
||||
}
|
||||
slot = (slot + 1) % UNSENT_QUEUE_CAPACITY;
|
||||
}
|
||||
let (slot, (_, timestamp, inflation)) = found.ok_or(Error::<T>::EraNotInUnsentQueue)?;
|
||||
let (slot, window) = found.ok_or(Error::<T>::WindowNotInUnsentQueue)?;
|
||||
|
||||
let reward_points = RewardPointsForEra::<T>::get(era_index);
|
||||
let info = reward_points
|
||||
.generate_era_rewards_info(era_index, inflation, timestamp)
|
||||
.ok_or(Error::<T>::RewardPointsPruned)?;
|
||||
let info = Self::window_rewards_info(
|
||||
window.window_start,
|
||||
window.window_index,
|
||||
window.duration,
|
||||
)
|
||||
.ok_or(Error::<T>::WindowRewardsMissing)?;
|
||||
|
||||
let message_id =
|
||||
Self::send_rewards_message(&info).ok_or(Error::<T>::MessageSendFailed)?;
|
||||
|
||||
Self::clear_window(window.window_start);
|
||||
Self::unsent_queue_remove_slot(slot);
|
||||
|
||||
Self::deposit_event(Event::RewardsMessageRetried {
|
||||
Self::deposit_event(Event::RewardsWindowRetried {
|
||||
message_id,
|
||||
era_index,
|
||||
window_start: window.window_start,
|
||||
window_index: window.window_index,
|
||||
total_points: info.total_points,
|
||||
inflation_amount: inflation,
|
||||
inflation_amount: info.inflation_amount,
|
||||
});
|
||||
|
||||
Ok(())
|
||||
|
|
@ -256,7 +260,7 @@ pub mod pallet {
|
|||
total_points: u128,
|
||||
inflation_amount: u128,
|
||||
},
|
||||
/// Window submission failed. Window data has been cleared.
|
||||
/// Window submission failed on the initial attempt.
|
||||
RewardsWindowSubmissionFailed {
|
||||
window_start: u32,
|
||||
window_index: u32,
|
||||
|
|
@ -266,27 +270,32 @@ pub mod pallet {
|
|||
window_start: u32,
|
||||
window_index: u32,
|
||||
},
|
||||
/// The rewards message failed to send; era queued for retry.
|
||||
RewardsMessageSendFailed { era_index: EraIndex },
|
||||
/// A previously failed rewards message was retried and sent successfully.
|
||||
RewardsMessageRetried {
|
||||
/// A previously failed rewards window was retried and sent successfully.
|
||||
RewardsWindowRetried {
|
||||
message_id: H256,
|
||||
era_index: EraIndex,
|
||||
window_start: u32,
|
||||
window_index: u32,
|
||||
total_points: u128,
|
||||
inflation_amount: u128,
|
||||
},
|
||||
/// An unsent era was dropped because its reward points have been pruned.
|
||||
UnsentEraExpired { era_index: EraIndex },
|
||||
/// The unsent queue is full; this era could not be enqueued for retry.
|
||||
UnsentQueueFull { era_index: EraIndex },
|
||||
/// A queued window was dropped because its stored rewards data is no longer available.
|
||||
UnsentWindowExpired {
|
||||
window_start: u32,
|
||||
window_index: u32,
|
||||
},
|
||||
/// The unsent queue is full; this failed window could not be enqueued for retry.
|
||||
UnsentWindowQueueFull {
|
||||
window_start: u32,
|
||||
window_index: u32,
|
||||
},
|
||||
}
|
||||
|
||||
#[pallet::error]
|
||||
pub enum Error<T> {
|
||||
/// The specified era is not in the unsent queue.
|
||||
EraNotInUnsentQueue,
|
||||
/// Reward points for the era have been pruned from storage.
|
||||
RewardPointsPruned,
|
||||
/// The specified window is not in the unsent queue.
|
||||
WindowNotInUnsentQueue,
|
||||
/// Rewards data for the window is no longer available.
|
||||
WindowRewardsMissing,
|
||||
/// The message delivery still failed on retry.
|
||||
MessageSendFailed,
|
||||
}
|
||||
|
|
@ -298,42 +307,6 @@ pub mod pallet {
|
|||
pub individual: BTreeMap<AccountId, RewardPoints>,
|
||||
}
|
||||
|
||||
impl<AccountId: Ord + sp_runtime::traits::Debug + Parameter> EraRewardPoints<AccountId> {
|
||||
/// Generate utils needed for EigenLayer rewards submission from era data.
|
||||
/// Used by the unsent-era retry mechanism to reconstruct a `RewardsPeriodUtils`
|
||||
/// from stored era reward points.
|
||||
pub fn generate_era_rewards_info(
|
||||
&self,
|
||||
era_index: EraIndex,
|
||||
inflation_amount: u128,
|
||||
era_start_timestamp: u32,
|
||||
) -> Option<RewardsPeriodUtils> {
|
||||
let mut individual_points = Vec::with_capacity(self.individual.len());
|
||||
|
||||
for (account_id, reward_points) in self.individual.iter() {
|
||||
// Convert AccountId to H160 for EigenLayer rewards submission.
|
||||
// In DataHaven, AccountId is H160, so encode() produces exactly 20 bytes.
|
||||
individual_points
|
||||
.push((H160::from_slice(&account_id.encode()[..20]), *reward_points));
|
||||
}
|
||||
|
||||
let total_points: u128 = individual_points.iter().map(|(_, pts)| *pts as u128).sum();
|
||||
|
||||
if total_points.is_zero() {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(RewardsPeriodUtils {
|
||||
period_index: era_index,
|
||||
period_start: era_start_timestamp,
|
||||
duration: 0,
|
||||
total_points,
|
||||
individual_points,
|
||||
inflation_amount,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<AccountId> Default for EraRewardPoints<AccountId> {
|
||||
fn default() -> Self {
|
||||
EraRewardPoints {
|
||||
|
|
@ -380,20 +353,18 @@ pub mod pallet {
|
|||
/// Maximum number of unsent reward entries in the ring buffer.
|
||||
pub const UNSENT_QUEUE_CAPACITY: u32 = 64;
|
||||
|
||||
/// Ring buffer of eras whose rewards messages failed to send.
|
||||
/// Each slot stores (era_index, era_start_timestamp, scaled_inflation).
|
||||
/// Metadata for a failed rewards window kept in the retry ring buffer.
|
||||
#[derive(RuntimeDebug, Encode, Decode, MaxEncodedLen, PartialEq, Eq, TypeInfo, Clone, Copy)]
|
||||
pub struct QueuedRewardsWindow {
|
||||
pub window_start: u32,
|
||||
pub window_index: u32,
|
||||
pub duration: u32,
|
||||
}
|
||||
|
||||
/// Ring buffer of windows whose rewards messages failed to send.
|
||||
/// Keyed by slot index [0, UNSENT_QUEUE_CAPACITY).
|
||||
#[pallet::storage]
|
||||
pub type UnsentRewardEra<T: Config> = StorageMap<
|
||||
_,
|
||||
Twox64Concat,
|
||||
u32,
|
||||
(
|
||||
EraIndex,
|
||||
/* era_start_timestamp */ u32,
|
||||
/* scaled_inflation */ u128,
|
||||
),
|
||||
>;
|
||||
pub type UnsentRewardWindow<T: Config> = StorageMap<_, Twox64Concat, u32, QueuedRewardsWindow>;
|
||||
|
||||
/// Ring buffer head: next slot to be processed by `on_initialize`.
|
||||
#[pallet::storage]
|
||||
|
|
@ -420,6 +391,10 @@ pub mod pallet {
|
|||
genesis + (timestamp.saturating_sub(genesis) / interval) * interval
|
||||
}
|
||||
|
||||
fn window_index_for(window_start: u32, genesis: u32, interval: u32) -> u32 {
|
||||
window_start.saturating_sub(genesis) / interval
|
||||
}
|
||||
|
||||
fn account_to_h160(account_id: &T::AccountId) -> H160 {
|
||||
H160::from_slice(&account_id.encode()[..20])
|
||||
}
|
||||
|
|
@ -442,6 +417,29 @@ pub mod pallet {
|
|||
earliest
|
||||
}
|
||||
|
||||
fn window_rewards_info(
|
||||
window_start: u32,
|
||||
window_index: u32,
|
||||
duration: u32,
|
||||
) -> Option<RewardsPeriodUtils> {
|
||||
let inflation_amount = WindowInflationAmount::<T>::get(window_start);
|
||||
let operator_points = WindowOperatorPoints::<T>::get(window_start);
|
||||
let total_points: u128 = operator_points.values().map(|p| *p as u128).sum();
|
||||
|
||||
if total_points == 0 || inflation_amount == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(RewardsPeriodUtils {
|
||||
period_index: window_index,
|
||||
period_start: window_start,
|
||||
duration,
|
||||
total_points,
|
||||
individual_points: operator_points.into_iter().collect(),
|
||||
inflation_amount,
|
||||
})
|
||||
}
|
||||
|
||||
fn allocate_era_inflation_to_windows(
|
||||
era_start: u32,
|
||||
era_end: u32,
|
||||
|
|
@ -503,7 +501,7 @@ pub mod pallet {
|
|||
let inflation_amount = WindowInflationAmount::<T>::get(next_window);
|
||||
let operator_points = WindowOperatorPoints::<T>::get(next_window);
|
||||
let total_points: u128 = operator_points.values().map(|p| *p as u128).sum();
|
||||
let window_index = next_window.saturating_sub(genesis) / interval;
|
||||
let window_index = Self::window_index_for(next_window, genesis, interval);
|
||||
|
||||
if total_points == 0 || inflation_amount == 0 {
|
||||
Self::clear_window(next_window);
|
||||
|
|
@ -539,6 +537,28 @@ pub mod pallet {
|
|||
window_start: next_window,
|
||||
window_index,
|
||||
});
|
||||
|
||||
let queued_window = QueuedRewardsWindow {
|
||||
window_start: next_window,
|
||||
window_index,
|
||||
duration: interval,
|
||||
};
|
||||
|
||||
if Self::unsent_queue_push(queued_window) {
|
||||
next_window = next_window.saturating_add(interval);
|
||||
continue;
|
||||
}
|
||||
|
||||
log::error!(
|
||||
target: "ext_validators_rewards",
|
||||
"Unsent reward queue full, cannot enqueue window {}",
|
||||
next_window,
|
||||
);
|
||||
Self::deposit_event(Event::UnsentWindowQueueFull {
|
||||
window_start: next_window,
|
||||
window_index,
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -618,9 +638,9 @@ pub mod pallet {
|
|||
tail.wrapping_sub(head) % UNSENT_QUEUE_CAPACITY
|
||||
}
|
||||
|
||||
/// Push a new entry into the ring buffer.
|
||||
/// Push a new window into the ring buffer.
|
||||
/// Returns `true` on success, `false` if the buffer is full.
|
||||
pub(crate) fn unsent_queue_push(entry: (EraIndex, u32, u128)) -> bool {
|
||||
pub(crate) fn unsent_queue_push(entry: QueuedRewardsWindow) -> bool {
|
||||
let head = UnsentRewardHead::<T>::get();
|
||||
let tail = UnsentRewardTail::<T>::get();
|
||||
let next_tail = (tail + 1) % UNSENT_QUEUE_CAPACITY;
|
||||
|
|
@ -628,7 +648,7 @@ pub mod pallet {
|
|||
// Buffer full
|
||||
return false;
|
||||
}
|
||||
UnsentRewardEra::<T>::insert(tail, entry);
|
||||
UnsentRewardWindow::<T>::insert(tail, entry);
|
||||
UnsentRewardTail::<T>::put(next_tail);
|
||||
true
|
||||
}
|
||||
|
|
@ -645,13 +665,13 @@ pub mod pallet {
|
|||
break;
|
||||
}
|
||||
// Move next → cur
|
||||
if let Some(entry) = UnsentRewardEra::<T>::get(next) {
|
||||
UnsentRewardEra::<T>::insert(cur, entry);
|
||||
if let Some(entry) = UnsentRewardWindow::<T>::get(next) {
|
||||
UnsentRewardWindow::<T>::insert(cur, entry);
|
||||
}
|
||||
cur = next;
|
||||
}
|
||||
// Remove the now-duplicate last entry and shrink tail
|
||||
UnsentRewardEra::<T>::remove(cur);
|
||||
UnsentRewardWindow::<T>::remove(cur);
|
||||
let new_tail = if tail == 0 {
|
||||
UNSENT_QUEUE_CAPACITY - 1
|
||||
} else {
|
||||
|
|
@ -677,9 +697,9 @@ pub mod pallet {
|
|||
|
||||
// ── Core retry logic ──────────────────────────────────────────────
|
||||
|
||||
/// Process at most one unsent reward era per block.
|
||||
/// Process at most one unsent reward window per block.
|
||||
/// On failure the head pointer advances to the next entry so a single
|
||||
/// stuck era does not block retries for subsequent eras.
|
||||
/// stuck window does not block retries for subsequent windows.
|
||||
pub(crate) fn process_unsent_reward_eras() -> Weight {
|
||||
let head = UnsentRewardHead::<T>::get();
|
||||
let tail = UnsentRewardTail::<T>::get();
|
||||
|
|
@ -688,55 +708,63 @@ pub mod pallet {
|
|||
return T::WeightInfo::process_unsent_reward_eras_empty();
|
||||
}
|
||||
|
||||
let Some((era_index, timestamp, inflation)) = UnsentRewardEra::<T>::get(head) else {
|
||||
let Some(window) = UnsentRewardWindow::<T>::get(head) else {
|
||||
// Slot unexpectedly empty — advance head past it
|
||||
UnsentRewardHead::<T>::put((head + 1) % UNSENT_QUEUE_CAPACITY);
|
||||
return T::WeightInfo::process_unsent_reward_eras_empty();
|
||||
};
|
||||
|
||||
// Check if reward points are still available
|
||||
let reward_points = RewardPointsForEra::<T>::get(era_index);
|
||||
let info =
|
||||
match reward_points.generate_era_rewards_info(era_index, inflation, timestamp) {
|
||||
Some(info) => info,
|
||||
None => {
|
||||
// Reward points have been pruned — discard this entry
|
||||
log::warn!(
|
||||
target: "ext_validators_rewards",
|
||||
"Unsent era {era_index} expired: reward points pruned",
|
||||
);
|
||||
UnsentRewardEra::<T>::remove(head);
|
||||
UnsentRewardHead::<T>::put((head + 1) % UNSENT_QUEUE_CAPACITY);
|
||||
Self::deposit_event(Event::UnsentEraExpired { era_index });
|
||||
return T::WeightInfo::process_unsent_reward_eras_expired();
|
||||
}
|
||||
};
|
||||
let info = match Self::window_rewards_info(
|
||||
window.window_start,
|
||||
window.window_index,
|
||||
window.duration,
|
||||
) {
|
||||
Some(info) => info,
|
||||
None => {
|
||||
log::warn!(
|
||||
target: "ext_validators_rewards",
|
||||
"Unsent window {} expired: rewards state missing",
|
||||
window.window_start,
|
||||
);
|
||||
Self::clear_window(window.window_start);
|
||||
UnsentRewardWindow::<T>::remove(head);
|
||||
UnsentRewardHead::<T>::put((head + 1) % UNSENT_QUEUE_CAPACITY);
|
||||
Self::deposit_event(Event::UnsentWindowExpired {
|
||||
window_start: window.window_start,
|
||||
window_index: window.window_index,
|
||||
});
|
||||
return T::WeightInfo::process_unsent_reward_eras_expired();
|
||||
}
|
||||
};
|
||||
|
||||
// Attempt to resend
|
||||
match Self::send_rewards_message(&info) {
|
||||
Some(message_id) => {
|
||||
UnsentRewardEra::<T>::remove(head);
|
||||
Self::clear_window(window.window_start);
|
||||
UnsentRewardWindow::<T>::remove(head);
|
||||
UnsentRewardHead::<T>::put((head + 1) % UNSENT_QUEUE_CAPACITY);
|
||||
Self::deposit_event(Event::RewardsMessageRetried {
|
||||
Self::deposit_event(Event::RewardsWindowRetried {
|
||||
message_id,
|
||||
era_index,
|
||||
window_start: window.window_start,
|
||||
window_index: window.window_index,
|
||||
total_points: info.total_points,
|
||||
inflation_amount: inflation,
|
||||
inflation_amount: info.inflation_amount,
|
||||
});
|
||||
T::WeightInfo::process_unsent_reward_eras_success()
|
||||
}
|
||||
None => {
|
||||
// Move the failed entry to the back of the queue so the
|
||||
// next block tries a different era (avoids head-of-line
|
||||
// next block tries a different window (avoids head-of-line
|
||||
// blocking). The entry is not lost — it will be retried
|
||||
// after all other pending entries.
|
||||
UnsentRewardEra::<T>::remove(head);
|
||||
UnsentRewardWindow::<T>::remove(head);
|
||||
UnsentRewardHead::<T>::put((head + 1) % UNSENT_QUEUE_CAPACITY);
|
||||
UnsentRewardEra::<T>::insert(tail, (era_index, timestamp, inflation));
|
||||
UnsentRewardWindow::<T>::insert(tail, window);
|
||||
UnsentRewardTail::<T>::put((tail + 1) % UNSENT_QUEUE_CAPACITY);
|
||||
log::warn!(
|
||||
target: "ext_validators_rewards",
|
||||
"Retry for unsent era {era_index} still failing, moved to back of queue",
|
||||
"Retry for unsent window {} still failing, moved to back of queue",
|
||||
window.window_start,
|
||||
);
|
||||
T::WeightInfo::process_unsent_reward_eras_failed()
|
||||
}
|
||||
|
|
@ -1060,17 +1088,27 @@ pub mod pallet {
|
|||
RewardPointsForEra::<T>::remove(era_index_to_delete);
|
||||
BlocksProducedInEra::<T>::remove(era_index_to_delete);
|
||||
|
||||
// Proactively clean up any unsent entries whose reward points
|
||||
// have been pruned (this era and any older ones still lingering).
|
||||
// Proactively clean up any unsent entries whose window state has
|
||||
// been removed while they were waiting for retry.
|
||||
let head = UnsentRewardHead::<T>::get();
|
||||
let mut tail = UnsentRewardTail::<T>::get();
|
||||
let mut slot = head;
|
||||
while slot != tail {
|
||||
if let Some((idx, _, _)) = UnsentRewardEra::<T>::get(slot) {
|
||||
if idx <= era_index_to_delete {
|
||||
if let Some(window) = UnsentRewardWindow::<T>::get(slot) {
|
||||
if Self::window_rewards_info(
|
||||
window.window_start,
|
||||
window.window_index,
|
||||
window.duration,
|
||||
)
|
||||
.is_none()
|
||||
{
|
||||
Self::clear_window(window.window_start);
|
||||
Self::unsent_queue_remove_slot(slot);
|
||||
tail = UnsentRewardTail::<T>::get();
|
||||
Self::deposit_event(Event::UnsentEraExpired { era_index: idx });
|
||||
Self::deposit_event(Event::UnsentWindowExpired {
|
||||
window_start: window.window_start,
|
||||
window_index: window.window_index,
|
||||
});
|
||||
// Don't advance slot — next entry slid into this position
|
||||
continue;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -424,15 +424,14 @@ fn window_mode_delivery_failure_emits_submission_failed_event() {
|
|||
"should not have emitted RewardsWindowSubmitted on delivery failure"
|
||||
);
|
||||
|
||||
// Storage should still be cleared even on failure
|
||||
// Failed window state should be retained for retry
|
||||
assert!(
|
||||
pallet_external_validators_rewards::WindowOperatorPoints::<Test>::get(30).is_empty(),
|
||||
"window points should be cleared after failed submission"
|
||||
!pallet_external_validators_rewards::WindowOperatorPoints::<Test>::get(30).is_empty(),
|
||||
"window points should be retained after failed submission"
|
||||
);
|
||||
assert_eq!(
|
||||
pallet_external_validators_rewards::WindowInflationAmount::<Test>::get(30),
|
||||
0,
|
||||
"window inflation should be cleared after failed submission"
|
||||
assert!(
|
||||
pallet_external_validators_rewards::WindowInflationAmount::<Test>::get(30) > 0,
|
||||
"window inflation should be retained after failed submission"
|
||||
);
|
||||
|
||||
// NextWindowToSubmit should still advance
|
||||
|
|
@ -440,6 +439,7 @@ fn window_mode_delivery_failure_emits_submission_failed_event() {
|
|||
pallet_external_validators_rewards::NextWindowToSubmit::<Test>::get(),
|
||||
40
|
||||
);
|
||||
assert_eq!(unsent_len(), 1, "failed window should be queued for retry");
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -4096,14 +4096,30 @@ fn test_era_end_uses_correct_era_blocks_not_session() {
|
|||
// Retry mechanism tests (ring-buffer storage)
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
/// Helper: push an entry into the unsent ring buffer via the pallet API.
|
||||
fn push_unsent(era_index: u32, timestamp: u32, inflation: u128) {
|
||||
/// Helper: push a window entry into the unsent ring buffer via the pallet API.
|
||||
fn push_unsent(window_start: u32, window_index: u32, duration: u32) {
|
||||
assert!(
|
||||
ExternalValidatorsRewards::unsent_queue_push((era_index, timestamp, inflation)),
|
||||
ExternalValidatorsRewards::unsent_queue_push(crate::QueuedRewardsWindow {
|
||||
window_start,
|
||||
window_index,
|
||||
duration,
|
||||
}),
|
||||
"unsent_queue_push should succeed"
|
||||
);
|
||||
}
|
||||
|
||||
/// Helper: seed persisted rewards state for a closed window.
|
||||
fn seed_window(window_start: u32, inflation_amount: u128, operator_points: &[(u64, u32)]) {
|
||||
crate::WindowInflationAmount::<Test>::insert(window_start, inflation_amount);
|
||||
crate::WindowOperatorPoints::<Test>::insert(
|
||||
window_start,
|
||||
operator_points
|
||||
.iter()
|
||||
.map(|(operator, points)| (H160::from_low_u64_be(*operator), *points))
|
||||
.collect::<BTreeMap<_, _>>(),
|
||||
);
|
||||
}
|
||||
|
||||
/// Helper: return the number of entries in the unsent ring buffer.
|
||||
fn unsent_len() -> u32 {
|
||||
ExternalValidatorsRewards::unsent_queue_len()
|
||||
|
|
@ -4115,7 +4131,7 @@ fn unsent_is_empty() -> bool {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn send_failure_emits_window_submission_failed() {
|
||||
fn send_failure_emits_window_submission_failed_and_queues_window() {
|
||||
new_test_ext().execute_with(|| {
|
||||
run_to_block(1);
|
||||
|
||||
|
|
@ -4144,9 +4160,6 @@ fn send_failure_emits_window_submission_failed() {
|
|||
System::reset_events();
|
||||
ExternalValidatorsRewards::on_era_end(1);
|
||||
|
||||
// Unsent queue should NOT be populated (window approach doesn't use it)
|
||||
assert!(unsent_is_empty());
|
||||
|
||||
// Verify window submission failure event
|
||||
let window_index = window_start.saturating_sub(genesis) / window_duration;
|
||||
System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards(
|
||||
|
|
@ -4155,11 +4168,21 @@ fn send_failure_emits_window_submission_failed() {
|
|||
window_index,
|
||||
},
|
||||
));
|
||||
|
||||
assert_eq!(unsent_len(), 1, "failed window should be queued for retry");
|
||||
assert!(
|
||||
!crate::WindowOperatorPoints::<Test>::get(window_start).is_empty(),
|
||||
"failed window points must be retained for retry"
|
||||
);
|
||||
assert!(
|
||||
crate::WindowInflationAmount::<Test>::get(window_start) > 0,
|
||||
"failed window inflation must be retained for retry"
|
||||
);
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn on_initialize_retries_and_succeeds() {
|
||||
fn failed_window_is_retried_on_initialize() {
|
||||
new_test_ext().execute_with(|| {
|
||||
run_to_block(1);
|
||||
|
||||
|
|
@ -4168,13 +4191,26 @@ fn on_initialize_retries_and_succeeds() {
|
|||
index: 1,
|
||||
start: Some(30_000),
|
||||
});
|
||||
mock.send_message_fails = true;
|
||||
});
|
||||
|
||||
// Set up reward points for era 1
|
||||
ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]);
|
||||
for _ in 0..600 {
|
||||
ExternalValidatorsRewards::note_block_author(H160::from_low_u64_be(1));
|
||||
}
|
||||
|
||||
// Manually populate the unsent queue
|
||||
push_unsent(1, 30, 42);
|
||||
let duration = RewardsWindowDuration::get();
|
||||
let genesis = RewardsWindowGenesisTimestamp::get();
|
||||
let now = (Timestamp::get() / 1000) as u32;
|
||||
let window_start = now - (now - genesis) % duration;
|
||||
let window_index = window_start.saturating_sub(genesis) / duration;
|
||||
|
||||
Timestamp::set_timestamp(((window_start + duration + 1) as u64) * 1000);
|
||||
ExternalValidatorsRewards::on_era_end(1);
|
||||
|
||||
assert_eq!(unsent_len(), 1);
|
||||
|
||||
Mock::mutate(|mock| mock.send_message_fails = false);
|
||||
|
||||
// Sending should succeed (send_message_fails is false by default)
|
||||
System::reset_events();
|
||||
|
|
@ -4185,9 +4221,49 @@ fn on_initialize_retries_and_succeeds() {
|
|||
|
||||
// Verify retry event
|
||||
System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards(
|
||||
crate::Event::RewardsMessageRetried {
|
||||
crate::Event::RewardsWindowRetried {
|
||||
message_id: Default::default(),
|
||||
era_index: 1,
|
||||
window_start,
|
||||
window_index,
|
||||
total_points: 100,
|
||||
inflation_amount: 30,
|
||||
},
|
||||
));
|
||||
|
||||
assert!(
|
||||
crate::WindowOperatorPoints::<Test>::get(window_start).is_empty(),
|
||||
"window points should be cleared after successful retry"
|
||||
);
|
||||
assert_eq!(
|
||||
crate::WindowInflationAmount::<Test>::get(window_start),
|
||||
0,
|
||||
"window inflation should be cleared after successful retry"
|
||||
);
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn on_initialize_retries_and_succeeds() {
|
||||
new_test_ext().execute_with(|| {
|
||||
run_to_block(1);
|
||||
|
||||
let window_start = 30;
|
||||
let window_index = 3;
|
||||
let duration = RewardsWindowDuration::get();
|
||||
|
||||
seed_window(window_start, 42, &[(1, 100)]);
|
||||
push_unsent(window_start, window_index, duration);
|
||||
|
||||
System::reset_events();
|
||||
ExternalValidatorsRewards::process_unsent_reward_eras();
|
||||
|
||||
assert!(unsent_is_empty());
|
||||
|
||||
System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards(
|
||||
crate::Event::RewardsWindowRetried {
|
||||
message_id: Default::default(),
|
||||
window_start,
|
||||
window_index,
|
||||
total_points: 100,
|
||||
inflation_amount: 42,
|
||||
},
|
||||
|
|
@ -4201,75 +4277,64 @@ fn on_initialize_moves_failed_entry_to_back() {
|
|||
run_to_block(1);
|
||||
|
||||
Mock::mutate(|mock| {
|
||||
mock.active_era = Some(ActiveEraInfo {
|
||||
index: 2,
|
||||
start: Some(30_000),
|
||||
});
|
||||
mock.send_message_fails = true;
|
||||
});
|
||||
|
||||
// Set up reward points for eras 1 and 2
|
||||
ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]);
|
||||
Mock::mutate(|mock| {
|
||||
mock.active_era = Some(ActiveEraInfo {
|
||||
index: 1,
|
||||
start: Some(30_000),
|
||||
});
|
||||
});
|
||||
ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 200)]);
|
||||
let duration = RewardsWindowDuration::get();
|
||||
seed_window(30, 42, &[(1, 100)]);
|
||||
seed_window(40, 84, &[(1, 200)]);
|
||||
|
||||
// Push two entries: era 1 then era 2
|
||||
push_unsent(1, 30, 42);
|
||||
push_unsent(2, 30, 84);
|
||||
push_unsent(30, 3, duration);
|
||||
push_unsent(40, 4, duration);
|
||||
|
||||
// First call: tries era 1, fails, moves era 1 to back of queue
|
||||
ExternalValidatorsRewards::process_unsent_reward_eras();
|
||||
// Queue length stays the same (entry moved, not removed)
|
||||
assert_eq!(unsent_len(), 2);
|
||||
|
||||
// Second call: tries era 2 (NOT era 1 again), fails, moves era 2 to back
|
||||
// First call: tries window 30, fails, moves it to the back of the queue
|
||||
ExternalValidatorsRewards::process_unsent_reward_eras();
|
||||
assert_eq!(unsent_len(), 2);
|
||||
|
||||
// Second call: tries window 40, fails, moves it to the back
|
||||
ExternalValidatorsRewards::process_unsent_reward_eras();
|
||||
assert_eq!(unsent_len(), 2);
|
||||
|
||||
// Re-enable sending
|
||||
Mock::mutate(|mock| mock.send_message_fails = false);
|
||||
|
||||
// Third call: era 1 (now at front again), succeeds
|
||||
// Third call: window 30 (now at front again), succeeds
|
||||
System::reset_events();
|
||||
ExternalValidatorsRewards::process_unsent_reward_eras();
|
||||
assert_eq!(unsent_len(), 1);
|
||||
System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards(
|
||||
crate::Event::RewardsMessageRetried {
|
||||
crate::Event::RewardsWindowRetried {
|
||||
message_id: Default::default(),
|
||||
era_index: 1,
|
||||
total_points: 200,
|
||||
window_start: 30,
|
||||
window_index: 3,
|
||||
total_points: 100,
|
||||
inflation_amount: 42,
|
||||
},
|
||||
));
|
||||
|
||||
// Fourth call: era 2, succeeds
|
||||
// Fourth call: window 40 succeeds
|
||||
ExternalValidatorsRewards::process_unsent_reward_eras();
|
||||
assert!(unsent_is_empty());
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn on_initialize_removes_expired_era() {
|
||||
fn on_initialize_removes_expired_window() {
|
||||
new_test_ext().execute_with(|| {
|
||||
run_to_block(1);
|
||||
|
||||
// Populate unsent queue with era 999 but do NOT add RewardPointsForEra for it
|
||||
push_unsent(999, 0, 42);
|
||||
let duration = RewardsWindowDuration::get();
|
||||
push_unsent(999, 99, duration);
|
||||
|
||||
System::reset_events();
|
||||
ExternalValidatorsRewards::process_unsent_reward_eras();
|
||||
|
||||
// Entry should be removed
|
||||
assert!(unsent_is_empty());
|
||||
|
||||
// Verify expired event
|
||||
System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards(
|
||||
crate::Event::UnsentEraExpired { era_index: 999 },
|
||||
crate::Event::UnsentWindowExpired {
|
||||
window_start: 999,
|
||||
window_index: 99,
|
||||
},
|
||||
));
|
||||
})
|
||||
}
|
||||
|
|
@ -4296,31 +4361,16 @@ fn on_initialize_processes_only_head() {
|
|||
new_test_ext().execute_with(|| {
|
||||
run_to_block(1);
|
||||
|
||||
Mock::mutate(|mock| {
|
||||
mock.active_era = Some(ActiveEraInfo {
|
||||
index: 3,
|
||||
start: Some(30_000),
|
||||
});
|
||||
});
|
||||
let duration = RewardsWindowDuration::get();
|
||||
seed_window(30, 42, &[(1, 100)]);
|
||||
seed_window(40, 84, &[(2, 200)]);
|
||||
|
||||
// Set up reward points for both eras
|
||||
ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]);
|
||||
Mock::mutate(|mock| {
|
||||
mock.active_era = Some(ActiveEraInfo {
|
||||
index: 2,
|
||||
start: Some(30_000),
|
||||
});
|
||||
});
|
||||
ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(2), 200)]);
|
||||
|
||||
// Push two entries
|
||||
push_unsent(3, 30, 42);
|
||||
push_unsent(2, 20, 84);
|
||||
push_unsent(30, 3, duration);
|
||||
push_unsent(40, 4, duration);
|
||||
|
||||
System::reset_events();
|
||||
ExternalValidatorsRewards::process_unsent_reward_eras();
|
||||
|
||||
// Only the head entry (era 3) should be processed (and removed on success)
|
||||
assert_eq!(unsent_len(), 1);
|
||||
})
|
||||
}
|
||||
|
|
@ -4330,33 +4380,25 @@ fn retry_extrinsic_success() {
|
|||
new_test_ext().execute_with(|| {
|
||||
run_to_block(1);
|
||||
|
||||
Mock::mutate(|mock| {
|
||||
mock.active_era = Some(ActiveEraInfo {
|
||||
index: 1,
|
||||
start: Some(30_000),
|
||||
});
|
||||
});
|
||||
|
||||
// Set up reward points
|
||||
ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]);
|
||||
|
||||
// Populate unsent queue
|
||||
push_unsent(1, 30, 42);
|
||||
let window_start = 30;
|
||||
let window_index = 3;
|
||||
let duration = RewardsWindowDuration::get();
|
||||
seed_window(window_start, 42, &[(1, 100)]);
|
||||
push_unsent(window_start, window_index, duration);
|
||||
|
||||
System::reset_events();
|
||||
assert_ok!(ExternalValidatorsRewards::retry_unsent_reward_era(
|
||||
assert_ok!(ExternalValidatorsRewards::retry_unsent_reward_window(
|
||||
RuntimeOrigin::root(),
|
||||
1
|
||||
window_start
|
||||
));
|
||||
|
||||
// Queue should be empty
|
||||
assert!(unsent_is_empty());
|
||||
|
||||
// Verify retry event
|
||||
System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards(
|
||||
crate::Event::RewardsMessageRetried {
|
||||
crate::Event::RewardsWindowRetried {
|
||||
message_id: Default::default(),
|
||||
era_index: 1,
|
||||
window_start,
|
||||
window_index,
|
||||
total_points: 100,
|
||||
inflation_amount: 42,
|
||||
},
|
||||
|
|
@ -4365,28 +4407,27 @@ fn retry_extrinsic_success() {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn retry_extrinsic_era_not_in_queue() {
|
||||
fn retry_extrinsic_window_not_in_queue() {
|
||||
new_test_ext().execute_with(|| {
|
||||
run_to_block(1);
|
||||
|
||||
assert_noop!(
|
||||
ExternalValidatorsRewards::retry_unsent_reward_era(RuntimeOrigin::root(), 1),
|
||||
crate::Error::<Test>::EraNotInUnsentQueue
|
||||
ExternalValidatorsRewards::retry_unsent_reward_window(RuntimeOrigin::root(), 1),
|
||||
crate::Error::<Test>::WindowNotInUnsentQueue
|
||||
);
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_extrinsic_pruned_data() {
|
||||
fn retry_extrinsic_missing_window_state() {
|
||||
new_test_ext().execute_with(|| {
|
||||
run_to_block(1);
|
||||
|
||||
// Queue an era but don't create reward points for it
|
||||
push_unsent(999, 0, 42);
|
||||
push_unsent(999, 99, RewardsWindowDuration::get());
|
||||
|
||||
assert_noop!(
|
||||
ExternalValidatorsRewards::retry_unsent_reward_era(RuntimeOrigin::root(), 999),
|
||||
crate::Error::<Test>::RewardPointsPruned
|
||||
ExternalValidatorsRewards::retry_unsent_reward_window(RuntimeOrigin::root(), 999),
|
||||
crate::Error::<Test>::WindowRewardsMissing
|
||||
);
|
||||
})
|
||||
}
|
||||
|
|
@ -4397,7 +4438,7 @@ fn retry_extrinsic_requires_root() {
|
|||
run_to_block(1);
|
||||
|
||||
assert_noop!(
|
||||
ExternalValidatorsRewards::retry_unsent_reward_era(
|
||||
ExternalValidatorsRewards::retry_unsent_reward_window(
|
||||
RuntimeOrigin::signed(H160::from_low_u64_be(1)),
|
||||
1
|
||||
),
|
||||
|
|
@ -4407,7 +4448,7 @@ fn retry_extrinsic_requires_root() {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn send_failure_clears_window_and_advances_pointer() {
|
||||
fn send_failure_retains_window_and_advances_pointer() {
|
||||
new_test_ext().execute_with(|| {
|
||||
run_to_block(1);
|
||||
|
||||
|
|
@ -4436,45 +4477,45 @@ fn send_failure_clears_window_and_advances_pointer() {
|
|||
System::reset_events();
|
||||
ExternalValidatorsRewards::on_era_end(1);
|
||||
|
||||
// Even though send failed, the window data should be cleared
|
||||
// and the next_window pointer should have advanced
|
||||
let next_window = crate::NextWindowToSubmit::<Test>::get();
|
||||
assert!(
|
||||
next_window > window_start,
|
||||
"NextWindowToSubmit should advance past the failed window"
|
||||
);
|
||||
|
||||
// Window storage should be cleared
|
||||
assert!(
|
||||
crate::WindowOperatorPoints::<Test>::get(window_start).is_empty(),
|
||||
"Window operator points should be cleared after failed submission"
|
||||
!crate::WindowOperatorPoints::<Test>::get(window_start).is_empty(),
|
||||
"Window operator points should be retained after failed submission"
|
||||
);
|
||||
assert!(
|
||||
crate::WindowInflationAmount::<Test>::get(window_start) > 0,
|
||||
"Window inflation should be retained after failed submission"
|
||||
);
|
||||
assert_eq!(
|
||||
crate::WindowInflationAmount::<Test>::get(window_start),
|
||||
0,
|
||||
"Window inflation should be cleared after failed submission"
|
||||
unsent_len(),
|
||||
1,
|
||||
"failed window should be present in retry queue"
|
||||
);
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn on_era_start_prunes_unsent_entry() {
|
||||
fn on_era_start_prunes_unsent_window_without_state() {
|
||||
new_test_ext().execute_with(|| {
|
||||
run_to_block(1);
|
||||
|
||||
// Set up: era 1 has an unsent entry
|
||||
push_unsent(1, 0, 42);
|
||||
push_unsent(30, 3, RewardsWindowDuration::get());
|
||||
|
||||
// HistoryDepth is 10, so era 11 should prune era 1
|
||||
System::reset_events();
|
||||
ExternalValidatorsRewards::on_era_start(11, 0, 11);
|
||||
|
||||
// Unsent entry should be removed
|
||||
assert!(unsent_is_empty());
|
||||
|
||||
// Verify expired event
|
||||
System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards(
|
||||
crate::Event::UnsentEraExpired { era_index: 1 },
|
||||
crate::Event::UnsentWindowExpired {
|
||||
window_start: 30,
|
||||
window_index: 3,
|
||||
},
|
||||
));
|
||||
})
|
||||
}
|
||||
|
|
@ -4484,26 +4525,15 @@ fn retry_extrinsic_send_still_fails() {
|
|||
new_test_ext().execute_with(|| {
|
||||
run_to_block(1);
|
||||
|
||||
Mock::mutate(|mock| {
|
||||
mock.active_era = Some(ActiveEraInfo {
|
||||
index: 1,
|
||||
start: Some(30_000),
|
||||
});
|
||||
mock.send_message_fails = true;
|
||||
});
|
||||
|
||||
// Set up reward points
|
||||
ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]);
|
||||
|
||||
// Populate unsent queue
|
||||
push_unsent(1, 30, 42);
|
||||
Mock::mutate(|mock| mock.send_message_fails = true);
|
||||
seed_window(30, 42, &[(1, 100)]);
|
||||
push_unsent(30, 3, RewardsWindowDuration::get());
|
||||
|
||||
assert_noop!(
|
||||
ExternalValidatorsRewards::retry_unsent_reward_era(RuntimeOrigin::root(), 1),
|
||||
ExternalValidatorsRewards::retry_unsent_reward_window(RuntimeOrigin::root(), 30),
|
||||
crate::Error::<Test>::MessageSendFailed
|
||||
);
|
||||
|
||||
// Queue should still have the entry
|
||||
assert_eq!(unsent_len(), 1);
|
||||
})
|
||||
}
|
||||
|
|
@ -4513,57 +4543,50 @@ fn head_of_line_blocking_avoided() {
|
|||
new_test_ext().execute_with(|| {
|
||||
run_to_block(1);
|
||||
|
||||
// Set up reward points for eras 1, 2, 3
|
||||
for era in 1..=3u32 {
|
||||
Mock::mutate(|mock| {
|
||||
mock.active_era = Some(ActiveEraInfo {
|
||||
index: era,
|
||||
start: Some(30_000),
|
||||
});
|
||||
});
|
||||
ExternalValidatorsRewards::reward_by_ids([(H160::from_low_u64_be(1), 100)]);
|
||||
}
|
||||
let duration = RewardsWindowDuration::get();
|
||||
seed_window(30, 10, &[(1, 100)]);
|
||||
seed_window(40, 20, &[(1, 100)]);
|
||||
seed_window(50, 30, &[(1, 100)]);
|
||||
|
||||
// Push eras 1, 2, 3 into the queue
|
||||
push_unsent(1, 30, 10);
|
||||
push_unsent(2, 30, 20);
|
||||
push_unsent(3, 30, 30);
|
||||
push_unsent(30, 3, duration);
|
||||
push_unsent(40, 4, duration);
|
||||
push_unsent(50, 5, duration);
|
||||
|
||||
// Make sending fail
|
||||
Mock::mutate(|mock| mock.send_message_fails = true);
|
||||
|
||||
// Block 1: tries era 1, fails, advances head → era 2
|
||||
// Block 1: tries window 30, fails, advances head → window 40
|
||||
ExternalValidatorsRewards::process_unsent_reward_eras();
|
||||
// Block 2: tries era 2, fails, advances head → era 3
|
||||
// Block 2: tries window 40, fails, advances head → window 50
|
||||
ExternalValidatorsRewards::process_unsent_reward_eras();
|
||||
|
||||
// Now re-enable sending
|
||||
Mock::mutate(|mock| mock.send_message_fails = false);
|
||||
|
||||
// Block 3: tries era 3, succeeds
|
||||
// Block 3: tries window 50, succeeds
|
||||
System::reset_events();
|
||||
ExternalValidatorsRewards::process_unsent_reward_eras();
|
||||
System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards(
|
||||
crate::Event::RewardsMessageRetried {
|
||||
crate::Event::RewardsWindowRetried {
|
||||
message_id: Default::default(),
|
||||
era_index: 3,
|
||||
window_start: 50,
|
||||
window_index: 5,
|
||||
total_points: 100,
|
||||
inflation_amount: 30,
|
||||
},
|
||||
));
|
||||
|
||||
// Block 4: wraps around to era 1, succeeds
|
||||
// Block 4: wraps around to window 30, succeeds
|
||||
ExternalValidatorsRewards::process_unsent_reward_eras();
|
||||
System::assert_has_event(RuntimeEvent::ExternalValidatorsRewards(
|
||||
crate::Event::RewardsMessageRetried {
|
||||
crate::Event::RewardsWindowRetried {
|
||||
message_id: Default::default(),
|
||||
era_index: 1,
|
||||
window_start: 30,
|
||||
window_index: 3,
|
||||
total_points: 100,
|
||||
inflation_amount: 10,
|
||||
},
|
||||
));
|
||||
|
||||
// Block 5: era 2, succeeds
|
||||
// Block 5: window 40 succeeds
|
||||
ExternalValidatorsRewards::process_unsent_reward_eras();
|
||||
assert!(unsent_is_empty());
|
||||
})
|
||||
|
|
|
|||
Loading…
Reference in a new issue