From 9fe972c95dc5f84e8ccb7338249c2ef41a876f57 Mon Sep 17 00:00:00 2001 From: Gonza Montiel Date: Mon, 30 Mar 2026 11:37:24 +0200 Subject: [PATCH] feat: retry ring on slash failure (#479) ## Summary This PR aligns the `external-validator-slashes` retry flow with the same general approach used by the rewards retry ring (https://github.com/datahaven-xyz/datahaven/pull/462). The main goal is to make slash retries preserve their original era identity, avoid head-of-line blocking by rotating failed work to the back of the queue, and add a governance escape hatch for manually retrying queued slash batches. ## Changes - Reworked `external-validator-slashes` to use a bounded ring-buffer queue for unsent slash batches instead of retrying from an unbounded raw slash queue. - Stored the original slash era together with each queued batch so retries keep the same outbound message identity across later eras. - Added a governance-only retry extrinsic for queued slash eras, mirroring the rewards retrial ring approach. - Added new retry-related errors and events for queued slash retries and queue-full handling. - Updated retry-path logging so expected send failures are treated as warnings instead of noisy errors during passing tests. - Extended pallet tests to cover: - preserved original era on retry - back-of-queue rotation on failed sends - governance retry success and failure cases - bounded queue capacity behavior - multi-era queue progression - Updated benchmarking and pallet weights for the new retry path. - Wired the new governance origin and weight changes into mainnet, testnet, and stagenet runtime configs. --- .../src/benchmarking.rs | 74 +++-- .../external-validator-slashes/src/lib.rs | 305 ++++++++++++++---- .../external-validator-slashes/src/mock.rs | 23 +- .../external-validator-slashes/src/tests.rs | 272 +++++++++++++++- .../external-validator-slashes/src/weights.rs | 10 + operator/runtime/mainnet/src/configs/mod.rs | 1 + .../pallet_external_validator_slashes.rs | 3 + operator/runtime/stagenet/src/configs/mod.rs | 1 + .../pallet_external_validator_slashes.rs | 3 + operator/runtime/testnet/src/configs/mod.rs | 1 + .../pallet_external_validator_slashes.rs | 3 + test/.papi/descriptors/package.json | 2 +- test/.papi/metadata/datahaven.scale | Bin 640566 -> 641589 bytes 13 files changed, 593 insertions(+), 105 deletions(-) diff --git a/operator/pallets/external-validator-slashes/src/benchmarking.rs b/operator/pallets/external-validator-slashes/src/benchmarking.rs index 7e8d7a00..5575c914 100644 --- a/operator/pallets/external-validator-slashes/src/benchmarking.rs +++ b/operator/pallets/external-validator-slashes/src/benchmarking.rs @@ -35,20 +35,24 @@ const MAX_SLASHES: u32 = 1000; mod benchmarks { use super::*; + fn dummy_slash(slash_id: T::SlashId) -> Slash { + let dummy = || T::AccountId::decode(&mut TrailingZeroInput::zeroes()).unwrap(); + Slash { + validator: dummy(), + reporters: vec![], + slash_id, + percentage: Perbill::from_percent(1), + confirmed: false, + offence_kind: OffenceKind::LivenessOffence, + } + } + #[benchmark] fn cancel_deferred_slash(s: Linear<1, MAX_SLASHES>) -> Result<(), BenchmarkError> { let mut existing_slashes = Vec::new(); let era = T::EraIndexProvider::active_era().index; - let dummy = || T::AccountId::decode(&mut TrailingZeroInput::zeroes()).unwrap(); for _ in 0..MAX_SLASHES { - existing_slashes.push(Slash { - validator: dummy(), - reporters: vec![], - slash_id: One::one(), - percentage: Perbill::from_percent(1), - confirmed: false, - offence_kind: OffenceKind::LivenessOffence, - }); + existing_slashes.push(dummy_slash::(One::one())); } Slashes::::insert( era.saturating_add(T::SlashDeferDuration::get()) @@ -102,35 +106,55 @@ mod benchmarks { #[benchmark] fn process_slashes_queue(s: Linear<1, 200>) -> Result<(), BenchmarkError> { - let mut queue = VecDeque::new(); - let dummy = || T::AccountId::decode(&mut TrailingZeroInput::zeroes()).unwrap(); + let first_batch = (0..s) + .map(|_| dummy_slash::(One::one())) + .collect::>(); + let second_batch = vec![dummy_slash::(One::one())]; - for _ in 0..(s + 1) { - queue.push_back(Slash { - validator: dummy(), - reporters: vec![], - slash_id: One::one(), - percentage: Perbill::from_percent(1), - confirmed: false, - offence_kind: OffenceKind::LivenessOffence, - }); - } - - UnreportedSlashesQueue::::set(queue); + assert!(ExternalValidatorSlashes::::unsent_queue_push(( + 1, + first_batch + ))); + assert!(ExternalValidatorSlashes::::unsent_queue_push(( + 2, + second_batch + ))); let processed; #[block] { - processed = Pallet::::process_slashes_queue(s).unwrap(); + processed = match Pallet::::process_slashes_queue() { + crate::ProcessSlashesQueueOutcome::Sent(count) => count, + crate::ProcessSlashesQueueOutcome::Empty + | crate::ProcessSlashesQueueOutcome::Requeued(_) => { + return Err(BenchmarkError::Stop("unexpected slashes queue outcome")) + } + }; } - assert_eq!(UnreportedSlashesQueue::::get().len(), 1); + assert_eq!(ExternalValidatorSlashes::::unsent_queue_len(), 1); assert_eq!(processed, s); Ok(()) } + #[benchmark] + fn retry_unsent_slash_era() -> Result<(), BenchmarkError> { + let batch = vec![dummy_slash::(One::one())]; + assert!(ExternalValidatorSlashes::::unsent_queue_push((1, batch))); + + let origin = + T::GovernanceOrigin::try_successful_origin().map_err(|_| BenchmarkError::Weightless)?; + + #[extrinsic_call] + _(origin as T::RuntimeOrigin, 1u32); + + assert!(ExternalValidatorSlashes::::unsent_queue_is_empty()); + + Ok(()) + } + #[benchmark] fn set_slashing_mode() -> Result<(), BenchmarkError> { #[extrinsic_call] diff --git a/operator/pallets/external-validator-slashes/src/lib.rs b/operator/pallets/external-validator-slashes/src/lib.rs index b24b85db..19da25c3 100644 --- a/operator/pallets/external-validator-slashes/src/lib.rs +++ b/operator/pallets/external-validator-slashes/src/lib.rs @@ -31,7 +31,7 @@ extern crate alloc; use pallet_external_validators::apply; use snowbridge_outbound_queue_primitives::SendError; use { - alloc::{collections::vec_deque::VecDeque, string::String, vec, vec::Vec}, + alloc::{string::String, vec, vec::Vec}, frame_support::{pallet_prelude::*, traits::DefensiveSaturating}, frame_system::pallet_prelude::*, log::log, @@ -132,10 +132,21 @@ pub mod pallet { }, /// The slashes message was sent correctly. SlashesMessageSent { message_id: H256 }, + /// The slashes message failed to send and the batch was moved to the back + /// of the queue for retry. + SlashesMessageSendFailed { era: EraIndex, count: u32 }, + /// A queued slashes batch was retried manually and sent successfully. + SlashesMessageRetried { + message_id: H256, + era: EraIndex, + count: u32, + }, /// We injected a slash SlashInjected { slash_id: T::SlashId, era: u32 }, /// Number of slashes processed SlashAddedToQueue { number: u32, era: u32 }, + /// The unsent queue is full; this slash era could not be enqueued. + UnsentQueueFull { era: EraIndex }, } #[pallet::config] @@ -199,6 +210,9 @@ pub mod pallet { /// The weight information of this pallet. type WeightInfo: WeightInfo; + + /// Origin for governance calls such as retrying an unsent slash batch. + type GovernanceOrigin: EnsureOrigin; } #[pallet::error] @@ -226,6 +240,10 @@ pub mod pallet { /// No PendingOffenceKind found for (session, validator) — offence was not /// reported through EquivocationReportWrapper, so the offence kind is unknown. MissingOffenceKind, + /// The specified era is not in the unsent slash queue. + EraNotInUnsentQueue, + /// The message delivery still failed on retry. + MessageSendFailed, } #[apply(derive_storage_traits)] @@ -269,12 +287,26 @@ pub mod pallet { pub type Slashes = StorageMap<_, Twox64Concat, EraIndex, Vec>, ValueQuery>; - /// All unreported slashes that will be processed in the future. + /// Maximum number of unsent slash batches in the retry ring buffer. + pub const UNSENT_QUEUE_CAPACITY: u32 = 64; + + /// Ring buffer of slash batches whose outbound message still needs to be sent. + /// Each slot stores the original slash era together with a bounded-size batch + /// of slash records. Retries keep the original era so the outbound message id + /// remains stable across later blocks and eras. #[pallet::storage] #[pallet::unbounded] - #[pallet::getter(fn unreported_slashes)] - pub type UnreportedSlashesQueue = - StorageValue<_, VecDeque>, ValueQuery>; + pub type UnsentSlashBatch = + StorageMap<_, Twox64Concat, u32, (EraIndex, Vec>)>; + + /// Ring buffer head: next slot to be processed by `on_initialize`. + #[pallet::storage] + pub type UnsentSlashHead = StorageValue<_, u32, ValueQuery>; + + /// Ring buffer tail: next slot to write a new entry into. + /// When head == tail the buffer is empty. + #[pallet::storage] + pub type UnsentSlashTail = StorageValue<_, u32, ValueQuery>; // Turns slashing on or off #[pallet::storage] @@ -415,6 +447,44 @@ pub mod pallet { Ok(()) } + #[pallet::call_index(2)] + #[pallet::weight(T::WeightInfo::retry_unsent_slash_era())] + pub fn retry_unsent_slash_era(origin: OriginFor, era_index: EraIndex) -> DispatchResult { + T::GovernanceOrigin::ensure_origin(origin)?; + + let head = UnsentSlashHead::::get(); + let tail = UnsentSlashTail::::get(); + let mut found = None; + let mut slot = head; + while slot != tail { + if let Some(entry @ (idx, _)) = UnsentSlashBatch::::get(slot) { + if idx == era_index { + found = Some((slot, entry)); + break; + } + } + slot = (slot + 1) % UNSENT_QUEUE_CAPACITY; + } + + let (slot, (era, slashes)) = found.ok_or(Error::::EraNotInUnsentQueue)?; + let count = slashes.len() as u32; + let slashes_to_send = slashes + .iter() + .map(Self::slash_to_send_data) + .collect::>(); + let message_id = Self::send_slashes_message(&slashes_to_send, era) + .ok_or(Error::::MessageSendFailed)?; + + Self::unsent_queue_remove_slot(slot); + Self::deposit_event(Event::::SlashesMessageRetried { + message_id, + era, + count, + }); + + Ok(()) + } + #[pallet::call_index(3)] #[pallet::weight(T::WeightInfo::set_slashing_mode())] pub fn set_slashing_mode(origin: OriginFor, mode: SlashingModeOption) -> DispatchResult { @@ -429,12 +499,12 @@ pub mod pallet { #[pallet::hooks] impl Hooks> for Pallet { fn on_initialize(_n: BlockNumberFor) -> Weight { - let processed = Self::process_slashes_queue(T::QueuedSlashesProcessedPerBlock::get()); - - if let Some(p) = processed { - T::WeightInfo::process_slashes_queue(p) - } else { - T::WeightInfo::process_slashes_queue(0) + match Self::process_slashes_queue() { + ProcessSlashesQueueOutcome::Empty => T::WeightInfo::process_slashes_queue(0), + ProcessSlashesQueueOutcome::Sent(count) + | ProcessSlashesQueueOutcome::Requeued(count) => { + T::WeightInfo::process_slashes_queue(count) + } } } } @@ -655,70 +725,65 @@ where impl Pallet { fn add_era_slashes_to_queue(active_era: EraIndex) { - let mut slashes: VecDeque<_> = Slashes::::get(active_era).into(); + let slashes = Slashes::::get(active_era); + if slashes.is_empty() { + return; + } - let len = slashes.len(); + let batch_size = T::QueuedSlashesProcessedPerBlock::get().max(1) as usize; + let mut enqueued = 0u32; - UnreportedSlashesQueue::::mutate(|queue| queue.append(&mut slashes)); + for batch in slashes.chunks(batch_size) { + if Self::unsent_queue_push((active_era, batch.to_vec())) { + enqueued = enqueued.saturating_add(batch.len() as u32); + } else { + log::warn!( + target: "ext_validators_slashes", + "Unsent slash queue full, cannot enqueue era {active_era}", + ); + Self::deposit_event(Event::::UnsentQueueFull { era: active_era }); + break; + } + } - if len > 0 { + if enqueued > 0 { Self::deposit_event(Event::::SlashAddedToQueue { - number: len as u32, + number: enqueued, era: active_era, }); } } - /// Returns number of slashes that were sent to ethereum. - fn process_slashes_queue(amount: u32) -> Option { - let mut slashes_to_send: Vec> = vec![]; - let era_index = T::EraIndexProvider::active_era().index; + fn slash_to_send_data(slash: &Slash) -> SlashData { + // Keep the original slash batch intact until delivery succeeds so failed + // batches can be moved to the back of the queue instead of being dropped. + let max_wad = T::MaxSlashWad::get(); + let wad_to_slash = (slash.percentage.deconstruct() as u128) + .saturating_mul(max_wad) + .checked_div(1_000_000_000u128) + .unwrap_or(0) + .min(max_wad); - UnreportedSlashesQueue::::mutate(|queue| { - for _ in 0..amount { - let Some(slash) = queue.pop_front() else { - // no more slashes to process in the queue - break; - }; - - // Convert Perbill to EigenLayer WAD format with linear mapping. - // Perbill(100%) → MaxSlashWad (e.g. 5% WAD = 5e16). - // Formula: perbill_inner * MaxSlashWad / 1e9 - // Clamp to MaxSlashWad to guard against overflow if governance - // sets MaxSlashWad high enough for saturating_mul to hit u128::MAX. - let max_wad = T::MaxSlashWad::get(); - let wad_to_slash = (slash.percentage.deconstruct() as u128) - .saturating_mul(max_wad) - .checked_div(1_000_000_000u128) - .unwrap_or(0) - .min(max_wad); - - slashes_to_send.push(SlashData { - validator: slash.validator, - wad_to_slash, - description: slash.offence_kind.to_description(), - }); - } - }); - - if slashes_to_send.is_empty() { - return None; + SlashData { + validator: slash.validator.clone(), + wad_to_slash, + description: slash.offence_kind.to_description(), } + } - let slashes_count = slashes_to_send.len() as u32; + fn send_slashes_message( + slashes_to_send: &[SlashData], + era_index: EraIndex, + ) -> Option { + let outbound = + T::SendMessage::build(&slashes_to_send.to_vec(), era_index).or_else(|| { + log::warn!(target: "ext_validators_slashes", "Failed to build outbound message"); + None + })?; - let outbound = match T::SendMessage::build(&slashes_to_send, era_index) { - Some(send_msg) => send_msg, - None => { - log::error!(target: "ext_validators_slashes", "Failed to build outbound message"); - return None; - } - }; - - // Validate and deliver the message let ticket = T::SendMessage::validate(outbound) .map_err(|e| { - log::error!( + log::warn!( target: "ext_validators_slashes", "Failed to validate outbound message: {:?}", e @@ -726,20 +791,126 @@ impl Pallet { }) .ok()?; - let message_id = T::SendMessage::deliver(ticket) + T::SendMessage::deliver(ticket) .map_err(|e| { - log::error!( + log::warn!( target: "ext_validators_slashes", "Failed to deliver outbound message: {:?}", e ); }) - .ok()?; - - Self::deposit_event(Event::::SlashesMessageSent { message_id }); - - Some(slashes_count) + .ok() } + + #[allow(dead_code)] + pub(crate) fn unsent_queue_is_empty() -> bool { + UnsentSlashHead::::get() == UnsentSlashTail::::get() + } + + #[allow(dead_code)] + pub(crate) fn unsent_queue_len() -> u32 { + let head = UnsentSlashHead::::get(); + let tail = UnsentSlashTail::::get(); + tail.wrapping_sub(head) % UNSENT_QUEUE_CAPACITY + } + + pub(crate) fn unsent_queue_push( + entry: (EraIndex, Vec>), + ) -> bool { + let head = UnsentSlashHead::::get(); + let tail = UnsentSlashTail::::get(); + let next_tail = (tail + 1) % UNSENT_QUEUE_CAPACITY; + if next_tail == head { + return false; + } + + UnsentSlashBatch::::insert(tail, entry); + UnsentSlashTail::::put(next_tail); + true + } + + fn unsent_queue_remove_slot(slot: u32) { + let tail = UnsentSlashTail::::get(); + let mut cur = slot; + loop { + let next = (cur + 1) % UNSENT_QUEUE_CAPACITY; + if next == tail { + break; + } + + if let Some(entry) = UnsentSlashBatch::::get(next) { + UnsentSlashBatch::::insert(cur, entry); + } + cur = next; + } + + UnsentSlashBatch::::remove(cur); + let new_tail = if tail == 0 { + UNSENT_QUEUE_CAPACITY - 1 + } else { + tail - 1 + }; + UnsentSlashTail::::put(new_tail); + + let head = UnsentSlashHead::::get(); + if head == tail { + UnsentSlashHead::::put(new_tail); + } + } + + /// Retry contract shared with rewards: + /// - process the current head batch, + /// - if send succeeds, remove it from the queue, + /// - if send fails, move the same batch to the back so later slash batches can progress. + pub(crate) fn process_slashes_queue() -> ProcessSlashesQueueOutcome { + let head = UnsentSlashHead::::get(); + let tail = UnsentSlashTail::::get(); + + if head == tail { + return ProcessSlashesQueueOutcome::Empty; + } + + let Some((era_index, slashes)) = UnsentSlashBatch::::get(head) else { + UnsentSlashHead::::put((head + 1) % UNSENT_QUEUE_CAPACITY); + return ProcessSlashesQueueOutcome::Empty; + }; + + let slashes_count = slashes.len() as u32; + let slashes_to_send = slashes + .iter() + .map(Self::slash_to_send_data) + .collect::>(); + + match Self::send_slashes_message(&slashes_to_send, era_index) { + Some(message_id) => { + UnsentSlashBatch::::remove(head); + UnsentSlashHead::::put((head + 1) % UNSENT_QUEUE_CAPACITY); + Self::deposit_event(Event::::SlashesMessageSent { message_id }); + ProcessSlashesQueueOutcome::Sent(slashes_count) + } + None => { + UnsentSlashBatch::::remove(head); + UnsentSlashHead::::put((head + 1) % UNSENT_QUEUE_CAPACITY); + UnsentSlashBatch::::insert(tail, (era_index, slashes)); + UnsentSlashTail::::put((tail + 1) % UNSENT_QUEUE_CAPACITY); + log::warn!( + target: "ext_validators_slashes", + "Failed to send {slashes_count} slash entries for era {era_index}, moved batch to back of queue", + ); + Self::deposit_event(Event::::SlashesMessageSendFailed { + era: era_index, + count: slashes_count, + }); + ProcessSlashesQueueOutcome::Requeued(slashes_count) + } + } + } +} + +pub(crate) enum ProcessSlashesQueueOutcome { + Empty, + Sent(u32), + Requeued(u32), } /// A pending slash record. The value of the slash has been computed but not applied yet, diff --git a/operator/pallets/external-validator-slashes/src/mock.rs b/operator/pallets/external-validator-slashes/src/mock.rs index 34ba198e..8ec92539 100644 --- a/operator/pallets/external-validator-slashes/src/mock.rs +++ b/operator/pallets/external-validator-slashes/src/mock.rs @@ -134,7 +134,9 @@ thread_local! { pub static SENT_ETHEREUM_MESSAGE_NONCE: RefCell = const { RefCell::new(0) }; pub static MOCK_REPORT_OFFENCE_SHOULD_FAIL: RefCell = const { RefCell::new(false) }; pub static MOCK_REPORT_OFFENCE_CALLED: RefCell = const { RefCell::new(false) }; + pub static MOCK_SEND_MESSAGE_SHOULD_FAIL: RefCell = const { RefCell::new(false) }; pub static LAST_SENT_SLASHES: RefCell>> = RefCell::new(Vec::new()); + pub static LAST_BUILT_ERA: RefCell> = const { RefCell::new(None) }; } impl MockEraIndexProvider { @@ -222,19 +224,32 @@ impl MockOkOutboundQueue { pub fn last_sent_slashes() -> Vec> { LAST_SENT_SLASHES.with(|r| r.borrow().clone()) } + + pub fn last_built_era() -> Option { + LAST_BUILT_ERA.with(|r| *r.borrow()) + } + + pub fn set_should_fail(fail: bool) { + MOCK_SEND_MESSAGE_SHOULD_FAIL.with(|r| *r.borrow_mut() = fail); + } } impl crate::SendMessage for MockOkOutboundQueue { type Ticket = (); type Message = (); - fn build(slashes: &Vec>, _: u32) -> Option { + fn build(slashes: &Vec>, era: u32) -> Option { LAST_SENT_SLASHES.with(|r| *r.borrow_mut() = slashes.clone()); + LAST_BUILT_ERA.with(|r| *r.borrow_mut() = Some(era)); Some(()) } fn validate(_: Self::Ticket) -> Result { Ok(()) } fn deliver(_: Self::Ticket) -> Result { - Ok(H256::zero()) + if MOCK_SEND_MESSAGE_SHOULD_FAIL.with(|r| *r.borrow()) { + Err(SendError::MessageTooLarge) + } else { + Ok(H256::zero()) + } } } @@ -271,6 +286,7 @@ impl external_validator_slashes::Config for Test { type QueuedSlashesProcessedPerBlock = ConstU32<20>; type WeightInfo = (); type SendMessage = MockOkOutboundQueue; + type GovernanceOrigin = frame_system::EnsureRoot; } pub struct FullIdentificationOf; @@ -286,6 +302,9 @@ impl pallet_session::historical::Config for Test { } // Build genesis storage according to the mock runtime. pub fn new_test_ext() -> sp_io::TestExternalities { + MOCK_SEND_MESSAGE_SHOULD_FAIL.with(|r| *r.borrow_mut() = false); + LAST_SENT_SLASHES.with(|r| r.borrow_mut().clear()); + LAST_BUILT_ERA.with(|r| *r.borrow_mut() = None); system::GenesisConfig::::default() .build_storage() .unwrap() diff --git a/operator/pallets/external-validator-slashes/src/tests.rs b/operator/pallets/external-validator-slashes/src/tests.rs index 126485b8..350af20f 100644 --- a/operator/pallets/external-validator-slashes/src/tests.rs +++ b/operator/pallets/external-validator-slashes/src/tests.rs @@ -28,6 +28,40 @@ use { sp_staking::offence::ReportOffence, }; +fn queued_slash_ids() -> Vec { + let mut queued = Vec::new(); + let mut slot = UnsentSlashHead::::get(); + let tail = UnsentSlashTail::::get(); + + while slot != tail { + if let Some((_, batch)) = UnsentSlashBatch::::get(slot) { + queued.extend(batch.into_iter().map(|slash| slash.slash_id)); + } + slot = (slot + 1) % UNSENT_QUEUE_CAPACITY; + } + + queued +} + +fn queued_batch_eras() -> Vec { + let mut queued = Vec::new(); + let mut slot = UnsentSlashHead::::get(); + let tail = UnsentSlashTail::::get(); + + while slot != tail { + if let Some((era, _)) = UnsentSlashBatch::::get(slot) { + queued.push(era); + } + slot = (slot + 1) % UNSENT_QUEUE_CAPACITY; + } + + queued +} + +fn unsent_queue_len() -> u32 { + ExternalValidatorSlashes::unsent_queue_len() +} + #[test] fn root_can_inject_manual_offence() { new_test_ext().execute_with(|| { @@ -574,14 +608,228 @@ fn test_on_offence_defer_period_0_messages_get_queued() { assert_eq!(Slashes::::get(get_slashing_era(1)).len(), 25); start_era(2, 2, 2); - assert_eq!(UnreportedSlashesQueue::::get().len(), 25); + assert_eq!(unsent_queue_len(), 2); + assert_eq!(queued_batch_eras(), vec![2, 2]); // this triggers on_initialize run_block(); - assert_eq!(UnreportedSlashesQueue::::get().len(), 5); + assert_eq!(unsent_queue_len(), 1); + assert_eq!(queued_slash_ids(), (20..25).collect::>()); run_block(); - assert_eq!(UnreportedSlashesQueue::::get().len(), 0); + assert!(ExternalValidatorSlashes::unsent_queue_is_empty()); + }); +} + +#[test] +fn failed_slashes_batch_is_moved_to_back_of_queue() { + new_test_ext().execute_with(|| { + crate::mock::DeferPeriodGetter::with_defer_period(0); + MockOkOutboundQueue::set_should_fail(true); + + start_era(0, 0, 0); + start_era(1, 1, 1); + + for i in 0..25 { + PendingOffenceKind::::insert(0, 3 + i, OffenceKind::LivenessOffence); + Pallet::::on_offence( + &[OffenceDetails { + offender: (3 + i, ()), + reporters: vec![], + }], + &[Perbill::from_percent(75)], + 0, + ); + } + + start_era(2, 2, 2); + assert_eq!(queued_slash_ids(), (0..25).collect::>()); + assert_eq!(queued_batch_eras(), vec![2, 2]); + + run_block(); + + assert_eq!(unsent_queue_len(), 2); + assert_eq!( + queued_slash_ids(), + (20..25).chain(0..20).collect::>() + ); + System::assert_has_event(RuntimeEvent::ExternalValidatorSlashes( + crate::Event::SlashesMessageSendFailed { era: 2, count: 20 }, + )); + }); +} + +#[test] +fn failed_slashes_batch_retries_after_send_is_reenabled() { + new_test_ext().execute_with(|| { + crate::mock::DeferPeriodGetter::with_defer_period(0); + MockOkOutboundQueue::set_should_fail(true); + + start_era(0, 0, 0); + start_era(1, 1, 1); + + for i in 0..25 { + PendingOffenceKind::::insert(0, 3 + i, OffenceKind::LivenessOffence); + Pallet::::on_offence( + &[OffenceDetails { + offender: (3 + i, ()), + reporters: vec![], + }], + &[Perbill::from_percent(75)], + 0, + ); + } + + start_era(2, 2, 2); + run_block(); + assert_eq!( + queued_slash_ids(), + (20..25).chain(0..20).collect::>() + ); + + start_era(3, 3, 3); + MockOkOutboundQueue::set_should_fail(false); + + run_block(); + assert_eq!(unsent_queue_len(), 1); + assert_eq!(queued_slash_ids(), (0..20).collect::>()); + assert_eq!(MockOkOutboundQueue::last_sent_slashes().len(), 5); + assert_eq!(MockOkOutboundQueue::last_built_era(), Some(2)); + System::assert_has_event(RuntimeEvent::ExternalValidatorSlashes( + crate::Event::SlashesMessageSent { + message_id: Default::default(), + }, + )); + + run_block(); + assert!(ExternalValidatorSlashes::unsent_queue_is_empty()); + }); +} + +#[test] +fn retry_extrinsic_succeeds_for_matching_era() { + new_test_ext().execute_with(|| { + crate::mock::DeferPeriodGetter::with_defer_period(0); + + start_era(0, 0, 0); + start_era(1, 1, 1); + + for i in 0..25 { + PendingOffenceKind::::insert(0, 3 + i, OffenceKind::LivenessOffence); + Pallet::::on_offence( + &[OffenceDetails { + offender: (3 + i, ()), + reporters: vec![], + }], + &[Perbill::from_percent(75)], + 0, + ); + } + + start_era(2, 2, 2); + start_era(5, 5, 5); + + assert_ok!(ExternalValidatorSlashes::retry_unsent_slash_era( + RuntimeOrigin::root(), + 2, + )); + + assert_eq!(unsent_queue_len(), 1); + assert_eq!(queued_slash_ids(), (20..25).collect::>()); + assert_eq!(MockOkOutboundQueue::last_built_era(), Some(2)); + }); +} + +#[test] +fn retry_extrinsic_errors_when_era_not_queued() { + new_test_ext().execute_with(|| { + assert_noop!( + ExternalValidatorSlashes::retry_unsent_slash_era(RuntimeOrigin::root(), 2), + Error::::EraNotInUnsentQueue + ); + }); +} + +#[test] +fn retry_extrinsic_requires_root() { + new_test_ext().execute_with(|| { + assert_noop!( + ExternalValidatorSlashes::retry_unsent_slash_era(RuntimeOrigin::signed(1), 2), + sp_runtime::DispatchError::BadOrigin + ); + }); +} + +#[test] +fn retry_extrinsic_preserves_failed_batch_when_send_still_fails() { + new_test_ext().execute_with(|| { + crate::mock::DeferPeriodGetter::with_defer_period(0); + MockOkOutboundQueue::set_should_fail(true); + + start_era(0, 0, 0); + start_era(1, 1, 1); + + for i in 0..25 { + PendingOffenceKind::::insert(0, 3 + i, OffenceKind::LivenessOffence); + Pallet::::on_offence( + &[OffenceDetails { + offender: (3 + i, ()), + reporters: vec![], + }], + &[Perbill::from_percent(75)], + 0, + ); + } + + start_era(2, 2, 2); + let before = queued_slash_ids(); + + assert_noop!( + ExternalValidatorSlashes::retry_unsent_slash_era(RuntimeOrigin::root(), 2), + Error::::MessageSendFailed + ); + + assert_eq!(queued_slash_ids(), before); + assert_eq!(unsent_queue_len(), 2); + }); +} + +#[test] +fn unsent_queue_full_emits_event() { + new_test_ext().execute_with(|| { + crate::mock::DeferPeriodGetter::with_defer_period(0); + + for i in 0..63u32 { + let slash = Slash { + validator: 1000 + i as u64, + reporters: vec![], + slash_id: i, + percentage: Perbill::from_percent(1), + confirmed: true, + offence_kind: OffenceKind::LivenessOffence, + }; + assert!(ExternalValidatorSlashes::unsent_queue_push(( + 1, + vec![slash] + ))); + } + + Slashes::::insert( + 2, + vec![Slash { + validator: 5000u64, + reporters: vec![], + slash_id: 999, + percentage: Perbill::from_percent(10), + confirmed: true, + offence_kind: OffenceKind::LivenessOffence, + }], + ); + + start_era(2, 2, 2); + + assert_eq!(unsent_queue_len(), 63); + assert_eq!(Slashes::::get(2).len(), 1); }); } @@ -628,14 +876,13 @@ fn test_on_offence_defer_period_0_messages_get_queued_across_eras() { } assert_eq!(Slashes::::get(get_slashing_era(1)).len(), 25); start_era(2, 2, 2); - assert_eq!(UnreportedSlashesQueue::::get().len(), 25); + assert_eq!(unsent_queue_len(), 2); // this triggers on_initialize run_block(); - assert_eq!(UnreportedSlashesQueue::::get().len(), 5); + assert_eq!(unsent_queue_len(), 1); + assert_eq!(queued_slash_ids(), (20..25).collect::>()); - // We have 5 non-dispatched, which should accumulate - // We shoulld have 30 after we initialie era 3 for i in 0..25 { PendingOffenceKind::::insert(2, 3 + i, OffenceKind::LivenessOffence); Pallet::::on_offence( @@ -651,15 +898,20 @@ fn test_on_offence_defer_period_0_messages_get_queued_across_eras() { } start_era(3, 3, 3); - assert_eq!(UnreportedSlashesQueue::::get().len(), 30); + assert_eq!(unsent_queue_len(), 3); + assert_eq!(queued_batch_eras(), vec![2, 3, 3]); // this triggers on_initialize run_block(); - assert_eq!(UnreportedSlashesQueue::::get().len(), 10); + assert_eq!(unsent_queue_len(), 2); + assert_eq!(queued_batch_eras(), vec![3, 3]); // this triggers on_initialize run_block(); - assert_eq!(UnreportedSlashesQueue::::get().len(), 0); + assert_eq!(unsent_queue_len(), 1); + + run_block(); + assert!(ExternalValidatorSlashes::unsent_queue_is_empty()); }); } diff --git a/operator/pallets/external-validator-slashes/src/weights.rs b/operator/pallets/external-validator-slashes/src/weights.rs index 011374bd..22971e79 100644 --- a/operator/pallets/external-validator-slashes/src/weights.rs +++ b/operator/pallets/external-validator-slashes/src/weights.rs @@ -57,6 +57,7 @@ pub trait WeightInfo { fn force_inject_slash() -> Weight; fn root_test_send_msg_to_eth() -> Weight; fn process_slashes_queue(s: u32, ) -> Weight; + fn retry_unsent_slash_era() -> Weight; fn set_slashing_mode() -> Weight; } @@ -136,6 +137,11 @@ impl WeightInfo for SubstrateWeight { .saturating_add(Weight::from_parts(0, 42).saturating_mul(s.into())) } + fn retry_unsent_slash_era() -> Weight { + // Same as the success path for one queued batch. + Self::process_slashes_queue(10) + } + fn set_slashing_mode() -> Weight { Weight::from_parts(7_402_000, 3601) .saturating_add(T::DbWeight::get().reads(1_u64)) @@ -221,6 +227,10 @@ impl WeightInfo for () { .saturating_add(Weight::from_parts(0, 42).saturating_mul(s.into())) } + fn retry_unsent_slash_era() -> Weight { + Self::process_slashes_queue(10) + } + fn set_slashing_mode() -> Weight { Weight::from_parts(7_402_000, 3601) .saturating_add(RocksDbWeight::get().reads(1_u64)) diff --git a/operator/runtime/mainnet/src/configs/mod.rs b/operator/runtime/mainnet/src/configs/mod.rs index e7ad43be..4da7995f 100644 --- a/operator/runtime/mainnet/src/configs/mod.rs +++ b/operator/runtime/mainnet/src/configs/mod.rs @@ -1735,6 +1735,7 @@ impl pallet_external_validator_slashes::Config for Runtime { type QueuedSlashesProcessedPerBlock = ConstU32<10>; type WeightInfo = mainnet_weights::pallet_external_validator_slashes::WeightInfo; type SendMessage = SlashesSendAdapter; + type GovernanceOrigin = EnsureRootWithSuccess; } parameter_types! { diff --git a/operator/runtime/mainnet/src/weights/pallet_external_validator_slashes.rs b/operator/runtime/mainnet/src/weights/pallet_external_validator_slashes.rs index 757af34c..d59f0197 100644 --- a/operator/runtime/mainnet/src/weights/pallet_external_validator_slashes.rs +++ b/operator/runtime/mainnet/src/weights/pallet_external_validator_slashes.rs @@ -113,6 +113,9 @@ impl pallet_external_validator_slashes::WeightInfo for .saturating_add(T::DbWeight::get().writes(4_u64)) .saturating_add(Weight::from_parts(0, 38).saturating_mul(s.into())) } + fn retry_unsent_slash_era() -> Weight { + Self::process_slashes_queue(10) + } /// Storage: `ExternalValidatorsSlashes::SlashingMode` (r:0 w:1) /// Proof: `ExternalValidatorsSlashes::SlashingMode` (`max_values`: Some(1), `max_size`: Some(1), added: 496, mode: `MaxEncodedLen`) fn set_slashing_mode() -> Weight { diff --git a/operator/runtime/stagenet/src/configs/mod.rs b/operator/runtime/stagenet/src/configs/mod.rs index a4b43eab..49f977a5 100644 --- a/operator/runtime/stagenet/src/configs/mod.rs +++ b/operator/runtime/stagenet/src/configs/mod.rs @@ -1731,6 +1731,7 @@ impl pallet_external_validator_slashes::Config for Runtime { type QueuedSlashesProcessedPerBlock = ConstU32<10>; type WeightInfo = stagenet_weights::pallet_external_validator_slashes::WeightInfo; type SendMessage = SlashesSendAdapter; + type GovernanceOrigin = EnsureRootWithSuccess; } parameter_types! { diff --git a/operator/runtime/stagenet/src/weights/pallet_external_validator_slashes.rs b/operator/runtime/stagenet/src/weights/pallet_external_validator_slashes.rs index 39796b7e..b463e931 100644 --- a/operator/runtime/stagenet/src/weights/pallet_external_validator_slashes.rs +++ b/operator/runtime/stagenet/src/weights/pallet_external_validator_slashes.rs @@ -113,6 +113,9 @@ impl pallet_external_validator_slashes::WeightInfo for .saturating_add(T::DbWeight::get().writes(4_u64)) .saturating_add(Weight::from_parts(0, 38).saturating_mul(s.into())) } + fn retry_unsent_slash_era() -> Weight { + Self::process_slashes_queue(10) + } /// Storage: `ExternalValidatorsSlashes::SlashingMode` (r:0 w:1) /// Proof: `ExternalValidatorsSlashes::SlashingMode` (`max_values`: Some(1), `max_size`: Some(1), added: 496, mode: `MaxEncodedLen`) fn set_slashing_mode() -> Weight { diff --git a/operator/runtime/testnet/src/configs/mod.rs b/operator/runtime/testnet/src/configs/mod.rs index fb73dd44..67eeeab3 100644 --- a/operator/runtime/testnet/src/configs/mod.rs +++ b/operator/runtime/testnet/src/configs/mod.rs @@ -1733,6 +1733,7 @@ impl pallet_external_validator_slashes::Config for Runtime { type QueuedSlashesProcessedPerBlock = ConstU32<10>; type WeightInfo = testnet_weights::pallet_external_validator_slashes::WeightInfo; type SendMessage = SlashesSendAdapter; + type GovernanceOrigin = EnsureRootWithSuccess; } parameter_types! { diff --git a/operator/runtime/testnet/src/weights/pallet_external_validator_slashes.rs b/operator/runtime/testnet/src/weights/pallet_external_validator_slashes.rs index 5dd4c60c..c3d9c314 100644 --- a/operator/runtime/testnet/src/weights/pallet_external_validator_slashes.rs +++ b/operator/runtime/testnet/src/weights/pallet_external_validator_slashes.rs @@ -113,6 +113,9 @@ impl pallet_external_validator_slashes::WeightInfo for .saturating_add(T::DbWeight::get().writes(4_u64)) .saturating_add(Weight::from_parts(0, 38).saturating_mul(s.into())) } + fn retry_unsent_slash_era() -> Weight { + Self::process_slashes_queue(10) + } /// Storage: `ExternalValidatorsSlashes::SlashingMode` (r:0 w:1) /// Proof: `ExternalValidatorsSlashes::SlashingMode` (`max_values`: Some(1), `max_size`: Some(1), added: 496, mode: `MaxEncodedLen`) fn set_slashing_mode() -> Weight { diff --git a/test/.papi/descriptors/package.json b/test/.papi/descriptors/package.json index aa8c9c70..8071f051 100644 --- a/test/.papi/descriptors/package.json +++ b/test/.papi/descriptors/package.json @@ -1,5 +1,5 @@ { - "version": "0.1.0-autogenerated.17592112782245438099", + "version": "0.1.0-autogenerated.14952921519994301429", "name": "@polkadot-api/descriptors", "files": [ "dist" diff --git a/test/.papi/metadata/datahaven.scale b/test/.papi/metadata/datahaven.scale index a3dd8d10877cbc65dbd7f7022f9c02ca954c801b..6a0d1d642b0f66148d58d783d9ea362f75975b90 100644 GIT binary patch delta 4936 zcmai2e^3)S0kaSgPUUi#gQz(gjbNk+5)e(y)SSz)z{zn3_d~^* z1WihfNtn?P^I|Hdw8JE3LJ}ItNk@{oD5lIv49z%G(~!ib{Glz)I1_Em)Un^)cPDId zrZac<_PzanKA-RR``*2M-~I5Vg5g&R5?a-e^0V{%61oylSAuCcf$y$G7Wzsp|9$!s z0W}opaD{zA)faLFy-Kjzr3Ms%D*xmcB2(nqP5iMwltpu{^3l%io3#6Cuu2UEl_qtQ z>T6i9c)V(Z8HoV0Ah{x-RQej!-5g4B`@_DF1&QJNR&XYB2C8dTWu0FQ%B_q*Zq(6o z$S;F@gRHOVKuDEjsAeFM_{Wn`c?~J zdDI3G6~?q&o(2c9Y*~=$BjbShS<|xkht*ec0<2=J}Bz{D&T#;qS(Jp6I(X z{Gy&O?#|u++BH7%+BIR+dlLZ1mS0LcZwiUX5qa^Zu#ZoF6tch3AFA}t z5#kbsv-GJSg#s1{3(HnMZw5J${_97fI;&3viNb#IxHupVio@cl7!}9FadA@Q`f0X+ z7tsbD+tG1${T`2((JTRH(yKf!K!e(-6AiPQAptK*91)QSZgJd**mKa75*ZQjQ41Pn z7Hz3`If~NWRJ@uW6U}sOzQ&Y_SJ1XJoI^LH;l+s)BC=HZ8&r;)pzUdR4Vq-wvuQXF zou^mQu+?-y=ce|BI2TRPP0*xHi)iH*)$Od<4U^>ydfY$fxGDO=Lfn`DAqw0@dTk+I zZ@S2kwdr`{UHZ6_D6r^kKZQY}I#n`oGJjp<=#30qK#yhMCFlkn%fPu= zNJ1Mn`&xYdUB0@2;tMKnO+n+iA=2$xkcGTO_-_bNR~F8t)=XRt%c&z1=OaSj$iz#F zi6qDm)df_wN@+6-kDw}E*n*r8p(KH8+K=f@iGvTUoGRYQrP7}=acwe|kiF8^tOj7+ zmMbBcpbSmo3469Q*Zmwx^B-b0SGNdUjz-DBdo-%soZdh(>DKA4lDyUV) z)-9C=aK(IZ1p`+Z;&ecmHDIfU>6neOh8fq;w`6>>bF%?spNp~2!)!GeZ_^TaOJN(( z#uFKn(1rxFB6%cEflHGrajIl454O2lTLZ4PfTz_%zstc^A5kUlLB#S37Q`eF%Q4f|740XQ;1OQEq18nQF2O@%i$*TaB|&xBWx zdYIN&aW;Lc0GFT<`gH+LhwbWPD_#n{+(PWMjvD-o&i6CQ{EX85h4`p#%pg8KSA2Y) z__!us_dmK5^s*I#nWT3uh2}h64NcMozzX7oG%1~zU}y8g*s6=#n0Z1QnKr1(5b-5& zRA1imY{ZwCrt5hcacu7i0z z!6-@3ur_46A#pco&vINy+lydlYqQ}5`rUH;7;JB!+Hjc(C(+M!fg=2(og~fOlym{< zb^;O;kZ83H?}{-M-)R6(o4~u&IC$DL>s=}Xr|QBo-Fz>8EIGq~)pm(&ipp>@$!5mc zG2bYrCT}yK zw8SfV;>`*(Q<#Fc1iNgi!B%t46Bn4{nuh^B4474lD{ZawfbF*dwliQm1AbVF_t?V* zU^o#ZJL1T2;%rPE4BA1XE8)4n|9*H{Jxx#Fj|=Sk3=HEK7`oyN<9L>#i!pRDhKE+- zYWo2LbczRPcN{v!&qBKyv|EEVCm%APv}f00dT|v_HXmkYhnd1Nt6}SC7)wr(oI1HJ>3%CITK{^l&#tG7AgBfH5gN$I$ z1K4RFHURbi^++5!_kTUYpd$<_Iq`b?=zM7OcF-t;Mj7;NCoWGOGeEV4InEa5II|sR zw(H8k!wG|}6KroCrL=RV@gy~wWbQ%LnLQ5*B=SWWUQC2u5uK(=EuXVL?O_bx1o$q1 zuQ`(TFzK|TyvVD&$hW9-4Lk4a1<*rU>BJ!cTaJhVXDLQ0sK!H~!6m3^w(5Y&pVdZ^ssoM$^u@mwK zfFab}2?PO8lgFocW9(210<1}eS}q{(@`RdYMb-t>hN7TnPprTLwp;!LRJT9S5VXmx zZVSpSs@kU4dAF!F(jY4Yjs8B^q{^O#A*cge6*%gHV@kbOl@)iu9}LP~B?L~?d;RW~ zpkAo~PtX>t#q^ZX>>Z?9G>mlYAgz%@#KH~hnpJlTocW(r1AeXc zRRjIv&>DSoQ=97Eq6UI+c0Vp!R_bGQyJI5*qQ$O{JgoZEpeHyeI+#V3KM+#9o>0fI z=qQY{yVX%~Xt~~dT=g_Hhoa)pcKTc=Nnbi94wdUSs-U26fK&ShD96XeuN=&vUi<=C z15MxsauBVeAH7M|M=D<=mrZ<`v^LW8GRZVWY7UZQ6JNpl)h1LK0khXpq~=vp!J{fl zkC0NdnSOSJw4kkY_3PwJq9UcV1^oU-kJKJX{0(^?8AgsBC10BaILw})$}y5md)^>z z3BWJO6?DhTWLd=VTjDgOKP@?1)D9OESgLl820GoQjM;qfJ95;-P}|tZ-hNUR^Dc!K zDkXCTjlM}zBA4GLIcV9GD0)?&WuL^A!y5v4bJD1)t_GM6k2h!*!@ENWyR!5*Ls9A{|4d7oq;h!Vk%1 z1c%szCrAoEAPv%t6J%$spFSXsNu0wS4rsNTOP?HWLK2!;7S5d@8N#F_w6X5Z6Qn>m zF9}Ckw`>H0xIkSaBpprD-6O;W1@m7<$QgcGycp^GBl!%$K-)>O4sH*fgvB#MXHLT0 zUZu8CXbAO>k|J~`?Hwg$D1-iWl-z@I=*>~G44P#hk^50m#QhO*AXo;keN1XmMdTkJ zlRN|yY>AQ;iB(D5^RO|v;6)PkMM)ZUM~M}ty*o;>&^G#Zl;k6oejX)fp~C6?gxFA+ zp8bT#@Z6lydi!YBDN+a(M#U*&heYl?MT(&mcuVW`(F>=@OPTOsacC0tP&>h1(4q?I z+9kaI8K8SllL~l0app8Bfm8i|P6P26%^oAADI=m&^=TW2OM6?U{SV{`+B8Pa@{>v2 Fe*q-jNyPvF delta 4208 zcmZvg4@?`^9mnt4evV&r=a3dQ6Cg1({BZ)rP=|!Xq#?y=Na`d&$e&GwIdBvY+kpSH zAzO-QLtCWK6}8DvRInCp*@h~n(!wsO;wf(HP`1(})1od_>B^*NBWqMeDr%!H)qdy0 z!6z5lzIX3_pYQMY-tXPJ-!b`W{+~|g>%C&owc=WjzE@9r^_oe&w!fC}@M^90D`}!m zJ3K;CpmsqUfof1`bDwN~fZ8}c3#{pR)-bHL*Yw2A?9hrhi^3o#yO0DFa(x!U4o{Y-vO0eo_2K4J`4*2EPU-UHZfk~e54!?`zCZRK(rb#Q7(V-M_ zL4hy-m==<-(iTkAtc~ynxRZzM9uqAkQFz8gw;85+lHzP{7U35r`Y`+(?Z}M6hPYfx0+kLX@l)84WsN?1lBOG@J?P|q9$P+Z6-Bq1~ETFAnUz?7*D#wX4iv(M}8uc{!8vbs}4foN*2u}6c)1RIy{(H zQcGIOY%RKSrO9DOBp=TOT*hYGGthAS1gS~Wx)j?js=v2@s$2Iy3b zo$!4Qz1Ur&VyuabHHoQL86T17@Vx){MHh0H&d?x;0@r{|jZ5HMf}twV(dhO1 z8$0}NuN%tOQVS8GX)P@yZg_1itvu?MZLOSF!ZNIHK|!P4pX0@*9=?X2A#RakEl8pN8#)IG7n|#REK9NR2RDNLLxJqlS*3 z7G{KN0&Zeayt-vp?lVlwJ##AWW*qcQ3^=yJ%}ay6sYHBJfX_>*8Fm)Y`VI3cG6l)o zaWZ9zyseP87s*>ibZYY*6}cdf_+A`cK>PSq-&0t*OLJG??!tUAEzsPHNo9j=1FgqR zEvtl9z{(A{Da~)7r>)eOxGBZ7nQ?JIiQ3E<%Oiu?Qbeb$_q9?5mBLL8Mz<2o+i0p` zrNXR~nQVBogzhg+QyI$}M0(s?!UmD9fawajyOcgcvf!CgYI-~yuW<5@CfCT}2h(CZ zvz)W2WQ&ZbTN2kTl99!&wFKLE4;D!n(Ln;A$z!Rv(Ym7>jXLv=1Ab9-x;hNHJ;bIv zNG>eLM_0K_%Rib5BZnW^Azr9dIUGWNj<~;}C3g$0XP-L z39oO$U7)Ot`m8l7@Z2&a>*M6PWlQer6|`PKzbT_9ijSyJdA=LsWWszmC}4vOK3nWk zF>;(@oJojNR2UKVZKj@LcM{|M0miE^UL`=joSv|Ds=&~4^cISfq2)_s3n^#_I=A5M zI$Ms9s4iHAZR<&u!ZbyNp18s^T~g>#6nbQZr>wmyYF3L>U!0oNE>V37)u;H~wv|>E z_p4C(fi(~(6CPLt3OJyEf8UCW|4b4%_ygde0uIK&vyW0ebld5k;$ammXJsS~CuC(r zfkzatYbxl0vUAv_2`%GBRRSgy7>f&-mc~1#2#hHLS1YJJbzB9;ZtzLv2A@=nvGA0W z(zK24Oudk799E3Oig7qTuBzgQ%2r;S(Rf4&i!-WlQN_vNqttGlR)KZ|?^k_tNvW=* zW{h(6njx9D;}cCc$deDiRjbW@!Bb7~tH}dA~ z#pXJtz^@63A(3zcypGd-ZLaiq%%S@x_g?~nt*)T?X*XUxCN8}ZnB6{euvIj-gn}Wz zC|y0tc#6;1*=Jc^_b^|u>#lG`bYSQ?Xljv4zcN==nr5ee2!1q z9P2x9e6eSgPqD^&9v6LLz#SOlZHk2x19y4c!LD)MRuETj5-;!*g|V|kqWj6#V3?oy z89e(OGp&p86IHPbPJA0`#>#aUzSBhc&uofet=QyoxxF*|#5MTfIkuNvhlftF&EzIL zaf-b|=Fj%BDso$L^s+Icy}{psh3DB7EfMBm>IL@m2d?uevL+E$!qykr?bI~E5IlZ^ zFF59DZ)$67MOU_T$==64(~@jR?`LO85xmvUy0LV%zQnfUq4Op7GT8{>H`z{U*RR+u zjka8Fhmq#nwD%`O`o zgj9T6Yi|(+uaNpRjJ&~CN%^ni_2mgyq~SN%=Ni%pzx)k*UXS{M*#YjeY`yfzZ<$?V z>Jsd2Vpn7P(>_ryB~dPBQ=!xT?;O=2Y&F!q#g1T&IQAAhL3*Tuah5|! zKkRv%eY*UNP#_ZXj*#2a3^kL?B<*^KT_I#pGEK5ZEg6=MpJ&5_j7UFvm)#;*&XJ2O zRXZw-f$1W9YVka6REP+ittsS}-(nhLIpU__{6)3_i>3cuWEJEZ=WUv^ymp$%45N5wKWJsspV-FHs0q{PnC6&^x_t{#4xm^)u zrFhs8W!0GC(@~a5j=*S?51kk`C<8d|2=N7@-9_ka6?MXLY+