diff --git a/operator/.cargo/audit.toml b/operator/.cargo/audit.toml new file mode 100644 index 00000000..6b107e46 --- /dev/null +++ b/operator/.cargo/audit.toml @@ -0,0 +1,12 @@ +[advisories] +# These advisories are currently pinned by upstream dependencies in the +# Frontier and polkadot-sdk stacks used by this branch. +ignore = [ + "RUSTSEC-2025-0009", # ring >= 0.17.12 required by upstream dependency graph + "RUSTSEC-2024-0363", # sqlx >= 0.8.1 required by Frontier's fc-db/fc-rpc stack + "RUSTSEC-2023-0091", # wasmtime fixes require moving off the 8.x line + "RUSTSEC-2026-0020", + "RUSTSEC-2026-0021", + "RUSTSEC-2024-0438", + "RUSTSEC-2025-0118", +] diff --git a/operator/Cargo.lock b/operator/Cargo.lock index 5ccff712..847abb1b 100644 --- a/operator/Cargo.lock +++ b/operator/Cargo.lock @@ -1268,7 +1268,7 @@ dependencies = [ "bitflags 2.9.4", "cexpr", "clang-sys", - "itertools 0.13.0", + "itertools 0.11.0", "proc-macro2", "quote", "regex", @@ -1701,9 +1701,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.10.1" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" dependencies = [ "serde", ] @@ -3948,7 +3948,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6147,7 +6147,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" dependencies = [ "equivalent", - "hashbrown 0.16.0", + "hashbrown 0.15.5", "serde", "serde_core", ] @@ -6489,9 +6489,9 @@ dependencies = [ [[package]] name = "keccak" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" +checksum = "cb26cec98cce3a3d96cbb7bced3c4b16e3d13f27ec56dbd62cbc8f39cfb9d653" dependencies = [ "cpufeatures", ] @@ -8074,9 +8074,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.1.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" [[package]] name = "num-derive" @@ -8166,7 +8166,7 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77e878c846a8abae00dd069496dbe8751b16ac1c3d6bd2a7283a938e8228f90d" dependencies = [ - "proc-macro-crate 3.4.0", + "proc-macro-crate 1.1.3", "proc-macro2", "quote", "syn 2.0.106", @@ -11167,7 +11167,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck 0.5.0", - "itertools 0.13.0", + "itertools 0.11.0", "log", "multimap", "once_cell", @@ -11200,7 +11200,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.11.0", "proc-macro2", "quote", "syn 2.0.106", @@ -11299,9 +11299,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.13" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ "bytes", "getrandom 0.3.3", @@ -11790,9 +11790,9 @@ dependencies = [ [[package]] name = "ruint" -version = "1.17.0" +version = "1.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a68df0380e5c9d20ce49534f292a36a7514ae21350726efe1865bdb1fa91d278" +checksum = "c141e807189ad38a07276942c6623032d3753c8859c146104ac2e4d68865945a" dependencies = [ "alloy-rlp", "ark-ff 0.3.0", @@ -11897,7 +11897,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -11910,7 +11910,7 @@ dependencies = [ "once_cell", "ring 0.17.14", "rustls-pki-types", - "rustls-webpki 0.103.7", + "rustls-webpki 0.103.10", "subtle 2.6.1", "zeroize", ] @@ -11960,7 +11960,7 @@ dependencies = [ "rustls", "rustls-native-certs", "rustls-platform-verifier-android", - "rustls-webpki 0.103.7", + "rustls-webpki 0.103.10", "security-framework 3.5.1", "security-framework-sys", "webpki-root-certs 0.26.11", @@ -11985,9 +11985,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.7" +version = "0.103.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10b3f4191e8a80e6b43eebabfac91e5dcecebb27a71f04e820c47ec41d314bf" +checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" dependencies = [ "ring 0.17.14", "rustls-pki-types", @@ -16743,7 +16743,7 @@ dependencies = [ "getrandom 0.3.3", "once_cell", "rustix 1.1.2", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -16858,30 +16858,30 @@ dependencies = [ [[package]] name = "time" -version = "0.3.44" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", "itoa", "num-conv", "powerfmt", - "serde", + "serde_core", "time-core", "time-macros", ] [[package]] name = "time-core" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" [[package]] name = "time-macros" -version = "0.2.24" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" dependencies = [ "num-conv", "time-core", @@ -18302,7 +18302,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] diff --git a/operator/pallets/external-validator-slashes/src/benchmarking.rs b/operator/pallets/external-validator-slashes/src/benchmarking.rs index 7e8d7a00..5575c914 100644 --- a/operator/pallets/external-validator-slashes/src/benchmarking.rs +++ b/operator/pallets/external-validator-slashes/src/benchmarking.rs @@ -35,20 +35,24 @@ const MAX_SLASHES: u32 = 1000; mod benchmarks { use super::*; + fn dummy_slash(slash_id: T::SlashId) -> Slash { + let dummy = || T::AccountId::decode(&mut TrailingZeroInput::zeroes()).unwrap(); + Slash { + validator: dummy(), + reporters: vec![], + slash_id, + percentage: Perbill::from_percent(1), + confirmed: false, + offence_kind: OffenceKind::LivenessOffence, + } + } + #[benchmark] fn cancel_deferred_slash(s: Linear<1, MAX_SLASHES>) -> Result<(), BenchmarkError> { let mut existing_slashes = Vec::new(); let era = T::EraIndexProvider::active_era().index; - let dummy = || T::AccountId::decode(&mut TrailingZeroInput::zeroes()).unwrap(); for _ in 0..MAX_SLASHES { - existing_slashes.push(Slash { - validator: dummy(), - reporters: vec![], - slash_id: One::one(), - percentage: Perbill::from_percent(1), - confirmed: false, - offence_kind: OffenceKind::LivenessOffence, - }); + existing_slashes.push(dummy_slash::(One::one())); } Slashes::::insert( era.saturating_add(T::SlashDeferDuration::get()) @@ -102,35 +106,55 @@ mod benchmarks { #[benchmark] fn process_slashes_queue(s: Linear<1, 200>) -> Result<(), BenchmarkError> { - let mut queue = VecDeque::new(); - let dummy = || T::AccountId::decode(&mut TrailingZeroInput::zeroes()).unwrap(); + let first_batch = (0..s) + .map(|_| dummy_slash::(One::one())) + .collect::>(); + let second_batch = vec![dummy_slash::(One::one())]; - for _ in 0..(s + 1) { - queue.push_back(Slash { - validator: dummy(), - reporters: vec![], - slash_id: One::one(), - percentage: Perbill::from_percent(1), - confirmed: false, - offence_kind: OffenceKind::LivenessOffence, - }); - } - - UnreportedSlashesQueue::::set(queue); + assert!(ExternalValidatorSlashes::::unsent_queue_push(( + 1, + first_batch + ))); + assert!(ExternalValidatorSlashes::::unsent_queue_push(( + 2, + second_batch + ))); let processed; #[block] { - processed = Pallet::::process_slashes_queue(s).unwrap(); + processed = match Pallet::::process_slashes_queue() { + crate::ProcessSlashesQueueOutcome::Sent(count) => count, + crate::ProcessSlashesQueueOutcome::Empty + | crate::ProcessSlashesQueueOutcome::Requeued(_) => { + return Err(BenchmarkError::Stop("unexpected slashes queue outcome")) + } + }; } - assert_eq!(UnreportedSlashesQueue::::get().len(), 1); + assert_eq!(ExternalValidatorSlashes::::unsent_queue_len(), 1); assert_eq!(processed, s); Ok(()) } + #[benchmark] + fn retry_unsent_slash_era() -> Result<(), BenchmarkError> { + let batch = vec![dummy_slash::(One::one())]; + assert!(ExternalValidatorSlashes::::unsent_queue_push((1, batch))); + + let origin = + T::GovernanceOrigin::try_successful_origin().map_err(|_| BenchmarkError::Weightless)?; + + #[extrinsic_call] + _(origin as T::RuntimeOrigin, 1u32); + + assert!(ExternalValidatorSlashes::::unsent_queue_is_empty()); + + Ok(()) + } + #[benchmark] fn set_slashing_mode() -> Result<(), BenchmarkError> { #[extrinsic_call] diff --git a/operator/pallets/external-validator-slashes/src/lib.rs b/operator/pallets/external-validator-slashes/src/lib.rs index b24b85db..19da25c3 100644 --- a/operator/pallets/external-validator-slashes/src/lib.rs +++ b/operator/pallets/external-validator-slashes/src/lib.rs @@ -31,7 +31,7 @@ extern crate alloc; use pallet_external_validators::apply; use snowbridge_outbound_queue_primitives::SendError; use { - alloc::{collections::vec_deque::VecDeque, string::String, vec, vec::Vec}, + alloc::{string::String, vec, vec::Vec}, frame_support::{pallet_prelude::*, traits::DefensiveSaturating}, frame_system::pallet_prelude::*, log::log, @@ -132,10 +132,21 @@ pub mod pallet { }, /// The slashes message was sent correctly. SlashesMessageSent { message_id: H256 }, + /// The slashes message failed to send and the batch was moved to the back + /// of the queue for retry. + SlashesMessageSendFailed { era: EraIndex, count: u32 }, + /// A queued slashes batch was retried manually and sent successfully. + SlashesMessageRetried { + message_id: H256, + era: EraIndex, + count: u32, + }, /// We injected a slash SlashInjected { slash_id: T::SlashId, era: u32 }, /// Number of slashes processed SlashAddedToQueue { number: u32, era: u32 }, + /// The unsent queue is full; this slash era could not be enqueued. + UnsentQueueFull { era: EraIndex }, } #[pallet::config] @@ -199,6 +210,9 @@ pub mod pallet { /// The weight information of this pallet. type WeightInfo: WeightInfo; + + /// Origin for governance calls such as retrying an unsent slash batch. + type GovernanceOrigin: EnsureOrigin; } #[pallet::error] @@ -226,6 +240,10 @@ pub mod pallet { /// No PendingOffenceKind found for (session, validator) — offence was not /// reported through EquivocationReportWrapper, so the offence kind is unknown. MissingOffenceKind, + /// The specified era is not in the unsent slash queue. + EraNotInUnsentQueue, + /// The message delivery still failed on retry. + MessageSendFailed, } #[apply(derive_storage_traits)] @@ -269,12 +287,26 @@ pub mod pallet { pub type Slashes = StorageMap<_, Twox64Concat, EraIndex, Vec>, ValueQuery>; - /// All unreported slashes that will be processed in the future. + /// Maximum number of unsent slash batches in the retry ring buffer. + pub const UNSENT_QUEUE_CAPACITY: u32 = 64; + + /// Ring buffer of slash batches whose outbound message still needs to be sent. + /// Each slot stores the original slash era together with a bounded-size batch + /// of slash records. Retries keep the original era so the outbound message id + /// remains stable across later blocks and eras. #[pallet::storage] #[pallet::unbounded] - #[pallet::getter(fn unreported_slashes)] - pub type UnreportedSlashesQueue = - StorageValue<_, VecDeque>, ValueQuery>; + pub type UnsentSlashBatch = + StorageMap<_, Twox64Concat, u32, (EraIndex, Vec>)>; + + /// Ring buffer head: next slot to be processed by `on_initialize`. + #[pallet::storage] + pub type UnsentSlashHead = StorageValue<_, u32, ValueQuery>; + + /// Ring buffer tail: next slot to write a new entry into. + /// When head == tail the buffer is empty. + #[pallet::storage] + pub type UnsentSlashTail = StorageValue<_, u32, ValueQuery>; // Turns slashing on or off #[pallet::storage] @@ -415,6 +447,44 @@ pub mod pallet { Ok(()) } + #[pallet::call_index(2)] + #[pallet::weight(T::WeightInfo::retry_unsent_slash_era())] + pub fn retry_unsent_slash_era(origin: OriginFor, era_index: EraIndex) -> DispatchResult { + T::GovernanceOrigin::ensure_origin(origin)?; + + let head = UnsentSlashHead::::get(); + let tail = UnsentSlashTail::::get(); + let mut found = None; + let mut slot = head; + while slot != tail { + if let Some(entry @ (idx, _)) = UnsentSlashBatch::::get(slot) { + if idx == era_index { + found = Some((slot, entry)); + break; + } + } + slot = (slot + 1) % UNSENT_QUEUE_CAPACITY; + } + + let (slot, (era, slashes)) = found.ok_or(Error::::EraNotInUnsentQueue)?; + let count = slashes.len() as u32; + let slashes_to_send = slashes + .iter() + .map(Self::slash_to_send_data) + .collect::>(); + let message_id = Self::send_slashes_message(&slashes_to_send, era) + .ok_or(Error::::MessageSendFailed)?; + + Self::unsent_queue_remove_slot(slot); + Self::deposit_event(Event::::SlashesMessageRetried { + message_id, + era, + count, + }); + + Ok(()) + } + #[pallet::call_index(3)] #[pallet::weight(T::WeightInfo::set_slashing_mode())] pub fn set_slashing_mode(origin: OriginFor, mode: SlashingModeOption) -> DispatchResult { @@ -429,12 +499,12 @@ pub mod pallet { #[pallet::hooks] impl Hooks> for Pallet { fn on_initialize(_n: BlockNumberFor) -> Weight { - let processed = Self::process_slashes_queue(T::QueuedSlashesProcessedPerBlock::get()); - - if let Some(p) = processed { - T::WeightInfo::process_slashes_queue(p) - } else { - T::WeightInfo::process_slashes_queue(0) + match Self::process_slashes_queue() { + ProcessSlashesQueueOutcome::Empty => T::WeightInfo::process_slashes_queue(0), + ProcessSlashesQueueOutcome::Sent(count) + | ProcessSlashesQueueOutcome::Requeued(count) => { + T::WeightInfo::process_slashes_queue(count) + } } } } @@ -655,70 +725,65 @@ where impl Pallet { fn add_era_slashes_to_queue(active_era: EraIndex) { - let mut slashes: VecDeque<_> = Slashes::::get(active_era).into(); + let slashes = Slashes::::get(active_era); + if slashes.is_empty() { + return; + } - let len = slashes.len(); + let batch_size = T::QueuedSlashesProcessedPerBlock::get().max(1) as usize; + let mut enqueued = 0u32; - UnreportedSlashesQueue::::mutate(|queue| queue.append(&mut slashes)); + for batch in slashes.chunks(batch_size) { + if Self::unsent_queue_push((active_era, batch.to_vec())) { + enqueued = enqueued.saturating_add(batch.len() as u32); + } else { + log::warn!( + target: "ext_validators_slashes", + "Unsent slash queue full, cannot enqueue era {active_era}", + ); + Self::deposit_event(Event::::UnsentQueueFull { era: active_era }); + break; + } + } - if len > 0 { + if enqueued > 0 { Self::deposit_event(Event::::SlashAddedToQueue { - number: len as u32, + number: enqueued, era: active_era, }); } } - /// Returns number of slashes that were sent to ethereum. - fn process_slashes_queue(amount: u32) -> Option { - let mut slashes_to_send: Vec> = vec![]; - let era_index = T::EraIndexProvider::active_era().index; + fn slash_to_send_data(slash: &Slash) -> SlashData { + // Keep the original slash batch intact until delivery succeeds so failed + // batches can be moved to the back of the queue instead of being dropped. + let max_wad = T::MaxSlashWad::get(); + let wad_to_slash = (slash.percentage.deconstruct() as u128) + .saturating_mul(max_wad) + .checked_div(1_000_000_000u128) + .unwrap_or(0) + .min(max_wad); - UnreportedSlashesQueue::::mutate(|queue| { - for _ in 0..amount { - let Some(slash) = queue.pop_front() else { - // no more slashes to process in the queue - break; - }; - - // Convert Perbill to EigenLayer WAD format with linear mapping. - // Perbill(100%) → MaxSlashWad (e.g. 5% WAD = 5e16). - // Formula: perbill_inner * MaxSlashWad / 1e9 - // Clamp to MaxSlashWad to guard against overflow if governance - // sets MaxSlashWad high enough for saturating_mul to hit u128::MAX. - let max_wad = T::MaxSlashWad::get(); - let wad_to_slash = (slash.percentage.deconstruct() as u128) - .saturating_mul(max_wad) - .checked_div(1_000_000_000u128) - .unwrap_or(0) - .min(max_wad); - - slashes_to_send.push(SlashData { - validator: slash.validator, - wad_to_slash, - description: slash.offence_kind.to_description(), - }); - } - }); - - if slashes_to_send.is_empty() { - return None; + SlashData { + validator: slash.validator.clone(), + wad_to_slash, + description: slash.offence_kind.to_description(), } + } - let slashes_count = slashes_to_send.len() as u32; + fn send_slashes_message( + slashes_to_send: &[SlashData], + era_index: EraIndex, + ) -> Option { + let outbound = + T::SendMessage::build(&slashes_to_send.to_vec(), era_index).or_else(|| { + log::warn!(target: "ext_validators_slashes", "Failed to build outbound message"); + None + })?; - let outbound = match T::SendMessage::build(&slashes_to_send, era_index) { - Some(send_msg) => send_msg, - None => { - log::error!(target: "ext_validators_slashes", "Failed to build outbound message"); - return None; - } - }; - - // Validate and deliver the message let ticket = T::SendMessage::validate(outbound) .map_err(|e| { - log::error!( + log::warn!( target: "ext_validators_slashes", "Failed to validate outbound message: {:?}", e @@ -726,20 +791,126 @@ impl Pallet { }) .ok()?; - let message_id = T::SendMessage::deliver(ticket) + T::SendMessage::deliver(ticket) .map_err(|e| { - log::error!( + log::warn!( target: "ext_validators_slashes", "Failed to deliver outbound message: {:?}", e ); }) - .ok()?; - - Self::deposit_event(Event::::SlashesMessageSent { message_id }); - - Some(slashes_count) + .ok() } + + #[allow(dead_code)] + pub(crate) fn unsent_queue_is_empty() -> bool { + UnsentSlashHead::::get() == UnsentSlashTail::::get() + } + + #[allow(dead_code)] + pub(crate) fn unsent_queue_len() -> u32 { + let head = UnsentSlashHead::::get(); + let tail = UnsentSlashTail::::get(); + tail.wrapping_sub(head) % UNSENT_QUEUE_CAPACITY + } + + pub(crate) fn unsent_queue_push( + entry: (EraIndex, Vec>), + ) -> bool { + let head = UnsentSlashHead::::get(); + let tail = UnsentSlashTail::::get(); + let next_tail = (tail + 1) % UNSENT_QUEUE_CAPACITY; + if next_tail == head { + return false; + } + + UnsentSlashBatch::::insert(tail, entry); + UnsentSlashTail::::put(next_tail); + true + } + + fn unsent_queue_remove_slot(slot: u32) { + let tail = UnsentSlashTail::::get(); + let mut cur = slot; + loop { + let next = (cur + 1) % UNSENT_QUEUE_CAPACITY; + if next == tail { + break; + } + + if let Some(entry) = UnsentSlashBatch::::get(next) { + UnsentSlashBatch::::insert(cur, entry); + } + cur = next; + } + + UnsentSlashBatch::::remove(cur); + let new_tail = if tail == 0 { + UNSENT_QUEUE_CAPACITY - 1 + } else { + tail - 1 + }; + UnsentSlashTail::::put(new_tail); + + let head = UnsentSlashHead::::get(); + if head == tail { + UnsentSlashHead::::put(new_tail); + } + } + + /// Retry contract shared with rewards: + /// - process the current head batch, + /// - if send succeeds, remove it from the queue, + /// - if send fails, move the same batch to the back so later slash batches can progress. + pub(crate) fn process_slashes_queue() -> ProcessSlashesQueueOutcome { + let head = UnsentSlashHead::::get(); + let tail = UnsentSlashTail::::get(); + + if head == tail { + return ProcessSlashesQueueOutcome::Empty; + } + + let Some((era_index, slashes)) = UnsentSlashBatch::::get(head) else { + UnsentSlashHead::::put((head + 1) % UNSENT_QUEUE_CAPACITY); + return ProcessSlashesQueueOutcome::Empty; + }; + + let slashes_count = slashes.len() as u32; + let slashes_to_send = slashes + .iter() + .map(Self::slash_to_send_data) + .collect::>(); + + match Self::send_slashes_message(&slashes_to_send, era_index) { + Some(message_id) => { + UnsentSlashBatch::::remove(head); + UnsentSlashHead::::put((head + 1) % UNSENT_QUEUE_CAPACITY); + Self::deposit_event(Event::::SlashesMessageSent { message_id }); + ProcessSlashesQueueOutcome::Sent(slashes_count) + } + None => { + UnsentSlashBatch::::remove(head); + UnsentSlashHead::::put((head + 1) % UNSENT_QUEUE_CAPACITY); + UnsentSlashBatch::::insert(tail, (era_index, slashes)); + UnsentSlashTail::::put((tail + 1) % UNSENT_QUEUE_CAPACITY); + log::warn!( + target: "ext_validators_slashes", + "Failed to send {slashes_count} slash entries for era {era_index}, moved batch to back of queue", + ); + Self::deposit_event(Event::::SlashesMessageSendFailed { + era: era_index, + count: slashes_count, + }); + ProcessSlashesQueueOutcome::Requeued(slashes_count) + } + } + } +} + +pub(crate) enum ProcessSlashesQueueOutcome { + Empty, + Sent(u32), + Requeued(u32), } /// A pending slash record. The value of the slash has been computed but not applied yet, diff --git a/operator/pallets/external-validator-slashes/src/mock.rs b/operator/pallets/external-validator-slashes/src/mock.rs index 34ba198e..8ec92539 100644 --- a/operator/pallets/external-validator-slashes/src/mock.rs +++ b/operator/pallets/external-validator-slashes/src/mock.rs @@ -134,7 +134,9 @@ thread_local! { pub static SENT_ETHEREUM_MESSAGE_NONCE: RefCell = const { RefCell::new(0) }; pub static MOCK_REPORT_OFFENCE_SHOULD_FAIL: RefCell = const { RefCell::new(false) }; pub static MOCK_REPORT_OFFENCE_CALLED: RefCell = const { RefCell::new(false) }; + pub static MOCK_SEND_MESSAGE_SHOULD_FAIL: RefCell = const { RefCell::new(false) }; pub static LAST_SENT_SLASHES: RefCell>> = RefCell::new(Vec::new()); + pub static LAST_BUILT_ERA: RefCell> = const { RefCell::new(None) }; } impl MockEraIndexProvider { @@ -222,19 +224,32 @@ impl MockOkOutboundQueue { pub fn last_sent_slashes() -> Vec> { LAST_SENT_SLASHES.with(|r| r.borrow().clone()) } + + pub fn last_built_era() -> Option { + LAST_BUILT_ERA.with(|r| *r.borrow()) + } + + pub fn set_should_fail(fail: bool) { + MOCK_SEND_MESSAGE_SHOULD_FAIL.with(|r| *r.borrow_mut() = fail); + } } impl crate::SendMessage for MockOkOutboundQueue { type Ticket = (); type Message = (); - fn build(slashes: &Vec>, _: u32) -> Option { + fn build(slashes: &Vec>, era: u32) -> Option { LAST_SENT_SLASHES.with(|r| *r.borrow_mut() = slashes.clone()); + LAST_BUILT_ERA.with(|r| *r.borrow_mut() = Some(era)); Some(()) } fn validate(_: Self::Ticket) -> Result { Ok(()) } fn deliver(_: Self::Ticket) -> Result { - Ok(H256::zero()) + if MOCK_SEND_MESSAGE_SHOULD_FAIL.with(|r| *r.borrow()) { + Err(SendError::MessageTooLarge) + } else { + Ok(H256::zero()) + } } } @@ -271,6 +286,7 @@ impl external_validator_slashes::Config for Test { type QueuedSlashesProcessedPerBlock = ConstU32<20>; type WeightInfo = (); type SendMessage = MockOkOutboundQueue; + type GovernanceOrigin = frame_system::EnsureRoot; } pub struct FullIdentificationOf; @@ -286,6 +302,9 @@ impl pallet_session::historical::Config for Test { } // Build genesis storage according to the mock runtime. pub fn new_test_ext() -> sp_io::TestExternalities { + MOCK_SEND_MESSAGE_SHOULD_FAIL.with(|r| *r.borrow_mut() = false); + LAST_SENT_SLASHES.with(|r| r.borrow_mut().clear()); + LAST_BUILT_ERA.with(|r| *r.borrow_mut() = None); system::GenesisConfig::::default() .build_storage() .unwrap() diff --git a/operator/pallets/external-validator-slashes/src/tests.rs b/operator/pallets/external-validator-slashes/src/tests.rs index 126485b8..350af20f 100644 --- a/operator/pallets/external-validator-slashes/src/tests.rs +++ b/operator/pallets/external-validator-slashes/src/tests.rs @@ -28,6 +28,40 @@ use { sp_staking::offence::ReportOffence, }; +fn queued_slash_ids() -> Vec { + let mut queued = Vec::new(); + let mut slot = UnsentSlashHead::::get(); + let tail = UnsentSlashTail::::get(); + + while slot != tail { + if let Some((_, batch)) = UnsentSlashBatch::::get(slot) { + queued.extend(batch.into_iter().map(|slash| slash.slash_id)); + } + slot = (slot + 1) % UNSENT_QUEUE_CAPACITY; + } + + queued +} + +fn queued_batch_eras() -> Vec { + let mut queued = Vec::new(); + let mut slot = UnsentSlashHead::::get(); + let tail = UnsentSlashTail::::get(); + + while slot != tail { + if let Some((era, _)) = UnsentSlashBatch::::get(slot) { + queued.push(era); + } + slot = (slot + 1) % UNSENT_QUEUE_CAPACITY; + } + + queued +} + +fn unsent_queue_len() -> u32 { + ExternalValidatorSlashes::unsent_queue_len() +} + #[test] fn root_can_inject_manual_offence() { new_test_ext().execute_with(|| { @@ -574,14 +608,228 @@ fn test_on_offence_defer_period_0_messages_get_queued() { assert_eq!(Slashes::::get(get_slashing_era(1)).len(), 25); start_era(2, 2, 2); - assert_eq!(UnreportedSlashesQueue::::get().len(), 25); + assert_eq!(unsent_queue_len(), 2); + assert_eq!(queued_batch_eras(), vec![2, 2]); // this triggers on_initialize run_block(); - assert_eq!(UnreportedSlashesQueue::::get().len(), 5); + assert_eq!(unsent_queue_len(), 1); + assert_eq!(queued_slash_ids(), (20..25).collect::>()); run_block(); - assert_eq!(UnreportedSlashesQueue::::get().len(), 0); + assert!(ExternalValidatorSlashes::unsent_queue_is_empty()); + }); +} + +#[test] +fn failed_slashes_batch_is_moved_to_back_of_queue() { + new_test_ext().execute_with(|| { + crate::mock::DeferPeriodGetter::with_defer_period(0); + MockOkOutboundQueue::set_should_fail(true); + + start_era(0, 0, 0); + start_era(1, 1, 1); + + for i in 0..25 { + PendingOffenceKind::::insert(0, 3 + i, OffenceKind::LivenessOffence); + Pallet::::on_offence( + &[OffenceDetails { + offender: (3 + i, ()), + reporters: vec![], + }], + &[Perbill::from_percent(75)], + 0, + ); + } + + start_era(2, 2, 2); + assert_eq!(queued_slash_ids(), (0..25).collect::>()); + assert_eq!(queued_batch_eras(), vec![2, 2]); + + run_block(); + + assert_eq!(unsent_queue_len(), 2); + assert_eq!( + queued_slash_ids(), + (20..25).chain(0..20).collect::>() + ); + System::assert_has_event(RuntimeEvent::ExternalValidatorSlashes( + crate::Event::SlashesMessageSendFailed { era: 2, count: 20 }, + )); + }); +} + +#[test] +fn failed_slashes_batch_retries_after_send_is_reenabled() { + new_test_ext().execute_with(|| { + crate::mock::DeferPeriodGetter::with_defer_period(0); + MockOkOutboundQueue::set_should_fail(true); + + start_era(0, 0, 0); + start_era(1, 1, 1); + + for i in 0..25 { + PendingOffenceKind::::insert(0, 3 + i, OffenceKind::LivenessOffence); + Pallet::::on_offence( + &[OffenceDetails { + offender: (3 + i, ()), + reporters: vec![], + }], + &[Perbill::from_percent(75)], + 0, + ); + } + + start_era(2, 2, 2); + run_block(); + assert_eq!( + queued_slash_ids(), + (20..25).chain(0..20).collect::>() + ); + + start_era(3, 3, 3); + MockOkOutboundQueue::set_should_fail(false); + + run_block(); + assert_eq!(unsent_queue_len(), 1); + assert_eq!(queued_slash_ids(), (0..20).collect::>()); + assert_eq!(MockOkOutboundQueue::last_sent_slashes().len(), 5); + assert_eq!(MockOkOutboundQueue::last_built_era(), Some(2)); + System::assert_has_event(RuntimeEvent::ExternalValidatorSlashes( + crate::Event::SlashesMessageSent { + message_id: Default::default(), + }, + )); + + run_block(); + assert!(ExternalValidatorSlashes::unsent_queue_is_empty()); + }); +} + +#[test] +fn retry_extrinsic_succeeds_for_matching_era() { + new_test_ext().execute_with(|| { + crate::mock::DeferPeriodGetter::with_defer_period(0); + + start_era(0, 0, 0); + start_era(1, 1, 1); + + for i in 0..25 { + PendingOffenceKind::::insert(0, 3 + i, OffenceKind::LivenessOffence); + Pallet::::on_offence( + &[OffenceDetails { + offender: (3 + i, ()), + reporters: vec![], + }], + &[Perbill::from_percent(75)], + 0, + ); + } + + start_era(2, 2, 2); + start_era(5, 5, 5); + + assert_ok!(ExternalValidatorSlashes::retry_unsent_slash_era( + RuntimeOrigin::root(), + 2, + )); + + assert_eq!(unsent_queue_len(), 1); + assert_eq!(queued_slash_ids(), (20..25).collect::>()); + assert_eq!(MockOkOutboundQueue::last_built_era(), Some(2)); + }); +} + +#[test] +fn retry_extrinsic_errors_when_era_not_queued() { + new_test_ext().execute_with(|| { + assert_noop!( + ExternalValidatorSlashes::retry_unsent_slash_era(RuntimeOrigin::root(), 2), + Error::::EraNotInUnsentQueue + ); + }); +} + +#[test] +fn retry_extrinsic_requires_root() { + new_test_ext().execute_with(|| { + assert_noop!( + ExternalValidatorSlashes::retry_unsent_slash_era(RuntimeOrigin::signed(1), 2), + sp_runtime::DispatchError::BadOrigin + ); + }); +} + +#[test] +fn retry_extrinsic_preserves_failed_batch_when_send_still_fails() { + new_test_ext().execute_with(|| { + crate::mock::DeferPeriodGetter::with_defer_period(0); + MockOkOutboundQueue::set_should_fail(true); + + start_era(0, 0, 0); + start_era(1, 1, 1); + + for i in 0..25 { + PendingOffenceKind::::insert(0, 3 + i, OffenceKind::LivenessOffence); + Pallet::::on_offence( + &[OffenceDetails { + offender: (3 + i, ()), + reporters: vec![], + }], + &[Perbill::from_percent(75)], + 0, + ); + } + + start_era(2, 2, 2); + let before = queued_slash_ids(); + + assert_noop!( + ExternalValidatorSlashes::retry_unsent_slash_era(RuntimeOrigin::root(), 2), + Error::::MessageSendFailed + ); + + assert_eq!(queued_slash_ids(), before); + assert_eq!(unsent_queue_len(), 2); + }); +} + +#[test] +fn unsent_queue_full_emits_event() { + new_test_ext().execute_with(|| { + crate::mock::DeferPeriodGetter::with_defer_period(0); + + for i in 0..63u32 { + let slash = Slash { + validator: 1000 + i as u64, + reporters: vec![], + slash_id: i, + percentage: Perbill::from_percent(1), + confirmed: true, + offence_kind: OffenceKind::LivenessOffence, + }; + assert!(ExternalValidatorSlashes::unsent_queue_push(( + 1, + vec![slash] + ))); + } + + Slashes::::insert( + 2, + vec![Slash { + validator: 5000u64, + reporters: vec![], + slash_id: 999, + percentage: Perbill::from_percent(10), + confirmed: true, + offence_kind: OffenceKind::LivenessOffence, + }], + ); + + start_era(2, 2, 2); + + assert_eq!(unsent_queue_len(), 63); + assert_eq!(Slashes::::get(2).len(), 1); }); } @@ -628,14 +876,13 @@ fn test_on_offence_defer_period_0_messages_get_queued_across_eras() { } assert_eq!(Slashes::::get(get_slashing_era(1)).len(), 25); start_era(2, 2, 2); - assert_eq!(UnreportedSlashesQueue::::get().len(), 25); + assert_eq!(unsent_queue_len(), 2); // this triggers on_initialize run_block(); - assert_eq!(UnreportedSlashesQueue::::get().len(), 5); + assert_eq!(unsent_queue_len(), 1); + assert_eq!(queued_slash_ids(), (20..25).collect::>()); - // We have 5 non-dispatched, which should accumulate - // We shoulld have 30 after we initialie era 3 for i in 0..25 { PendingOffenceKind::::insert(2, 3 + i, OffenceKind::LivenessOffence); Pallet::::on_offence( @@ -651,15 +898,20 @@ fn test_on_offence_defer_period_0_messages_get_queued_across_eras() { } start_era(3, 3, 3); - assert_eq!(UnreportedSlashesQueue::::get().len(), 30); + assert_eq!(unsent_queue_len(), 3); + assert_eq!(queued_batch_eras(), vec![2, 3, 3]); // this triggers on_initialize run_block(); - assert_eq!(UnreportedSlashesQueue::::get().len(), 10); + assert_eq!(unsent_queue_len(), 2); + assert_eq!(queued_batch_eras(), vec![3, 3]); // this triggers on_initialize run_block(); - assert_eq!(UnreportedSlashesQueue::::get().len(), 0); + assert_eq!(unsent_queue_len(), 1); + + run_block(); + assert!(ExternalValidatorSlashes::unsent_queue_is_empty()); }); } diff --git a/operator/pallets/external-validator-slashes/src/weights.rs b/operator/pallets/external-validator-slashes/src/weights.rs index 011374bd..22971e79 100644 --- a/operator/pallets/external-validator-slashes/src/weights.rs +++ b/operator/pallets/external-validator-slashes/src/weights.rs @@ -57,6 +57,7 @@ pub trait WeightInfo { fn force_inject_slash() -> Weight; fn root_test_send_msg_to_eth() -> Weight; fn process_slashes_queue(s: u32, ) -> Weight; + fn retry_unsent_slash_era() -> Weight; fn set_slashing_mode() -> Weight; } @@ -136,6 +137,11 @@ impl WeightInfo for SubstrateWeight { .saturating_add(Weight::from_parts(0, 42).saturating_mul(s.into())) } + fn retry_unsent_slash_era() -> Weight { + // Same as the success path for one queued batch. + Self::process_slashes_queue(10) + } + fn set_slashing_mode() -> Weight { Weight::from_parts(7_402_000, 3601) .saturating_add(T::DbWeight::get().reads(1_u64)) @@ -221,6 +227,10 @@ impl WeightInfo for () { .saturating_add(Weight::from_parts(0, 42).saturating_mul(s.into())) } + fn retry_unsent_slash_era() -> Weight { + Self::process_slashes_queue(10) + } + fn set_slashing_mode() -> Weight { Weight::from_parts(7_402_000, 3601) .saturating_add(RocksDbWeight::get().reads(1_u64)) diff --git a/operator/runtime/mainnet/src/configs/mod.rs b/operator/runtime/mainnet/src/configs/mod.rs index 5842383f..22f830ad 100644 --- a/operator/runtime/mainnet/src/configs/mod.rs +++ b/operator/runtime/mainnet/src/configs/mod.rs @@ -1738,6 +1738,7 @@ impl pallet_external_validator_slashes::Config for Runtime { type QueuedSlashesProcessedPerBlock = ConstU32<10>; type WeightInfo = mainnet_weights::pallet_external_validator_slashes::WeightInfo; type SendMessage = SlashesSendAdapter; + type GovernanceOrigin = EnsureRootWithSuccess; } parameter_types! { diff --git a/operator/runtime/mainnet/src/weights/pallet_external_validator_slashes.rs b/operator/runtime/mainnet/src/weights/pallet_external_validator_slashes.rs index 757af34c..d59f0197 100644 --- a/operator/runtime/mainnet/src/weights/pallet_external_validator_slashes.rs +++ b/operator/runtime/mainnet/src/weights/pallet_external_validator_slashes.rs @@ -113,6 +113,9 @@ impl pallet_external_validator_slashes::WeightInfo for .saturating_add(T::DbWeight::get().writes(4_u64)) .saturating_add(Weight::from_parts(0, 38).saturating_mul(s.into())) } + fn retry_unsent_slash_era() -> Weight { + Self::process_slashes_queue(10) + } /// Storage: `ExternalValidatorsSlashes::SlashingMode` (r:0 w:1) /// Proof: `ExternalValidatorsSlashes::SlashingMode` (`max_values`: Some(1), `max_size`: Some(1), added: 496, mode: `MaxEncodedLen`) fn set_slashing_mode() -> Weight { diff --git a/operator/runtime/stagenet/src/configs/mod.rs b/operator/runtime/stagenet/src/configs/mod.rs index 8406c5e1..8f85d611 100644 --- a/operator/runtime/stagenet/src/configs/mod.rs +++ b/operator/runtime/stagenet/src/configs/mod.rs @@ -1734,6 +1734,7 @@ impl pallet_external_validator_slashes::Config for Runtime { type QueuedSlashesProcessedPerBlock = ConstU32<10>; type WeightInfo = stagenet_weights::pallet_external_validator_slashes::WeightInfo; type SendMessage = SlashesSendAdapter; + type GovernanceOrigin = EnsureRootWithSuccess; } parameter_types! { diff --git a/operator/runtime/stagenet/src/weights/pallet_external_validator_slashes.rs b/operator/runtime/stagenet/src/weights/pallet_external_validator_slashes.rs index 39796b7e..b463e931 100644 --- a/operator/runtime/stagenet/src/weights/pallet_external_validator_slashes.rs +++ b/operator/runtime/stagenet/src/weights/pallet_external_validator_slashes.rs @@ -113,6 +113,9 @@ impl pallet_external_validator_slashes::WeightInfo for .saturating_add(T::DbWeight::get().writes(4_u64)) .saturating_add(Weight::from_parts(0, 38).saturating_mul(s.into())) } + fn retry_unsent_slash_era() -> Weight { + Self::process_slashes_queue(10) + } /// Storage: `ExternalValidatorsSlashes::SlashingMode` (r:0 w:1) /// Proof: `ExternalValidatorsSlashes::SlashingMode` (`max_values`: Some(1), `max_size`: Some(1), added: 496, mode: `MaxEncodedLen`) fn set_slashing_mode() -> Weight { diff --git a/operator/runtime/testnet/src/configs/mod.rs b/operator/runtime/testnet/src/configs/mod.rs index f2f98590..3c40349d 100644 --- a/operator/runtime/testnet/src/configs/mod.rs +++ b/operator/runtime/testnet/src/configs/mod.rs @@ -1736,6 +1736,7 @@ impl pallet_external_validator_slashes::Config for Runtime { type QueuedSlashesProcessedPerBlock = ConstU32<10>; type WeightInfo = testnet_weights::pallet_external_validator_slashes::WeightInfo; type SendMessage = SlashesSendAdapter; + type GovernanceOrigin = EnsureRootWithSuccess; } parameter_types! { diff --git a/operator/runtime/testnet/src/weights/pallet_external_validator_slashes.rs b/operator/runtime/testnet/src/weights/pallet_external_validator_slashes.rs index 5dd4c60c..c3d9c314 100644 --- a/operator/runtime/testnet/src/weights/pallet_external_validator_slashes.rs +++ b/operator/runtime/testnet/src/weights/pallet_external_validator_slashes.rs @@ -113,6 +113,9 @@ impl pallet_external_validator_slashes::WeightInfo for .saturating_add(T::DbWeight::get().writes(4_u64)) .saturating_add(Weight::from_parts(0, 38).saturating_mul(s.into())) } + fn retry_unsent_slash_era() -> Weight { + Self::process_slashes_queue(10) + } /// Storage: `ExternalValidatorsSlashes::SlashingMode` (r:0 w:1) /// Proof: `ExternalValidatorsSlashes::SlashingMode` (`max_values`: Some(1), `max_size`: Some(1), added: 496, mode: `MaxEncodedLen`) fn set_slashing_mode() -> Weight { diff --git a/test/.papi/descriptors/package.json b/test/.papi/descriptors/package.json index 479f5a5e..b90127de 100644 --- a/test/.papi/descriptors/package.json +++ b/test/.papi/descriptors/package.json @@ -1,5 +1,5 @@ { - "version": "0.1.0-autogenerated.816556291038383388", + "version": "0.1.0-autogenerated.14138049732278430947", "name": "@polkadot-api/descriptors", "files": [ "dist" diff --git a/test/.papi/metadata/datahaven.scale b/test/.papi/metadata/datahaven.scale index 240771b7..7c1874f6 100644 Binary files a/test/.papi/metadata/datahaven.scale and b/test/.papi/metadata/datahaven.scale differ