diff --git a/operator/runtime/common/src/migrations.rs b/operator/runtime/common/src/migrations.rs index a0a7a507..6b342519 100644 --- a/operator/runtime/common/src/migrations.rs +++ b/operator/runtime/common/src/migrations.rs @@ -45,8 +45,19 @@ pub type MultiBlockMigrationList = pallet_migrations::mock_helpers::MockedMigrat /// Placeholder handler for migration status notifications. We do not emit any extra signals yet. pub type MigrationStatusHandler = (); -/// Default handler triggered on migration failures. -pub type FailedMigrationHandler = frame_support::migrations::FreezeChainOnFailedMigration; +/// Handler triggered on migration failures. +/// +/// This handler attempts to enter SafeMode when a migration fails, allowing governance to +/// intervene and fix the issue while preventing regular user transactions from interacting +/// with potentially inconsistent storage state. +/// +/// The handler is parameterized by the SafeMode pallet type from each runtime, with a fallback +/// to freezing the chain if SafeMode cannot be entered. +pub type FailedMigrationHandler = + frame_support::migrations::EnterSafeModeOnFailedMigration< + SafeMode, + frame_support::migrations::FreezeChainOnFailedMigration, + >; /// Multi-block migration for updating the EVM chain ID to the new value. pub mod evm_chain_id { diff --git a/operator/runtime/mainnet/src/configs/mod.rs b/operator/runtime/mainnet/src/configs/mod.rs index 335a409c..344b5154 100644 --- a/operator/runtime/mainnet/src/configs/mod.rs +++ b/operator/runtime/mainnet/src/configs/mod.rs @@ -79,8 +79,8 @@ use datahaven_runtime_common::{ }, gas::WEIGHT_PER_GAS, migrations::{ - FailedMigrationHandler as DefaultFailedMigrationHandler, MigrationCursorMaxLen, - MigrationIdentifierMaxLen, MigrationStatusHandler, + FailedMigrationHandler, MigrationCursorMaxLen, MigrationIdentifierMaxLen, + MigrationStatusHandler, }, safe_mode::{ ReleaseDelayNone, RuntimeCallFilter, SafeModeDuration, SafeModeEnterDeposit, @@ -854,8 +854,7 @@ impl pallet_migrations::Config for Runtime { type CursorMaxLen = MigrationCursorMaxLen; type IdentifierMaxLen = MigrationIdentifierMaxLen; type MigrationStatusHandler = MigrationStatusHandler; - // TODO: Remove this once we have a proper failed migration handler (Safe mode) - type FailedMigrationHandler = DefaultFailedMigrationHandler; + type FailedMigrationHandler = FailedMigrationHandler; type MaxServiceWeight = MaxServiceWeight; type WeightInfo = mainnet_weights::pallet_migrations::WeightInfo; } diff --git a/operator/runtime/mainnet/tests/migrations.rs b/operator/runtime/mainnet/tests/migrations.rs index 288abb68..34292d9b 100644 --- a/operator/runtime/mainnet/tests/migrations.rs +++ b/operator/runtime/mainnet/tests/migrations.rs @@ -18,7 +18,9 @@ mod common; use common::*; -use datahaven_mainnet_runtime::{Runtime, RuntimeCall, RuntimeOrigin}; +use datahaven_mainnet_runtime::{ + Runtime, RuntimeCall, RuntimeEvent, RuntimeOrigin, SafeMode, System, +}; use frame_support::{assert_noop, assert_ok}; use pallet_migrations::{Call as MigrationsCall, HistoricCleanupSelector}; use sp_runtime::{traits::Dispatchable, DispatchError}; @@ -69,3 +71,76 @@ fn migrations_force_calls_are_root_only() { assert_ok!(clear_historic.dispatch(RuntimeOrigin::root())); }); } + +#[test] +fn failed_migration_enters_safe_mode() { + ExtBuilder::default().build().execute_with(|| { + // Verify SafeMode is not active initially + assert!( + !SafeMode::is_entered(), + "SafeMode should not be active initially" + ); + + // Simulate a failed migration by directly calling the FailedMigrationHandler + // This tests that when migrations fail, the system enters SafeMode + use frame_support::migrations::FailedMigrationHandler; + type Handler = ::FailedMigrationHandler; + + // Call the failed handler (simulating a migration failure) + let result = Handler::failed(Some(0)); + + // The handler should indicate that SafeMode was entered + assert_eq!( + result, + frame_support::migrations::FailedMigrationHandling::KeepStuck, + "Handler should keep the chain stuck in SafeMode" + ); + + // Verify that SafeMode is now active + assert!( + SafeMode::is_entered(), + "SafeMode should be active after migration failure" + ); + + // Get the block number when SafeMode should expire + let entered_until = pallet_safe_mode::EnteredUntil::::get(); + assert!( + entered_until.is_some(), + "SafeMode should have an expiry block" + ); + + // Verify that the SafeMode event was emitted + let events = System::events(); + assert!( + events.iter().any(|e| matches!( + e.event, + RuntimeEvent::SafeMode(pallet_safe_mode::Event::Entered { .. }) + )), + "SafeMode::Entered event should be emitted" + ); + }); +} + +#[test] +fn safe_mode_allows_governance_during_migration_failure() { + ExtBuilder::default().build().execute_with(|| { + // Simulate a failed migration + use frame_support::migrations::FailedMigrationHandler; + type Handler = ::FailedMigrationHandler; + Handler::failed(Some(0)); + + // Verify SafeMode is active + assert!(SafeMode::is_entered(), "SafeMode should be active"); + + // Test that SafeMode management calls are still allowed + let force_exit_call = RuntimeCall::SafeMode(pallet_safe_mode::Call::force_exit {}); + let result = force_exit_call.dispatch(RuntimeOrigin::root()); + assert_ok!(result); + + // Verify SafeMode is now inactive + assert!( + !SafeMode::is_entered(), + "SafeMode should be inactive after force exit" + ); + }); +} diff --git a/operator/runtime/stagenet/src/configs/mod.rs b/operator/runtime/stagenet/src/configs/mod.rs index 55352a33..893d2feb 100644 --- a/operator/runtime/stagenet/src/configs/mod.rs +++ b/operator/runtime/stagenet/src/configs/mod.rs @@ -79,8 +79,8 @@ use datahaven_runtime_common::{ }, gas::WEIGHT_PER_GAS, migrations::{ - FailedMigrationHandler as DefaultFailedMigrationHandler, MigrationCursorMaxLen, - MigrationIdentifierMaxLen, MigrationStatusHandler, + FailedMigrationHandler, MigrationCursorMaxLen, MigrationIdentifierMaxLen, + MigrationStatusHandler, }, safe_mode::{ ReleaseDelayNone, RuntimeCallFilter, SafeModeDuration, SafeModeEnterDeposit, @@ -857,8 +857,7 @@ impl pallet_migrations::Config for Runtime { type CursorMaxLen = MigrationCursorMaxLen; type IdentifierMaxLen = MigrationIdentifierMaxLen; type MigrationStatusHandler = MigrationStatusHandler; - // TODO: Remove this once we have a proper failed migration handler (Safe mode) - type FailedMigrationHandler = DefaultFailedMigrationHandler; + type FailedMigrationHandler = FailedMigrationHandler; type MaxServiceWeight = MaxServiceWeight; type WeightInfo = stagenet_weights::pallet_migrations::WeightInfo; } diff --git a/operator/runtime/stagenet/tests/migrations.rs b/operator/runtime/stagenet/tests/migrations.rs new file mode 100644 index 00000000..961f7cc6 --- /dev/null +++ b/operator/runtime/stagenet/tests/migrations.rs @@ -0,0 +1,146 @@ +// Copyright 2025 DataHaven +// This file is part of DataHaven. + +// DataHaven is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// DataHaven is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with DataHaven. If not, see . + +#[path = "common.rs"] +mod common; + +use common::*; +use datahaven_stagenet_runtime::{ + Runtime, RuntimeCall, RuntimeEvent, RuntimeOrigin, SafeMode, System, +}; +use frame_support::{assert_noop, assert_ok}; +use pallet_migrations::{Call as MigrationsCall, HistoricCleanupSelector}; +use sp_runtime::{traits::Dispatchable, DispatchError}; + +#[test] +fn migrations_force_calls_are_root_only() { + ExtBuilder::default().build().execute_with(|| { + let signed_origin = RuntimeOrigin::signed(account_id(ALICE)); + + let force_onboard = + RuntimeCall::MultiBlockMigrations(MigrationsCall::::force_onboard_mbms {}); + assert_noop!( + force_onboard.clone().dispatch(signed_origin.clone()), + DispatchError::BadOrigin + ); + assert_ok!(force_onboard.dispatch(RuntimeOrigin::root())); + + let force_set_cursor = + RuntimeCall::MultiBlockMigrations(MigrationsCall::::force_set_cursor { + cursor: None, + }); + assert_noop!( + force_set_cursor.clone().dispatch(signed_origin.clone()), + DispatchError::BadOrigin + ); + assert_ok!(force_set_cursor.dispatch(RuntimeOrigin::root())); + + let force_set_active = + RuntimeCall::MultiBlockMigrations(MigrationsCall::::force_set_active_cursor { + index: 0, + inner_cursor: None, + started_at: None, + }); + assert_noop!( + force_set_active.clone().dispatch(signed_origin.clone()), + DispatchError::BadOrigin + ); + assert_ok!(force_set_active.dispatch(RuntimeOrigin::root())); + + let clear_historic = + RuntimeCall::MultiBlockMigrations(MigrationsCall::::clear_historic { + selector: HistoricCleanupSelector::Specific(Vec::new()), + }); + assert_noop!( + clear_historic.clone().dispatch(signed_origin), + DispatchError::BadOrigin + ); + assert_ok!(clear_historic.dispatch(RuntimeOrigin::root())); + }); +} + +#[test] +fn failed_migration_enters_safe_mode() { + ExtBuilder::default().build().execute_with(|| { + // Verify SafeMode is not active initially + assert!( + !SafeMode::is_entered(), + "SafeMode should not be active initially" + ); + + // Simulate a failed migration by directly calling the FailedMigrationHandler + // This tests that when migrations fail, the system enters SafeMode + use frame_support::migrations::FailedMigrationHandler; + type Handler = ::FailedMigrationHandler; + + // Call the failed handler (simulating a migration failure) + let result = Handler::failed(Some(0)); + + // The handler should indicate that SafeMode was entered + assert_eq!( + result, + frame_support::migrations::FailedMigrationHandling::KeepStuck, + "Handler should keep the chain stuck in SafeMode" + ); + + // Verify that SafeMode is now active + assert!( + SafeMode::is_entered(), + "SafeMode should be active after migration failure" + ); + + // Get the block number when SafeMode should expire + let entered_until = pallet_safe_mode::EnteredUntil::::get(); + assert!( + entered_until.is_some(), + "SafeMode should have an expiry block" + ); + + // Verify that the SafeMode event was emitted + let events = System::events(); + assert!( + events.iter().any(|e| matches!( + e.event, + RuntimeEvent::SafeMode(pallet_safe_mode::Event::Entered { .. }) + )), + "SafeMode::Entered event should be emitted" + ); + }); +} + +#[test] +fn safe_mode_allows_governance_during_migration_failure() { + ExtBuilder::default().build().execute_with(|| { + // Simulate a failed migration + use frame_support::migrations::FailedMigrationHandler; + type Handler = ::FailedMigrationHandler; + Handler::failed(Some(0)); + + // Verify SafeMode is active + assert!(SafeMode::is_entered(), "SafeMode should be active"); + + // Test that SafeMode management calls are still allowed + let force_exit_call = RuntimeCall::SafeMode(pallet_safe_mode::Call::force_exit {}); + let result = force_exit_call.dispatch(RuntimeOrigin::root()); + assert_ok!(result); + + // Verify SafeMode is now inactive + assert!( + !SafeMode::is_entered(), + "SafeMode should be inactive after force exit" + ); + }); +} diff --git a/operator/runtime/stagenet/tests/safe_mode_tx_pause.rs b/operator/runtime/stagenet/tests/safe_mode_tx_pause.rs index fd185f82..6abd142f 100644 --- a/operator/runtime/stagenet/tests/safe_mode_tx_pause.rs +++ b/operator/runtime/stagenet/tests/safe_mode_tx_pause.rs @@ -21,8 +21,7 @@ mod common; use common::{account_id, ExtBuilder, ALICE, BOB}; use datahaven_stagenet_runtime::{ - Runtime, RuntimeCall, RuntimeEvent, RuntimeOrigin, SafeMode, System, TxPause, - UncheckedExtrinsic, + Runtime, RuntimeCall, RuntimeEvent, RuntimeOrigin, System, UncheckedExtrinsic, }; use frame_support::{assert_noop, assert_ok, BoundedVec}; use pallet_safe_mode::EnteredUntil; diff --git a/operator/runtime/testnet/src/configs/mod.rs b/operator/runtime/testnet/src/configs/mod.rs index a45515ba..2f7bc6e7 100644 --- a/operator/runtime/testnet/src/configs/mod.rs +++ b/operator/runtime/testnet/src/configs/mod.rs @@ -79,8 +79,8 @@ use datahaven_runtime_common::{ }, gas::WEIGHT_PER_GAS, migrations::{ - FailedMigrationHandler as DefaultFailedMigrationHandler, MigrationCursorMaxLen, - MigrationIdentifierMaxLen, MigrationStatusHandler, + FailedMigrationHandler, MigrationCursorMaxLen, MigrationIdentifierMaxLen, + MigrationStatusHandler, }, safe_mode::{ ReleaseDelayNone, RuntimeCallFilter, SafeModeDuration, SafeModeEnterDeposit, @@ -860,8 +860,7 @@ impl pallet_migrations::Config for Runtime { type CursorMaxLen = MigrationCursorMaxLen; type IdentifierMaxLen = MigrationIdentifierMaxLen; type MigrationStatusHandler = MigrationStatusHandler; - // TODO: Remove this once we have a proper failed migration handler (Safe mode) - type FailedMigrationHandler = DefaultFailedMigrationHandler; + type FailedMigrationHandler = FailedMigrationHandler; type MaxServiceWeight = MaxServiceWeight; type WeightInfo = testnet_weights::pallet_migrations::WeightInfo; } diff --git a/operator/runtime/testnet/tests/migrations.rs b/operator/runtime/testnet/tests/migrations.rs new file mode 100644 index 00000000..6e76a85e --- /dev/null +++ b/operator/runtime/testnet/tests/migrations.rs @@ -0,0 +1,146 @@ +// Copyright 2025 DataHaven +// This file is part of DataHaven. + +// DataHaven is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// DataHaven is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with DataHaven. If not, see . + +#[path = "common.rs"] +mod common; + +use common::*; +use datahaven_testnet_runtime::{ + Runtime, RuntimeCall, RuntimeEvent, RuntimeOrigin, SafeMode, System, +}; +use frame_support::{assert_noop, assert_ok}; +use pallet_migrations::{Call as MigrationsCall, HistoricCleanupSelector}; +use sp_runtime::{traits::Dispatchable, DispatchError}; + +#[test] +fn migrations_force_calls_are_root_only() { + ExtBuilder::default().build().execute_with(|| { + let signed_origin = RuntimeOrigin::signed(account_id(ALICE)); + + let force_onboard = + RuntimeCall::MultiBlockMigrations(MigrationsCall::::force_onboard_mbms {}); + assert_noop!( + force_onboard.clone().dispatch(signed_origin.clone()), + DispatchError::BadOrigin + ); + assert_ok!(force_onboard.dispatch(RuntimeOrigin::root())); + + let force_set_cursor = + RuntimeCall::MultiBlockMigrations(MigrationsCall::::force_set_cursor { + cursor: None, + }); + assert_noop!( + force_set_cursor.clone().dispatch(signed_origin.clone()), + DispatchError::BadOrigin + ); + assert_ok!(force_set_cursor.dispatch(RuntimeOrigin::root())); + + let force_set_active = + RuntimeCall::MultiBlockMigrations(MigrationsCall::::force_set_active_cursor { + index: 0, + inner_cursor: None, + started_at: None, + }); + assert_noop!( + force_set_active.clone().dispatch(signed_origin.clone()), + DispatchError::BadOrigin + ); + assert_ok!(force_set_active.dispatch(RuntimeOrigin::root())); + + let clear_historic = + RuntimeCall::MultiBlockMigrations(MigrationsCall::::clear_historic { + selector: HistoricCleanupSelector::Specific(Vec::new()), + }); + assert_noop!( + clear_historic.clone().dispatch(signed_origin), + DispatchError::BadOrigin + ); + assert_ok!(clear_historic.dispatch(RuntimeOrigin::root())); + }); +} + +#[test] +fn failed_migration_enters_safe_mode() { + ExtBuilder::default().build().execute_with(|| { + // Verify SafeMode is not active initially + assert!( + !SafeMode::is_entered(), + "SafeMode should not be active initially" + ); + + // Simulate a failed migration by directly calling the FailedMigrationHandler + // This tests that when migrations fail, the system enters SafeMode + use frame_support::migrations::FailedMigrationHandler; + type Handler = ::FailedMigrationHandler; + + // Call the failed handler (simulating a migration failure) + let result = Handler::failed(Some(0)); + + // The handler should indicate that SafeMode was entered + assert_eq!( + result, + frame_support::migrations::FailedMigrationHandling::KeepStuck, + "Handler should keep the chain stuck in SafeMode" + ); + + // Verify that SafeMode is now active + assert!( + SafeMode::is_entered(), + "SafeMode should be active after migration failure" + ); + + // Get the block number when SafeMode should expire + let entered_until = pallet_safe_mode::EnteredUntil::::get(); + assert!( + entered_until.is_some(), + "SafeMode should have an expiry block" + ); + + // Verify that the SafeMode event was emitted + let events = System::events(); + assert!( + events.iter().any(|e| matches!( + e.event, + RuntimeEvent::SafeMode(pallet_safe_mode::Event::Entered { .. }) + )), + "SafeMode::Entered event should be emitted" + ); + }); +} + +#[test] +fn safe_mode_allows_governance_during_migration_failure() { + ExtBuilder::default().build().execute_with(|| { + // Simulate a failed migration + use frame_support::migrations::FailedMigrationHandler; + type Handler = ::FailedMigrationHandler; + Handler::failed(Some(0)); + + // Verify SafeMode is active + assert!(SafeMode::is_entered(), "SafeMode should be active"); + + // Test that SafeMode management calls are still allowed + let force_exit_call = RuntimeCall::SafeMode(pallet_safe_mode::Call::force_exit {}); + let result = force_exit_call.dispatch(RuntimeOrigin::root()); + assert_ok!(result); + + // Verify SafeMode is now inactive + assert!( + !SafeMode::is_entered(), + "SafeMode should be inactive after force exit" + ); + }); +} diff --git a/operator/runtime/testnet/tests/safe_mode_tx_pause.rs b/operator/runtime/testnet/tests/safe_mode_tx_pause.rs index acb7e8f5..7f236113 100644 --- a/operator/runtime/testnet/tests/safe_mode_tx_pause.rs +++ b/operator/runtime/testnet/tests/safe_mode_tx_pause.rs @@ -21,8 +21,7 @@ mod common; use common::{account_id, ExtBuilder, ALICE, BOB}; use datahaven_testnet_runtime::{ - Runtime, RuntimeCall, RuntimeEvent, RuntimeOrigin, SafeMode, System, TxPause, - UncheckedExtrinsic, + Runtime, RuntimeCall, RuntimeEvent, RuntimeOrigin, System, UncheckedExtrinsic, }; use frame_support::{assert_noop, assert_ok, BoundedVec}; use pallet_safe_mode::EnteredUntil;