diff --git a/tests/src/cucumber/steps/manual_cluster.rs b/tests/src/cucumber/steps/cluster.rs similarity index 98% rename from tests/src/cucumber/steps/manual_cluster.rs rename to tests/src/cucumber/steps/cluster.rs index 2def5465e..161d7bdf7 100644 --- a/tests/src/cucumber/steps/manual_cluster.rs +++ b/tests/src/cucumber/steps/cluster.rs @@ -111,9 +111,7 @@ pub fn build_manual_cluster_deployment( if let Some(genesis_block) = deployment.config.genesis_block.clone() { world.chain.genesis_block_utxos = - crate::cucumber::steps::manual_nodes::utils::genesis_block_utxos( - &genesis_block.genesis_tx(), - ); + crate::cucumber::steps::nodes::genesis_block_utxos(&genesis_block.genesis_tx()); world.chain.genesis_block_id = Some(genesis_block.header().id()); } @@ -373,7 +371,7 @@ fn compose_dial_addr(listen_addr: Option<&Multiaddr>, peer_id: PeerId) -> Option pub fn build_user_wallets( world: &CucumberWorld, node_name: &str, - wallet_start_info: &[crate::cucumber::steps::manual_nodes::utils::WalletStartInfo], + wallet_start_info: &[crate::cucumber::steps::nodes::WalletStartInfo], ) -> Result, StepError> { let mut wallet_info = HashMap::new(); for wallet in wallet_start_info { diff --git a/tests/src/cucumber/steps/manual_k8s.rs b/tests/src/cucumber/steps/k8s.rs similarity index 99% rename from tests/src/cucumber/steps/manual_k8s.rs rename to tests/src/cucumber/steps/k8s.rs index 4b840e22f..dfcebafb6 100644 --- a/tests/src/cucumber/steps/manual_k8s.rs +++ b/tests/src/cucumber/steps/k8s.rs @@ -7,12 +7,12 @@ use testing_framework_core::scenario::{PeerSelection, StartNodeOptions}; use crate::cucumber::{ error::{StepError, StepResult}, steps::{ - manual_cluster::{ + cluster::{ assert_manual_node_has_peers, build_manual_cluster_deployment, build_user_wallets, insert_started_node_info, start_manual_node, stop_active_manual_cluster, wait_manual_node_ready, }, - manual_nodes::utils::{ + nodes::{ NodesToStartUnordered, parse_wallet_resources_table_row, start_nodes_order_respecting_dependencies, verify_node_wallet_resources_table_indexes, }, diff --git a/tests/src/cucumber/steps/manual_nodes/mod.rs b/tests/src/cucumber/steps/manual_nodes/mod.rs deleted file mode 100644 index 1d71cef31..000000000 --- a/tests/src/cucumber/steps/manual_nodes/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub(crate) mod config_override; -pub(crate) mod diagnostics; -pub mod lib_assertions; -pub mod parameters; -pub(crate) mod snapshots; -pub mod steps; -pub mod utils; diff --git a/tests/src/cucumber/steps/manual_nodes/steps.rs b/tests/src/cucumber/steps/manual_nodes/steps.rs deleted file mode 100644 index 4d429f2fa..000000000 --- a/tests/src/cucumber/steps/manual_nodes/steps.rs +++ /dev/null @@ -1,1656 +0,0 @@ -use std::{ - collections::{HashMap, HashSet}, - path::PathBuf, - time::Duration, -}; - -use cucumber::{gherkin::Step, given, then, when}; -use lb_common_http_client::CommonHttpClient; -use lb_core::{codec::DeserializeOp as _, mantle::GenesisTime}; -use lb_key_management_system_service::keys::ZkPublicKey; -use lb_libp2p::{Multiaddr, PeerId}; -use lb_testing_framework::{ - USER_CONFIG_FILE, - configs::{ - deployment::{NodeBinaryProfile, SdpFundingConfig}, - wallet::WalletAccount, - }, - ensure_node_binary_built, -}; -use time::{Duration as TimeDuration, OffsetDateTime}; -use tokio::time::{Instant, sleep}; -use tracing::{info, warn}; - -use crate::{ - common::wallet::WalletUtxos, - cucumber::{ - error::{StepError, StepResult}, - steps::{ - TARGET, - manual_cluster::{ - assert_manual_node_has_peers, connect_manual_node_to_node, - install_local_manual_cluster, rebuild_pending_local_manual_cluster, - stop_active_manual_cluster, - }, - manual_nodes::{ - config_override::{set_deployment_config_override, set_user_config_override}, - diagnostics::set_blend_diagnostic_parameter_set, - snapshots::validate_snapshot_path_component, - utils::{ - NodesToStartUnordered, create_snapshot_all_nodes_with_wallet_state, - create_snapshot_node_with_wallet_state, create_snapshots_all_nodes, - ensure_all_nodes_agree_on_lib, - ensure_fee_sponsorship_and_fork_groups_are_not_mixed, - get_cryptarchia_info_all_nodes, nodes_converged, - parse_genesis_wallet_tokens_row, parse_mining_wallet_resources_table_row, - parse_url, parse_wallet_resources_table_row, - poll_all_nodes_and_update_consensus_cache, restart_node, start_node, - start_nodes_order_respecting_dependencies, stop_node, - verify_genesis_wallet_resources_table_indexes, - verify_mining_node_wallet_resources_table_indexes, - verify_node_wallet_resources_table_indexes, - verify_reponsive_and_network_ready_with_timeout, wait_all_nodes_responive, - wait_for_all_nodes_to_be_synced_to_chain, - }, - }, - manual_transactions::utils::{ - create_and_submit_transaction_hashes_with_utxo_cache, - wait_for_transactions_inclusion, - }, - }, - utils::{ - blend_core_locator_from_node_yaml, blend_core_zk_pk_from_node_yaml, - resolve_literal_or_env, - }, - wallet::{ - snapshot::{ - prepare_all_wallets_snapshot, prepare_wallet_snapshot_restore_if_present, - save_prepared_all_wallets_snapshot, - }, - sync::{WalletSendReadiness, wait_wallet_send_ready}, - }, - world::{ - ConfigOverride, CucumberWorld, GenesisTokens, ManualClusterKind, ManualClusterSpec, - NodeSnapshot, PublicCryptarchiaEndpointPeer, - }, - }, - non_zero, -}; - -const PUBLIC_CRYPTARCHIA_ENDPOINT: &str = "public_cryptarchia_endpoint"; -const PUBLIC_CRYPTARCHIA_ENDPOINT_USERNAME: &str = "username"; -const PUBLIC_CRYPTARCHIA_ENDPOINT_PASSWORD: &str = "password"; - -fn resolve_step_genesis_time( - step_value: &str, - now: OffsetDateTime, - seconds: i64, -) -> Result { - if seconds < 0 { - return Err(StepError::InvalidArgument { - message: format!("step `{step_value}` requires a non-negative offset"), - }); - } - - let genesis_datetime = now - .checked_add(TimeDuration::seconds(seconds)) - .ok_or_else(|| StepError::InvalidArgument { - message: format!( - "step `{step_value}` has an invalid genesis time: offset is out of range" - ), - })?; - - GenesisTime::try_from(genesis_datetime).map_err(|error| StepError::InvalidArgument { - message: format!("step `{step_value}` has an invalid genesis time: {error}"), - }) -} - -fn validate_genesis_time_change( - existing_genesis_time: Option, - nodes_started: bool, - requested_genesis_time: GenesisTime, -) -> StepResult { - if nodes_started && existing_genesis_time != Some(requested_genesis_time) { - return Err(StepError::LogicalError { - message: "cannot change genesis time after nodes have started".to_owned(), - }); - } - - Ok(()) -} - -#[given(expr = "the chain starts {int} seconds from now")] -#[when(expr = "the chain starts {int} seconds from now")] -async fn step_chain_starts_from_now( - world: &mut CucumberWorld, - step: &Step, - seconds: i64, -) -> StepResult { - let node_binary_profile = if world.tokio_console_profile_enabled() { - NodeBinaryProfile::TokioConsole - } else { - NodeBinaryProfile::default() - }; - ensure_node_binary_built(&node_binary_profile) - .await - .map_err(|error| StepError::Preflight { - message: format!("failed to resolve/build node binary: {error}"), - })?; - - let genesis_time = resolve_step_genesis_time(&step.value, OffsetDateTime::now_utc(), seconds)?; - validate_genesis_time_change( - world.lifecycle.genesis_time, - !world.nodes_info.is_empty(), - genesis_time, - )?; - - world.set_genesis_time(genesis_time); - if world.nodes_info.is_empty() && world.cluster.manual_cluster_spec.is_some() { - rebuild_pending_local_manual_cluster(world)?; - } - - Ok(()) -} - -#[then(expr = "the configured genesis time has not passed for {int} seconds")] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require the world as the first `&mut` argument" -)] -async fn step_genesis_time_has_not_passed( - world: &mut CucumberWorld, - step: &Step, - seconds: u64, -) -> StepResult { - let genesis_time = world - .lifecycle - .genesis_time - .ok_or_else(|| StepError::LogicalError { - message: "the scenario has no configured genesis time".to_owned(), - })?; - let genesis_datetime = OffsetDateTime::from(genesis_time); - let timeout = Duration::from_secs(seconds); - let started_waiting = Instant::now(); - - loop { - let now = OffsetDateTime::now_utc(); - if now >= genesis_datetime { - return Err(StepError::StepFail { - message: format!( - "Step `{}` failed: configured genesis time {genesis_datetime} had passed at {now}", - step.value - ), - }); - } - - if started_waiting.elapsed() >= timeout { - info!( - target: TARGET, - "Configured genesis time {genesis_datetime} remained in the future for {seconds} seconds" - ); - return Ok(()); - } - - sleep(Duration::from_millis(250)).await; - } -} - -#[given(expr = "I have a cluster with capacity of {int} nodes")] -#[when(expr = "I have a cluster with capacity of {int} nodes")] -fn step_manual_cluster(world: &mut CucumberWorld, step: &Step, nodes_count: usize) -> StepResult { - install_local_manual_cluster( - world, - ManualClusterSpec { - kind: ManualClusterKind::Generated, - capacity: nodes_count, - }, - ) - .inspect_err(|e| { - warn!(target: TARGET, "Step '{step}' error: {e}"); - }) -} - -#[given(expr = "I have a devnet cluster with capacity of {int} nodes")] -#[when(expr = "I have a devnet cluster with capacity of {int} nodes")] -fn step_manual_devnet_cluster( - world: &mut CucumberWorld, - step: &Step, - nodes_count: usize, -) -> StepResult { - install_local_manual_cluster( - world, - ManualClusterSpec { - kind: ManualClusterKind::Devnet, - capacity: nodes_count, - }, - ) - .inspect_err(|e| { - warn!(target: TARGET, "Step '{step}' error: {e}"); - }) -} - -#[given("the genesis block has the following wallet resources:")] -#[when("the genesis block has the following wallet resources:")] -fn step_cluster_has_wallet_resources(world: &mut CucumberWorld, step: &Step) -> StepResult { - let table = step - .table - .as_ref() - .ok_or(StepError::MissingTable) - .inspect_err(|e| { - warn!(target: TARGET, "Step `{}` error: {e}", step.value); - })?; - - verify_genesis_wallet_resources_table_indexes(table, &step.value)?; - world.chain.genesis_tokens.clear(); - for row in table.rows.iter().skip(1) { - let (account_index, token_count, token_amount) = - parse_genesis_wallet_tokens_row(&step.value, row)?; - - world.chain.genesis_tokens.push(GenesisTokens { - account_index, - token_count, - token_amount, - }); - } - - Ok(()) -} - -#[given(expr = "we have a sponsored genesis fee account with {int} tokens of {int} value each")] -#[when(expr = "we have a sponsored genesis fee account with {int} tokens of {int} value each")] -fn step_sponsored_genesis_fee_account( - world: &mut CucumberWorld, - step: &Step, - token_count: usize, - token_value: u64, -) -> StepResult { - ensure_fee_sponsorship_and_fork_groups_are_not_mixed(world, step.value.as_str())?; - - let token_count = non_zero!("genesis fee token count", token_count)?; - let token_value = non_zero!("genesis fee token value", token_value)?; - - world - .wallet_registry - .fee_state - .set_sponsored_genesis_account(token_count, token_value); - Ok(()) -} - -#[given("I start nodes with wallet resources:")] -#[when("I start nodes with wallet resources:")] -async fn step_start_nodes_with_wallet_resources( - world: &mut CucumberWorld, - step: &Step, -) -> StepResult { - let table = step - .table - .as_ref() - .ok_or(StepError::MissingTable) - .inspect_err(|e| { - warn!(target: TARGET, "Step `{}` error: {e}", step.value); - })?; - - // Map wallet start info and connected peers to node name - verify_node_wallet_resources_table_indexes(table, &step.value)?; - let mut nodes_to_start: NodesToStartUnordered = HashMap::new(); - for row in table.rows.iter().skip(1) { - let (node_name, wallet_start_info, connected_to) = - parse_wallet_resources_table_row(&step.value, row)?; - let entry = nodes_to_start - .entry(node_name) - .or_insert_with(|| (Vec::new(), Vec::new())); - entry.0.push(wallet_start_info); - if let Some(peer) = connected_to { - entry.1.push(peer); - } - } - - let nodes_to_start_ordered = start_nodes_order_respecting_dependencies( - nodes_to_start, - world.nodes_info.keys().cloned().collect(), - ) - .inspect_err(|e| { - warn!(target: TARGET, "Step `{}` error: {e}", step.value); - })?; - for (node_name, wallet_start_info, mut initial_peers) in nodes_to_start_ordered { - initial_peers.sort(); - initial_peers.dedup(); - start_node( - world, - &step.value, - &node_name, - &wallet_start_info, - &initial_peers, - false, - &[], - ) - .await?; - } - - world.ensure_wallet_scanner_started().await?; - - Ok(()) -} - -/// Starts mining nodes, each carrying one or more wallet resources of which -/// exactly one is flagged `is_mining_wallet`. That wallet's public key is -/// derived and applied as the node's `pow.claim_address`, so mined rewards are -/// paid to a wallet the test tracks. Multiple mining nodes are supported; each -/// gets its own single mining wallet / claim address. -#[given("I start mining nodes with wallet resources:")] -#[when("I start mining nodes with wallet resources:")] -async fn step_start_mining_nodes_with_wallet_resources( - world: &mut CucumberWorld, - step: &Step, -) -> StepResult { - let table = step - .table - .as_ref() - .ok_or(StepError::MissingTable) - .inspect_err(|e| { - warn!(target: TARGET, "Step `{}` error: {e}", step.value); - })?; - - verify_mining_node_wallet_resources_table_indexes(table, &step.value)?; - - let mut nodes_to_start: NodesToStartUnordered = HashMap::new(); - let mut node_claim_overrides: HashMap = HashMap::new(); - let mut node_mining_wallet_count: HashMap = HashMap::new(); - for row in table.rows.iter().skip(1) { - let (node_name, wallet_start_info, is_mining_wallet, connected_to) = - parse_mining_wallet_resources_table_row(&step.value, row)?; - - if is_mining_wallet { - *node_mining_wallet_count - .entry(node_name.clone()) - .or_insert(0) += 1; - let account = - WalletAccount::deterministic(wallet_start_info.account_index as u64, 0, true) - .map_err(|source| StepError::InvalidArgument { - message: format!( - "Step `{}` error: failed to derive mining wallet public key for \ - `{}`: {source}", - step.value, wallet_start_info.wallet_name - ), - })?; - let value = serde_yaml::to_value(account.public_key()).map_err(|source| { - StepError::InvalidArgument { - message: format!( - "Step `{}` error: failed to serialize mining wallet public key for \ - `{}`: {source}", - step.value, wallet_start_info.wallet_name - ), - } - })?; - node_claim_overrides.insert( - node_name.clone(), - ConfigOverride { - path: "pow.claim_address".to_owned(), - value, - }, - ); - } - - let entry = nodes_to_start - .entry(node_name) - .or_insert_with(|| (Vec::new(), Vec::new())); - entry.0.push(wallet_start_info); - if let Some(peer) = connected_to { - entry.1.push(peer); - } - } - - // Every mining node must configure exactly one mining wallet. - for node_name in nodes_to_start.keys() { - let count = node_mining_wallet_count - .get(node_name) - .copied() - .unwrap_or(0); - if count != 1 { - return Err(StepError::InvalidArgument { - message: format!( - "Step `{}` error: mining node `{node_name}` must have exactly one \ - is_mining_wallet row, found {count}", - step.value - ), - }); - } - } - - let nodes_to_start_ordered = start_nodes_order_respecting_dependencies( - nodes_to_start, - world.nodes_info.keys().cloned().collect(), - ) - .inspect_err(|e| { - warn!(target: TARGET, "Step `{}` error: {e}", step.value); - })?; - for (node_name, wallet_start_info, mut initial_peers) in nodes_to_start_ordered { - initial_peers.sort(); - initial_peers.dedup(); - let extra_user_overrides = node_claim_overrides - .get(&node_name) - .map_or(&[][..], std::slice::from_ref); - start_node( - world, - &step.value, - &node_name, - &wallet_start_info, - &initial_peers, - false, - extra_user_overrides, - ) - .await?; - } - - world.ensure_wallet_scanner_started().await?; - - Ok(()) -} - -#[given(expr = "I start node {string}")] -#[when(expr = "I start node {string}")] -async fn step_start_manual_stand_alone_node( - world: &mut CucumberWorld, - step: &Step, - node_name: String, -) -> StepResult { - start_node( - world, - &step.value, - &node_name, - &Vec::new(), - &Vec::new(), - false, - &[], - ) - .await -} - -#[given(expr = "I immediate start node {string}")] -#[when(expr = "I immediate start node {string}")] -async fn step_start_manual_network_ready_only_stand_alone_node( - world: &mut CucumberWorld, - step: &Step, - node_name: String, -) -> StepResult { - start_node( - world, - &step.value, - &node_name, - &Vec::new(), - &Vec::new(), - true, - &[], - ) - .await -} - -#[when(expr = "I start node {string} to be ready between {int} and {int} seconds")] -async fn step_start_manual_stand_alone_node_not_ready_before( - world: &mut CucumberWorld, - step: &Step, - node_name: String, - min_wait_seconds: u64, - max_wait_seconds: u64, -) -> StepResult { - let start = Instant::now(); - start_node( - world, - &step.value, - &node_name, - &Vec::new(), - &Vec::new(), - false, - &[], - ) - .await?; - - let elapsed = start.elapsed(); - if elapsed < Duration::from_secs(min_wait_seconds) { - return Err(StepError::StepFail { - message: format!( - "Step `{}` error: Node '{node_name}' became ready too early: elapsed {:.2?}, \ - expected at least {min_wait_seconds}s", - step.value, elapsed, - ), - }); - } - if elapsed > Duration::from_secs(max_wait_seconds) { - return Err(StepError::StepFail { - message: format!( - "Step `{}` error: Node '{node_name}' took too long to become ready: elapsed {:.2?}, \ - expected at most {max_wait_seconds}s", - step.value, elapsed, - ), - }); - } - - Ok(()) -} - -#[when( - expr = "I start peer node {string} connected to node {string} to be ready between {int} and {int} seconds" -)] -async fn step_start_manual_peer_node_not_ready_before( - world: &mut CucumberWorld, - step: &Step, - node_name: String, - peer_name: String, - min_wait_seconds: u64, - max_wait_seconds: u64, -) -> StepResult { - let start = Instant::now(); - start_node( - world, - &step.value, - &node_name, - &Vec::new(), - &[peer_name], - false, - &[], - ) - .await?; - - let elapsed = start.elapsed(); - if elapsed < Duration::from_secs(min_wait_seconds) { - return Err(StepError::StepFail { - message: format!( - "Step `{}` error: Node '{node_name}' became ready too early: elapsed {:.2?}, \ - expected at least {min_wait_seconds}s", - step.value, elapsed, - ), - }); - } - if elapsed > Duration::from_secs(max_wait_seconds) { - return Err(StepError::StepFail { - message: format!( - "Step `{}` error: Node '{node_name}' took too long to become ready: elapsed {:.2?}, \ - expected at most {max_wait_seconds}s", - step.value, elapsed, - ), - }); - } - - Ok(()) -} - -#[when(expr = "I connect node {string} to node {string} at runtime")] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step entrypoints must take `&mut World`" -)] -async fn step_connect_nodes_at_runtime( - world: &mut CucumberWorld, - source_node_name: String, - target_node_name: String, -) -> StepResult { - connect_manual_node_to_node(world, &source_node_name, &target_node_name).await -} - -#[then(expr = "node {string} has at least {int} peers within {int} seconds")] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step entrypoints must take `&mut World`" -)] -async fn step_node_has_peers( - world: &mut CucumberWorld, - node_name: String, - min_peers: usize, - timeout_secs: u64, -) -> StepResult { - assert_manual_node_has_peers(world, &node_name, min_peers, timeout_secs).await -} - -#[when(expr = "I restart node {string}")] -async fn step_restart_node( - world: &mut CucumberWorld, - step: &Step, - node_name: String, -) -> StepResult { - restart_node(world, &step.value, &node_name).await?; - if world.blend_diagnostics.observation_count > 0 { - world.blend_diagnostics.stopped_nodes.remove(&node_name); - world.blend_diagnostics.phase = - Some(crate::cucumber::world::BlendDiagnosticPhase::Recovery); - } - Ok(()) -} - -#[when(expr = "I stop node {string}")] -async fn step_stop_node(world: &mut CucumberWorld, step: &Step, node_name: String) -> StepResult { - if world.blend_diagnostics.observation_count > 0 { - world.blend_diagnostics.phase = Some(crate::cucumber::world::BlendDiagnosticPhase::Outage); - } - stop_node(world, &step.value, &node_name).await?; - if world.blend_diagnostics.observation_count > 0 { - world.blend_diagnostics.stopped_nodes.insert(node_name); - } - Ok(()) -} - -#[given(expr = "the cluster uses Blend diagnostic parameter set {string}")] -#[when(expr = "the cluster uses Blend diagnostic parameter set {string}")] -#[expect( - clippy::needless_pass_by_value, - reason = "Cucumber step arguments must use owned types" -)] -fn step_set_blend_diagnostic_parameter_set( - world: &mut CucumberWorld, - step: &Step, - parameter_set_name: String, -) -> StepResult { - set_blend_diagnostic_parameter_set(world, &step.value, ¶meter_set_name) -} - -#[given(expr = "we use IBD peers")] -#[when(expr = "we use IBD peers")] -const fn step_we_use_ibd_peers(world: &mut CucumberWorld) { - world.startup.populate_ibd_peers_from_initial_peers = Some(true); -} - -#[given(expr = "we join an external network")] -#[when(expr = "we join an external network")] -const fn step_we_join_external_network(world: &mut CucumberWorld) { - world.startup.join_external_network = Some(true); -} - -#[given(expr = "we will have distinct node groups to query wallet balances:")] -#[when(expr = "we will have distinct node groups to query wallet balances:")] -fn step_define_node_groups(world: &mut CucumberWorld, step: &Step) -> Result<(), StepError> { - ensure_fee_sponsorship_and_fork_groups_are_not_mixed(world, &step.value)?; - - let table = step.table.as_ref().ok_or(StepError::LogicalError { - message: "Expected a data table".to_owned(), - })?; - - if table.rows.is_empty() || table.rows[0].len() != 2 { - return Err(StepError::LogicalError { - message: "Expected table columns: | group_name | node_name |".to_owned(), - }); - } - - if table.rows[0][0].trim() != "group_name" || table.rows[0][1].trim() != "node_name" { - return Err(StepError::LogicalError { - message: "Expected table columns: | group_name | node_name |".to_owned(), - }); - } - - let assignments = table - .rows - .iter() - .skip(1) - .map(|row| { - if row.len() != 2 { - return Err(StepError::LogicalError { - message: "Each node-group row must have exactly two columns".to_owned(), - }); - } - - Ok((row[0].trim().to_owned(), row[1].trim().to_owned())) - }) - .collect::, _>>()?; - - world.fork_groups.replace_all(assignments) -} - -#[given(expr = "I have user config override {string} as {string}")] -#[when(expr = "I have user config override {string} as {string}")] -#[expect( - clippy::needless_pass_by_value, - reason = "Required by cucumber expression" -)] -fn step_set_user_config_setting( - world: &mut CucumberWorld, - step: &Step, - setting_path: String, - setting_value: String, -) -> StepResult { - set_user_config_override(world, &step.value, &setting_path, &setting_value) -} - -#[given(expr = "I have deployment config override {string} as {string}")] -#[when(expr = "I have deployment config override {string} as {string}")] -#[expect( - clippy::needless_pass_by_value, - reason = "Required by cucumber expression" -)] -fn step_set_deployment_config_setting( - world: &mut CucumberWorld, - step: &Step, - setting_path: String, - setting_value: String, -) -> StepResult { - set_deployment_config_override(world, &step.value, &setting_path, &setting_value) -} - -#[given(expr = "the first {int} nodes are declared as blend providers")] -#[when(expr = "the first {int} nodes are declared as blend providers")] -fn step_blend_provider_count(world: &mut CucumberWorld, provider_count: usize) -> StepResult { - world.cluster.blend_core_nodes = Some(provider_count); - rebuild_pending_local_manual_cluster(world) -} - -#[given(expr = "the cluster uses SDP funding of {int} per provider split across {int} notes")] -#[when(expr = "the cluster uses SDP funding of {int} per provider split across {int} notes")] -fn step_set_sdp_funding( - world: &mut CucumberWorld, - total_value_per_node: u64, - target_notes_per_node: usize, -) -> StepResult { - if target_notes_per_node == 0 { - return Err(StepError::InvalidArgument { - message: "SDP funding note count must be greater than zero".to_owned(), - }); - } - - world.set_sdp_funding_config(SdpFundingConfig::new( - total_value_per_node, - target_notes_per_node, - )); - rebuild_pending_local_manual_cluster(world) -} - -#[given(expr = "no nodes are declared as blend providers")] -#[when(expr = "no nodes are declared as blend providers")] -fn step_no_blend_providers(world: &mut CucumberWorld) -> StepResult { - world.cluster.blend_core_nodes = Some(0); - rebuild_pending_local_manual_cluster(world) -} - -#[given(expr = "I will create a snapshot {string} of all nodes when stopping")] -#[when(expr = "I will create a snapshot {string} of all nodes when stopping")] -#[expect( - clippy::needless_pass_by_value, - reason = "Required by cucumber expression" -)] -fn step_set_snapshot_all_nodes_on_stop( - world: &mut CucumberWorld, - snapshot_name: String, -) -> StepResult { - if snapshot_name.trim().is_empty() { - return Err(StepError::InvalidArgument { - message: "Snapshot name cannot be empty".to_owned(), - }); - } - validate_snapshot_path_component(&snapshot_name, "Snapshot name")?; - let snapshot_name = snapshot_name.trim().to_owned(); - world.snapshots.save.node_state = Some(snapshot_name.clone()); - world.snapshots.save.extensions = Some(snapshot_name); - Ok(()) -} - -#[given(expr = "I will initialize started nodes from snapshot {string} source node {string}")] -#[when(expr = "I will initialize started nodes from snapshot {string} source node {string}")] -#[expect( - clippy::needless_pass_by_value, - reason = "Required by cucumber expression" -)] -fn step_set_node_snapshot_on_startup( - world: &mut CucumberWorld, - snapshot_name: String, - node_name: String, -) -> StepResult { - validate_snapshot_path_component(&snapshot_name, "Snapshot name")?; - validate_snapshot_path_component(&node_name, "Node name")?; - - let snapshot_name = snapshot_name.trim().to_owned(); - world.snapshots.node_snapshot_on_startup = Some(NodeSnapshot { - name: snapshot_name.clone(), - node: node_name.trim().to_owned(), - }); - world.snapshots.restore.extensions = Some(snapshot_name); - if let Some(snapshot_name) = world.snapshots.restore.extensions.clone() { - prepare_wallet_snapshot_restore_if_present(&snapshot_name, world)?; - } - Ok(()) -} - -#[given(expr = "I create a snapshot {string} of all nodes")] -#[when(expr = "I create a snapshot {string} of all nodes")] -async fn step_create_snapshot_all_nodes_now( - world: &mut CucumberWorld, - snapshot_name: String, -) -> StepResult { - create_snapshot_all_nodes_with_wallet_state(world, &snapshot_name).await -} - -#[given(expr = "I create a snapshot {string} of node {string}")] -#[when(expr = "I create a snapshot {string} of node {string}")] -#[then(expr = "I create a snapshot {string} of node {string}")] -async fn step_create_snapshot_node_now( - world: &mut CucumberWorld, - snapshot_name: String, - node_name: String, -) -> StepResult { - create_snapshot_node_with_wallet_state(world, &snapshot_name, &node_name).await -} - -#[given("I have public cryptarchia endpoint peers:")] -#[when("I have public cryptarchia endpoint peers:")] -fn step_set_public_cryptarchia_endpoint_peers( - world: &mut CucumberWorld, - step: &Step, -) -> StepResult { - let table = step.table.as_ref().ok_or(StepError::MissingTable)?; - - if table.rows.is_empty() { - return Err(StepError::InvalidArgument { - message: format!( - "Step `{step}` error: public cryptarchia endpoint peers table cannot be empty" - ), - }); - } - if table.rows.iter().any(|row| row.len() != 3) { - return Err(StepError::InvalidArgument { - message: format!( - "Step `{step}` error: public cryptarchia endpoint peers table must have exactly three columns" - ), - }); - } - if !matches!(table.rows[0][0].trim(), PUBLIC_CRYPTARCHIA_ENDPOINT) - || !matches!( - table.rows[0][1].trim(), - PUBLIC_CRYPTARCHIA_ENDPOINT_USERNAME - ) - || !matches!( - table.rows[0][2].trim(), - PUBLIC_CRYPTARCHIA_ENDPOINT_PASSWORD - ) - { - return Err(StepError::InvalidArgument { - message: format!( - "Step `{step}` error: public cryptarchia endpoint peers table header row must be \ - '{PUBLIC_CRYPTARCHIA_ENDPOINT}', '{PUBLIC_CRYPTARCHIA_ENDPOINT_USERNAME}', \ - '{PUBLIC_CRYPTARCHIA_ENDPOINT_PASSWORD}'" - ), - }); - } - - let mut endpoint_peers = Vec::with_capacity(table.rows.len().saturating_sub(1)); - for row in table.rows.iter().skip(1) { - let url = parse_url(&row[0]).map_err(|e| StepError::InvalidArgument { - message: format!( - "Step `{}` error: invalid public cryptarchia endpoint '{}': {e}", - step.value, row[0] - ), - })?; - - let username = - resolve_literal_or_env(row[1].trim(), "public cryptarchia endpoint username").map_err( - |e| StepError::InvalidArgument { - message: format!("Step `{}` error: {e}", step.value), - }, - )?; - if username.is_empty() { - return Err(StepError::InvalidArgument { - message: format!( - "Step `{}` error: username cannot be empty for public cryptarchia endpoint '{}'", - step.value, url - ), - }); - } - - let password = - resolve_literal_or_env(row[2].trim(), "public cryptarchia endpoint password").map_err( - |e| StepError::InvalidArgument { - message: format!("Step `{}` error: {e}", step.value), - }, - )?; - if password.is_empty() { - return Err(StepError::InvalidArgument { - message: format!( - "Step `{}` error: password cannot be empty for public cryptarchia endpoint '{}'", - step.value, url - ), - }); - } - - endpoint_peers.push(PublicCryptarchiaEndpointPeer { - base_url: url, - username, - password, - }); - } - - if endpoint_peers.is_empty() { - return Err(StepError::InvalidArgument { - message: format!( - "Step `{step}` error: at least one public cryptarchia endpoint peer is required" - ), - }); - } - world.startup.public_cryptarchia_endpoint_peers = Some(endpoint_peers); - - Ok(()) -} - -#[given(expr = "all peers must be mode online after startup in {int} seconds")] -#[when(expr = "all peers must be mode online after startup in {int} seconds")] -const fn step_all_nodes_to_be_mode_online(world: &mut CucumberWorld, on_line_time_out: u64) { - world.startup.require_all_peers_mode_online_at_startup = - Some(Duration::from_secs(on_line_time_out)); -} - -#[given("I have initial peers:")] -#[when("I have initial peers:")] -fn step_set_initial_peers(world: &mut CucumberWorld, step: &Step) -> StepResult { - let table = step.table.as_ref().ok_or(StepError::MissingTable)?; - if table.rows.is_empty() || table.rows[0].len() != 1 || table.rows[0][0] != "initial_peer" { - return Err(StepError::InvalidArgument { - message: format!( - "Step `{}` error: initial peers table header must be `initial_peer`", - step.value - ), - }); - } - - let mut peers = Vec::with_capacity(table.rows.len().saturating_sub(1)); - for row in table.rows.iter().skip(1) { - let peer = row[0] - .trim() - .parse::() - .map_err(|e| StepError::InvalidArgument { - message: format!( - "Step `{}` error: invalid initial peer '{}': {e}", - step.value, row[0] - ), - })?; - peers.push(peer); - } - - world.startup.initial_peers_override = Some(peers); - Ok(()) -} - -#[given("I have IBD peers:")] -#[when("I have IBD peers:")] -fn step_set_ibd_peers(world: &mut CucumberWorld, step: &Step) -> StepResult { - let table = step.table.as_ref().ok_or(StepError::MissingTable)?; - if table.rows.is_empty() || table.rows[0].len() != 1 || table.rows[0][0] != "ibd_peer" { - return Err(StepError::InvalidArgument { - message: format!( - "Step `{}` error: IBD peers table header must be `ibd_peer`", - step.value - ), - }); - } - - let mut peers = HashSet::with_capacity(table.rows.len().saturating_sub(1)); - for row in table.rows.iter().skip(1) { - let peer = row[0] - .trim() - .parse::() - .map_err(|e| StepError::InvalidArgument { - message: format!( - "Step `{}` error: invalid IBD peer '{}': {e}", - step.value, row[0] - ), - })?; - peers.insert(peer); - } - - world.startup.ibd_peers_override = Some(peers); - world.startup.populate_ibd_peers_from_initial_peers = Some(true); - Ok(()) -} - -#[given(expr = "I start peer node {string} connected to node {string}")] -#[when(expr = "I start peer node {string} connected to node {string}")] -async fn step_start_manual_connected_node( - world: &mut CucumberWorld, - step: &Step, - node_name: String, - peer_name: String, -) -> StepResult { - start_node( - world, - &step.value, - &node_name, - &Vec::new(), - &[peer_name], - false, - &[], - ) - .await -} - -#[given(expr = "I immediate start peer node {string} connected to node {string}")] -#[when(expr = "I immediate start peer node {string} connected to node {string}")] -async fn step_immediate_start_manual_connected_node( - world: &mut CucumberWorld, - step: &Step, - node_name: String, - peer_name: String, -) -> StepResult { - start_node( - world, - &step.value, - &node_name, - &Vec::new(), - &[peer_name], - true, - &[], - ) - .await -} - -#[given(expr = "I start peer node {string} connected to node {string} and node {string}")] -#[when(expr = "I start peer node {string} connected to node {string} and node {string}")] -async fn step_start_manual_two_connected_nodes( - world: &mut CucumberWorld, - step: &Step, - node_name: String, - peer_name1: String, - peer_name2: String, -) -> StepResult { - start_node( - world, - &step.value, - &node_name, - &Vec::new(), - &[peer_name1, peer_name2], - false, - &[], - ) - .await -} - -#[given(expr = "I immediate start peer node {string} connected to node {string} and node {string}")] -#[when(expr = "I immediate start peer node {string} connected to node {string} and node {string}")] -async fn step_immediate_start_manual_two_connected_nodes( - world: &mut CucumberWorld, - step: &Step, - node_name: String, - peer_name1: String, - peer_name2: String, -) -> StepResult { - start_node( - world, - &step.value, - &node_name, - &Vec::new(), - &[peer_name1, peer_name2], - true, - &[], - ) - .await -} - -#[when(expr = "I wait for all nodes to be responsive in {int} seconds")] -#[then(expr = "I wait for all nodes to be responsive in {int} seconds")] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require the world as the first `&mut` argument" -)] -async fn step_wait_all_nodes_responsive( - world: &mut CucumberWorld, - step: &Step, - time_out_seconds: u64, -) -> StepResult { - let cluster = world - .cluster - .local_cluster - .as_ref() - .ok_or(StepError::LogicalError { - message: "No local cluster available".into(), - })?; - if let Err(e) = wait_all_nodes_responive(cluster, Duration::from_secs(time_out_seconds)).await { - return Err(StepError::StepFail { - message: format!("Step `{}` error: {e}", step.value), - }); - } - - let wait_tasks: Vec<_> = world - .nodes_info - .values() - .map(|node| { - let fut = verify_reponsive_and_network_ready_with_timeout( - &node.started_node.client, - &node.name, - &node.started_node.name, - Duration::from_secs(time_out_seconds), - ); - let step_value = step.value.clone(); - async move { - fut.await.map_err(|e| StepError::StepFail { - message: format!("Step `{step_value}` error: {e}"), - }) - } - }) - .collect(); - - futures::future::try_join_all(wait_tasks).await?; - - Ok(()) -} - -#[when(expr = "node {string} is at height {int} in {int} seconds")] -#[then(expr = "node {string} is at height {int} in {int} seconds")] -async fn step_node_is_at_height( - world: &mut CucumberWorld, - step: &Step, - node_name: String, - height: u64, - time_out_seconds: u64, -) -> StepResult { - let start = Instant::now(); - let time_out = Duration::from_secs(time_out_seconds); - - let mut count = 0usize; - loop { - poll_all_nodes_and_update_consensus_cache(&step.value, &mut world.nodes_info).await?; - let best_height = world.node_best_height(&node_name)?.unwrap_or_default(); - if best_height >= height { - info!( - target: TARGET, - "Node '{node_name}' reached height {height} in {:.2?}", - start.elapsed() - ); - return Ok(()); - } else if count.is_multiple_of(50) { - info!( - target: TARGET, - "Waiting for '{node_name}' to reach height {height} - elapsed: {:.2?}, current \ - height: {}", start.elapsed(), best_height - ); - } - - if start.elapsed() >= time_out { - return Err(StepError::StepFail { - message: format!( - "Step `{}` error: Node '{node_name}' did not reach height {height} in {time_out_seconds} s", - step.value - ), - }); - } - sleep(Duration::from_millis(100)).await; - count += 1; - } -} - -#[when(expr = "node {string} is exactly at height")] -#[then(expr = "node {string} is exactly at height")] -async fn step_node_is_exactly_at_height( - world: &mut CucumberWorld, - step: &Step, - node_name: String, - height: u64, -) -> StepResult { - poll_all_nodes_and_update_consensus_cache(&step.value, &mut world.nodes_info).await?; - let node_height = world.node_best_height(&node_name)?.unwrap_or_default(); - if node_height != height { - return Err(StepError::StepFail { - message: format!( - "Step `{}` error: Node '{node_name}' is at height {node_height}, required {height}", - step.value - ), - }); - } - Ok(()) -} - -#[when(expr = "all nodes converged to within {int} blocks in {int} seconds")] -#[then(expr = "all nodes converged to within {int} blocks in {int} seconds")] -async fn step_all_nodes_converged( - world: &mut CucumberWorld, - step: &Step, - max_diff_height: u64, - time_out_seconds: u64, -) -> StepResult { - nodes_converged(world, &step.value, None, max_diff_height, time_out_seconds).await -} - -#[when( - expr = "all nodes have at least {int} blocks and converged to within {int} blocks in {int} seconds" -)] -#[then( - expr = "all nodes have at least {int} blocks and converged to within {int} blocks in {int} seconds" -)] -async fn step_all_nodes_reached_min_height_and_converged( - world: &mut CucumberWorld, - step: &Step, - min_height: u64, - max_diff_height: u64, - time_out_seconds: u64, -) -> StepResult { - nodes_converged( - world, - &step.value, - Some(min_height), - max_diff_height, - time_out_seconds, - ) - .await -} - -#[when(expr = "all nodes agree on LIB in {int} seconds")] -#[then(expr = "all nodes agree on LIB in {int} seconds")] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require the world as the first `&mut` argument" -)] -async fn step_all_nodes_agree_on_lib( - world: &mut CucumberWorld, - step: &Step, - time_out_seconds: u64, -) -> StepResult { - ensure_all_nodes_agree_on_lib(world, &step.value, time_out_seconds).await -} - -#[when("I wait for all nodes to be synced to the chain")] -#[then("I wait for all nodes to be synced to the chain")] -async fn step_wait_for_all_nodes_to_be_synced_to_the_chain( - world: &mut CucumberWorld, - step: &Step, -) -> StepResult { - wait_for_all_nodes_to_be_synced_to_chain(world, &step.value).await -} - -#[when("I query cryptarchia info for all nodes")] -#[then("I query cryptarchia info for all nodes")] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require the world as the first `&mut` argument" -)] -async fn step_query_cryptarchia_info_all_nodes(world: &mut CucumberWorld, step: &Step) { - get_cryptarchia_info_all_nodes(world, &step.value).await; -} - -#[then(expr = "I stop all nodes")] -async fn step_stop_all_nodes(world: &mut CucumberWorld) -> StepResult { - let runtime_dir_by_node_name: Vec<(String, String)> = world - .nodes_info - .iter() - .map(|(node_name, info)| (node_name.clone(), info.started_node.name.clone())) - .collect(); - - if world.snapshots.save.extensions.is_some() { - prepare_all_wallets_snapshot(world).await?; - } - - world.reset_wallet_scanner_after_current_iteration().await; - world.zone.clear(); - stop_active_manual_cluster(world)?; - - if let Some(snapshot_name) = world.snapshots.save.node_state.take() { - create_snapshots_all_nodes(world, &snapshot_name)?; - } - - if let Some(snapshot_name) = world.snapshots.save.extensions.take() { - save_prepared_all_wallets_snapshot(&snapshot_name, world)?; - } - - for (node_name, _) in &runtime_dir_by_node_name { - info!(target: TARGET, "Stopping node '{node_name}'"); - } - world.nodes_info.clear(); - - Ok(()) -} - -#[when( - expr = "I send {int} transactions of {int} LGO each from wallet {string} to blend core zk key of node {string}" -)] -async fn step_send_multiple_transactions_to_blend_core_zk_key( - world: &mut CucumberWorld, - step: &Step, - number_of_transactions: usize, - output_value: u64, - sender_wallet_name: String, - receiver_node_name: String, -) -> StepResult { - let receiver_blend_zk_pk = blend_zk_pk_for_node(world, &receiver_node_name)?; - let sender_node_name = world.resolve_wallet(&sender_wallet_name)?.node_name; - let sender_node_client = world - .nodes_info - .get(&sender_node_name) - .ok_or_else(|| StepError::LogicalError { - message: format!("Node '{sender_node_name}' not found in world state"), - })? - .started_node - .client - .clone(); - - let mut available_utxos = WalletUtxos::new(); - let best_node_info = wait_wallet_send_ready( - world, - &step.value, - &sender_wallet_name, - 180, - number_of_transactions as u64 * output_value, - WalletSendReadiness::TotalValueOnly, - &mut available_utxos, - &HashSet::new(), - ) - .await?; - - for _ in 0..number_of_transactions { - let tx_hashes = create_and_submit_transaction_hashes_with_utxo_cache( - world, - &step.value, - &sender_wallet_name, - &[(receiver_blend_zk_pk, output_value)], - Some(&best_node_info), - Some(&mut available_utxos), - ) - .await - .inspect_err(|error| { - warn!(target: TARGET, "Step `{}` error: {error}", step.value); - })?; - - wait_for_transactions_inclusion(&sender_node_client, &tx_hashes, Duration::from_mins(2)) - .await - .inspect_err(|error| { - warn!(target: TARGET, "Step `{}` error: {error}", step.value); - })?; - - info!( - target: TARGET, - "Sent and included normal transaction from `{sender_wallet_name}` to blend zk key of {receiver_node_name}, value: {output_value}, tx count: {}", - tx_hashes.len(), - ); - } - - Ok(()) -} - -fn blend_zk_pk_for_node(world: &CucumberWorld, node_name: &str) -> Result { - let node_info = world - .nodes_info - .get(node_name) - .ok_or_else(|| StepError::LogicalError { - message: format!("Node '{node_name}' not found in world state"), - })?; - - let user_config_path = node_info.runtime_dir.join(USER_CONFIG_FILE); - let blend_zk_pk_hex = blend_core_zk_pk_from_node_yaml(&user_config_path)?; - let blend_zk_pk = ZkPublicKey::from_bytes(&hex::decode(blend_zk_pk_hex)?)?; - - Ok(blend_zk_pk) -} - -/// Wait for the node-local wallet API to expose a funded note for a Blend key. -/// -/// Blend ZK keys are read from node configuration and are not scenario wallets, -/// so the wallet scanner does not currently track them. Keep this exception -/// node-local because the returned note is immediately consumed by that node's -/// SDP declaration endpoint. -async fn wait_for_blend_funded_note( - world: &CucumberWorld, - node_name: &str, - blend_zk_pk: ZkPublicKey, -) -> Result { - let base_url = world - .nodes_info - .get(node_name) - .ok_or_else(|| StepError::LogicalError { - message: format!("Node '{node_name}' not found in world state"), - })? - .started_node - .client - .base_url() - .clone(); - let timeout = Duration::from_secs(30); - let started = Instant::now(); - let client = CommonHttpClient::new(None); - - loop { - let last_error = match client - .get_wallet_balance(base_url.clone(), blend_zk_pk, None) - .await - { - Ok(wallet_balance) => { - if let Some(note_id) = wallet_balance.notes.keys().next().copied() { - return Ok(note_id); - } - "wallet has no notes yet".to_owned() - } - Err(error) => error.to_string(), - }; - - if started.elapsed() >= timeout { - return Err(StepError::Timeout { - message: format!( - "Timed out waiting for a funded note on Blend ZK key of '{node_name}' via \ - '{base_url}' (last error: {last_error})" - ), - }); - } - - sleep(Duration::from_millis(250)).await; - } -} - -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Required to be mutable by cucumber step function signature" -)] -#[expect(unused_variables, reason = "Cucumber step function signature")] -#[then(expr = "I declare node {string} as blend core node via the CLI binary")] -async fn step_run_blend_sdp_declaration_cli( - world: &mut CucumberWorld, - step: &Step, - declarer_node_name: String, -) -> StepResult { - let user_config_path = node_user_config_path(world, &declarer_node_name)?; - let locator = blend_core_locator_from_node_yaml(&user_config_path)?; - let blend_zk_pk = blend_zk_pk_for_node(world, &declarer_node_name)?; - let service_note_id = - wait_for_blend_funded_note(world, &declarer_node_name, blend_zk_pk).await?; - let service_note_id_json = - serde_json::to_string(&service_note_id).map_err(|error| StepError::LogicalError { - message: format!("Failed to serialize service note ID: {error}"), - })?; - let service_note_id_hex = service_note_id_json.trim_matches('"').to_owned(); - - let declarer_api_base_url = world - .nodes_info - .get(&declarer_node_name) - .ok_or_else(|| StepError::LogicalError { - message: format!("Node '{declarer_node_name}' not found in world state"), - })? - .started_node - .client - .base_url() - .clone(); - - let workspace_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(".."); - let output = tokio::process::Command::new("cargo") - .current_dir(workspace_root) - .arg("run") - .arg("-p") - .arg("logos-blockchain-tools") - .arg("--bin") - .arg("logos-blockchain-tools-api") - .arg("--") - .arg("sdp") - .arg("post-blend-declaration") - .arg("--user-config-path") - .arg(user_config_path) - .arg("--blend-addr") - .arg(format!("{locator}")) - .arg("--service-note-id") - .arg(service_note_id_hex) - .arg("--node-address") - .arg(declarer_api_base_url.to_string()) - .output() - .await?; - - if !output.status.success() { - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(StepError::StepFail { - message: format!( - "Blend declaration CLI failed for node '{declarer_node_name}'\nstdout:\n{stdout}\nstderr:\n{stderr}" - ), - }); - } - - Ok(()) -} - -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Required to be mutable by cucumber step function signature" -)] -#[then(expr = "I declare node {string} as blend core node via the API")] -async fn step_run_blend_sdp_declaration_api( - world: &mut CucumberWorld, - step: &Step, - declarer_node_name: String, -) -> StepResult { - let user_config_path = node_user_config_path(world, &declarer_node_name)?; - let locator = blend_core_locator_from_node_yaml(&user_config_path)?; - let blend_zk_pk = blend_zk_pk_for_node(world, &declarer_node_name)?; - let service_note_id = - wait_for_blend_funded_note(world, &declarer_node_name, blend_zk_pk).await?; - - let declarer_node_client = world - .nodes_info - .get(&declarer_node_name) - .ok_or_else(|| StepError::LogicalError { - message: format!("Node '{declarer_node_name}' not found in world state"), - })? - .started_node - .client - .clone(); - - let declaration_id = declarer_node_client - .join_blend_network(locator, service_note_id) - .await - .inspect_err(|error| { - warn!(target: TARGET, "Step `{}` error: {error}", step.value); - })?; - - info!( - target: TARGET, - "Node '{declarer_node_name}' joined blend core via API, declaration id: {declaration_id}" - ); - - Ok(()) -} - -fn node_user_config_path(world: &CucumberWorld, node_name: &str) -> Result { - let node_info = world - .nodes_info - .get(node_name) - .ok_or_else(|| StepError::LogicalError { - message: format!("Node '{node_name}' not found in world state"), - })?; - - Ok(node_info.runtime_dir.join(USER_CONFIG_FILE)) -} - -#[expect( - clippy::cognitive_complexity, - reason = "TODO: Address this at some point." -)] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Required to be mutable by cucumber step function signature" -)] -#[then(expr = "blend core SDP declaration for node {string} is included on node {string}")] -async fn step_verify_blend_sdp_declaration_included( - world: &mut CucumberWorld, - step: &Step, - declarer_node_name: String, - api_node_name: String, -) -> StepResult { - let blend_zk_pk = blend_zk_pk_for_node(world, &declarer_node_name)?; - let service_note_id = - wait_for_blend_funded_note(world, &declarer_node_name, blend_zk_pk).await?; - - let step_timeout = Duration::from_secs(30); - let start_time = Instant::now(); - loop { - let declarations_result = world - .nodes_info - .get(&api_node_name) - .ok_or_else(|| StepError::LogicalError { - message: format!("Node '{api_node_name}' not found in world state"), - })? - .started_node - .client - .get_sdp_declarations() - .await; - - let declarations = match declarations_result { - Ok(declarations) => declarations, - Err(error) => { - let error_message = error.to_string(); - if error_message.contains("404 Not Found") { - info!( - target: TARGET, - "Skipping declaration visibility assertion on '{api_node_name}' because testing SDP endpoint is unavailable: {error_message}", - ); - return Ok(()); - } - - warn!(target: TARGET, "Step `{}` error: {error}", step.value); - return Err(error.into()); - } - }; - - if declarations.values().any(|declaration| { - declaration.service_note_id == service_note_id && declaration.zk_id == blend_zk_pk - }) { - info!( - target: TARGET, - "Blend declaration observed for node '{declarer_node_name}'" - ); - break; - } - - if start_time.elapsed() >= step_timeout { - return Err(StepError::Timeout { - message: format!( - "Timed out waiting for declaration submitted by '{declarer_node_name}' to appear on node '{api_node_name}'" - ), - }); - } - - sleep(Duration::from_millis(250)).await; - } - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn step_relative_genesis_time_uses_now_plus_offset() { - let now = OffsetDateTime::from_unix_timestamp(1_000).expect("valid timestamp"); - - let genesis_time = - resolve_step_genesis_time("the chain starts 60 seconds from now", now, 60) - .expect("offset should produce a valid genesis time"); - - assert_eq!(genesis_time, GenesisTime::new(1_060)); - } - - #[test] - fn overflowing_step_relative_genesis_time_is_invalid_argument() { - let now = time::Date::MAX.midnight().assume_utc(); - - let error = resolve_step_genesis_time("the chain starts 1 seconds from now", now, 1) - .expect_err("overflowing offset should fail"); - - assert!(matches!(error, StepError::InvalidArgument { .. })); - } - - #[test] - fn genesis_time_change_after_nodes_started_is_rejected() { - let error = validate_genesis_time_change( - Some(GenesisTime::new(1_000)), - true, - GenesisTime::new(1_001), - ) - .expect_err("genesis time should not change after nodes start"); - - assert!( - matches!(error, StepError::LogicalError { message } if message == "cannot change genesis time after nodes have started") - ); - } -} diff --git a/tests/src/cucumber/steps/manual_nodes/utils.rs b/tests/src/cucumber/steps/manual_nodes/utils.rs deleted file mode 100644 index 091644bad..000000000 --- a/tests/src/cucumber/steps/manual_nodes/utils.rs +++ /dev/null @@ -1,2335 +0,0 @@ -use std::{ - collections::{HashMap, HashSet}, - fs, - net::{IpAddr, Ipv4Addr, SocketAddr, TcpStream}, - num::NonZero, - path::{Path, PathBuf}, - time::{Duration, Instant}, -}; - -use cucumber::gherkin::Table; -use futures::future::try_join_all; -use hex::ToHex as _; -use lb_chain_service::{ChainServiceInfo, CryptarchiaInfo, PhaseTag}; -use lb_config::kms::key_id_for_preload_backend; -use lb_core::mantle::{Utxo, ops::OpId as _, traits::GenesisTx as _}; -use lb_http_api_common::paths::CRYPTARCHIA_INFO; -use lb_key_management_system_service::{backend::preload::KeyId, keys::Key}; -use lb_libp2p::PeerId; -use lb_node::config::{ - DeploymentSettings, RunConfig, - tracing::serde::console::{Layer as ConsoleLayer, TokioConfig}, -}; -use lb_testing_framework::{ - LbcEnv, LbcManualCluster, NodeHttpClient, USER_CONFIG_FILE, configs::wallet::WalletAccount, -}; -use libp2p::Multiaddr; -use reqwest::{Client, Url}; -use testing_framework_core::scenario::{PeerSelection, StartNodeOptions, StartedNode}; -use tokio::time::{Instant as TokioInstant, sleep, timeout}; -use tracing::{info, warn}; - -use crate::cucumber::{ - error::{StepError, StepResult}, - steps::{ - TARGET, - manual_nodes::{ - config_override::{apply_deployment_config_overrides, apply_user_config_overrides}, - diagnostics::log_node_lifecycle_marker, - snapshots::{ - reset_named_snapshot, restore_node_state_from_snapshot, - save_named_node_state_snapshot, validate_snapshot_path_component, - }, - }, - tokio_console::profile::TokioConsoleProfileNode, - }, - utils::{ - display_last_path_components, extract_child_dir_name, matching_child_dirs, - node_wallet_keys_from_node_yaml, peer_id_from_node_yaml, track_progress, truncate_hash, - }, - wallet::snapshot::{create_and_save_all_wallets_snapshot, restore_wallet_snapshot_if_present}, - world::{ - ChainInfoMap, ConfigOverride, CucumberWorld, ManualNodeConfigOverrides, NodeInfo, - NodeWalletKey, NodeWalletKeyRole, PublicCryptarchiaEndpointPeer, WalletInfo, WalletInfoMap, - WalletType, - }, -}; - -pub(crate) type NodesToStartUnordered = HashMap, Vec)>; -type NodesToStartOrdered = Vec<(String, Vec, Vec)>; - -const CHAIN_SYNC_POLL_INTERVAL: Duration = Duration::from_secs(5); -const CHAIN_SYNC_STATUS_LOG_INTERVAL: Duration = Duration::from_mins(2); - -// Returns the root directory for a named snapshot. - -enum AlignmentStatus { - MissingChainInfo, - Fork, - Aligned, -} - -#[derive(Debug, Clone)] -struct ConsensusSnapshot { - node_name: String, - height: u64, - header_hash: String, -} - -#[derive(Debug, Clone)] -struct MaybeSnapshot { - height: u64, - header_hash: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -struct SyncTargetStats { - lib: String, - tip: String, - slot: u64, - height: u64, -} - -#[derive(Debug, Clone)] -struct PublicPeerConsensusSnapshot { - peer_url: String, - stats: SyncTargetStats, -} - -#[derive(Debug, Clone)] -struct MajorityPublicSyncTarget { - peer_urls: Vec, - stats: SyncTargetStats, -} - -impl SyncTargetStats { - fn from_cryptarchia_info(info: &CryptarchiaInfo) -> Self { - Self { - lib: info.lib.encode_hex::(), - tip: info.tip.encode_hex::(), - slot: info.slot.into_inner(), - height: info.height, - } - } -} - -#[must_use] -pub(crate) fn genesis_block_utxos( - genesis_tx: &lb_core::mantle::transactions::GenesisTx, -) -> Vec { - let transfer_op = genesis_tx.genesis_transfer().clone(); - let transfer_id = transfer_op.op_id(); - - transfer_op - .outputs - .iter() - .enumerate() - .map(|(idx, note)| Utxo::new(transfer_id, idx, *note)) - .collect() -} - -const ACCOUNT_INDEX: &str = "account_index"; -const ACCOUNT_INDEX_IDX_T1: usize = 0; -const TOKEN_COUNT: &str = "token_count"; -const TOKEN_COUNT_IDX: usize = 1; -const TOKEN_AMOUNT: &str = "token_amount"; -const TOKEN_AMOUNT_IDX: usize = 2; - -pub(crate) fn verify_genesis_wallet_resources_table_indexes( - table: &Table, - step: &str, -) -> Result<(), StepError> { - if table.rows.is_empty() - || table.rows[0].len() != 3 - || table.rows[0][ACCOUNT_INDEX_IDX_T1] != ACCOUNT_INDEX - || table.rows[0][TOKEN_COUNT_IDX] != TOKEN_COUNT - || table.rows[0][TOKEN_AMOUNT_IDX] != TOKEN_AMOUNT - { - return Err(StepError::InvalidArgument { - message: format!( - "Step `{step}` error: Wallet resources table must have a header row with columns: \ - {ACCOUNT_INDEX}, {TOKEN_COUNT}, {TOKEN_AMOUNT}" - ), - }); - } - // All wallet account indexes must be unique - let wallet_accounts: HashSet<_> = table - .rows - .iter() - .map(|row| &row[ACCOUNT_INDEX_IDX_T1]) - .collect(); - if wallet_accounts.len() != table.rows.len() { - return Err(StepError::InvalidArgument { - message: format!( - "Step `{step}` error: Duplicate {ACCOUNT_INDEX} indexes found in the table" - ), - }); - } - - Ok(()) -} - -pub(crate) fn parse_genesis_wallet_tokens_row( - step: &str, - row: &[String], -) -> Result<(usize, usize, u64), StepError> { - let account_index = - row[ACCOUNT_INDEX_IDX_T1] - .parse::() - .map_err(|_| StepError::InvalidArgument { - message: format!("Step `{step}` error: {ACCOUNT_INDEX} must be a valid number"), - })?; - let token_count = - row[TOKEN_COUNT_IDX] - .parse::() - .map_err(|_| StepError::InvalidArgument { - message: format!("Step `{step}` error: {TOKEN_COUNT} must be a valid number"), - })?; - let token_amount = - row[TOKEN_AMOUNT_IDX] - .parse::() - .map_err(|_| StepError::InvalidArgument { - message: format!("Step `{step}` error: {TOKEN_AMOUNT} must be a valid number"), - })?; - Ok((account_index, token_count, token_amount)) -} - -const NODE_NAME: &str = "node_name"; -const NODE_NAME_IDX: usize = 0; -const ACCOUNT_INDEX_IDX_T2: usize = 1; -const WALLET_NAME: &str = "wallet_name"; -const WALLET_NAME_IDX: usize = 2; -const CONNECTED_TO: &str = "connected_to"; -const CONNECTED_TO_IDX: usize = 3; - -// Mining-node wallet-resources table adds an `is_mining_wallet` column between -// `wallet_name` and `connected_to`. -const IS_MINING_WALLET: &str = "is_mining_wallet"; -const IS_MINING_WALLET_IDX: usize = 3; -const MINING_CONNECTED_TO_IDX: usize = 4; - -pub(crate) fn verify_node_wallet_resources_table_indexes( - table: &Table, - step: &str, -) -> Result<(), StepError> { - if table.rows.is_empty() - || table.rows[0].len() != 4 - || table.rows[0][NODE_NAME_IDX] != NODE_NAME - || table.rows[0][ACCOUNT_INDEX_IDX_T2] != ACCOUNT_INDEX - || table.rows[0][WALLET_NAME_IDX] != WALLET_NAME - || table.rows[0][CONNECTED_TO_IDX] != CONNECTED_TO - { - return Err(StepError::InvalidArgument { - message: format!( - "Step `{step}` error: Wallet resources table must have a header row with columns: {NODE_NAME}, {ACCOUNT_INDEX}, {WALLET_NAME}, {CONNECTED_TO}" - ), - }); - } - // All wallet indexes must be unique - let account_indexes: HashSet<_> = table - .rows - .iter() - .map(|row| &row[ACCOUNT_INDEX_IDX_T2]) - .collect(); - if account_indexes.len() != table.rows.len() { - return Err(StepError::InvalidArgument { - message: format!( - "Step `{step}` error: Duplicate {ACCOUNT_INDEX} indexes found in the table" - ), - }); - } - // All wallet names must be unique - let wallet_names: HashSet<_> = table.rows.iter().map(|row| &row[WALLET_NAME_IDX]).collect(); - if wallet_names.len() != table.rows.len() { - return Err(StepError::InvalidArgument { - message: format!( - "Step `{step}` error: Duplicate {WALLET_NAME} indexes found in the table" - ), - }); - } - // node_name and connected_to must be different - for row in table.rows.iter().skip(1) { - let node_name = row[NODE_NAME_IDX].trim(); - let connected_to = row - .get(CONNECTED_TO_IDX) - .map(|s| s.trim()) - .filter(|s| !s.is_empty()); - if let Some(peer) = connected_to - && peer == node_name - { - return Err(StepError::InvalidArgument { - message: format!( - "Step `{step}` error: {NODE_NAME} and {CONNECTED_TO} cannot be the same" - ), - }); - } - } - - Ok(()) -} - -pub(crate) fn parse_wallet_resources_table_row( - step: &str, - row: &[String], -) -> Result<(String, WalletStartInfo, Option), StepError> { - let node_name = row[NODE_NAME_IDX].trim().to_owned(); - if node_name.is_empty() { - return Err(StepError::InvalidArgument { - message: format!("Step `{step}` error: {NODE_NAME} cannot be empty"), - }); - } - let account_index = row[ACCOUNT_INDEX_IDX_T2] - .trim() - .parse::() - .map_err(|_| StepError::InvalidArgument { - message: format!( - "Step `{step}` error: {ACCOUNT_INDEX} '{}' must be a valid number", - row[ACCOUNT_INDEX_IDX_T2] - ), - })?; - let wallet_name = row[WALLET_NAME_IDX].trim().to_owned(); - if wallet_name.is_empty() { - return Err(StepError::InvalidArgument { - message: format!("Step `{step}` error: {WALLET_NAME} cannot be empty"), - }); - } - let connected_to = row - .get(CONNECTED_TO_IDX) - .map(|s| s.trim()) - .filter(|s| !s.is_empty()) - .map(str::to_owned); - - Ok(( - node_name, - WalletStartInfo { - wallet_name, - account_index, - }, - connected_to, - )) -} - -pub(crate) fn verify_mining_node_wallet_resources_table_indexes( - table: &Table, - step: &str, -) -> Result<(), StepError> { - if table.rows.is_empty() - || table.rows[0].len() != 5 - || table.rows[0][NODE_NAME_IDX] != NODE_NAME - || table.rows[0][ACCOUNT_INDEX_IDX_T2] != ACCOUNT_INDEX - || table.rows[0][WALLET_NAME_IDX] != WALLET_NAME - || table.rows[0][IS_MINING_WALLET_IDX] != IS_MINING_WALLET - || table.rows[0][MINING_CONNECTED_TO_IDX] != CONNECTED_TO - { - return Err(StepError::InvalidArgument { - message: format!( - "Step `{step}` error: Mining wallet resources table must have a header row with columns: {NODE_NAME}, {ACCOUNT_INDEX}, {WALLET_NAME}, {IS_MINING_WALLET}, {CONNECTED_TO}" - ), - }); - } - // All wallet indexes must be unique. - let account_indexes: HashSet<_> = table - .rows - .iter() - .map(|row| &row[ACCOUNT_INDEX_IDX_T2]) - .collect(); - if account_indexes.len() != table.rows.len() { - return Err(StepError::InvalidArgument { - message: format!( - "Step `{step}` error: Duplicate {ACCOUNT_INDEX} indexes found in the table" - ), - }); - } - // All wallet names must be unique. - let wallet_names: HashSet<_> = table.rows.iter().map(|row| &row[WALLET_NAME_IDX]).collect(); - if wallet_names.len() != table.rows.len() { - return Err(StepError::InvalidArgument { - message: format!( - "Step `{step}` error: Duplicate {WALLET_NAME} indexes found in the table" - ), - }); - } - - Ok(()) -} - -pub(crate) fn parse_mining_wallet_resources_table_row( - step: &str, - row: &[String], -) -> Result<(String, WalletStartInfo, bool, Option), StepError> { - let node_name = row[NODE_NAME_IDX].trim().to_owned(); - if node_name.is_empty() { - return Err(StepError::InvalidArgument { - message: format!("Step `{step}` error: {NODE_NAME} cannot be empty"), - }); - } - let account_index = row[ACCOUNT_INDEX_IDX_T2] - .trim() - .parse::() - .map_err(|_| StepError::InvalidArgument { - message: format!( - "Step `{step}` error: {ACCOUNT_INDEX} '{}' must be a valid number", - row[ACCOUNT_INDEX_IDX_T2] - ), - })?; - let wallet_name = row[WALLET_NAME_IDX].trim().to_owned(); - if wallet_name.is_empty() { - return Err(StepError::InvalidArgument { - message: format!("Step `{step}` error: {WALLET_NAME} cannot be empty"), - }); - } - let is_mining_wallet = match row[IS_MINING_WALLET_IDX].trim() { - "true" => true, - "false" => false, - other => { - return Err(StepError::InvalidArgument { - message: format!( - "Step `{step}` error: {IS_MINING_WALLET} '{other}' must be 'true' or 'false'" - ), - }); - } - }; - let connected_to = row - .get(MINING_CONNECTED_TO_IDX) - .map(|s| s.trim()) - .filter(|s| !s.is_empty()) - .map(str::to_owned); - - Ok(( - node_name, - WalletStartInfo { - wallet_name, - account_index, - }, - is_mining_wallet, - connected_to, - )) -} - -pub(crate) fn ensure_fee_sponsorship_and_fork_groups_are_not_mixed( - world: &CucumberWorld, - step_value: &str, -) -> StepResult { - if world - .wallet_registry - .fee_state - .sponsored_genesis_account - .is_some() - && !world.fork_groups.groups().is_empty() - { - return Err(StepError::InvalidArgument { - message: format!( - "Step `{step_value}` error: sponsored fee accounts cannot be combined with distinct node groups in the same scenario" - ), - }); - } - - Ok(()) -} - -pub(crate) async fn wait_for_all_nodes_to_be_synced_to_chain( - world: &mut CucumberWorld, - step: &str, -) -> StepResult { - let public_cryptarchia_endpoint_peers = world - .startup - .public_cryptarchia_endpoint_peers - .clone() - .unwrap_or_default(); - if public_cryptarchia_endpoint_peers.is_empty() { - return Err(StepError::InvalidArgument { - message: format!( - "Step `{step}` error: no public cryptarchia endpoint peers configured" - ), - }); - } - if world.nodes_info.is_empty() { - return Err(StepError::InvalidArgument { - message: format!("Step `{step}` error: no local nodes are available to check sync"), - }); - } - - let client = Client::new(); - let start = Instant::now(); - let mut last_status_log_at = None; - - loop { - let public_snapshots = - fetch_public_peer_consensus_snapshots(&client, &public_cryptarchia_endpoint_peers) - .await; - let majority_target = select_majority_public_sync_target(&public_snapshots); - - if let Some(target) = majority_target.as_ref() - && all_local_nodes_match_sync_target(world, target).await - { - get_cryptarchia_info_all_nodes(world, step).await; - info!( - target: TARGET, - "All nodes synced to the chain in {:.2?}", - start.elapsed() - ); - - catch_up_known_wallet_tracking_after_chain_sync(world, step).await?; - - return Ok(()); - } - - if should_log_chain_sync_status(last_status_log_at) { - log_chain_sync_progress( - start.elapsed(), - public_cryptarchia_endpoint_peers.len(), - &public_snapshots, - majority_target.as_ref(), - ); - get_cryptarchia_info_all_nodes(world, step).await; - last_status_log_at = Some(Instant::now()); - } - - sleep(CHAIN_SYNC_POLL_INTERVAL).await; - } -} - -async fn catch_up_known_wallet_tracking_after_chain_sync( - world: &mut CucumberWorld, - step: &str, -) -> StepResult { - if world.wallet_registry.wallet_info.is_empty() { - return Ok(()); - } - - let started_at = Instant::now(); - // The wallet scanner timeout is moderate here because the contract is that the - // majority nodes have been synced prior just to this step. - world - .wait_for_wallet_scanner_catch_up(Duration::from_secs(30)) - .await?; - - info!( - target: TARGET, - "Wallet scanner caught up after chain sync for step `{step}` in {:.2?}", - started_at.elapsed() - ); - - Ok(()) -} - -pub(crate) fn parse_url(raw: &str) -> Result { - let mut trimmed = raw.trim(); - trimmed = trimmed.trim_end_matches('/'); - if trimmed.is_empty() { - return Err("url cannot be empty".to_owned()); - } - - Url::parse(trimmed).map_err(|e| format!("invalid url '{trimmed}': {e}"))?; - - Ok(trimmed.to_owned()) -} - -async fn fetch_public_peer_consensus_snapshots( - client: &Client, - peers: &[PublicCryptarchiaEndpointPeer], -) -> Vec { - let mut snapshots = Vec::new(); - - for peer in peers { - match fetch_public_peer_consensus(client, peer).await { - Ok(info) => snapshots.push(PublicPeerConsensusSnapshot { - peer_url: peer.base_url.clone(), - stats: SyncTargetStats::from_cryptarchia_info(&info), - }), - Err(e) => warn!( - target: TARGET, - "Failed to fetch public cryptarchia info from '{}': {e}", - peer.base_url - ), - } - } - - snapshots -} - -/// Fetch the current consensus info from a public cryptarchia endpoint peer. -/// Returns an error if the request fails or the response is invalid. -pub async fn fetch_public_peer_consensus( - client: &Client, - peer: &PublicCryptarchiaEndpointPeer, -) -> Result { - let request_url = Url::parse(&format!( - "{peer_url}/{path}", - peer_url = peer.base_url.as_str(), - path = CRYPTARCHIA_INFO.trim_start_matches('/') - )) - .map_err(|e| StepError::InvalidArgument { - message: format!( - "Invalid public cryptarchia info URL for '{}': {e}", - peer.base_url.as_str() - ), - })?; - - Ok(client - .get(request_url) - .basic_auth(&peer.username, Some(&peer.password)) - .send() - .await? - .error_for_status()? - .json::() - .await - .map_err(StepError::from)? - .cryptarchia_info) -} - -fn select_majority_public_sync_target( - snapshots: &[PublicPeerConsensusSnapshot], -) -> Option { - let mut groups = HashMap::>::new(); - for snapshot in snapshots { - groups - .entry(snapshot.stats.clone()) - .or_default() - .push(snapshot.peer_url.clone()); - } - - let best = groups - .into_iter() - .max_by(|(left_stats, left_peers), (right_stats, right_peers)| { - left_peers - .len() - .cmp(&right_peers.len()) - .then_with(|| left_stats.height.cmp(&right_stats.height)) - .then_with(|| left_stats.slot.cmp(&right_stats.slot)) - }) - .map(|(stats, peer_urls)| MajorityPublicSyncTarget { peer_urls, stats })?; - - if best.peer_urls.len() * 2 <= snapshots.len() { - return None; - } - - Some(best) -} - -async fn all_local_nodes_match_sync_target( - world: &CucumberWorld, - target: &MajorityPublicSyncTarget, -) -> bool { - let mut node_names = world.nodes_info.keys().cloned().collect::>(); - node_names.sort(); - - for node_name in node_names { - let Some(node_info) = world.nodes_info.get(&node_name) else { - return false; - }; - - let Ok(consensus) = node_info.started_node.client.consensus_info().await else { - return false; - }; - if SyncTargetStats::from_cryptarchia_info(&consensus.cryptarchia_info) != target.stats { - return false; - } - } - - true -} - -fn should_log_chain_sync_status(last_status_log_at: Option) -> bool { - last_status_log_at.is_none_or(|last| last.elapsed() >= CHAIN_SYNC_STATUS_LOG_INTERVAL) -} - -fn log_chain_sync_progress( - elapsed: Duration, - total_public_peers: usize, - public_snapshots: &[PublicPeerConsensusSnapshot], - majority_target: Option<&MajorityPublicSyncTarget>, -) { - if let Some(target) = majority_target { - info!( - target: TARGET, - "Waiting to be synced - elapsed {:.2?}, height {}/{}, public peers {}/{}, majority {}/{}, tip '{} ...', lib '{} ...'", - elapsed, - target.stats.height, - target.stats.slot, - public_snapshots.len(), - total_public_peers, - target.peer_urls.len(), - public_snapshots.len(), - truncate_hash(&target.stats.tip, 16), - truncate_hash(&target.stats.lib, 16), - ); - } else { - info!( - target: TARGET, - "Waiting to be synced - elapsed {:.2?}, no majority public peer consensus ({}/{} reachable)", - elapsed, - public_snapshots.len(), - total_public_peers, - ); - } -} - -// Sort nodes_to_start with empty peers first to ensure standalone nodes start -// before connected nodes, then by dependency order to ensure all peers of a -// node are started before the node itself is started. If there is a circular -// dependency, return an error. -pub(crate) fn start_nodes_order_respecting_dependencies( - nodes_to_start: NodesToStartUnordered, - already_started: HashSet, -) -> Result { - let mut remaining = nodes_to_start; - // Peers that are already running (started by earlier steps) count as - // satisfied dependencies, so a node in this batch may connect to them. - let mut started = already_started; - let mut ordered = Vec::new(); - - // Step 1: Find all nodes whose peer dependencies are already satisfied - // (no in-batch peers, or all peers already running). - let nodes_without_peers: Vec = remaining - .iter() - .filter(|&(_, (_, peers))| peers.iter().all(|peer| started.contains(peer))) - .map(|(node_name, (_, _))| node_name.clone()) - .collect(); - - if nodes_without_peers.is_empty() && !remaining.is_empty() { - return Err(StepError::InvalidArgument { - message: "No nodes without peer dependencies found. Possible circular dependency." - .to_owned(), - }); - } - - // Update start list with all nodes without peers - for node_name in nodes_without_peers { - if let Some((wallet_infos, initial_peers)) = remaining.remove(&node_name) { - ordered.push((node_name.clone(), wallet_infos, initial_peers)); - started.insert(node_name); - } - } - - // Step 2: Iteratively find nodes whose peer dependencies are already included - // in the start list - while !remaining.is_empty() { - let mut made_progress = false; - - let ready_nodes: Vec = remaining - .iter() - .filter_map(|(node_name, (_, peers))| { - let all_peers_started = peers.iter().all(|peer| started.contains(peer)); - all_peers_started.then(|| node_name.clone()) - }) - .collect(); - - for node_name in ready_nodes { - if let Some((wallet_infos, mut peers)) = remaining.remove(&node_name) { - peers.sort(); - peers.dedup(); - ordered.push((node_name.clone(), wallet_infos, peers)); - started.insert(node_name); - made_progress = true; - } - } - - if !made_progress { - let remaining_nodes: Vec = remaining.keys().cloned().collect(); - return Err(StepError::InvalidArgument { - message: format!("Circular dependency detected among nodes: {remaining_nodes:?}"), - }); - } - } - - Ok(ordered) -} - -#[expect( - clippy::too_many_lines, - reason = "Covers startup, optional snapshot seeding, wallet wiring, and readiness in one path" -)] -#[expect( - clippy::cognitive_complexity, - reason = "Singular fn with multiple branches to handle different events and futures." -)] -pub async fn start_node( - world: &mut CucumberWorld, - step: &str, - node_name: &str, - wallet_start_info: &[WalletStartInfo], - initial_peers: &[String], - immediate_start: bool, - extra_user_overrides: &[ConfigOverride], -) -> StepResult { - if world.cluster.local_cluster.is_none() { - return Err(StepError::LogicalError { - message: "No local cluster available".into(), - }); - } - let mut startup_settings = - get_startup_settings(world, initial_peers, node_name).inspect_err(|e| { - warn!(target: TARGET, "Step `{step}` error: {e}"); - })?; - // Merge per-node user config overrides (e.g. a mining node's derived - // `pow.claim_address`) on top of the scenario-wide ones, upserting by path. - for extra in extra_user_overrides { - if let Some(existing) = startup_settings - .user_config_overrides - .iter_mut() - .find(|item| item.path == extra.path) - { - existing.value = extra.value.clone(); - } else { - startup_settings.user_config_overrides.push(extra.clone()); - } - } - let is_bootstrap_node = startup_settings.is_bootstrap_node; - let join_external_network = startup_settings.join_external_network; - let persist_dir = world.lifecycle.scenario_base_dir.join(node_name); - let runtime_dir_prefix = format!("{node_name}_"); - let final_dir_ignore_list = matching_child_dirs(&persist_dir, &runtime_dir_prefix); - let tokio_console_node = startup_settings.tokio_console_node.clone(); - let scenario_wallet_key_ids = world - .wallet_registry - .wallet_accounts - .values() - .map(wallet_account_key_id) - .chain( - world - .wallet_registry - .fee_state - .wallet_account - .iter() - .map(wallet_account_key_id), - ) - .collect(); - let start_options = StartNodeOptions::default() - .with_peers(startup_settings.peer_selection) - .with_persist_dir(persist_dir) - .create_patch(move |mut config: RunConfig| { - prepare_config_patch( - &mut config, - startup_settings.join_external_network, - startup_settings.deployment_settings_override.as_ref(), - &startup_settings.manual_node_config_overrides, - startup_settings.initial_peers_override.as_ref(), - &startup_settings.ibd_peers, - &startup_settings.user_config_overrides, - &startup_settings.deployment_config_overrides, - startup_settings.tokio_console_node.as_ref(), - &scenario_wallet_key_ids, - )?; - Ok(config) - }); - - let started_node = { - let cluster = world - .cluster - .local_cluster - .as_ref() - .expect("local cluster checked"); - Box::pin(cluster.start_node_with(node_name, start_options)) - .await - .inspect_err(|e| { - warn!(target: TARGET, "Step `{step}` error: {e}"); - })? - }; - - let node_final_dir = extract_child_dir_name( - &world.lifecycle.scenario_base_dir, - &runtime_dir_prefix, - &final_dir_ignore_list, - ) - .inspect_err(|e| { - warn!(target: TARGET, "Step `{step}` error: {e}"); - })?; - let node_runtime_dir = world - .lifecycle - .scenario_base_dir - .join(node_final_dir.clone()); - populate_slots_per_epoch_from_deployment(world, &node_runtime_dir)?; - let started_node_name = started_node.name.clone(); - info!( - target: TARGET, - "Starting node `{node_name}` with runtime_dir='{}'", - display_last_path_components(&node_runtime_dir, 4) - ); - - // `StartNodeOptions::with_persist_dir` currently creates a fresh runtime - // directory for each launch. Seed that runtime directory and restart once - // to effectively initialize from a named snapshot. - let restored_node_snapshot = if let Some(node_snapshot) = - world.snapshots.node_snapshot_on_startup.clone() - { - let stop_result = { - let cluster = world - .cluster - .local_cluster - .as_ref() - .expect("local cluster checked"); - cluster - .stop_node(&started_node_name) - .await - .inspect_err(|e| { - warn!(target: TARGET, "Step `{step}` error: {e}"); - }) - }; - stop_result?; - - restore_node_state_from_snapshot(&node_snapshot, &node_runtime_dir).inspect_err(|e| { - warn!(target: TARGET, "Step `{step}` error: {e}"); - })?; - populate_slots_per_epoch_from_deployment(world, &node_runtime_dir)?; - - let restart_result = { - let cluster = world - .cluster - .local_cluster - .as_ref() - .expect("local cluster checked"); - cluster - .restart_node(&started_node_name) - .await - .inspect_err(|e| { - warn!(target: TARGET, "Step `{step}` error: {e}"); - }) - }; - restart_result?; - info!( - target: TARGET, - "Node {node_name} started from snapshot {}/{}", - node_snapshot.name, node_snapshot.node - ); - Some(node_snapshot) - } else { - None - }; - - // Scrape the final node directory name to get the correct path to the node's - // YAML file for extracting the peer ID, since the actual directory name has - // a random suffix added by the deployer. - world.cluster.node_peer_ids.insert( - node_name.to_owned(), - peer_id_from_node_yaml(&node_runtime_dir.join(USER_CONFIG_FILE))?, - ); - - let wallet_info = add_wallets( - world, - step, - node_name, - wallet_start_info, - &started_node, - &node_runtime_dir, - join_external_network, - ) - .inspect_err(|e| { - warn!(target: TARGET, "Step `{step}` error: {e}"); - })?; - - world - .wallet_registry - .wallet_info - .extend(wallet_info.iter().map(|(k, v)| (k.clone(), v.clone()))); - - let client = started_node.client.clone(); - // Move `started_node` into the world's NodeInfo (no clone required) - world.nodes_info.insert( - node_name.to_owned(), - NodeInfo { - name: node_name.to_owned(), - started_node, - run_config: None, - chain_info: HashMap::default(), - wallet_info, - runtime_dir: node_runtime_dir, - immediate_start, - }, - ); - - if let Some(node_snapshot) = restored_node_snapshot - && let Some(snapshot_name) = world.snapshots.restore.extensions.clone() - { - restore_wallet_snapshot_if_present( - &snapshot_name, - &node_snapshot.node, - node_name, - &client, - world, - ) - .await - .inspect_err(|e| { - warn!(target: TARGET, "Step `{step}` error: {e}"); - })?; - } - - // All nodes are required to be network ready responsive, and bootstrap nodes - // must be `Mode::OnLine` for IBD of other peers to succeed - if !immediate_start { - let cluster = world - .cluster - .local_cluster - .as_ref() - .expect("local cluster checked"); - ensure_node_ready( - cluster, - &client, - node_name, - &started_node_name, - is_bootstrap_node, - world.startup.require_all_peers_mode_online_at_startup, - startup_settings.join_external_network, - ) - .await - .inspect_err(|e| { - warn!(target: TARGET, "Step `{step}` error: {e}"); - })?; - } - - if world.snapshots.node_snapshot_on_startup.is_some() { - match client.consensus_info().await { - Ok(info) => { - info!( - target: TARGET, - "Node `{node_name}` snapshot state - height: {}/{}, tip: {}, lib: {}", - info.cryptarchia_info.height, - info.cryptarchia_info.slot.into_inner(), - truncate_hash(&info.cryptarchia_info.tip.encode_hex::(), 16), - truncate_hash(&info.cryptarchia_info.lib.encode_hex::(), 16) - ); - } - Err(e) => { - warn!( - target: TARGET, - "Node `{node_name}` failed to fetch post-start consensus after snapshot init: {e}" - ); - } - } - } - - if let Some(tokio_console) = tokio_console_node { - check_tokio_console_port(node_name, tokio_console.port); - } - - Ok(()) -} - -fn check_tokio_console_port(node_name: &str, port: u16) { - let addr = SocketAddr::from((Ipv4Addr::LOCALHOST, port)); - - match TcpStream::connect_timeout(&addr, Duration::from_secs(2)) { - Ok(_) => info!( - target: TARGET, - "Tokio console endpoint for `{node_name}` is listening at port `{port}`, connect with \ - `tokio-console http://127.0.0.1:{port}`" - ), - Err(error) => warn!( - target: TARGET, - "Tokio console endpoint for `{node_name}` is not reachable at \ - `http://127.0.0.1:{port}`: {error}. Refer to the repo root `README.md -> Tokio task \ - profiling` for general instructions." - ), - } -} - -/// Stop a node and leave it down. -/// -/// Unlike [`restart_node`], which brings it back up and waits for readiness, -/// this leaves the node down, useful to exercise reconnect behavior while the -/// node is down. -pub async fn stop_node(world: &CucumberWorld, step: &str, node_name: &str) -> StepResult { - let cluster = world - .cluster - .local_cluster - .as_ref() - .ok_or(StepError::LogicalError { - message: "No local cluster available".into(), - })?; - let started_node_name = world - .resolve_node_runtime_name(node_name) - .inspect_err(|e| { - warn!(target: TARGET, "Step `{step}` error: {e}"); - })?; - - log_node_lifecycle_marker(world, "node_stop", node_name, "before").await; - - cluster - .stop_node(&started_node_name) - .await - .inspect_err(|e| { - warn!(target: TARGET, "Step `{step}` error: {e}"); - })?; - - log_node_lifecycle_marker(world, "node_stop", node_name, "after").await; - - info!( - target: TARGET, - "Stopped node `{node_name}` (runtime name `{started_node_name}`)" - ); - Ok(()) -} - -pub async fn restart_node(world: &CucumberWorld, step: &str, node_name: &str) -> StepResult { - let cluster = world - .cluster - .local_cluster - .as_ref() - .ok_or(StepError::LogicalError { - message: "No local cluster available".into(), - })?; - let started_node_name = world - .resolve_node_runtime_name(node_name) - .inspect_err(|e| { - warn!(target: TARGET, "Step `{step}` error: {e}"); - })?; - - log_node_lifecycle_marker(world, "node_restart", node_name, "before").await; - - cluster - .restart_node(&started_node_name) - .await - .inspect_err(|e| { - warn!(target: TARGET, "Step `{step}` error: {e}"); - })?; - - log_node_lifecycle_marker(world, "node_restart", node_name, "after").await; - let client = world.resolve_node_http_client(node_name).inspect_err(|e| { - warn!(target: TARGET, "Step `{step}` error: {e}"); - })?; - ensure_node_ready( - cluster, - &client, - node_name, - &started_node_name, - // TODO: Add `is_bootstrap_node` to world - false, - None, - world.startup.join_external_network.unwrap_or_default(), - ) - .await - .inspect_err(|e| { - warn!(target: TARGET, "Step `{step}` error: {e}"); - })?; - - info!( - target: TARGET, - "Restarted node `{node_name}` (runtime name `{started_node_name}`)" - ); - - Ok(()) -} - -fn add_wallets( - world: &CucumberWorld, - step: &str, - node_name: &str, - wallet_start_info: &[WalletStartInfo], - started_node: &StartedNode, - node_runtime_dir: &Path, - join_external_network: bool, -) -> Result { - let wallet_info = compile_wallet_in_map( - wallet_start_info, - node_name, - world, - step, - node_runtime_dir, - join_external_network, - )?; - for (wallet_name, info) in &wallet_info { - let wallet_type = match info.wallet_type.clone() { - WalletType::User { .. } => "User", - WalletType::Funding { .. } => "Funding", - }; - info!(target: TARGET, "{wallet_type} wallet `{}/{node_name}` created: {}", - wallet_name, - format!("{}wallet/{}/balance", started_node.client.base_url(), info.public_key_hex()) - ); - } - - Ok(wallet_info) -} - -struct StartupSettings { - peer_selection: PeerSelection, - ibd_peers: HashSet, - is_bootstrap_node: bool, - initial_peers_override: Option>, - join_external_network: bool, - user_config_overrides: Vec, - deployment_config_overrides: Vec, - deployment_settings_override: Option, - manual_node_config_overrides: ManualNodeConfigOverrides, - tokio_console_node: Option, -} - -fn get_startup_settings( - world: &CucumberWorld, - initial_peers: &[String], - node_name: &str, -) -> Result { - let peer_selection = if initial_peers.is_empty() { - PeerSelection::None - } else { - let named = initial_peers - .iter() - .map(|peer| world.resolve_node_runtime_name(peer)) - .collect::, StepError>>()?; - PeerSelection::Named(named) - }; - let mut ibd_peers = world.startup.ibd_peers_override.clone().unwrap_or_default(); - let populate_ibd_peers_from_initial_peers = world - .startup - .populate_ibd_peers_from_initial_peers - .unwrap_or_default(); - if populate_ibd_peers_from_initial_peers { - for peer in initial_peers { - if let Some(peer_id) = world.cluster.node_peer_ids.get(peer) { - ibd_peers.insert(*peer_id); - } - } - } - let is_bootstrap_node = initial_peers.is_empty(); - let initial_peers_override = world.startup.initial_peers_override.clone(); - let join_external_network = world.startup.join_external_network.unwrap_or_default(); - let deployment_settings_override = world - .startup - .deployment_config_override_path - .clone() - .map(|path| load_run_config(&path)) - .transpose()?; - let user_config_overrides = world.startup.user_config_overrides.clone(); - let deployment_config_overrides = world.startup.deployment_config_overrides.clone(); - let tokio_console_node = world.tokio_console_profile.node(node_name).cloned(); - - Ok(StartupSettings { - peer_selection, - ibd_peers, - is_bootstrap_node, - initial_peers_override, - join_external_network, - deployment_settings_override, - manual_node_config_overrides: world.startup.manual_node_config_overrides.clone(), - user_config_overrides, - deployment_config_overrides, - tokio_console_node, - }) -} - -#[expect(clippy::too_many_arguments, reason = "all needed")] -fn prepare_config_patch( - config: &mut RunConfig, - join_external_network: bool, - deployment_override: Option<&DeploymentSettings>, - config_overrides: &ManualNodeConfigOverrides, - initial_peers_override: Option<&Vec>, - ibd_peers: &HashSet, - user_config_overrides: &[ConfigOverride], - deployment_config_overrides: &[ConfigOverride], - tokio_console_node: Option<&TokioConsoleProfileNode>, - scenario_wallet_key_ids: &HashSet, -) -> Result<(), StepError> { - if join_external_network { - config.deployment = deployment_override - .cloned() - .unwrap_or_else(DeploymentSettings::default); - } else if let Some(deployment_override) = deployment_override { - config.deployment = deployment_override.clone(); - } - - config_overrides.apply_to(config); - - if let Some(initial_peers) = &initial_peers_override { - config - .user - .network - .backend - .initial_peers - .clone_from(initial_peers); - } - config - .user - .cryptarchia - .network - .bootstrap - .ibd - .peers - .clone_from(ibd_peers); - if let Some(node) = &tokio_console_node { - config.user.tracing.console = ConsoleLayer::Console(TokioConfig { - bind_address: IpAddr::V4(Ipv4Addr::LOCALHOST), - port: node.port, - recording_path: node - .record_raw - .then(|| PathBuf::from("tokio-console-raw.jsonl")), - }); - } - - apply_user_config_overrides(config, user_config_overrides)?; - apply_deployment_config_overrides(config, deployment_config_overrides)?; - if join_external_network { - remove_external_scenario_wallet_keys(config, scenario_wallet_key_ids); - } - Ok(()) -} - -fn wallet_account_key_id(account: &WalletAccount) -> KeyId { - let key: Key = account.secret_key.clone().into(); - key_id_for_preload_backend(&key) -} - -fn remove_external_scenario_wallet_keys( - config: &mut RunConfig, - scenario_wallet_key_ids: &HashSet, -) { - remove_external_scenario_wallet_keys_from_maps( - &mut config.user.wallet.known_keys, - &mut config.user.kms.backend.keys, - scenario_wallet_key_ids, - ); -} - -fn remove_external_scenario_wallet_keys_from_maps( - known_keys: &mut HashMap, - kms_keys: &mut HashMap, - scenario_wallet_key_ids: &HashSet, -) { - for key_id in scenario_wallet_key_ids { - known_keys.remove(key_id); - kms_keys.remove(key_id); - } -} - -fn load_run_config(path: &Path) -> Result { - let text = fs::read_to_string(path).map_err(|e| StepError::LogicalError { - message: format!("Failed to read '{}': {e}", path.display()), - })?; - serde_yaml::from_str::(&text).map_err(|e| StepError::LogicalError { - message: format!("Failed to parse '{}': {e}", path.display()), - }) -} - -fn populate_slots_per_epoch_from_deployment( - world: &mut CucumberWorld, - node_runtime_dir: &Path, -) -> Result<(), StepError> { - let path = node_runtime_dir.join("deployment.yaml"); - let text = fs::read_to_string(&path).map_err(|source| StepError::LogicalError { - message: format!( - "failed to read effective deployment config '{}': {source}", - path.display() - ), - })?; - let deployment = serde_yaml::from_str::(&text).map_err(|source| { - StepError::LogicalError { - message: format!( - "failed to parse effective deployment config '{}': {source}", - path.display() - ), - } - })?; - let slots_per_epoch = deployment.cryptarchia.slots_per_epoch(); - let slots_per_epoch = NonZero::new(slots_per_epoch).ok_or_else(|| StepError::LogicalError { - message: format!( - "effective deployment config '{}' has zero slots per epoch", - path.display() - ), - })?; - world.chain.slots_per_epoch = slots_per_epoch; - info!( - target: TARGET, - "Loaded effective epoch configuration from '{}': slots_per_epoch={slots_per_epoch}", - path.display() - ); - Ok(()) -} - -// Ensure this node is ready, and achieved `Mode::OnLine` if it is a bootstrap -// node. -async fn ensure_node_ready( - cluster: &LbcManualCluster, - client: &NodeHttpClient, - node_name: &str, - started_node_name: &str, - is_bootstrap_node: bool, - require_all_peers_mode_online_at_startup: Option, - join_external_network: bool, -) -> StepResult { - // General readiness check to ensure the node is responsive. - let operation = format!("node '{started_node_name}' readiness"); - track_progress(&operation, Duration::from_secs(5), async { - cluster - .wait_node_ready(started_node_name) - .await - .map_err(|source| StepError::StepFail { - message: format!( - "node '{started_node_name}' did not become ready after start: {source}" - ), - }) - }) - .await?; - - verify_reponsive_and_network_ready(client, node_name, started_node_name).await?; - - if !is_bootstrap_node && require_all_peers_mode_online_at_startup.is_none() - || join_external_network - { - return Ok(()); - } - - verify_online( - client, - node_name, - started_node_name, - require_all_peers_mode_online_at_startup, - ) - .await?; - Ok(()) -} - -async fn verify_online( - client: &NodeHttpClient, - node_name: &str, - started_node_name: &str, - time_out: Option, -) -> StepResult { - let time_out = time_out.unwrap_or_else(|| Duration::from_mins(1)); - let start = Instant::now(); - let mut count = 0usize; - loop { - let mut mode_online = false; - match client.consensus_info().await { - Ok(val) => { - if matches!(val.phase, PhaseTag::Following) { - mode_online = true; - } - } - Err(e) if start.elapsed() < time_out => { - if count.is_multiple_of(20) { - info!( - target: TARGET, - "Waiting for node `{node_name}/{started_node_name}` to be `Mode::OnLine` - \ - elapsed: {:.2?} ({e})", - start.elapsed() - ); - } - } - Err(e) => { - return Err(StepError::StepFail { - message: format!( - "Node `{node_name}/{started_node_name}` failed `Mode::OnLine` - elapsed \ - {:.2?}: {e}", - start.elapsed() - ), - }); - } - } - if mode_online { - info!( - target: TARGET, - "Node `{node_name}/{started_node_name}` achieved `Mode::OnLine` and listen \ - addresses in {:.2?}", - start.elapsed() - ); - return Ok(()); - } - sleep(Duration::from_millis(100)).await; - count += 1; - } -} - -/// Wait for all nodes to become responsive -pub async fn wait_all_nodes_responive( - cluster: &LbcManualCluster, - time_out: Duration, -) -> StepResult { - timeout(time_out, cluster.wait_network_ready()) - .await - .map_err(|_| StepError::StepFail { - message: format!("Not all nodes became responsive after {time_out:?}"), - })? - .map_err(|e| StepError::StepFail { - message: format!("Failed to check all nodes ready: {e}"), - }) -} - -async fn verify_reponsive_and_network_ready( - client: &NodeHttpClient, - node_name: &str, - started_node_name: &str, -) -> StepResult { - verify_reponsive_and_network_ready_with_timeout( - client, - node_name, - started_node_name, - Duration::from_mins(1), - ) - .await -} - -/// Wait for the node to be responsive and network ready, with a timeout. -#[expect( - clippy::cognitive_complexity, - reason = "Singular fn with multiple branches to handle different events and futures." -)] -pub async fn verify_reponsive_and_network_ready_with_timeout( - client: &NodeHttpClient, - node_name: &str, - started_node_name: &str, - time_out: Duration, -) -> StepResult { - let start = Instant::now(); - let mut count = 0usize; - let mut can_provide_consensus_info; - let mut is_network_ready; - - loop { - can_provide_consensus_info = false; - match client.consensus_info().await { - Ok(_) => { - can_provide_consensus_info = true; - } - Err(e) if start.elapsed() < time_out => { - if count.is_multiple_of(20) { - info!( - target: TARGET, - "Waiting for node `{node_name}/{started_node_name}` to be responsive - \ - elapsed: {:.2?} ({e})", - start.elapsed() - ); - } - } - Err(e) => { - return Err(StepError::StepFail { - message: format!( - "Node `{node_name}/{started_node_name}` failed to be responsive - elapsed \ - {:.2?}: {e}", - start.elapsed() - ), - }); - } - } - is_network_ready = false; - match client.network_info().await { - Ok(val) => { - is_network_ready = !val.listen_addresses.is_empty(); - } - Err(e) if start.elapsed() < time_out => { - if count.is_multiple_of(20) { - info!( - target: TARGET, - "Waiting for node `{node_name}/{started_node_name}` to be network ready - \ - elapsed: {:.2?} ({e})", - start.elapsed() - ); - } - } - Err(e) => { - return Err(StepError::StepFail { - message: format!( - "Node `{node_name}/{started_node_name}` failed to be network ready - elapsed \ - {:.2?}: {e}", - start.elapsed() - ), - }); - } - } - if can_provide_consensus_info && is_network_ready { - info!( - target: TARGET, - "Node `{node_name}/{started_node_name}` is responsive and network ready in {:.2?}", - start.elapsed() - ); - return Ok(()); - } - sleep(Duration::from_millis(100)).await; - count += 1; - } -} - -fn compile_wallet_in_map( - wallet_start_info: &[WalletStartInfo], - node_name: &str, - world: &CucumberWorld, - step: &str, - node_runtime_dir: &Path, - join_external_network: bool, -) -> Result { - let mut wallet_info: WalletInfoMap = HashMap::new(); - for wallet in wallet_start_info { - let wallet_account = match world - .wallet_registry - .wallet_accounts - .get(&wallet.account_index) - { - Some(wallet_account) => wallet_account.clone(), - None => { - if join_external_network { - WalletAccount::random() - .map_err(|source| StepError::LogicalError { - message: format!( - "Step `{step}` error: Failed to derive random wallet account for index {}: {source}", - wallet.account_index - ), - })? - } else { - WalletAccount::deterministic( - wallet.account_index as u64, - 0, - true, - ) - .map_err(|source| StepError::LogicalError { - message: format!( - "Step `{step}` error: Failed to derive deterministic wallet account for index {}: {source}", - wallet.account_index - ), - })? - } - } - }; - - wallet_info.insert( - wallet.wallet_name.clone(), - WalletInfo { - wallet_name: wallet.wallet_name.clone(), - node_name: node_name.to_owned(), - wallet_type: WalletType::User { wallet_account }, - }, - ); - } - - let node_wallet_keys = - node_wallet_keys_from_node_yaml(&node_runtime_dir.join(USER_CONFIG_FILE))?; - let user_wallets_by_pk = world - .wallet_registry - .wallet_accounts - .values() - .map(|account| (account.public_key_hex(), account.label.clone())) - .chain( - world - .wallet_registry - .fee_state - .wallet_account - .iter() - .map(|account| (account.public_key_hex(), account.label.clone())), - ) - .collect::>(); - let mut generic_key_index = 0usize; - - for node_wallet_key in node_wallet_keys { - if let Some(user_wallet_name) = user_wallets_by_pk.get(&node_wallet_key.wallet_pk) { - if node_wallet_key.role != NodeWalletKeyRole::General { - return Err(StepError::LogicalError { - message: format!( - "Scenario wallet `{user_wallet_name}` public key conflicts with the \ - {role:?} key owned by `{node_name}`", - role = node_wallet_key.role, - ), - }); - } - info!( - target: TARGET, - "Scenario wallet `{user_wallet_name}` is registered with `{node_name}`; \ - excluding its public key from node-wallet aliases" - ); - continue; - } - - let wallet_name = node_wallet_name(node_name, &node_wallet_key, &mut generic_key_index); - wallet_info.insert( - wallet_name.clone(), - WalletInfo { - wallet_name, - node_name: node_name.to_owned(), - wallet_type: WalletType::Funding { - key: node_wallet_key, - }, - }, - ); - } - - Ok(wallet_info) -} - -fn node_wallet_name(node_name: &str, key: &NodeWalletKey, generic_key_index: &mut usize) -> String { - let role = match key.role { - NodeWalletKeyRole::Funding => "FUNDING".to_owned(), - NodeWalletKeyRole::VoucherMaster => "VOUCHER_MASTER".to_owned(), - NodeWalletKeyRole::BlendZk => "BLEND_ZK".to_owned(), - NodeWalletKeyRole::General => { - *generic_key_index += 1; - format!("GENERAL_{}", *generic_key_index) - } - }; - format!("{node_name}_WALLET_{role}") -} - -fn tips_aligned_at_min_difference( - nodes_chain_info: &HashMap, - all_nodes_min: u64, -) -> (AlignmentStatus, Vec) { - // Always return per-node view at min_height for logging - let mut anchor_hashes: Vec = Vec::with_capacity(nodes_chain_info.len()); - - for node_name in nodes_chain_info.keys() { - let peer_chain = nodes_chain_info - .get(node_name) - .expect("nodes_chain_info must be pre-initialized"); - anchor_hashes.push(MaybeSnapshot { - height: all_nodes_min, - header_hash: peer_chain.get(&all_nodes_min).cloned(), - }); - } - - let all_have = anchor_hashes.iter().all(|snap| snap.header_hash.is_some()); - if !all_have { - return (AlignmentStatus::MissingChainInfo, anchor_hashes); - } - - let compare_hash = anchor_hashes[0].header_hash.as_ref().unwrap(); - let all_same = anchor_hashes - .iter() - .all(|snap| snap.header_hash.as_ref().unwrap() == compare_hash); - - if all_same { - (AlignmentStatus::Aligned, anchor_hashes) - } else { - (AlignmentStatus::Fork, anchor_hashes) - } -} - -async fn fetch_and_update_chain_info( - step: &str, - nodes_info: &mut HashMap, - nodes_chain_info: &mut HashMap, -) -> Result<(u64, u64, Vec), StepError> { - poll_all_nodes_and_update_consensus_cache(step, nodes_info).await?; - - let mut best_node_heights: Vec = Vec::with_capacity(nodes_info.len()); - - for node_info in nodes_info.values() { - let max_height = node_info.best_height().unwrap_or_default(); - best_node_heights.push(max_height); - - let started_node_name = node_info.started_node.name.clone(); - let chain = - nodes_chain_info - .get_mut(&started_node_name) - .ok_or(StepError::LogicalError { - message: format!( - "Started node '{started_node_name}' not found in chain info map", - ), - })?; - let chain_info = node_info.chain_info(); - for (height, hash) in chain_info { - chain.insert(*height, hash.clone()); - } - } - - let all_nodes_min = *best_node_heights.iter().min().unwrap_or(&0); - let all_nodes_max = *best_node_heights.iter().max().unwrap_or(&0); - let diff = all_nodes_max - all_nodes_min; - - Ok((all_nodes_min, diff, best_node_heights)) -} - -fn log_waiting_status( - status: &AlignmentStatus, - min_height: Option, - diff: u64, - peer_heights: &[u64], - peer_min: u64, - anchor_hashes: &[MaybeSnapshot], - start: Instant, -) { - match status { - AlignmentStatus::Aligned => { - let converge = min_height.map_or_else( - || "Waiting for all nodes to converge".to_owned(), - |min_height| format!("Waiting for at least {min_height} blocks converged"), - ); - info!( - target: TARGET, - "{converge} - elapsed: {:.2?}, diff: {diff}, heights: {peer_heights:?}", - start.elapsed() - ); - } - AlignmentStatus::MissingChainInfo => { - info!( - target: TARGET, - "Waiting for all node's hashes at height {peer_min} - elapsed: {:.2?}, diff: \ - {diff}, heights: {peer_heights:?}, anchors: {:?}", - start.elapsed(), - anchor_hashes.iter().map(|snap| &snap.header_hash).collect::>() - ); - } - AlignmentStatus::Fork => { - let fork_hashes: HashSet<_> = anchor_hashes - .iter() - .filter_map(|snap| snap.header_hash.as_ref()) - .collect(); - info!( - target: TARGET, - "{} fork chains detected!!! Elapsed: {:.2?}, diff: {diff}, heights: {peer_heights:?}, \ - fork hashes at height {}: {:?}", - fork_hashes.len(), start.elapsed(), anchor_hashes[0].height, fork_hashes - ); - } - } -} - -pub async fn nodes_converged( - world: &mut CucumberWorld, - step: &str, - min_height: Option, - max_diff_height: u64, - time_out_seconds: u64, -) -> StepResult { - let nodes_info = &world.nodes_info.values().collect::>(); - let start = Instant::now(); - let time_out = Duration::from_secs(time_out_seconds); - - // node_name -> (height -> header_id) (overwrites on reorg) - let mut nodes_chain_info: HashMap = - HashMap::with_capacity(nodes_info.len()); - - // Pre-initialize so lookups are deterministic - for node_info in nodes_info { - nodes_chain_info - .entry(node_info.started_node.name.clone()) - .or_default(); - } - - let mut count = 0usize; - loop { - let (all_nodes_min, diff, peer_heights) = - fetch_and_update_chain_info(step, &mut world.nodes_info, &mut nodes_chain_info).await?; - let (status, anchor_hashes) = - tips_aligned_at_min_difference(&nodes_chain_info, all_nodes_min); - - if diff <= max_diff_height - && matches!(status, AlignmentStatus::Aligned) - && all_nodes_min >= min_height.unwrap_or_default() - { - if let Some(min_height) = min_height { - info!( - target: TARGET, - "All nodes have at least {min_height} blocks, converged in {:.2?} - max diff: \ - {diff}, heights: {peer_heights:?}", - start.elapsed() - ); - } else { - info!( - target: TARGET, - "All nodes converged in {:.2?} - max diff: {diff}, heights: {peer_heights:?}", - start.elapsed() - ); - } - return Ok(()); - } - - if count.is_multiple_of(50) { - log_waiting_status( - &status, - min_height, - diff, - &peer_heights, - all_nodes_min, - &anchor_hashes, - start, - ); - } - - if start.elapsed() >= time_out { - let err = min_height.map_or_else(|| StepError::StepFail { - message: format!( - "Step `{step}` error: Nodes did not converge to {max_diff_height} blocks at in \ - {time_out_seconds} s" - ), - }, |min_height| StepError::StepFail { - message: format!( - "Step `{step}` error: Nodes did not converge to {max_diff_height} blocks at minimum height \ - {min_height} in {time_out_seconds} s" - ), - }); - return Err(err); - } - - sleep(Duration::from_millis(100)).await; - count += 1; - } -} - -pub async fn ensure_all_nodes_agree_on_lib( - world: &CucumberWorld, - step: &str, - time_out_seconds: u64, -) -> StepResult { - let start = Instant::now(); - let time_out = Duration::from_secs(time_out_seconds); - let mut count = 0usize; - - loop { - let snapshots = try_join_all(world.nodes_info.values().map(async |node| { - let consensus = node.started_node.client.consensus_info().await?; - Ok::<_, StepError>(( - node.name.clone(), - consensus.cryptarchia_info.height, - consensus.cryptarchia_info.lib.encode_hex::(), - )) - })) - .await?; - - let libs = snapshots - .iter() - .map(|(_, _, lib)| lib.clone()) - .collect::>(); - - if libs.len() == 1 { - info!( - target: TARGET, - "All nodes agree on LIB in {:.2?}", - start.elapsed() - ); - return Ok(()); - } - - if count.is_multiple_of(50) { - let status = format_lib_agreement_status(&snapshots); - - info!( - target: TARGET, - "Waiting for all nodes to agree on LIB - elapsed {:.2?}, {status}", - start.elapsed() - ); - } - - if start.elapsed() >= time_out { - let status = format_lib_agreement_status(&snapshots); - - return Err(StepError::StepFail { - message: format!( - "Step `{step}` error: Nodes did not agree on LIB in {time_out_seconds} s ({status})" - ), - }); - } - - sleep(Duration::from_millis(100)).await; - count += 1; - } -} - -fn format_lib_agreement_status(snapshots: &[(String, u64, String)]) -> String { - snapshots - .iter() - .map(|(node_name, height, lib)| format!("{node_name}: {height}/{}", truncate_hash(lib, 16))) - .collect::>() - .join(", ") -} - -pub async fn poll_all_nodes_and_update_consensus_cache( - step: &str, - nodes_info: &mut HashMap, -) -> Result<(), StepError> { - use futures_util::future::join_all; - - let nodes = nodes_info.values().collect::>(); - - // Query every node, but do not fail-fast on the first error. - let info_futures = nodes.iter().map(async |node| { - let node_name = node.name.clone(); - let result = node.started_node.client.consensus_info().await; - (node_name, result) - }); - - let results = join_all(info_futures).await; - - let mut snapshots = Vec::::new(); - let mut failed_nodes = Vec::::new(); - - for (node_name, result) in results { - match result { - Ok(info) => snapshots.push(ConsensusSnapshot { - node_name, - height: info.cryptarchia_info.height, - header_hash: info.cryptarchia_info.tip.encode_hex(), - }), - Err(e) => { - // If both `consensus_info` and `network_info` fail, assume the node is no - // longer responsive. - if let Err(e2) = poll_network_info( - nodes_info.get_mut(&node_name).expect("Failed to get node"), - &node_name, - 5, - ) - .await - { - return Err(StepError::StepFail { - message: format!( - "Step `{step}` error: {node_name} is not responsive anymore: {e} / {e2}" - ), - }); - } - warn!( - target: TARGET, - "Step `{step}` error: node `{node_name}` did not respond with consensus_info: {e}", - ); - failed_nodes.push(node_name); - } - } - } - - // If all nodes failed in this poll, surface a hard error. - // If at least one succeeded, update cache for those and let caller keep - // polling. - if snapshots.is_empty() { - let failed = if failed_nodes.is_empty() { - "none".to_owned() - } else { - failed_nodes.join(", ") - }; - return Err(StepError::StepFail { - message: format!( - "Step `{step}` error: all nodes failed to respond with consensus_info in this poll \ - (failed: [{failed}])" - ), - }); - } - - for snap in &snapshots { - let node = nodes_info - .get_mut(&snap.node_name) - .ok_or(StepError::LogicalError { - message: format!( - "Step `{step}` error: Runtime node '{}' not found in world.nodes_info", - snap.node_name - ), - })?; - node.upsert_tip(snap.height, snap.header_hash.clone()); - } - - if !failed_nodes.is_empty() { - warn!( - target: TARGET, - "Step `{step}` warning: partial consensus poll failure; updated {}/{} node(s), failed: [{}]", - snapshots.len(), - snapshots.len() + failed_nodes.len(), - failed_nodes.join(", "), - ); - } - - Ok(()) -} - -async fn poll_network_info( - node_info: &NodeInfo, - node_name: &str, - time_out_seconds: u64, -) -> Result<(), String> { - let start = TokioInstant::now(); - let time_out = Duration::from_secs(time_out_seconds); - while start.elapsed() <= time_out { - if node_info.started_node.client.network_info().await.is_ok() { - return Ok(()); - } - sleep(Duration::from_millis(250)).await; - } - Err(format!( - "Node `{node_name}` did not respond to network_info after {time_out_seconds:.2?}" - )) -} - -/// This struct represents the wallet resources to be associated with a node at -/// startup. -pub struct WalletStartInfo { - // Logical name of the wallet resource, used for referencing in steps. - pub wallet_name: String, - // The account index in the genesis tokens that this resource corresponds to. - pub account_index: usize, -} - -/// Saves the current node state of all nodes into a named snapshot location for -/// later use. -pub fn create_snapshots_all_nodes( - world: &CucumberWorld, - snapshot_name: &str, -) -> Result<(), StepError> { - validate_snapshot_path_component(snapshot_name, "Snapshot name")?; - reset_named_snapshot(snapshot_name)?; - - let runtime_dir_by_node_name: Vec<(String, PathBuf)> = world - .nodes_info - .iter() - .map(|(node_name, info)| (node_name.clone(), info.runtime_dir.clone())) - .collect(); - - for (node_name, node_runtime_dir) in &runtime_dir_by_node_name { - save_named_node_state_snapshot(snapshot_name, node_name, node_runtime_dir)?; - info!( - target: TARGET, - "Saved snapshot `{snapshot_name}` for node `{node_name}`", - ); - } - Ok(()) -} - -pub async fn create_snapshot_all_nodes_with_wallet_state( - world: &mut CucumberWorld, - snapshot_name: &str, -) -> StepResult { - if world.nodes_info.is_empty() { - return Err(StepError::InvalidArgument { - message: "cannot create snapshot: no running nodes".to_owned(), - }); - } - - create_and_save_all_wallets_snapshot(snapshot_name, world).await?; - create_snapshots_all_nodes(world, snapshot_name) -} - -pub async fn create_snapshot_node_with_wallet_state( - world: &mut CucumberWorld, - snapshot_name: &str, - node_name: &str, -) -> StepResult { - if world.nodes_info.is_empty() { - return Err(StepError::InvalidArgument { - message: "cannot create snapshot: no running nodes".to_owned(), - }); - } - - if let Some(runtime_dir) = world - .nodes_info - .get(node_name) - .map(|info| info.runtime_dir.clone()) - { - reset_named_snapshot(snapshot_name)?; - create_and_save_all_wallets_snapshot(snapshot_name, world).await?; - save_named_node_state_snapshot(snapshot_name, node_name, &runtime_dir)?; - info!( - target: TARGET, - "Saved snapshot `{snapshot_name}` for node {}", - runtime_dir.display() - ); - Ok(()) - } else { - Err(StepError::InvalidArgument { - message: format!("Node {node_name} does not exist"), - }) - } -} - -/// Fetches and logs the consensus info of all nodes, for debugging purposes. -/// Does not require the nodes to be aligned or have any specific state, and is -/// resilient to some nodes being offline or unresponsive. -#[expect( - clippy::cognitive_complexity, - reason = "Singular fn with multiple branches to handle different events and futures." -)] -pub(crate) async fn get_cryptarchia_info_all_nodes(world: &CucumberWorld, step: &str) { - let mut node_names = world.nodes_info.keys().cloned().collect::>(); - node_names.sort(); - - if node_names.is_empty() { - warn!( - target: TARGET, - "Step `{step}` no nodes found for CRYPTARCHIA_INFO_ALL_NODES" - ); - return; - } - - for node_name in node_names { - let Some(node_info) = world.nodes_info.get(&node_name) else { - continue; - }; - match node_info.started_node.client.consensus_info().await { - Ok(consensus) => { - let mode = if matches!(consensus.phase, PhaseTag::Following) { - "Online" - } else { - "Bootstrapping" - }; - info!( - target: TARGET, - "cryptarchia/info - '{}', '{}', {}/{}, tip '{} ...', lib '{} ...'", - node_name, - mode, - consensus.cryptarchia_info.height, - consensus.cryptarchia_info.slot.into_inner(), - truncate_hash(&consensus.cryptarchia_info.tip.encode_hex::(), 16), - truncate_hash(&consensus.cryptarchia_info.lib.encode_hex::(), 16), - ); - } - Err(e) => { - warn!( - target: TARGET, - "Step `{step}` CRYPTARCHIA_INFO failed for node `{node_name}`: {e}", - ); - } - } - } -} - -#[cfg(test)] -mod wallet_name_tests { - use super::{NodeWalletKey, NodeWalletKeyRole, node_wallet_name}; - - fn node_wallet_key(role: NodeWalletKeyRole) -> NodeWalletKey { - NodeWalletKey { - wallet_pk: "00".repeat(32), - role, - } - } - - #[test] - fn node_wallet_names_use_semantic_roles_before_numbered_fallbacks() { - let mut generic_key_index = 0; - - assert_eq!( - node_wallet_name( - "NODE_1", - &node_wallet_key(NodeWalletKeyRole::Funding), - &mut generic_key_index - ), - "NODE_1_WALLET_FUNDING" - ); - assert_eq!( - node_wallet_name( - "NODE_1", - &node_wallet_key(NodeWalletKeyRole::VoucherMaster), - &mut generic_key_index - ), - "NODE_1_WALLET_VOUCHER_MASTER" - ); - assert_eq!( - node_wallet_name( - "NODE_1", - &node_wallet_key(NodeWalletKeyRole::BlendZk), - &mut generic_key_index - ), - "NODE_1_WALLET_BLEND_ZK" - ); - assert_eq!( - node_wallet_name( - "NODE_1", - &node_wallet_key(NodeWalletKeyRole::General), - &mut generic_key_index - ), - "NODE_1_WALLET_GENERAL_1" - ); - assert_eq!( - node_wallet_name( - "NODE_1", - &node_wallet_key(NodeWalletKeyRole::General), - &mut generic_key_index - ), - "NODE_1_WALLET_GENERAL_2" - ); - } - - #[test] - fn semantic_wallet_names_do_not_consume_general_indexes() { - let mut generic_key_index = 0; - - assert_eq!( - node_wallet_name( - "NODE_1", - &node_wallet_key(NodeWalletKeyRole::Funding), - &mut generic_key_index - ), - "NODE_1_WALLET_FUNDING" - ); - assert_eq!(generic_key_index, 0); - } -} - -#[cfg(test)] -mod scenario_wallet_key_tests { - use lb_key_management_system_service::keys::{Ed25519Key, secured_key::SecuredKey as _}; - - use super::*; - - #[test] - fn removes_only_external_scenario_wallet_keys() { - let mut kms_keys = HashMap::new(); - let mut known_keys = HashMap::new(); - let scenario_accounts = [ - WalletAccount::deterministic(100, 0, true).expect("account"), - WalletAccount::deterministic(101, 0, true).expect("account"), - ]; - let sponsored_fee_account = - WalletAccount::deterministic(103, 0, true).expect("fee account"); - let scenario_key_ids = scenario_accounts - .iter() - .map(wallet_account_key_id) - .chain(std::iter::once(wallet_account_key_id( - &sponsored_fee_account, - ))) - .collect::>(); - - for account in scenario_accounts - .iter() - .chain(std::iter::once(&sponsored_fee_account)) - { - let key: Key = account.secret_key.clone().into(); - let key_id = key_id_for_preload_backend(&key); - kms_keys.insert(key_id.clone(), key); - known_keys.insert(key_id, account.secret_key.as_public_key()); - } - - let unrelated = WalletAccount::deterministic(102, 0, true).expect("account"); - let unrelated_key: Key = unrelated.secret_key.clone().into(); - let unrelated_key_id = key_id_for_preload_backend(&unrelated_key); - kms_keys.insert(unrelated_key_id.clone(), unrelated_key); - known_keys.insert( - unrelated_key_id.clone(), - unrelated.secret_key.as_public_key(), - ); - - let ed25519_key: Key = Ed25519Key::from_bytes(&[9; 32]).into(); - let ed25519_key_id = key_id_for_preload_backend(&ed25519_key); - kms_keys.insert(ed25519_key_id.clone(), ed25519_key); - let voucher_master = WalletAccount::deterministic(104, 0, true).expect("account"); - let voucher_key: Key = voucher_master.secret_key.clone().into(); - let voucher_master_key_id = key_id_for_preload_backend(&voucher_key); - kms_keys.insert(voucher_master_key_id.clone(), voucher_key); - known_keys.insert( - voucher_master_key_id.clone(), - voucher_master.secret_key.as_public_key(), - ); - - remove_external_scenario_wallet_keys_from_maps( - &mut known_keys, - &mut kms_keys, - &scenario_key_ids, - ); - remove_external_scenario_wallet_keys_from_maps( - &mut known_keys, - &mut kms_keys, - &scenario_key_ids, - ); - - for key_id in &scenario_key_ids { - assert!(!kms_keys.contains_key(key_id)); - assert!(!known_keys.contains_key(key_id)); - } - assert!(kms_keys.contains_key(&unrelated_key_id)); - assert!(known_keys.contains_key(&unrelated_key_id)); - assert!(kms_keys.contains_key(&ed25519_key_id)); - assert!(kms_keys.contains_key(&voucher_master_key_id)); - assert!(known_keys.contains_key(&voucher_master_key_id)); - } - - #[test] - fn empty_scenario_wallet_set_changes_nothing() { - let mut kms_keys = HashMap::new(); - let mut known_keys = HashMap::new(); - let account = WalletAccount::deterministic(105, 0, true).expect("account"); - let key: Key = account.secret_key.clone().into(); - let key_id = key_id_for_preload_backend(&key); - kms_keys.insert(key_id.clone(), key); - known_keys.insert(key_id, account.secret_key.as_public_key()); - let before_kms = kms_keys.clone(); - let before_known_keys = known_keys.clone(); - - remove_external_scenario_wallet_keys_from_maps( - &mut known_keys, - &mut kms_keys, - &HashSet::new(), - ); - - assert_eq!(kms_keys, before_kms); - assert_eq!(known_keys, before_known_keys); - } -} diff --git a/tests/src/cucumber/steps/manual_transactions/command_file_utils.rs b/tests/src/cucumber/steps/manual_transactions/command_file_utils.rs deleted file mode 100644 index eb124d6c7..000000000 --- a/tests/src/cucumber/steps/manual_transactions/command_file_utils.rs +++ /dev/null @@ -1,1818 +0,0 @@ -//! This module executes manual commands for Cucumber scenarios. -//! -//! External command controller: -//! - Set `CUCUMBER_MANUAL_COMMAND_FILE=/tmp/cucumber-manual-commands.txt`. -//! - Start the scenario. -//! - Prepare the command file beforehand, or append commands while the test -//! runs. -//! -//! Supported commands (one per line): -//! -//! ```text -//! COIN_SPLIT, wallet '', outputs , value -//! VERIFY, wallet '', outputs , time_out -//! BALANCE, wallet '' -//! EXPORT_FUNDS, wallet '', value , output '', include_secret true|false -//! BALANCE_ALL_WALLETS -//! BALANCE_ALL_USER_WALLETS -//! BALANCE_ALL_FUNDING_WALLETS -//! CLEAR_ENCUMBRANCES, wallet '' -//! CLEAR_ENCUMBRANCES_ALL_WALLETS -//! SEND, num_transactions , value , from '', to '' -//! DRAIN, from '', to '' -//! DRAIN_ALL_NODE_WALLETS, node_name '', to '' -//! VERIFY_MAX, wallet '', wallet_state_type 'on-chain'/'encumbered'/'available', outputs , value 14000, time_out -//! VERIFY_MIN, wallet '', wallet_state_type 'on-chain'/'encumbered'/'available', outputs , value 14000, time_out -//! CONTINUOUS_ROUND_ROBIN_USER_WALLETS, coin_split_outputs , coin_split_value , num_transactions , value , cycles , epochs_headroom -//! COIN_SPLIT_ALL_USER_WALLETS, splits_per_wallet , outputs , value -//! VERIFY_MIN_AVAILABLE_OUTPUTS_ALL_USER_WALLETS, min_outputs , timeout_seconds -//! CONTINUOUS_NEXT_WALLET_USER_WALLETS, cycles , num_transactions , value , epochs_headroom -//! FAUCET_ALL_USER_WALLETS, rounds -//! FAUCET_ALL_FUNDING_WALLETS, rounds -//! CREATE_SNAPSHOT_ALL_NODES, snapshot_name '' -//! CREATE_SNAPSHOT_NODE, snapshot_name '', node_name '' -//! RESTART_NODE, node_name '' -//! CRYPTARCHIA_INFO_ALL_NODES -//! WAIT_ALL_NODES_SYNCED_TO_CHAIN -//! STOP -//! ``` - -use std::{ - collections::{BTreeMap, HashSet}, - env, fs, - hash::BuildHasher, - num::NonZero, - path::Path, - time::Duration, -}; - -use lb_core::mantle::{ - NoteId, Utxo, - transactions::{GasPrices, hash::TxHash}, -}; -use lb_key_management_system_service::keys::ZkPublicKey; -use lb_wallet::WalletError; -use serde::Serialize; -use tokio::time::{Instant, sleep}; -use tracing::{info, warn}; - -use crate::{ - common::wallet::{TransactionFeePolicy, WalletStateView, WalletUtxos, build_fee_horizon}, - cucumber::{ - error::{StepError, StepResult}, - steps::{ - TARGET, manual_nodes, - manual_nodes::utils::{ - create_snapshot_all_nodes_with_wallet_state, - create_snapshot_node_with_wallet_state, restart_node, - wait_for_all_nodes_to_be_synced_to_chain, - }, - manual_transactions::{ - command_file_parsing::{ManualCommand, take_next_command}, - drain_wallets::{drain_all_node_wallets, drain_node_wallet, drain_user_wallet}, - utils, - utils::{BestNodeInfo, WalletOutputState, extend_note_id_set, extend_tx_hash_set}, - }, - }, - wallet::{ - best_node::get_best_node_info, - checks::wait_for_observed_transaction_hashes, - submissions::{SignedUserWalletSubmission, validate_fee_horizon_after_wallet_batch}, - sync, - sync::{WalletSendReadiness, current_available_utxos_for_user_wallets}, - }, - world::{CucumberWorld, WalletInfo, WalletType}, - }, -}; - -const MANUAL_COMMAND_FILE_ENV: &str = "CUCUMBER_MANUAL_COMMAND_FILE"; -const MANUAL_COMMAND_POLL_INTERVAL_ENV: &str = "CUCUMBER_MANUAL_COMMAND_POLL_INTERVAL_MS"; -const MAX_TEST_EPOCH_HEADROOM: u32 = 16; - -pub(crate) async fn build_cycle_fee_policy( - world: &CucumberWorld, - step: &str, - representative_wallet: &str, - epochs_headroom: u32, -) -> Result { - if epochs_headroom > MAX_TEST_EPOCH_HEADROOM { - return Err(StepError::InvalidArgument { - message: format!("epochs headroom per cycle must be at most {MAX_TEST_EPOCH_HEADROOM}"), - }); - } - - let best = get_best_node_info(world, representative_wallet, None).await?; - let best_node = best.best_node_for_wallet(world, representative_wallet)?; - let client = world.resolve_node_http_client(&best_node)?; - let consensus = client - .consensus_info() - .await - .map_err(|source| StepError::StepFail { - message: format!("Step `{step}` error: consensus query failed: {source}"), - })?; - let tip = consensus.cryptarchia_info.tip; - let prices = client - .gas_prices(Some(tip)) - .await - .map_err(|source| StepError::StepFail { - message: format!("Step `{step}` error: gas prices query failed: {source}"), - })?; - if prices.tip != tip { - return Err(StepError::StepFail { - message: format!( - "Step `{step}` error: gas prices response referenced tip {:?}, requested {:?}", - prices.tip, tip - ), - }); - } - let live_prices = GasPrices { - execution_base_gas_price: prices.execution_base_gas_price, - storage_gas_price: prices.storage_gas_price, - }; - let horizon = build_fee_horizon( - tip, - u64::from(consensus.cryptarchia_info.slot), - world.chain.slots_per_epoch, - epochs_headroom, - live_prices, - ) - .map_err(|source| StepError::LogicalError { - message: format!("failed to build transaction fee horizon: {source}"), - })?; - let policy = TransactionFeePolicy::new(horizon).map_err(|source| StepError::LogicalError { - message: format!("failed to build transaction fee policy: {source}"), - })?; - info!( - target: TARGET, - "Cycle fee horizon: prepared tip {:?}, slot {:?}, epoch {:?}, valid through {:?}, execution {}->{}, storage {}->{}, priority reserve {}%, representative wallet `{representative_wallet}`", - policy.horizon.prepared_at_tip, - consensus.cryptarchia_info.slot, - policy.horizon.prepared_at_epoch, - policy.horizon.valid_through_epoch, - policy.horizon.live_prices.execution_base_gas_price.into_inner(), - policy.horizon.ceiling_prices.execution_base_gas_price.into_inner(), - policy.horizon.live_prices.storage_gas_price.into_inner(), - policy.horizon.ceiling_prices.storage_gas_price.into_inner(), - policy.priority_fee_percent, - ); - Ok(policy) -} - -pub(crate) async fn execute_manual_command( - world: &mut CucumberWorld, - step: &str, - command: &ManualCommand, -) -> Result { - if matches!(command, ManualCommand::Stop) { - return Ok(true); - } - - execute_non_stop_manual_command(world, step, command).await?; - Ok(false) -} - -#[expect( - clippy::too_many_arguments, - reason = "Cucumber transaction shape and fee policy inputs" -)] -pub(crate) async fn execute_continuous_round_robin_user_wallets( - world: &mut CucumberWorld, - step: &str, - coin_split_outputs: usize, - coin_split_value: u64, - num_transactions: usize, - value: u64, - cycles: usize, - epochs_headroom: u32, -) -> Result<(), StepError> { - let command = ManualCommand::ContinuousRoundRobinUserWallets { - coin_split_outputs, - coin_split_value, - num_transactions, - value, - cycles, - epochs_headroom, - }; - execute_non_stop_manual_command(world, step, &command).await -} - -pub(crate) async fn execute_coin_splits_all_user_wallets( - world: &mut CucumberWorld, - step: &str, - splits_per_wallet: usize, - outputs: usize, - value: u64, -) -> Result<(), StepError> { - let mut wallet_names: Vec<_> = world - .all_user_wallets() - .iter() - .map(|w| w.wallet_name.clone()) - .collect(); - if wallet_names.len() < 2 { - return Err(StepError::InvalidArgument { - message: "coin split for all user wallets requires at least two wallets".to_owned(), - }); - } - wallet_names.sort(); - let mut available_utxos = current_available_utxos_for_user_wallets(world, step).await?; - - for wallet_name in &wallet_names { - let best_node_info = sync::wait_wallet_send_ready( - world, - step, - wallet_name, - 180, - splits_per_wallet as u64 * outputs as u64 * value, - WalletSendReadiness::TotalValueOnly, - &mut available_utxos, - &HashSet::new(), - ) - .await?; - - for _ in 0..splits_per_wallet { - execute_coin_split_with_utxo_cache( - world, - step, - wallet_name, - outputs, - value, - Some(&best_node_info), - &mut available_utxos, - ) - .await?; - } - } - - Ok(()) -} - -pub(crate) async fn verify_min_outputs_all_user_wallets( - world: &mut CucumberWorld, - step: &str, - min_outputs: usize, - timeout_seconds: u64, - wallet_state_type: WalletOutputState, -) -> Result<(), StepError> { - let mut wallet_names: Vec<_> = world - .all_user_wallets() - .iter() - .map(|w| w.wallet_name.clone()) - .collect(); - wallet_names.sort(); - - for wallet_name in &wallet_names { - utils::wait_for_wallet_output_state( - world, - step, - wallet_name.clone(), - Some(&min_outputs), - None, - None, - None, - timeout_seconds, - wallet_state_type, - ) - .await?; - } - - Ok(()) -} - -fn destructure_next_wallet_command( - command: &ManualCommand, -) -> Result<(usize, usize, u64, u32), StepError> { - let ManualCommand::ContinuousNextWalletUserWallets { - cycles, - num_transactions, - value, - epochs_headroom, - } = command - else { - return Err(StepError::LogicalError { - message: "expected ContinuousNextWalletUserWallets command".to_owned(), - }); - }; - Ok((*cycles, *num_transactions, *value, *epochs_headroom)) -} - -pub(crate) async fn execute_continuous_next_wallet_user_wallet( - world: &mut CucumberWorld, - step: &str, - command: &ManualCommand, -) -> Result<(), StepError> { - execute_continuous_next_wallet_user_wallet_inner(world, step, command).await -} - -async fn execute_continuous_next_wallet_user_wallet_inner( - world: &mut CucumberWorld, - step: &str, - command: &ManualCommand, -) -> Result<(), StepError> { - let (cycles, transactions_per_wallet, value, epochs_headroom) = - destructure_next_wallet_command(command)?; - let wallet_names = all_user_wallets(world)?; - - let mut used_input_note_ids: HashSet = HashSet::new(); - let mut all_next_wallet_tx_hashes = HashSet::new(); - for cycle in 0..cycles { - let mut available_utxos = current_available_utxos_for_user_wallets(world, step).await?; - - let (cycle_tx_hashes, cycle_used_input_note_ids) = execute_ring_send_round_with_utxo_cache( - world, - step, - &wallet_names, - transactions_per_wallet, - value, - cycle, - epochs_headroom, - &mut available_utxos, - &used_input_note_ids, - ) - .await?; - verify_no_duplicate_transactions( - &cycle_tx_hashes, - &all_next_wallet_tx_hashes, - cycle, - "CONTINUOUS NEXT WALLET", - )?; - extend_note_id_set(&mut used_input_note_ids, &cycle_used_input_note_ids); - - verify_transactions_mined( - world, - step, - &cycle_tx_hashes, - wallet_names.len() * transactions_per_wallet, - Some(cycle + 1), - "CONTINUOUS NEXT WALLET", - "D", - ) - .await?; - extend_tx_hash_set(&mut all_next_wallet_tx_hashes, &cycle_tx_hashes); - } - - let expected_total = cycles * wallet_names.len() * transactions_per_wallet; - if all_next_wallet_tx_hashes.len() != expected_total { - return Err(StepError::StepFail { - message: format!( - "CONTINUOUS NEXT WALLET submitted {} unique transaction hash(es), expected \ - {expected_total}", - all_next_wallet_tx_hashes.len(), - ), - }); - } - - info!( - target: TARGET, - "CONTINUOUS NEXT WALLET scenario complete: {} unique submitted transaction(s) verified \ - across {} cycle(s)", - all_next_wallet_tx_hashes.len(), - cycles, - ); - - Ok(()) -} - -async fn verify_transactions_mined( - world: &mut CucumberWorld, - step: &str, - tx_hashes: &HashSet, - expected_tx_count: usize, - cycle: Option, - tag: &str, - phase: &str, -) -> Result<(), StepError> { - if tx_hashes.len() != expected_tx_count { - return Err(StepError::StepFail { - message: format!( - "{tag}{} submitted {} transaction hash(es), expected {expected_tx_count}", - cycle.map_or_else(String::new, |cycle| format!(" cycle {cycle}")), - tx_hashes.len(), - ), - }); - } - - info!( - target: TARGET, - "{tag}{} {phase}: Wait for {} submitted transaction hashes to be observed in chain blocks", - cycle.map_or_else(String::new, |cycle| format!(" cycle {cycle}")), - tx_hashes.len(), - ); - - wait_for_observed_transaction_hashes(world, step, tx_hashes, Duration::from_mins(10)).await -} - -fn log_phase_counts( - tag: &str, - cycle: usize, - phase: &str, - kind: &str, - counts: &BTreeMap, -) { - let counts = counts - .iter() - .map(|(wallet, count)| format!("{wallet}={count}")) - .collect::>() - .join(", "); - info!( - target: TARGET, - "{tag} cycle {} {phase}: {kind} tx counts by sender wallet: {counts}", - cycle + 1, - ); -} - -#[expect(clippy::too_many_arguments, reason = "Need all args")] -async fn execute_ring_send_round_with_utxo_cache( - world: &mut CucumberWorld, - step: &str, - wallet_names: &[String], - transactions_per_wallet: usize, - value: u64, - cycle: usize, - epochs_headroom: u32, - available_utxos: &mut WalletUtxos, - used_input_note_ids: &HashSet, -) -> Result<(HashSet, HashSet), StepError> { - let policy = build_cycle_fee_policy(world, step, &wallet_names[0], epochs_headroom).await?; - - let mut signed_submissions = Vec::with_capacity(wallet_names.len() * transactions_per_wallet); - let mut prepared_counts = BTreeMap::new(); - - for i in 0..wallet_names.len() { - info!( - target: TARGET, - "CONTINUOUS NEXT WALLET cycle {} A: Await funds", - cycle + 1 - ); - let from = &wallet_names[i]; - let to = &wallet_names[(i + 1) % wallet_names.len()]; - - let required_available = transactions_per_wallet as u64 * value; - sync::wait_wallet_send_ready( - world, - step, - from, - 180, - required_available, - WalletSendReadiness::EligibleUtxoBatch { - min_required_outputs: transactions_per_wallet, - min_value_per_transaction: value, - }, - available_utxos, - used_input_note_ids, - ) - .await?; - - info!( - target: TARGET, - "CONTINUOUS NEXT WALLET cycle {} B: Prepare transactions to next wallet concurrently", - cycle + 1 - ); - let mut prepared = prepare_ring_send_round_send_with_utxo_cache( - world, - step, - transactions_per_wallet, - value, - from, - to, - available_utxos, - Some(policy.horizon.ceiling_prices.clone()), - policy.priority_fee_percent, - ) - .await?; - prepared_counts.insert(from.clone(), prepared.len()); - validate_fee_horizon_after_wallet_batch(world, &policy, from, prepared.len()).await?; - signed_submissions.append(&mut prepared); - } - - let mut cycle_used_input_note_ids: HashSet = HashSet::new(); - for submission in &signed_submissions { - extend_note_id_set( - &mut cycle_used_input_note_ids, - &submission.reserved_inputs().input_note_ids_list(), - ); - } - - log_phase_counts( - "CONTINUOUS NEXT WALLET", - cycle, - "C", - "prepared", - &prepared_counts, - ); - - let submitted_hashes = utils::submit_signed_user_wallet_submissions_concurrently( - world, - signed_submissions, - Some(&policy), - ) - .await?; - let mut submitted_counts = BTreeMap::new(); - for (sender, _) in &submitted_hashes { - *submitted_counts.entry(sender.clone()).or_insert(0usize) += 1; - } - log_phase_counts( - "CONTINUOUS NEXT WALLET", - cycle, - "D", - "submitted", - &submitted_counts, - ); - let cycle_tx_hashes = submitted_hashes - .into_iter() - .map(|(_, tx_hash)| tx_hash) - .collect::>(); - - Ok((cycle_tx_hashes, cycle_used_input_note_ids)) -} - -#[expect(clippy::too_many_lines, reason = "Test function.")] -async fn execute_non_stop_manual_command( - world: &mut CucumberWorld, - step: &str, - command: &ManualCommand, -) -> Result<(), StepError> { - match command { - ManualCommand::CreateSnapshotAllNodes { snapshot_name } => { - create_snapshot_all_nodes_with_wallet_state(world, snapshot_name).await - } - ManualCommand::CreateSnapshotNode { - snapshot_name, - node_name, - } => create_snapshot_node_with_wallet_state(world, snapshot_name, node_name).await, - ManualCommand::CoinSplit { - wallet, - outputs, - value, - } => execute_coin_split(world, step, wallet, *outputs, *value) - .await - .map(|_| ()), - ManualCommand::Verify { .. } => handle_verify_command(world, step, command).await, - ManualCommand::WalletBalance { wallet_name } => { - log_wallet_balance(world, step, wallet_name).await - } - ManualCommand::WalletBalanceAllUserWallets => { - log_wallet_balances(world, step, world.all_user_wallets()).await - } - ManualCommand::WalletBalanceAllFundingWallets => { - log_wallet_balances(world, step, world.all_node_wallets()).await - } - ManualCommand::WalletBalanceAllWallets => { - let mut wallets = world.all_user_wallets(); - wallets.extend(world.all_node_wallets()); - - log_wallet_balances(world, step, wallets).await - } - ManualCommand::ExportFunds { - wallet_name, - value, - output_path, - include_secret, - } => { - export_funds( - world, - step, - wallet_name, - *value, - output_path, - *include_secret, - ) - .await - } - ManualCommand::ClearEncumbrances { wallet_name } => { - clear_wallet_encumbrances(world, step, wallet_name) - } - ManualCommand::ClearEncumbrancesAllWallets => clear_all_wallet_encumbrances(world, step), - ManualCommand::Send { - num_transactions, - value, - from, - to, - } => execute_send(world, step, *num_transactions, *value, from, to).await, - ManualCommand::Drain { from, to } => execute_drain(world, step, from, to).await, - ManualCommand::DrainAllNodeWallets { node_name, to } => { - drain_all_node_wallets(world, node_name, to).await - } - ManualCommand::ContinuousRoundRobinUserWallets { .. } => { - execute_continuous_round_robin(world, step, command).await - } - ManualCommand::FaucetFundsAllUserWallets { rounds } => { - request_faucet_funds_all_user_wallets(world, step, *rounds) - } - ManualCommand::FaucetFundsAllFundingWallets { rounds } => { - request_faucet_funds_all_funding_wallets(world, step, *rounds) - } - ManualCommand::RestartNode { node_name } => restart_node(world, step, node_name).await, - ManualCommand::CryptarchiaInfoAllNodes => { - manual_nodes::utils::get_cryptarchia_info_all_nodes(world, step).await; - Ok(()) - } - ManualCommand::WaitAllNodesSyncedToChain => { - wait_for_all_nodes_to_be_synced_to_chain(world, step).await - } - ManualCommand::CoinSplitAllUserWallets { - splits_per_wallet, - outputs, - value, - } => { - execute_coin_splits_all_user_wallets(world, step, *splits_per_wallet, *outputs, *value) - .await - } - ManualCommand::VerifyMinAvailableOutputsAllUserWallets { - min_outputs, - timeout_seconds, - } => { - verify_min_outputs_all_user_wallets( - world, - step, - *min_outputs, - *timeout_seconds, - WalletOutputState::Available, - ) - .await - } - ManualCommand::ContinuousNextWalletUserWallets { .. } => { - execute_continuous_next_wallet_user_wallet(world, step, command).await - } - ManualCommand::Stop => Ok(()), - } -} - -async fn execute_drain(world: &mut CucumberWorld, step: &str, from: &str, to: &str) -> StepResult { - let sender = world.resolve_wallet(from)?; - let receiver_pk = world.resolve_recipient(to)?.public_key; - - if sender.public_key()? == receiver_pk { - return Err(StepError::InvalidArgument { - message: format!("Cannot drain wallet `{from}` into itself"), - }); - } - - match sender.wallet_type { - WalletType::User { .. } => drain_user_wallet(world, step, &sender, receiver_pk).await, - WalletType::Funding { .. } => drain_node_wallet(world, &sender, receiver_pk).await, - } -} - -pub async fn log_wallet_balances( - world: &mut CucumberWorld, - step: &str, - wallets: Vec, -) -> StepResult { - let tracked_wallets = wallets - .iter() - .filter(|wallet| wallet.is_user_wallet()) - .cloned() - .collect::>(); - let states = if tracked_wallets.is_empty() { - BTreeMap::new() - } else { - utils::current_wallet_states_for_wallets(world, step, &tracked_wallets).await? - }; - - for wallet in &wallets { - if wallet.is_node_wallet() { - log_node_wallet_balance(world, wallet).await?; - continue; - } - let state = - states - .get(wallet.wallet_name.as_str()) - .ok_or_else(|| StepError::LogicalError { - message: format!( - "Wallet `{}` balance state is not tracked", - wallet.wallet_name - ), - })?; - log_wallet_state_balance(&wallet.wallet_name, &wallet.public_key_hex(), state); - } - Ok(()) -} - -async fn log_node_wallet_balance(world: &CucumberWorld, wallet: &WalletInfo) -> StepResult { - let node = world - .nodes_info - .get(&wallet.node_name) - .ok_or_else(|| StepError::LogicalError { - message: format!( - "Node '{}' for wallet '{}' not found", - wallet.node_name, wallet.wallet_name - ), - })?; - - let balance_response = node - .started_node - .client - .wallet_balance(wallet.public_key()?, None) - .await; - match balance_response { - Ok(balance) => { - info!( - target: TARGET, - "Wallet `{}` [On-chain] {}/{} LGO, {}", - wallet.wallet_name, - balance.notes.len(), - balance.balance, - wallet.public_key_hex(), - ); - } - Err(_) => { - info!( - target: TARGET, - "Wallet `{}` [On-chain] no funds yet, {}", - wallet.wallet_name, - wallet.public_key_hex(), - ); - } - } - - Ok(()) -} - -async fn log_wallet_balance( - world: &mut CucumberWorld, - step: &str, - wallet_name: &str, -) -> StepResult { - let wallet = world.resolve_wallet(wallet_name)?; - log_wallet_balances(world, step, vec![wallet]).await -} - -fn log_wallet_state_balance(wallet_name: &str, public_key_hex: &str, state: &WalletStateView) { - let available = state.balance(WalletOutputState::Available); - let reserved = state.balance(WalletOutputState::Reserved); - let on_chain = state.balance(WalletOutputState::OnChain); - - info!( - target: TARGET, - "Wallet `{wallet_name}` [Available] {}/{} LGO, [Encumbered] {}/{} LGO, \ - [On-chain] {}/{} LGO, {}", - available.output_count, - available.value, - reserved.output_count, - reserved.value, - on_chain.output_count, - on_chain.value, - public_key_hex, - ); -} - -#[derive(Serialize)] -struct WalletFundsExport { - wallet: String, - node_url: String, - public_key: String, - #[serde(skip_serializing_if = "Option::is_none")] - secret_key: Option, - requested_value: u64, - selected_value: u64, - #[serde(skip_serializing_if = "Option::is_none")] - height: Option, - utxos: Vec, -} - -#[derive(Serialize)] -struct ExportedUtxo { - utxo_id: String, - value: u64, - encoded_utxo: String, -} - -async fn export_funds( - world: &mut CucumberWorld, - step: &str, - wallet_name: &str, - value: u64, - output_path: &str, - include_secret: bool, -) -> Result<(), StepError> { - let wallet = world.resolve_wallet(wallet_name)?.clone(); - let available_utxos = current_available_utxos_for_user_wallets(world, step) - .await? - .get(wallet_name) - .cloned() - .ok_or(StepError::LogicalError { - message: format!("Wallet '{wallet_name}' not found in updated balances"), - })?; - let selected = select_utxos_covering(available_utxos.clone(), value)?; - let selected_value = selected.iter().map(|utxo| utxo.note.value).sum(); - let export = WalletFundsExport { - wallet: wallet.wallet_name.clone(), - node_url: format!( - "{}", - world - .resolve_node_http_client(&wallet.node_name)? - .base_url() - ), - public_key: wallet.public_key_hex(), - secret_key: exported_secret_key(&wallet, include_secret)?, - requested_value: value, - selected_value, - height: best_known_wallet_node_height(world, &wallet).await, - utxos: selected - .iter() - .map(exported_utxo) - .collect::, _>>()?, - }; - - let path = Path::new(output_path); - if let Some(parent) = path.parent() - && !parent.as_os_str().is_empty() - { - fs::create_dir_all(parent).map_err(|error| StepError::StepFail { - message: format!( - "Failed to create EXPORT_FUNDS output directory '{}': {error}", - parent.display() - ), - })?; - } - let json = serde_json::to_string_pretty(&export).map_err(|error| StepError::StepFail { - message: format!("Failed to serialize EXPORT_FUNDS JSON: {error}"), - })?; - fs::write(path, json).map_err(|error| StepError::StepFail { - message: format!( - "Failed to write EXPORT_FUNDS output '{}': {error}", - path.display() - ), - })?; - - info!( - target: TARGET, - "EXPORT_FUNDS wrote {} UTXO(s), selected value {}, requested value {}, output '{}'", - export.utxos.len(), - selected_value, - value, - path.display() - ); - Ok(()) -} - -fn select_utxos_covering(mut utxos: Vec, value: u64) -> Result, StepError> { - utxos.sort_by_key(|utxo| std::cmp::Reverse(utxo.note.value)); - let available = utxos.iter().map(|utxo| utxo.note.value).sum(); - let mut selected = Vec::new(); - let mut selected_value = 0u64; - - for utxo in utxos { - selected_value = selected_value.saturating_add(utxo.note.value); - selected.push(utxo); - if selected_value >= value { - return Ok(selected); - } - } - - Err(StepError::WalletError(WalletError::InsufficientFunds { - available, - })) -} - -fn exported_secret_key( - wallet: &WalletInfo, - include_secret: bool, -) -> Result, StepError> { - if !include_secret { - return Ok(None); - } - - let WalletType::User { wallet_account } = &wallet.wallet_type else { - return Err(StepError::InvalidArgument { - message: format!( - "EXPORT_FUNDS include_secret true requires a user wallet; '{}' is a funding wallet", - wallet.wallet_name - ), - }); - }; - - bincode::serialize(&wallet_account.secret_key) - .map(hex::encode) - .map(Some) - .map_err(|error| StepError::StepFail { - message: format!("Failed to encode wallet secret key for EXPORT_FUNDS: {error}"), - }) -} - -fn exported_utxo(utxo: &Utxo) -> Result { - Ok(ExportedUtxo { - utxo_id: hex::encode(utxo.id().as_bytes()), - value: utxo.note.value, - encoded_utxo: bincode::serialize(utxo).map(hex::encode).map_err(|error| { - StepError::StepFail { - message: format!("Failed to encode UTXO for EXPORT_FUNDS: {error}"), - } - })?, - }) -} - -async fn best_known_wallet_node_height(world: &CucumberWorld, wallet: &WalletInfo) -> Option { - let node = world.nodes_info.get(&wallet.node_name)?; - node.started_node - .client - .consensus_info() - .await - .ok() - .map(|info| info.cryptarchia_info.height) -} - -fn clear_wallet_encumbrances( - world: &mut CucumberWorld, - step: &str, - wallet_name: &str, -) -> StepResult { - if world.resolve_wallet(wallet_name).is_err() { - warn!(target: TARGET, "Step `{}` error: wallet '{wallet_name}' not found in world state", step); - return Err(StepError::LogicalError { - message: format!("wallet '{wallet_name}' not found in world state"), - }); - } - - world.with_wallets_mut(|wallets| wallets.clear_encumbrances(wallet_name))?; - world - .wallet_registry - .fee_state - .clear_wallet_reservations(wallet_name); - info!(target: TARGET, "Cleared encumbrances for wallet '{wallet_name}'"); - Ok(()) -} - -fn clear_all_wallet_encumbrances(world: &mut CucumberWorld, step: &str) -> StepResult { - let wallet_names: Vec = world.wallet_registry.wallet_info.keys().cloned().collect(); - - for wallet_name in wallet_names { - clear_wallet_encumbrances(world, step, &wallet_name)?; - } - info!(target: TARGET, "Cleared encumbrances for all wallets"); - Ok(()) -} - -async fn handle_verify_command( - world: &mut CucumberWorld, - step: &str, - command: &ManualCommand, -) -> Result<(), StepError> { - let ManualCommand::Verify { - wallet, - outputs, - value, - time_out, - wallet_state_type, - verify_max, - } = command - else { - unreachable!("handle_verify_command must be called with ManualCommand::Verify") - }; - - let verify_min = !*verify_max; - utils::wait_for_wallet_output_state( - world, - step, - wallet.clone(), - if verify_min { outputs.as_ref() } else { None }, - if *verify_max { outputs.as_ref() } else { None }, - if verify_min { value.as_ref() } else { None }, - if *verify_max { value.as_ref() } else { None }, - *time_out, - *wallet_state_type, - ) - .await -} - -fn request_faucet_funds_all_user_wallets( - world: &mut CucumberWorld, - step: &str, - rounds: usize, -) -> Result<(), StepError> { - let number_of_rounds = NonZero::new(rounds).ok_or_else(|| StepError::InvalidArgument { - message: "Invalid value for 'rounds': '0'".to_owned(), - })?; - let all_wallets_pk_hex = world - .wallet_registry - .wallet_info - .values() - .filter(|w| w.is_user_wallet()) - .map(WalletInfo::public_key_hex) - .collect::>(); - utils::request_faucet_funds(world, step, number_of_rounds, &all_wallets_pk_hex) -} - -fn request_faucet_funds_all_funding_wallets( - world: &mut CucumberWorld, - step: &str, - rounds: usize, -) -> Result<(), StepError> { - let number_of_rounds = NonZero::new(rounds).ok_or_else(|| StepError::InvalidArgument { - message: "Invalid value for 'rounds': '0'".to_owned(), - })?; - let all_wallets_pk_hex = world - .wallet_registry - .wallet_info - .values() - .filter(|wallet| wallet.is_node_funding_wallet()) - .map(WalletInfo::public_key_hex) - .collect::>(); - utils::request_faucet_funds(world, step, number_of_rounds, &all_wallets_pk_hex) -} - -async fn execute_coin_split( - world: &mut CucumberWorld, - step: &str, - wallet_name: &str, - outputs: usize, - value: u64, -) -> Result, StepError> { - let wallet = world.resolve_wallet(wallet_name)?; - let self_pk = wallet.public_key()?; - let receivers = vec![(self_pk, value); outputs]; - - let mut available_utxos = WalletUtxos::new(); - let best_node_info = sync::wait_wallet_send_ready( - world, - step, - wallet_name, - 180, - outputs as u64 * value, - WalletSendReadiness::TotalValueOnly, - &mut available_utxos, - &HashSet::new(), - ) - .await?; - - utils::create_and_submit_transaction_hashes_with_utxo_cache( - world, - step, - wallet_name, - &receivers, - Some(&best_node_info), - Some(&mut available_utxos), - ) - .await -} - -async fn execute_coin_split_with_utxo_cache( - world: &mut CucumberWorld, - step: &str, - wallet_name: &str, - outputs: usize, - value: u64, - best_node_info: Option<&BestNodeInfo>, - available_utxos: &mut WalletUtxos, -) -> Result, StepError> { - let wallet = world.resolve_wallet(wallet_name)?; - let self_pk = wallet.public_key()?; - let receivers = vec![(self_pk, value); outputs]; - utils::create_and_submit_transaction_hashes_with_utxo_cache( - world, - step, - wallet_name, - &receivers, - best_node_info, - Some(available_utxos), - ) - .await -} - -async fn prepare_signed_submissions_with_utxo_cache( - world: &mut CucumberWorld, - step: &str, - requests: Vec<(String, Vec<(ZkPublicKey, u64)>)>, - available_utxos: &mut WalletUtxos, - gas_prices: Option, - priority_fee_percent: u64, -) -> Result, StepError> { - let mut reserved_submissions = Vec::with_capacity(requests.len()); - - for (sender, receivers) in requests { - let reserved_submission = - utils::reserve_user_wallet_transaction_submission_with_utxo_cache( - world, - step, - &sender, - &receivers, - available_utxos, - gas_prices.clone(), - priority_fee_percent, - ) - .await?; - reserved_submissions.push(reserved_submission); - } - - utils::finalize_reserved_user_wallet_submissions_concurrently(step, reserved_submissions).await -} - -#[expect(clippy::too_many_arguments, reason = "Coin-split preparation inputs")] -async fn prepare_coin_splits_all_wallets_with_utxo_cache( - world: &mut CucumberWorld, - step: &str, - wallet_names: &[String], - outputs: usize, - value: u64, - available_utxos: &mut WalletUtxos, - gas_prices: Option, - priority_fee_percent: u64, -) -> Result<(Vec, BTreeMap), StepError> { - let mut requests = Vec::with_capacity(wallet_names.len()); - let mut prepared_counts = BTreeMap::new(); - - for wallet_name in wallet_names { - let wallet = world.resolve_wallet(wallet_name)?; - let self_pk = wallet.public_key()?; - let receivers = vec![(self_pk, value); outputs]; - *prepared_counts.entry(wallet_name.clone()).or_insert(0usize) += 1; - requests.push((wallet_name.clone(), receivers)); - } - - let signed_submissions = prepare_signed_submissions_with_utxo_cache( - world, - step, - requests, - available_utxos, - gas_prices, - priority_fee_percent, - ) - .await?; - Ok((signed_submissions, prepared_counts)) -} - -async fn execute_send( - world: &mut CucumberWorld, - step: &str, - number_of_transactions: usize, - value: u64, - from: &str, - to: &str, -) -> Result<(), StepError> { - let receiver = world.resolve_recipient(to)?; - let receiver_pk = receiver.public_key; - - let mut available_utxos = WalletUtxos::new(); - let best_node_info = sync::wait_wallet_send_ready( - world, - step, - from, - 180, - number_of_transactions as u64 * value, - WalletSendReadiness::EligibleUtxoBatch { - min_required_outputs: number_of_transactions, - min_value_per_transaction: value, - }, - &mut available_utxos, - &HashSet::new(), - ) - .await?; - - for i in 0..number_of_transactions { - let result = utils::create_and_submit_transaction( - world, - step, - from, - &[(receiver_pk, value)], - Some(&best_node_info), - Some(&mut available_utxos), - ) - .await; - - if let Err(StepError::WalletError(WalletError::InsufficientFunds { available })) = result { - return Err(StepError::FundsDeficit { - available, - num_utxos_required: number_of_transactions - i, - value_per_utxos_required: value, - }); - } - result?; - } - Ok(()) -} - -#[expect(clippy::too_many_arguments, reason = "Transaction preparation inputs")] -async fn prepare_ring_send_round_send_with_utxo_cache( - world: &mut CucumberWorld, - step: &str, - transactions: usize, - value: u64, - from: &str, - to: &str, - available_utxos: &mut WalletUtxos, - gas_prices: Option, - priority_fee_percent: u64, -) -> Result, StepError> { - let receiver = world.resolve_recipient(to)?; - let receiver_pk = receiver.public_key; - let mut reserved_submissions = Vec::with_capacity(transactions); - - for i in 0..transactions { - let sender_utxo_count_before = available_utxos.get(from).map_or(0usize, Vec::len); - - let receivers = vec![(receiver_pk, value)]; - let reserved_submission = - utils::reserve_user_wallet_transaction_submission_with_utxo_cache( - world, - step, - from, - &receivers, - available_utxos, - gas_prices.clone(), - priority_fee_percent, - ) - .await - .map_err(|error| match error { - StepError::WalletError(WalletError::InsufficientFunds { available }) => { - StepError::FundsDeficit { - available, - num_utxos_required: transactions - i, - value_per_utxos_required: value, - } - } - error => error, - })?; - let sender_utxo_count_after = available_utxos.get(from).map_or(0usize, Vec::len); - - if transactions > 1 && sender_utxo_count_after >= sender_utxo_count_before { - return Err(StepError::LogicalError { - message: format!( - "Batch cache accounting failed for '{from}': expected available input count to \ - decrease between submissions ({sender_utxo_count_before} -> {sender_utxo_count_after})" - ), - }); - } - - reserved_submissions.push(reserved_submission); - } - - utils::finalize_reserved_user_wallet_submissions_concurrently(step, reserved_submissions).await -} - -fn destructure_round_robin_command( - command: &ManualCommand, -) -> Result<(usize, u64, usize, u64, usize, u32), StepError> { - let ManualCommand::ContinuousRoundRobinUserWallets { - coin_split_outputs, - coin_split_value, - num_transactions, - value, - cycles, - epochs_headroom, - } = command - else { - return Err(StepError::LogicalError { - message: "expected ContinuousRoundRobinUserWallets command".to_owned(), - }); - }; - Ok(( - *coin_split_outputs, - *coin_split_value, - *num_transactions, - *value, - *cycles, - *epochs_headroom, - )) -} - -fn all_user_wallets(world: &CucumberWorld) -> Result, StepError> { - let mut wallet_names = world - .all_user_wallets() - .iter() - .map(|w| w.wallet_name.clone()) - .collect::>(); - if wallet_names.len() < 2 { - return Err(StepError::InvalidArgument { - message: "This command requires at least two user wallets".to_owned(), - }); - } - wallet_names.sort(); - Ok(wallet_names) -} - -#[expect(clippy::too_many_arguments, reason = "Transaction preparation inputs")] -async fn prepare_and_submit_round_robin_transactions( - world: &mut CucumberWorld, - step: &str, - cycle: usize, - wallet_names: &[String], - num_transactions: usize, - value: u64, - available_utxos: &mut WalletUtxos, - epochs_headroom: u32, -) -> Result<(HashSet, HashSet), StepError> { - let policy = build_cycle_fee_policy(world, step, &wallet_names[0], epochs_headroom).await?; - - let mut signed_submissions = Vec::with_capacity(wallet_names.len() * num_transactions); - let mut prepared_counts = BTreeMap::new(); - for sender in wallet_names { - let recipients = recipient_wallets(wallet_names, sender)?; - let mut prepared = prepare_round_robin_with_utxo_cache( - world, - step, - sender, - &recipients, - num_transactions, - value, - available_utxos, - Some(policy.horizon.ceiling_prices.clone()), - policy.priority_fee_percent, - ) - .await - .map_err(|e| StepError::StepFail { - message: format!( - "CONTINUOUS ROUND ROBIN cycle {} failed to prepare transactions for sender \ - '{sender}': {e}", - cycle + 1, - ), - })?; - - prepared_counts.insert(sender.clone(), prepared.len()); - validate_fee_horizon_after_wallet_batch(world, &policy, sender, prepared.len()).await?; - signed_submissions.append(&mut prepared); - } - - let mut cycle_used_input_note_ids: HashSet = HashSet::new(); - for submission in &signed_submissions { - extend_note_id_set( - &mut cycle_used_input_note_ids, - &submission.reserved_inputs().input_note_ids_list(), - ); - } - - log_phase_counts( - "CONTINUOUS ROUND ROBIN", - cycle, - "D", - "prepared", - &prepared_counts, - ); - - let submitted_hashes = utils::submit_signed_user_wallet_submissions_concurrently( - world, - signed_submissions, - Some(&policy), - ) - .await?; - let mut submitted_counts = BTreeMap::new(); - for (sender, _) in &submitted_hashes { - *submitted_counts.entry(sender.clone()).or_insert(0usize) += 1; - } - log_phase_counts( - "CONTINUOUS ROUND ROBIN", - cycle, - "D", - "submitted", - &submitted_counts, - ); - - let cycle_tx_hashes = submitted_hashes - .into_iter() - .map(|(_, tx_hash)| tx_hash) - .collect::>(); - - Ok((cycle_tx_hashes, cycle_used_input_note_ids)) -} - -/// Manages the coin split process for a round robin cycle, including performing -/// the coin splits, waiting for them to be mined, and verifying that the -/// transactions were successfully mined. Returns a set of used input note IDs -/// from the coin split transactions. Note: This function needs a readiness -/// prepared UTXO cache. -#[expect(clippy::too_many_arguments, reason = "Round-robin split inputs")] -async fn manage_round_robin_coin_splits_with_utxo_cache( - world: &mut CucumberWorld, - step: &str, - cycle: usize, - wallet_names: &[String], - coin_split_outputs: usize, - coin_split_value: u64, - epochs_headroom: u32, - available_utxos: &mut WalletUtxos, -) -> Result, StepError> { - let policy = build_cycle_fee_policy(world, step, &wallet_names[0], epochs_headroom).await?; - - let (split_tx_hashes, split_used_input_note_ids) = - perform_coin_splits_for_round_robin_with_utxo_cache( - world, - step, - wallet_names, - coin_split_outputs, - coin_split_value, - cycle, - available_utxos, - &policy, - ) - .await?; - - wait_for_n_blocks_or_warn(world, step, wallet_names, Duration::from_mins(3), 2, cycle).await?; - - verify_transactions_mined( - world, - step, - &split_tx_hashes, - wallet_names.len(), - Some(cycle + 1), - "CONTINUOUS ROUND ROBIN", - "B", - ) - .await?; - - Ok(split_used_input_note_ids) -} - -async fn refresh_round_robin_sender_cache_entries( - world: &mut CucumberWorld, - step: &str, - wallet_names: &[String], - required_available: u64, - readiness: WalletSendReadiness, - available_utxos: &mut WalletUtxos, - used_input_note_ids: &HashSet, -) -> Result<(), StepError> { - for sender in wallet_names { - sync::wait_wallet_send_ready( - world, - step, - sender, - 180, - required_available, - readiness, - available_utxos, - used_input_note_ids, - ) - .await?; - } - - Ok(()) -} - -fn verify_no_duplicate_transactions( - cycle_tx_hashes: &HashSet, - all_tx_hashes: &HashSet, - cycle: usize, - scenario_tag: &str, -) -> Result<(), StepError> { - let duplicate_hashes = cycle_tx_hashes - .intersection(all_tx_hashes) - .copied() - .collect::>(); - if duplicate_hashes.is_empty() { - Ok(()) - } else { - Err(StepError::StepFail { - message: format!( - "{scenario_tag} cycle {} prepared/submitted {} duplicate transaction hash(es) from \ - previous cycles", - cycle + 1, - duplicate_hashes.len(), - ), - }) - } -} - -#[expect(clippy::too_many_lines, reason = "Test function.")] -#[expect( - clippy::cognitive_complexity, - reason = "This function has multiple steps that are logically distinct." -)] -async fn execute_continuous_round_robin( - world: &mut CucumberWorld, - step: &str, - command: &ManualCommand, -) -> Result<(), StepError> { - let (coin_split_outputs, coin_split_value, num_transactions, value, cycles, epochs_headroom) = - destructure_round_robin_command(command)?; - let wallet_names = all_user_wallets(world)?; - - let mut used_input_note_ids: HashSet = HashSet::new(); - let mut all_round_robin_tx_hashes = HashSet::new(); - let mut available_utxos = WalletUtxos::new(); - for cycle in 0..cycles { - info!( - target: TARGET, - "CONTINUOUS ROUND ROBIN cycle {} A: Wait for available coin-split funds all wallets", - cycle + 1 - ); - - refresh_round_robin_sender_cache_entries( - world, - step, - &wallet_names, - coin_split_outputs as u64 * coin_split_value, - WalletSendReadiness::TotalValueOnly, - &mut available_utxos, - &used_input_note_ids, - ) - .await?; - - info!( - target: TARGET, - "CONTINUOUS ROUND ROBIN cycle {} B: Perform coin splits all wallets and wait mined", - cycle + 1 - ); - - let split_used_input_note_ids = manage_round_robin_coin_splits_with_utxo_cache( - world, - step, - cycle, - &wallet_names, - coin_split_outputs, - coin_split_value, - epochs_headroom, - &mut available_utxos, - ) - .await?; - extend_note_id_set(&mut used_input_note_ids, &split_used_input_note_ids); - - info!( - target: TARGET, - "CONTINUOUS ROUND ROBIN cycle {} C: Wait all wallets ready with coin-split outputs", - cycle + 1 - ); - - refresh_round_robin_sender_cache_entries( - world, - step, - &wallet_names, - num_transactions as u64 * value, - WalletSendReadiness::EligibleUtxoBatch { - min_required_outputs: num_transactions, - min_value_per_transaction: value, - }, - &mut available_utxos, - &used_input_note_ids, - ) - .await?; - - info!( - target: TARGET, - "CONTINUOUS ROUND ROBIN cycle {} D: Prepare and send all round robin transactions", - cycle + 1 - ); - - let (cycle_tx_hashes, cycle_used_input_note_ids) = - prepare_and_submit_round_robin_transactions( - world, - step, - cycle, - &wallet_names, - num_transactions, - value, - &mut available_utxos, - epochs_headroom, - ) - .await?; - verify_no_duplicate_transactions( - &cycle_tx_hashes, - &all_round_robin_tx_hashes, - cycle, - "CONTINUOUS ROUND ROBIN", - )?; - extend_note_id_set(&mut used_input_note_ids, &cycle_used_input_note_ids); - - // Assert transaction count - verify_transactions_mined( - world, - step, - &cycle_tx_hashes, - wallet_names.len() * num_transactions, - Some(cycle + 1), - "CONTINUOUS ROUND ROBIN", - "E", - ) - .await?; - - // Collect hashes for final drain verification - extend_tx_hash_set(&mut all_round_robin_tx_hashes, &cycle_tx_hashes); - } - - // Final drain: verify submitted D-phase transaction hashes are observed in - // chain blocks. - info!( - target: TARGET, - "CONTINUOUS ROUND ROBIN final: Verify {} submitted round-robin transaction(s) were observed \ - in chain blocks", - all_round_robin_tx_hashes.len(), - ); - - wait_for_observed_transaction_hashes( - world, - step, - &all_round_robin_tx_hashes, - Duration::from_mins(10), - ) - .await?; - - info!( - target: TARGET, - "CONTINUOUS ROUND ROBIN scenario complete: {} transaction(s) verified from observed chain block transaction hashes across {} cycle(s)", - all_round_robin_tx_hashes.len(), - cycles - ); - - Ok(()) -} - -async fn wait_for_n_blocks_or_warn( - world: &CucumberWorld, - step: &str, - wallet_names: &[String], - time_out: Duration, - blocks_to_wait: u64, - cycle: usize, -) -> Result<(), StepError> { - if wallet_names.is_empty() { - return Err(StepError::InvalidArgument { - message: "No wallet names provided for wait_for_n_blocks".to_owned(), - }); - } - let mut last_msg = String::new(); - let best_node_info = get_best_node_info(world, &wallet_names[0], Some(&mut last_msg)).await?; - let node = world - .resolve_node_http_client(&best_node_info.best_node_for_wallet(world, &wallet_names[0])?)?; - let start_height = node.consensus_info().await?.cryptarchia_info.height; - let start = Instant::now(); - loop { - sleep(Duration::from_secs(1)).await; - let best_node_info = - get_best_node_info(world, &wallet_names[0], Some(&mut last_msg)).await?; - let node = world.resolve_node_http_client( - &best_node_info.best_node_for_wallet(world, &wallet_names[0])?, - )?; - let height = node.consensus_info().await?.cryptarchia_info.height; - if height >= start_height + blocks_to_wait { - return Ok(()); - } - if start.elapsed() > time_out { - warn!( - target: TARGET, - "Step `{step}` cycle {}: Chain could not grow by {blocks_to_wait} blocks in {time_out:.2?}", - cycle + 1 - ); - return Ok(()); - } - } -} - -#[expect(clippy::too_many_arguments, reason = "Round-robin split inputs")] -async fn perform_coin_splits_for_round_robin_with_utxo_cache( - world: &mut CucumberWorld, - step: &str, - wallet_names: &[String], - coin_split_outputs: usize, - coin_split_value: u64, - cycle: usize, - available_utxos: &mut WalletUtxos, - policy: &TransactionFeePolicy, -) -> Result<(HashSet, HashSet), StepError> { - info!(target: TARGET, "CONTINUOUS ROUND ROBIN cycle {} B: Perform coin splits all wallets", cycle + 1); - - let (signed_submissions, prepared_split_counts) = - prepare_coin_splits_all_wallets_with_utxo_cache( - world, - step, - wallet_names, - coin_split_outputs, - coin_split_value, - available_utxos, - Some(policy.horizon.ceiling_prices.clone()), - policy.priority_fee_percent, - ) - .await?; - log_phase_counts( - "CONTINUOUS ROUND ROBIN", - cycle, - "B", - "split prepared", - &prepared_split_counts, - ); - - let mut split_used_input_note_ids: HashSet = HashSet::new(); - for submission in &signed_submissions { - extend_note_id_set( - &mut split_used_input_note_ids, - &submission.reserved_inputs().input_note_ids_list(), - ); - } - - let submitted_split_hashes = utils::submit_signed_user_wallet_submissions_concurrently( - world, - signed_submissions, - Some(policy), - ) - .await?; - let mut submitted_split_counts = BTreeMap::new(); - for (sender, _) in &submitted_split_hashes { - *submitted_split_counts - .entry(sender.clone()) - .or_insert(0usize) += 1; - } - log_phase_counts( - "CONTINUOUS ROUND ROBIN", - cycle, - "B", - "split submitted", - &submitted_split_counts, - ); - - let split_tx_hashes = submitted_split_hashes - .into_iter() - .map(|(_, tx_hash)| tx_hash) - .collect::>(); - - Ok((split_tx_hashes, split_used_input_note_ids)) -} - -fn recipient_wallets(wallet_names: &[String], sender: &str) -> Result, StepError> { - let recipients: Vec<_> = wallet_names - .iter() - .filter(|wallet| wallet.as_str() != sender) - .cloned() - .collect(); - if recipients.is_empty() { - return Err(StepError::InvalidArgument { - message: format!("No recipient wallets available for sender '{sender}'"), - }); - } - - Ok(recipients) -} - -#[expect(clippy::too_many_arguments, reason = "Transaction preparation inputs")] -async fn prepare_round_robin_with_utxo_cache( - world: &mut CucumberWorld, - step: &str, - sender: &str, - recipients: &[String], - transactions: usize, - value: u64, - available_utxos: &mut WalletUtxos, - gas_prices: Option, - priority_fee_percent: u64, -) -> Result, StepError> { - let mut reserved_submissions = Vec::with_capacity(transactions); - - for i in 0..transactions { - let receiver_name = &recipients[i % recipients.len()]; - let receiver = world.resolve_recipient(receiver_name)?; - let receiver_pk = receiver.public_key; - - let receivers = vec![(receiver_pk, value)]; - let reserved_submission = - utils::reserve_user_wallet_transaction_submission_with_utxo_cache( - world, - step, - sender, - &receivers, - available_utxos, - gas_prices.clone(), - priority_fee_percent, - ) - .await?; - - reserved_submissions.push(reserved_submission); - } - - utils::finalize_reserved_user_wallet_submissions_concurrently(step, reserved_submissions).await -} - -#[expect( - clippy::cognitive_complexity, - reason = "Singular fn with multiple branches to handle different events and futures." -)] -pub async fn perform_manual_step_control( - world: &mut CucumberWorld, - step: &str, - timeout_seconds: u64, -) -> Result<(), StepError> { - let command_file = - env::var(MANUAL_COMMAND_FILE_ENV).map_err(|_| StepError::InvalidArgument { - message: format!( - "Step `{step}` requires environment variable '{MANUAL_COMMAND_FILE_ENV}' to be set", - ), - })?; - let poll_interval_ms = env::var(MANUAL_COMMAND_POLL_INTERVAL_ENV) - .ok() - .and_then(|v| v.parse::().ok()) - .unwrap_or(300); - - info!( - target: TARGET, - "Manual control step started. Monitoring command file: `{command_file}`" - ); - - let time_out = Duration::from_secs(timeout_seconds); - let start = Instant::now(); - while start.elapsed() < time_out { - if let Some(command) = take_next_command(Path::new(&command_file))? { - info!(target: TARGET, "====> manual command: {command:?}"); - if matches!( - execute_manual_command(world, step, &command).await, - Ok(true) - ) { - info!( - target: TARGET, - "Manual command loop stopped by STOP command after {:.2?}", - start.elapsed() - ); - return Ok(()); - } - } else { - sleep(Duration::from_millis(poll_interval_ms)).await; - } - } - info!(target: TARGET, "Manual command loop stopped by tine-out after {:.2?}", start.elapsed()); - - Ok(()) -} diff --git a/tests/src/cucumber/steps/manual_transactions/steps.rs b/tests/src/cucumber/steps/manual_transactions/steps.rs deleted file mode 100644 index 7a489fb54..000000000 --- a/tests/src/cucumber/steps/manual_transactions/steps.rs +++ /dev/null @@ -1,933 +0,0 @@ -use std::{collections::HashSet, time::Duration}; - -use cucumber::{gherkin::Step, given, then, when}; -use tokio::time::timeout; -use tracing::{info, warn}; - -use crate::{ - common::wallet::WalletUtxos, - cucumber::{ - error::{StepError, StepResult}, - steps::{ - TARGET, - manual_transactions::{ - command_file_parsing::ManualCommand, - command_file_utils::{ - execute_coin_splits_all_user_wallets, - execute_continuous_next_wallet_user_wallet, - execute_continuous_round_robin_user_wallets, log_wallet_balances, - perform_manual_step_control, verify_min_outputs_all_user_wallets, - }, - drain_wallets::{drain_all_node_wallets, drain_node_wallet, drain_user_wallet}, - tracked_transactions::{ - submit_funded_transfer_transaction, submit_invalid_transfer_transaction, - submit_stateless_invalid_transfer_transaction, - transaction_is_not_included_in_seconds, - transaction_is_rejected_during_preverification, - }, - utils, - utils::{ - WalletOutputState, - assert_tracked_wallet_fees_equal_sponsored_fee_account_spend, - create_and_submit_transaction, parse_wallet_output_state, - wait_for_wallet_output_state, wait_for_wallet_submitted_transactions_inclusion, - }, - }, - }, - wallet::{ - submissions::create_and_submit_transaction_hashes_with_utxo_cache, - sync::{WalletSendReadiness, wait_wallet_send_ready}, - }, - world::{CucumberWorld, WalletInfo, WalletType}, - }, - non_zero, -}; - -#[when(expr = "I do a coin split for {string} of {int} UTXOs valued at {int} LGO tokens each")] -async fn step_do_coin_split( - world: &mut CucumberWorld, - step: &Step, - wallet_name: String, - number_of_outputs: usize, - output_value: u64, -) -> StepResult { - let wallet = world.resolve_wallet(&wallet_name).inspect_err(|e| { - warn!(target: TARGET, "Step `{}` error: {e}", step.value); - })?; - - let mut available_utxos = WalletUtxos::new(); - let best_node_info = wait_wallet_send_ready( - world, - &step.value, - &wallet_name, - 180, - number_of_outputs as u64 * output_value, - WalletSendReadiness::TotalValueOnly, - &mut available_utxos, - &HashSet::new(), - ) - .await?; - - let self_pk = wallet.public_key().inspect_err(|e| { - warn!(target: TARGET, "Step `{}` error: {e}", step.value); - })?; - let receivers = vec![(self_pk, output_value); number_of_outputs]; - let tx_hash_hex = create_and_submit_transaction( - world, - &step.value, - &wallet_name, - &receivers, - Some(&best_node_info), - Some(&mut available_utxos), - ) - .await - .inspect_err(|e| { - warn!(target: TARGET, "Step `{}` error: {e}", step.value); - })?; - - info!( - target: TARGET, - "Submitted coin split transaction for `{wallet_name}/{}`, outputs: {number_of_outputs}, \ - value: {output_value}, tx hash: {tx_hash_hex}", - wallet.node_name - ); - - Ok(()) -} - -/// Tops up a node's funding wallet from a user wallet with `note_count` -/// notes of `note_value` LGO each, in a single transaction. -/// -/// Funding a transaction reserves one wallet note until the transaction is -/// mined, so a funding wallet supports at most `note count` concurrent -/// in-flight funded transactions — and its single `10_000` genesis note cannot -/// pay the fees of scenarios that publish many messages at non-zero gas -/// prices. This mirrors the workflow of a real node operator provisioning -/// their sequencer's funding wallet. -#[when( - expr = "wallet {string} sends {int} notes of {int} LGO to node {string} funding wallet as {string}" -)] -async fn step_topup_node_funding_wallet( - world: &mut CucumberWorld, - step: &Step, - wallet_name: String, - note_count: usize, - note_value: u64, - node_name: String, - transaction_alias: String, -) -> StepResult { - let funding_wallet = world.funding_wallet(&node_name).inspect_err(|e| { - warn!(target: TARGET, "Step `{}` error: {e}", step.value); - })?; - let funding_pk = funding_wallet.public_key().inspect_err(|e| { - warn!(target: TARGET, "Step `{}` error: {e}", step.value); - })?; - - let mut available_utxos = WalletUtxos::new(); - let best_node_info = wait_wallet_send_ready( - world, - &step.value, - &wallet_name, - 180, - note_count as u64 * note_value, - WalletSendReadiness::TotalValueOnly, - &mut available_utxos, - &HashSet::new(), - ) - .await?; - - let receivers = vec![(funding_pk, note_value); note_count]; - let tx_hashes = create_and_submit_transaction_hashes_with_utxo_cache( - world, - &step.value, - &wallet_name, - &receivers, - Some(&best_node_info), - Some(&mut available_utxos), - ) - .await - .inspect_err(|e| { - warn!(target: TARGET, "Step `{}` error: {e}", step.value); - })?; - let tx_hash = tx_hashes - .first() - .copied() - .ok_or_else(|| StepError::LogicalError { - message: "funding wallet top-up produced no transaction".to_owned(), - })?; - world.remember_submitted_transaction(transaction_alias.clone(), tx_hash); - - info!( - target: TARGET, - "Submitted funding wallet top-up `{transaction_alias}`: {note_count} notes of \ - {note_value} LGO from `{wallet_name}` to `{}`", - funding_wallet.wallet_name, - ); - - Ok(()) -} - -#[when(expr = "wallet {string} has {int} or more outputs in {int} seconds")] -#[then(expr = "wallet {string} has {int} or more outputs in {int} seconds")] -async fn step_wallet_has_at_least_coins( - world: &mut CucumberWorld, - step: &Step, - wallet_name: String, - min_coin_count: usize, - time_out_seconds: u64, -) -> StepResult { - wait_for_wallet_output_state( - world, - &step.value, - wallet_name, - Some(&min_coin_count), - None, - None, - None, - time_out_seconds, - WalletOutputState::OnChain, - ) - .await -} - -#[when(expr = "wallet {string} has {int} or less outputs in {int} seconds")] -#[then(expr = "wallet {string} has {int} or less outputs in {int} seconds")] -async fn step_wallet_has_at_most_coins( - world: &mut CucumberWorld, - step: &Step, - wallet_name: String, - max_coin_count: usize, - time_out_seconds: u64, -) -> StepResult { - wait_for_wallet_output_state( - world, - &step.value, - wallet_name, - None, - Some(&max_coin_count), - None, - None, - time_out_seconds, - WalletOutputState::OnChain, - ) - .await -} - -#[when(expr = "wallet {string} has exactly {int} outputs in {int} seconds")] -#[then(expr = "wallet {string} has exactly {int} outputs in {int} seconds")] -async fn step_wallet_has_exact_coins( - world: &mut CucumberWorld, - step: &Step, - wallet_name: String, - coin_count: usize, - time_out_seconds: u64, -) -> StepResult { - wait_for_wallet_output_state( - world, - &step.value, - wallet_name, - Some(&coin_count), - Some(&coin_count), - None, - None, - time_out_seconds, - WalletOutputState::OnChain, - ) - .await -} - -#[when(expr = "wallet {string} has {int} or less encumbered outputs in {int} seconds")] -#[then(expr = "wallet {string} has {int} or less encumbered outputs in {int} seconds")] -async fn step_wallet_has_at_most_encumbered_coins( - world: &mut CucumberWorld, - step: &Step, - wallet_name: String, - max_coin_count: usize, - time_out_seconds: u64, -) -> StepResult { - wait_for_wallet_output_state( - world, - &step.value, - wallet_name, - None, - Some(&max_coin_count), - None, - None, - time_out_seconds, - WalletOutputState::Reserved, - ) - .await -} - -#[when(expr = "wallet {string} has {int} or more LGO in {int} seconds")] -#[then(expr = "wallet {string} has {int} or more LGO in {int} seconds")] -async fn step_wallet_has_at_least_value( - world: &mut CucumberWorld, - step: &Step, - wallet_name: String, - min_token_value: u64, - time_out_seconds: u64, -) -> StepResult { - wait_for_wallet_output_state( - world, - &step.value, - wallet_name, - None, - None, - Some(&min_token_value), - None, - time_out_seconds, - WalletOutputState::OnChain, - ) - .await -} - -#[when(expr = "wallet {string} has exactly {int} LGO in {int} seconds")] -#[then(expr = "wallet {string} has exactly {int} LGO in {int} seconds")] -async fn step_wallet_has_exact_value( - world: &mut CucumberWorld, - step: &Step, - wallet_name: String, - token_value: u64, - time_out_seconds: u64, -) -> StepResult { - wait_for_wallet_output_state( - world, - &step.value, - wallet_name, - None, - None, - Some(&token_value), - Some(&token_value), - time_out_seconds, - WalletOutputState::OnChain, - ) - .await -} - -#[when(expr = "wallet {string} has {int} or less LGO in {int} seconds")] -#[then(expr = "wallet {string} has {int} or less LGO in {int} seconds")] -async fn step_wallet_has_at_most_value( - world: &mut CucumberWorld, - step: &Step, - wallet_name: String, - max_token_value: u64, - time_out_seconds: u64, -) -> StepResult { - wait_for_wallet_output_state( - world, - &step.value, - wallet_name, - None, - None, - None, - Some(&max_token_value), - time_out_seconds, - WalletOutputState::OnChain, - ) - .await -} - -#[when(expr = "wallet {string} has {int} or more outputs and {int} or more LGO in {int} seconds")] -#[then(expr = "wallet {string} has {int} or more outputs and {int} or more LGO in {int} seconds")] -async fn step_wallet_has_at_least_coins_and_value( - world: &mut CucumberWorld, - step: &Step, - wallet_name: String, - min_coin_count: usize, - min_token_value: u64, - time_out_seconds: u64, -) -> StepResult { - wait_for_wallet_output_state( - world, - &step.value, - wallet_name, - Some(&min_coin_count), - None, - Some(&min_token_value), - None, - time_out_seconds, - WalletOutputState::OnChain, - ) - .await -} - -#[when(expr = "wallet {string} has {int} or less outputs and {int} or less LGO in {int} seconds")] -#[then(expr = "wallet {string} has {int} or less outputs and {int} or less LGO in {int} seconds")] -async fn step_wallet_has_at_most_coins_and_value( - world: &mut CucumberWorld, - step: &Step, - wallet_name: String, - max_coin_count: usize, - max_token_value: u64, - time_out_seconds: u64, -) -> StepResult { - wait_for_wallet_output_state( - world, - &step.value, - wallet_name, - None, - Some(&max_coin_count), - None, - Some(&max_token_value), - time_out_seconds, - WalletOutputState::OnChain, - ) - .await -} - -#[when(expr = "wallet {string} has exactly {int} outputs and {int} LGO in {int} seconds")] -#[then(expr = "wallet {string} has exactly {int} outputs and {int} LGO in {int} seconds")] -async fn step_wallet_has_exact_coins_and_value( - world: &mut CucumberWorld, - step: &Step, - wallet_name: String, - coin_count: usize, - token_value: u64, - time_out_seconds: u64, -) -> StepResult { - wait_for_wallet_output_state( - world, - &step.value, - wallet_name, - Some(&coin_count), - Some(&coin_count), - Some(&token_value), - Some(&token_value), - time_out_seconds, - WalletOutputState::OnChain, - ) - .await -} - -#[when(expr = "I log wallet balances for all wallets")] -#[then(expr = "I log wallet balances for all wallets")] -async fn step_wallet_balance_all_wallets(world: &mut CucumberWorld, step: &Step) -> StepResult { - let mut wallets = world.all_user_wallets(); - wallets.extend(world.all_node_wallets()); - - log_wallet_balances(world, &step.value, wallets).await -} - -#[when(expr = "I drain wallet {string} into {string}")] -#[then(expr = "I drain wallet {string} into {string}")] -async fn step_drain_wallet( - world: &mut CucumberWorld, - step: &Step, - sender_wallet_name: String, - receiver_wallet_name: String, -) -> StepResult { - let sender = world.resolve_wallet(&sender_wallet_name)?; - let receiver = world.resolve_recipient(&receiver_wallet_name)?; - let sender_pk = sender.public_key()?; - let receiver_pk = receiver.public_key; - - if sender_pk == receiver_pk { - return Err(StepError::InvalidArgument { - message: format!("Cannot drain wallet `{sender_wallet_name}` into itself"), - }); - } - - match sender.wallet_type { - WalletType::User { .. } => { - drain_user_wallet(world, &step.value, &sender, receiver_pk).await - } - WalletType::Funding { .. } => drain_node_wallet(world, &sender, receiver_pk).await, - } -} - -#[when(expr = "I drain all node {string} wallets into {string}")] -#[then(expr = "I drain all node {string} wallets into {string}")] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require a mutable World reference" -)] -async fn step_drain_all_node_wallets( - world: &mut CucumberWorld, - node_name: String, - receiver_wallet_name: String, -) -> StepResult { - drain_all_node_wallets(world, &node_name, &receiver_wallet_name).await -} - -#[when(expr = "wallet {string} has all submitted transactions settled in {int} seconds")] -#[then(expr = "wallet {string} has all submitted transactions settled in {int} seconds")] -#[when(expr = "wallet {string} has all submitted transactions included in {int} seconds")] -#[then(expr = "wallet {string} has all submitted transactions included in {int} seconds")] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require the world as the first `&mut` argument" -)] -async fn step_wallet_has_all_submitted_transactions_settled( - world: &mut CucumberWorld, - step: &Step, - wallet_name: String, - time_out_seconds: u64, -) -> StepResult { - wait_for_wallet_submitted_transactions_inclusion( - world, - &wallet_name, - Duration::from_secs(time_out_seconds), - ) - .await - .inspect_err(|e| { - warn!(target: TARGET, "Step `{}` error: {e}", step.value); - }) -} - -#[when(expr = "tracked wallet fees equal sponsored fee account spent fees")] -#[then(expr = "tracked wallet fees equal sponsored fee account spent fees")] -async fn step_tracked_wallet_fees_equal_sponsored_fee_account_spend( - world: &mut CucumberWorld, - step: &Step, -) -> StepResult { - assert_tracked_wallet_fees_equal_sponsored_fee_account_spend(world, &step.value).await -} - -#[when( - expr = "I send {int} transactions of {int} LGO each from wallet {string} to wallet {string}" -)] -async fn step_send_multiple_transactions_to_single_wallet( - world: &mut CucumberWorld, - step: &Step, - number_of_transactions: usize, - output_value: u64, - sender_wallet_name: String, - receiver_wallet_name: String, -) -> StepResult { - let sender_wallet = world.resolve_wallet(&sender_wallet_name).inspect_err(|e| { - warn!(target: TARGET, "Step `{}` error: {e}", step.value); - })?; - let receiver = world.resolve_recipient(&receiver_wallet_name)?; - let receiver_wallet_pk = receiver.public_key; - - let mut available_utxos = WalletUtxos::new(); - let best_node_info = wait_wallet_send_ready( - world, - &step.value, - &sender_wallet_name, - 180, - number_of_transactions as u64 * output_value, - WalletSendReadiness::TotalValueOnly, - &mut available_utxos, - &HashSet::new(), - ) - .await?; - - for _ in 0..number_of_transactions { - let tx_hash_hex = create_and_submit_transaction( - world, - &step.value, - &sender_wallet_name, - &[(receiver_wallet_pk, output_value)], - Some(&best_node_info), - Some(&mut available_utxos), - ) - .await - .inspect_err(|e| { - warn!(target: TARGET, "Step `{}` error: {e}", step.value); - })?; - - info!( - target: TARGET, - "Sent normal transaction from `{sender_wallet_name}/{}` to {}, \ - value: {output_value}, tx hash: {tx_hash_hex}", - sender_wallet.node_name, - receiver.label - ); - } - - Ok(()) -} - -#[when(expr = "I submit invalid transfer transaction {string} to node {string}")] -async fn step_submit_invalid_transfer_transaction( - world: &mut CucumberWorld, - step: &Step, - transaction_alias: String, - node_name: String, -) -> StepResult { - submit_invalid_transfer_transaction(world, &step.value, transaction_alias, node_name).await -} - -#[when(expr = "I submit a stateless-invalid transfer transaction {string} to node {string}")] -async fn step_submit_stateless_invalid_transfer_transaction( - world: &mut CucumberWorld, - step: &Step, - transaction_alias: String, - node_name: String, -) -> StepResult { - submit_stateless_invalid_transfer_transaction(world, &step.value, transaction_alias, node_name) - .await -} - -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require `&mut World` as the first parameter" -)] -#[expect( - clippy::needless_pass_by_value, - reason = "Cucumber step captures are always owned `String`s, even when the step only needs to borrow them" -)] -#[then(expr = "transaction {string} is rejected during preverification")] -fn step_transaction_is_rejected_during_preverification( - world: &mut CucumberWorld, - transaction_alias: String, -) -> StepResult { - transaction_is_rejected_during_preverification(world, &transaction_alias) -} - -#[when( - expr = "I submit funded transfer transaction {string} of {int} LGO from wallet {string} to wallet {string}" -)] -async fn step_submit_funded_transfer_transaction( - world: &mut CucumberWorld, - step: &Step, - transaction_alias: String, - amount: u64, - sender_wallet_name: String, - receiver_wallet_name: String, -) -> StepResult { - submit_funded_transfer_transaction( - world, - &step.value, - transaction_alias, - amount, - sender_wallet_name, - receiver_wallet_name, - ) - .await -} - -#[when(expr = "transaction {string} is not included in {int} seconds")] -#[then(expr = "transaction {string} is not included in {int} seconds")] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require `&mut World` as the first parameter" -)] -async fn step_transaction_is_not_included_in_seconds( - world: &mut CucumberWorld, - step: &Step, - transaction_alias: String, - timeout_seconds: u64, -) -> StepResult { - transaction_is_not_included_in_seconds(world, &step.value, transaction_alias, timeout_seconds) - .await -} - -#[when( - expr = "I send one transaction with {int} outputs of {int} LGO each from wallet {string} to wallet {string}" -)] -async fn step_send_single_transaction_multiple_outputs_to_single_wallet( - world: &mut CucumberWorld, - step: &Step, - number_of_outputs: usize, - output_value: u64, - sender_wallet_name: String, - receiver_wallet_name: String, -) -> StepResult { - let sender_wallet = world.resolve_wallet(&sender_wallet_name).inspect_err(|e| { - warn!(target: TARGET, "Step `{}` error: {e}", step.value); - })?; - let receiver = world.resolve_recipient(&receiver_wallet_name)?; - let receiver_wallet_pk = receiver.public_key; - - let receivers = vec![(receiver_wallet_pk, output_value); number_of_outputs]; - let tx_hash_hex = create_and_submit_transaction( - world, - &step.value, - &sender_wallet_name, - &receivers, - None, - None, - ) - .await - .inspect_err(|e| { - warn!(target: TARGET, "Step `{}` error: {e}", step.value); - })?; - - info!( - target: TARGET, - "Sent normal transaction from `{sender_wallet_name}/{}` to {}, \ - number_of_outputs: {number_of_outputs}, value: {output_value}, tx hash: {tx_hash_hex}", - sender_wallet.node_name, - receiver.label - ); - - Ok(()) -} - -#[when(expr = "I perform manual control of transactions for all wallets for {int} seconds")] -async fn step_manual_control_transactions( - world: &mut CucumberWorld, - step: &Step, - timeout_seconds: u64, -) -> StepResult { - perform_manual_step_control(world, &step.value, timeout_seconds).await -} - -#[when(expr = "I perform manual control of transactions for all wallets no time-out")] -async fn step_manual_control_transactions_no_time_out( - world: &mut CucumberWorld, - step: &Step, -) -> StepResult { - perform_manual_step_control(world, &step.value, u64::MAX).await -} - -#[when( - expr = "I perform continuous transactions on user wallets with {int} coin split outputs of {int} LGO, {int} transactions of {int} LGO each for {int} cycles with {int} epochs headroom" -)] -#[expect( - clippy::too_many_arguments, - reason = "Cucumber step captures map directly to step arguments" -)] -async fn step_continuous_user_wallets( - world: &mut CucumberWorld, - step: &Step, - coin_split_outputs: usize, - coin_split_value: u64, - transactions: usize, - value: u64, - cycles: usize, - epochs_headroom: u32, -) -> StepResult { - info!( - target: TARGET, - "Starting continuous user wallet transactions: coin_split_outputs={coin_split_outputs}, coin_split_value={coin_split_value}, transactions={transactions}, value={value}, cycles={cycles}" - ); - - execute_continuous_round_robin_user_wallets( - world, - &step.value, - coin_split_outputs, - coin_split_value, - transactions, - value, - cycles, - epochs_headroom, - ) - .await - .inspect_err(|e| { - warn!(target: TARGET, "Step `{}` error: {e}", step.value); - })?; - - info!(target: TARGET, "Completed continuous user wallet transactions step"); - - Ok(()) -} - -#[when( - expr = "I perform continuous transactions on user wallets with {int} coin split outputs of {int} LGO, {int} transactions of {int} LGO each for {int} cycles and timeout of {int} seconds with {int} epochs headroom" -)] -#[expect( - clippy::too_many_arguments, - reason = "Cucumber step captures map directly to step function arguments." -)] -async fn step_continuous_user_wallets_with_timeout( - world: &mut CucumberWorld, - step: &Step, - coin_split_outputs: usize, - coin_split_value: u64, - transactions: usize, - value: u64, - cycles: usize, - timeout_seconds: u64, - epochs_headroom: u32, -) -> StepResult { - timeout( - Duration::from_secs(timeout_seconds), - step_continuous_user_wallets( - world, - step, - coin_split_outputs, - coin_split_value, - transactions, - value, - cycles, - epochs_headroom, - ), - ) - .await - .map_err(|_| StepError::Timeout { - message: format!( - "continuous user wallet transactions did not finish within {timeout_seconds} seconds" - ), - })? -} - -#[when( - expr = "I perform {int} coin split transactions for each user wallet with {int} outputs of {int} LGO each" -)] -async fn step_coin_split_transactions_for_each_user_wallet( - world: &mut CucumberWorld, - step: &Step, - splits_per_wallet: usize, - outputs: usize, - value: u64, -) -> StepResult { - execute_coin_splits_all_user_wallets(world, &step.value, splits_per_wallet, outputs, value) - .await - .inspect_err(|e| { - warn!(target: TARGET, "Step `{}` error: {e}", step.value); - })?; - - Ok(()) -} - -#[when(expr = "I verify each wallet has minimum {int} outputs {string} in {int} seconds")] -async fn step_verify_each_wallet_minimum_outputs( - world: &mut CucumberWorld, - step: &Step, - min_outputs: usize, - wallet_state_type: String, - timeout_seconds: u64, -) -> StepResult { - verify_min_outputs_all_user_wallets( - world, - &step.value, - min_outputs, - timeout_seconds, - parse_wallet_output_state(&wallet_state_type) - .inspect_err(|e| { - warn!(target: TARGET, "Step `{}` error: {e}", step.value); - }) - .map_err(|e| StepError::InvalidArgument { - message: e.to_string(), - })?, - ) - .await - .inspect_err(|e| { - warn!(target: TARGET, "Step `{}` error: {e}", step.value); - })?; - - Ok(()) -} - -#[when( - expr = "I perform {int} stress continuous cycles with {int} transactions of {int} LGO to the next user wallet with {int} epochs headroom" -)] -async fn step_perform_stress_continuous_cycles_next_user_wallet( - world: &mut CucumberWorld, - step: &Step, - cycles: usize, - num_transactions: usize, - value: u64, - epochs_headroom: u32, -) -> StepResult { - execute_continuous_next_wallet_user_wallet( - world, - &step.value, - &ManualCommand::ContinuousNextWalletUserWallets { - cycles, - num_transactions, - value, - epochs_headroom, - }, - ) - .await - .inspect_err(|e| { - warn!(target: TARGET, "Step `{}` error: {e}", step.value); - })?; - - Ok(()) -} - -#[given(expr = "I have a faucet with URL {string}")] -#[when(expr = "I have a faucet with URL {string}")] -fn step_faucet_details(world: &mut CucumberWorld, base_url: String) { - world.wallet_registry.faucet_base_url = Some(base_url); -} - -#[given(expr = "I request {int} rounds of faucet funds for wallet {string}")] -#[when(expr = "I request {int} rounds of faucet funds for wallet {string}")] -#[expect( - clippy::needless_pass_by_value, - reason = "Required by cucumber expression" -)] -fn step_request_faucet_funds_for_wallet( - world: &mut CucumberWorld, - step: &Step, - number_of_rounds: usize, - wallet_name: String, -) -> StepResult { - let wallet = world.resolve_wallet(&wallet_name).inspect_err(|error| { - warn!(target: TARGET, "Step `{}` error: {error}", step.value); - })?; - - let wallet_pk_hex = wallet.public_key_hex(); - - utils::request_faucet_funds( - world, - &step.value, - non_zero!("number of rounds", number_of_rounds)?, - &[wallet_pk_hex], - ) -} - -#[given(expr = "I request {int} rounds of faucet funds for all wallets")] -#[when(expr = "I request {int} rounds of faucet funds for all wallets")] -fn step_request_faucet_funds_for_all_wallets( - world: &mut CucumberWorld, - step: &Step, - number_of_rounds: usize, -) -> StepResult { - let all_wallets_pk_hex = world - .wallet_registry - .wallet_info - .values() - .map(WalletInfo::public_key_hex) - .collect::>(); - - utils::request_faucet_funds( - world, - &step.value, - non_zero!("number of rounds", number_of_rounds)?, - &all_wallets_pk_hex, - ) -} - -#[given(expr = "I request {int} rounds of faucet funds for all user wallets")] -#[when(expr = "I request {int} rounds of faucet funds for all user wallets")] -fn step_request_faucet_funds_for_all_user_wallets( - world: &mut CucumberWorld, - step: &Step, - number_of_rounds: usize, -) -> StepResult { - let all_wallets_pk_hex = world - .wallet_registry - .wallet_info - .values() - .filter(|w| w.is_user_wallet()) - .map(WalletInfo::public_key_hex) - .collect::>(); - - utils::request_faucet_funds( - world, - &step.value, - non_zero!("number of rounds", number_of_rounds)?, - &all_wallets_pk_hex, - ) -} - -#[given(expr = "I request {int} rounds of faucet funds for all funding wallets")] -#[when(expr = "I request {int} rounds of faucet funds for all funding wallets")] -fn step_request_faucet_funds_for_all_funding_wallets( - world: &mut CucumberWorld, - step: &Step, - number_of_rounds: usize, -) -> StepResult { - let all_wallets_pk_hex = world - .wallet_registry - .wallet_info - .values() - .filter(|wallet| wallet.is_node_funding_wallet()) - .map(WalletInfo::public_key_hex) - .collect::>(); - - utils::request_faucet_funds( - world, - &step.value, - non_zero!("number of rounds", number_of_rounds)?, - &all_wallets_pk_hex, - ) -} diff --git a/tests/src/cucumber/steps/manual_zone/actions.rs b/tests/src/cucumber/steps/manual_zone/actions.rs deleted file mode 100644 index 2c1f5737b..000000000 --- a/tests/src/cucumber/steps/manual_zone/actions.rs +++ /dev/null @@ -1,1015 +0,0 @@ -use std::{collections::HashMap, num::NonZero, time::Duration}; - -use cucumber::gherkin::Step; -use futures::future::join_all; -use lb_common_http_client::CommonHttpClient; -use lb_core::mantle::{ - TxHash, Utxo, - gas::GasCost, - ops::channel::{config::Keys, deposit::Metadata, inscribe::Inscription}, -}; -use lb_key_management_system_service::keys::Ed25519Key; -use lb_testing_framework::NodeHttpClient; -use lb_zone_sdk::{ - adapter::NodeHttpClient as ZoneNodeHttpClient, - sequencer::{FundingConfig, ZoneSequencer}, -}; -use tokio::{ - sync::broadcast, - task::JoinHandle, - time::{error::Elapsed, timeout}, -}; -use tracing::warn; - -use super::{ - errors::{log_step_error, zone_step_error}, - runner::{Event, PublishResult, SequencerCheckpoint, SequencerClient}, - steps::DEFAULT_ZONE_SEQUENCER, - support::{ - AtomicZoneDepositRequest, CustomRepublishDeps, DiscardedPayloads, PublishDeadline, - ZoneAccountBalances, ZoneDeposit, build_zone_deposit, build_zone_deposit_from_values, - ensure_zone_transactions_included, keygen, publish_atomic_zone_withdraw, - publish_message_with_retry, sequencer_config, sequencer_config_with_pending_submit_depth, - start_balance_aware_policy, start_custom_republish_policy, start_republish_lineage_policy, - start_sequencer_event_loop, start_sorted_conflict_policy, submit_atomic_zone_deposit, - submit_zone_channel_split, submit_zone_deposit, submit_zone_withdraw, - }, - tables::{ConcurrentZoneMessageRow, ZoneNodeResourcesRow, group_zone_messages_by_sequencer}, -}; -use crate::{ - common::{ - mantle_inscription::make_inscription, manual_cluster::wait_for_height, - wallet::WalletReservedInputs, - }, - cucumber::{ - error::{StepError, StepResult}, - steps::{ - TARGET, - manual_nodes::{ - config_override::set_deployment_config_override, - utils::{ - NodesToStartUnordered, WalletStartInfo, start_node, - start_nodes_order_respecting_dependencies, - }, - }, - }, - wallet::sync::current_available_utxos_for_wallet, - world::{CucumberWorld, WalletInfo, ZoneReaderConfig}, - }, -}; - -const ZONE_CHANNEL_WITHDRAW_THRESHOLD: u16 = 1; -const ZONE_CHANNEL_DEPOSIT_THRESHOLD: u16 = 1; -const SEQUENCER_READY_TIMEOUT: Duration = Duration::from_mins(2); -const SEQUENCER_READY_POLL_TIMEOUT: Duration = Duration::from_secs(10); -const SEQUENCER_READY_HEIGHT_ADVANCE_TIMEOUT: Duration = Duration::from_secs(30); -const ZONE_SECURITY_PARAM: u32 = 5; -// These high-volume scenarios can move storage prices before all queued -// publishes are included. Keep their test funding comfortably above the -// public 12% default so they exercise sequencing rather than fee starvation. -const ZONE_TEST_PRIORITY_FEE_PERCENT: u64 = 50; - -pub(super) enum DriveMode { - Passive { - republish_orphans: bool, - }, - RepublishLineage { - planned: Vec, - }, - Sorted { - discarded: DiscardedPayloads, - }, - BalanceAware { - initial_balances: ZoneAccountBalances, - planned_payloads: Vec, - }, - CustomRepublish { - deps: Box, - }, -} - -impl DriveMode { - pub(super) const fn passive() -> Self { - Self::Passive { - republish_orphans: false, - } - } - - pub(super) const fn passive_republish_orphans() -> Self { - Self::Passive { - republish_orphans: true, - } - } -} - -struct PublishedZoneMessage { - alias: String, - payload: Inscription, - result: PublishResult, -} - -struct StartedSequencerRuntime { - task: JoinHandle<()>, - client: SequencerClient, - events: broadcast::Receiver, - checkpoint_rx: tokio::sync::watch::Receiver>, - ready_rx: tokio::sync::watch::Receiver, - channel_view_rx: tokio::sync::watch::Receiver, - turn_to_write_rx: tokio::sync::watch::Receiver, - tx_status_rx: broadcast::Receiver, - discarded_payloads: Option, -} - -pub(super) fn register_zone_sequencers_with_shared_key( - world: &mut CucumberWorld, - source_alias: &str, - aliases: Vec, -) -> StepResult { - let signing_key = world.zone.sequencer_signing_key(source_alias)?.clone(); - - for alias in aliases { - world.zone.register_sequencer(alias, signing_key.clone()); - } - - Ok(()) -} - -pub(super) async fn start_nodes_with_zone_resources( - world: &mut CucumberWorld, - step: &Step, - rows: Vec, -) -> StepResult { - apply_zone_timing_defaults(world, &step.value)?; - - let nodes = collect_zone_nodes_to_start(&rows); - let nodes = start_nodes_order_respecting_dependencies( - nodes, - world.nodes_info.keys().cloned().collect(), - ) - .inspect_err(|error| { - warn!(target: TARGET, "Step `{}` error: {error}", step.value); - })?; - - for (node_name, wallet_start_info, mut initial_peers) in nodes { - initial_peers.sort(); - initial_peers.dedup(); - - start_node( - world, - &step.value, - &node_name, - &wallet_start_info, - &initial_peers, - false, - &[], - ) - .await?; - } - - register_zone_resources(world, rows) -} - -fn apply_zone_timing_defaults(world: &mut CucumberWorld, step: &str) -> StepResult { - world.set_cryptarchia_security_param( - NonZero::new(ZONE_SECURITY_PARAM).expect("zone security parameter is non-zero"), - ); - world.set_prolonged_bootstrap_period(Duration::ZERO); - - set_deployment_config_override(world, step, "time.slot_duration", "seconds(1)")?; - set_deployment_config_override( - world, - step, - "cryptarchia.slot_activation_coeff.numerator", - "1", - )?; - set_deployment_config_override( - world, - step, - "cryptarchia.slot_activation_coeff.denominator", - "2", - ) -} - -fn collect_zone_nodes_to_start(rows: &[ZoneNodeResourcesRow]) -> NodesToStartUnordered { - let mut nodes = HashMap::new(); - - for row in rows { - let entry = nodes - .entry(row.node_name.clone()) - .or_insert_with(|| (Vec::new(), Vec::new())); - - entry.0.push(WalletStartInfo { - wallet_name: row.wallet_name.clone(), - account_index: row.account_index, - }); - - if let Some(peer) = &row.connected_to { - entry.1.push(peer.clone()); - } - } - - nodes -} - -fn register_zone_resources( - world: &mut CucumberWorld, - rows: Vec, -) -> StepResult { - for row in rows { - for alias in row.sequencers { - if !world.zone.has_sequencer(&alias) { - world.zone.register_sequencer(alias.clone(), keygen()); - } - - world.zone.attach_sequencer_resources( - &alias, - row.node_name.clone(), - row.wallet_name.clone(), - )?; - } - } - - Ok(()) -} - -pub(super) async fn submit_zone_channel_config( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: &str, - transaction_alias: String, - authorized_aliases: Vec, - posting_timeframe: u32, - posting_timeout: u32, -) -> StepResult { - let handle = log_step_error(step, world.zone.sequencer_client(sequencer_alias))?; - let mut ordered_aliases = vec![sequencer_alias.to_owned()]; - - for alias in authorized_aliases { - if ordered_aliases.iter().any(|existing| existing == &alias) { - continue; - } - - ordered_aliases.push(alias); - } - - let authorized_keys = ordered_aliases - .into_iter() - .map(|alias| { - world - .zone - .sequencer_signing_key(&alias) - .map(Ed25519Key::public_key) - }) - .collect::, _>>()?; - - let mut checkpoint_rx = world - .zone - .checkpoint_receiver(sequencer_alias) - .ok_or_else(|| StepError::LogicalError { - message: format!("Zone sequencer '{sequencer_alias}' has no checkpoint watch"), - })?; - checkpoint_rx.mark_unchanged(); - - let ((result, post_call_checkpoint), _signed_tx) = handle - .channel_config( - Keys::new_unchecked(authorized_keys), - posting_timeframe.into(), - posting_timeout.into(), - ZONE_CHANNEL_WITHDRAW_THRESHOLD, - ZONE_CHANNEL_DEPOSIT_THRESHOLD, - ) - .await - .map_err(|error| StepError::LogicalError { - message: format!("Zone channel_config failed: {error}"), - })?; - - // Sanity-check the inline checkpoint already mentions our tx; the - // event-stream watcher below also catches it once the drive task - // re-publishes its checkpoint after the next block. - let tx_hash = result.inscription_id(); - let checkpoint = if post_call_checkpoint - .pending_txs - .iter() - .any(|(hash, _)| *hash == tx_hash) - { - post_call_checkpoint - } else { - timeout( - Duration::from_secs(30), - wait_for_checkpoint_with_tx(&mut checkpoint_rx, tx_hash), - ) - .await - .map_err(|_| StepError::LogicalError { - message: format!( - "timed out waiting for sequencer '{sequencer_alias}' checkpoint to include {tx_hash:?}", - ), - })? - .map_err(|message| StepError::LogicalError { message })? - }; - - world - .zone - .set_latest_checkpoint_for(sequencer_alias, checkpoint.clone()); - world - .zone - .remember_checkpoint(format!("{transaction_alias}_CHECKPOINT"), checkpoint); - world.remember_submitted_transaction(transaction_alias, tx_hash); - - Ok(()) -} - -pub(super) fn stop_zone_sequencer( - world: &mut CucumberWorld, - sequencer_alias: impl AsRef, -) -> StepResult { - world.zone.stop_sequencer(sequencer_alias.as_ref())?; - - Ok(()) -} - -pub(super) fn save_zone_checkpoint( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: impl AsRef, - checkpoint_alias: String, -) -> StepResult { - let sequencer_alias = sequencer_alias.as_ref(); - let checkpoint = log_step_error(step, world.zone.current_checkpoint_for(sequencer_alias))?; - - world.zone.remember_checkpoint(checkpoint_alias, checkpoint); - - Ok(()) -} - -pub(super) fn remember_published_zone_message( - world: &mut CucumberWorld, - sequencer_alias: &str, - message_alias: String, - payload: Inscription, - result: &PublishResult, -) { - let checkpoint = world.zone.current_checkpoint_for(sequencer_alias).ok(); - world.zone.remember_zone_message( - message_alias, - payload, - Some(result.inscription_id()), - Some(sequencer_alias), - checkpoint, - ); -} - -async fn wait_for_checkpoint_with_tx( - rx: &mut tokio::sync::watch::Receiver>, - tx_hash: TxHash, -) -> Result { - loop { - let snapshot = rx.borrow_and_update().clone(); - if let Some(checkpoint) = snapshot - && checkpoint - .pending_txs - .iter() - .any(|(hash, _)| *hash == tx_hash) - { - return Ok(checkpoint); - } - - rx.changed() - .await - .map_err(|error| format!("checkpoint watch closed: {error}"))?; - } -} - -fn resolve_zone_wallet( - world: &CucumberWorld, - sequencer_alias: &str, -) -> Result { - let wallet_name = world.zone.sequencer_default_wallet_name(sequencer_alias)?; - - world.resolve_wallet(wallet_name) -} - -fn record_zone_wallet_submission( - world: &CucumberWorld, - wallet_name: &str, - tx_hash: TxHash, - reserved_inputs: Vec, -) -> StepResult { - world.with_wallets_mut(|wallets| { - wallets.record_wallet_reservation( - wallet_name.to_owned(), - tx_hash, - WalletReservedInputs::new(reserved_inputs, Vec::new()), - 0, - ); - }) -} - -pub(super) async fn submit_zone_deposit_transaction( - world: &mut CucumberWorld, - step: &Step, - transaction_alias: String, - channel_alias: String, - amount: u64, - metadata: Metadata, -) -> StepResult { - let node_url = log_step_error(step, world.zone_node_url_for_sequencer(&channel_alias))?; - let wallet = log_step_error(step, resolve_zone_wallet(world, &channel_alias))?; - let public_key = log_step_error(step, wallet.public_key())?; - let available_utxos = log_step_error( - step, - current_available_utxos_for_wallet(world, &step.value, &wallet.wallet_name).await, - )?; - let ZoneDeposit { - deposit, - reserved_inputs, - channel_notes, - } = build_zone_deposit( - available_utxos, - world.zone.sequencer_channel_id(&channel_alias)?, - amount, - metadata, - ) - .map_err(|error| zone_step_error(step, &error))?; - - let response = submit_zone_deposit(&node_url, &deposit, public_key) - .await - .map_err(|error| zone_step_error(step, &error))?; - - world - .zone - .remember_deposit_channel_notes(transaction_alias.clone(), channel_notes); - world - .zone - .remember_submitted_deposit(transaction_alias.clone(), deposit, amount); - record_zone_wallet_submission(world, &wallet.wallet_name, response, reserved_inputs)?; - world.remember_submitted_transaction(transaction_alias, response); - - Ok(()) -} - -/// Submits a multi-input channel deposit that consumes one wallet note per -/// listed value, exercising the channel wallet's per-note value tracking. -pub(super) async fn submit_zone_multi_deposit_transaction( - world: &mut CucumberWorld, - step: &Step, - transaction_alias: String, - channel_alias: String, - input_values: Vec, - metadata: Metadata, -) -> StepResult { - let node_url = log_step_error(step, world.zone_node_url_for_sequencer(&channel_alias))?; - let wallet = log_step_error(step, resolve_zone_wallet(world, &channel_alias))?; - let public_key = log_step_error(step, wallet.public_key())?; - let available_utxos = log_step_error( - step, - current_available_utxos_for_wallet(world, &step.value, &wallet.wallet_name).await, - )?; - let amount: u64 = input_values.iter().sum(); - let ZoneDeposit { - deposit, - reserved_inputs, - channel_notes, - } = build_zone_deposit_from_values( - available_utxos, - world.zone.sequencer_channel_id(&channel_alias)?, - &input_values, - metadata, - ) - .map_err(|error| zone_step_error(step, &error))?; - - let response = submit_zone_deposit(&node_url, &deposit, public_key) - .await - .map_err(|error| zone_step_error(step, &error))?; - - world - .zone - .remember_deposit_channel_notes(transaction_alias.clone(), channel_notes); - world - .zone - .remember_submitted_deposit(transaction_alias.clone(), deposit, amount); - record_zone_wallet_submission(world, &wallet.wallet_name, response, reserved_inputs)?; - world.remember_submitted_transaction(transaction_alias, response); - - Ok(()) -} - -pub(super) async fn submit_zone_channel_split_transaction( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: &str, - deposit_alias: &str, - dust_count: usize, - transaction_alias: String, -) -> StepResult { - let node_url = log_step_error(step, world.zone_node_url_for_sequencer(sequencer_alias))?; - let wallet = log_step_error(step, resolve_zone_wallet(world, sequencer_alias))?; - let public_key = log_step_error(step, wallet.public_key())?; - let channel_id = world.zone.sequencer_channel_id(sequencer_alias)?; - let signing_key = - log_step_error(step, world.zone.sequencer_signing_key(sequencer_alias))?.clone(); - let input_note = *log_step_error( - step, - world.zone.resolve_deposit_channel_notes(deposit_alias), - )? - .first() - .ok_or_else(|| StepError::LogicalError { - message: format!("Zone deposit '{deposit_alias}' created no channel notes to split"), - })?; - - let tx_hash = submit_zone_channel_split( - &node_url, - channel_id, - &signing_key, - public_key, - input_note, - dust_count, - ) - .await - .map_err(|error| zone_step_error(step, &error))?; - - world.remember_submitted_transaction(transaction_alias, tx_hash); - - Ok(()) -} - -pub(super) async fn submit_atomic_zone_deposit_transaction( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: &str, - transaction_alias: String, - message_alias: String, - amount: u64, - metadata: Metadata, -) -> StepResult { - let node_url = log_step_error(step, world.zone_node_url_for_sequencer(sequencer_alias))?; - let wallet = log_step_error(step, resolve_zone_wallet(world, sequencer_alias))?; - let public_key = log_step_error(step, wallet.public_key())?; - let available_utxos = log_step_error( - step, - current_available_utxos_for_wallet(world, &step.value, &wallet.wallet_name).await, - )?; - let sequencer = log_step_error(step, world.zone.sequencer_client(sequencer_alias))?; - let inscription_data = make_inscription(&format!("Mint {amount} to Alice")); - - let submission = submit_atomic_zone_deposit( - &node_url, - sequencer, - AtomicZoneDepositRequest { - channel_id: world.zone.sequencer_channel_id(sequencer_alias)?, - funding_public_key: public_key, - available_utxos, - amount, - metadata, - inscription_data: inscription_data.clone(), - }, - ) - .await - .map_err(|error| zone_step_error(step, &error))?; - - world - .zone - .remember_submitted_deposit(transaction_alias.clone(), submission.deposit, amount); - remember_published_zone_message( - world, - sequencer_alias, - message_alias, - inscription_data, - &submission.publish, - ); - record_zone_wallet_submission( - world, - &wallet.wallet_name, - submission.publish.inscription_id(), - submission.reserved_inputs, - )?; - world.remember_submitted_transaction(transaction_alias, submission.publish.inscription_id()); - - Ok(()) -} - -pub(super) async fn submit_zone_withdraw_transaction( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: &str, - transaction_alias: String, - message_alias: String, - amount: u64, -) -> StepResult { - let wallet = log_step_error(step, resolve_zone_wallet(world, sequencer_alias))?; - let public_key = log_step_error(step, wallet.public_key())?; - let sequencer = log_step_error(step, world.zone.sequencer_client(sequencer_alias))?; - let inscription_data = make_inscription(&format!("Burn {amount}")); - - let submission = submit_zone_withdraw( - sequencer, - world.zone.sequencer_channel_id(sequencer_alias)?, - public_key, - amount, - inscription_data.clone(), - ) - .await - .map_err(|error| zone_step_error(step, &error))?; - - world - .zone - .remember_submitted_withdraw(transaction_alias.clone(), submission.withdraw); - remember_published_zone_message( - world, - sequencer_alias, - message_alias, - inscription_data, - &submission.publish, - ); - world.remember_submitted_transaction(transaction_alias, submission.publish.inscription_id()); - - Ok(()) -} - -/// Action wrapper for the new `publish_atomic_withdraw` SDK API. Mirrors -/// [`submit_zone_withdraw_transaction`] but uses the high-level fire-and-forget -/// flow: SDK fills the withdraw nonce, locates its own accredited-key index, -/// builds the bundled `MantleTx`, signs locally, and submits. -/// -/// `withdraw_rows` carries one `(alias, outputs)` per `WithdrawArg`; each -/// withdraw is remembered under its own alias so multi-withdraw bundles can -/// be asserted per-withdraw via the indexer step. `bundle_alias` is remembered -/// as the bundle's tx hash for `zone transaction "..." is finalized`. -pub(super) async fn publish_atomic_zone_withdraw_transaction( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: &str, - bundle_alias: String, - message_alias: String, - withdraw_rows: Vec<(String, Vec)>, -) -> StepResult { - let wallet = log_step_error(step, resolve_zone_wallet(world, sequencer_alias))?; - let public_key = log_step_error(step, wallet.public_key())?; - let total: u64 = withdraw_rows - .iter() - .flat_map(|(_, outputs)| outputs.iter()) - .sum(); - let inscription_data = make_inscription(&format!("Burn {total}")); - let outputs_per_arg: Vec> = withdraw_rows - .iter() - .map(|(_, outputs)| outputs.clone()) - .collect(); - - let submission = { - let sequencer = log_step_error(step, world.zone.sequencer_client(sequencer_alias))?.clone(); - - publish_atomic_zone_withdraw( - &sequencer, - public_key, - outputs_per_arg, - inscription_data.clone(), - PublishDeadline::from_now(Duration::from_mins(3)), - ) - .await - .map_err(|error| zone_step_error(step, &error))? - }; - - // A bundle carries a single `ChannelWithdrawOp` that releases every - // recipient note the transfer created, regardless of how many withdraw args - // were passed. Remember that one op under each row alias so per-withdraw - // indexer assertions all resolve to the same finalized op. - let [withdraw_op] = submission.withdraws.as_slice() else { - return Err(zone_step_error( - step, - &super::support::ZoneTestError::SubmitWithdraw { - message: format!( - "atomic withdraw bundle produced {} withdraw ops, expected exactly 1", - submission.withdraws.len(), - ), - }, - )); - }; - for (alias, _) in &withdraw_rows { - world - .zone - .remember_submitted_withdraw(alias.clone(), withdraw_op.clone()); - } - remember_published_zone_message( - world, - sequencer_alias, - message_alias, - inscription_data, - &submission.publish, - ); - world.remember_submitted_transaction(bundle_alias, submission.publish.inscription_id()); - - Ok(()) -} - -pub(super) fn initialize_zone_indexer( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: impl AsRef, -) -> StepResult { - let sequencer_alias = sequencer_alias.as_ref(); - let node_url = log_step_error(step, world.zone_node_url_for_sequencer(sequencer_alias))?; - let indexer = ZoneReaderConfig { - channel_id: world.zone.sequencer_channel_id(sequencer_alias)?, - node_url, - }; - - world.zone.set_indexer(indexer); - - Ok(()) -} - -pub(super) async fn publish_zone_messages( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: impl AsRef, - rows: Vec<(String, Inscription)>, -) -> StepResult { - let sequencer_alias = sequencer_alias.as_ref().to_owned(); - let node = log_step_error( - step, - world.zone_node_http_client_for_sequencer(&sequencer_alias), - )?; - - let published = { - let sequencer = - log_step_error(step, world.zone.sequencer_client(&sequencer_alias))?.clone(); - - let publish_deadline = PublishDeadline::from_now(Duration::from_mins(3)); - let mut published = Vec::with_capacity(rows.len()); - - for (alias, payload) in &rows { - let result = publish_message_with_retry(&sequencer, payload, publish_deadline) - .await - .map_err(|error| zone_step_error(step, &error))?; - - ensure_zone_transactions_included( - &node, - &[result.inscription_id()], - Duration::from_mins(3), - ) - .await - .map_err(|error| zone_step_error(step, &error))?; - - published.push(PublishedZoneMessage { - alias: alias.clone(), - payload: payload.clone(), - result, - }); - } - - published - }; - - for message in published { - remember_published_zone_message( - world, - &sequencer_alias, - message.alias, - message.payload, - &message.result, - ); - } - - Ok(()) -} - -pub(super) async fn publish_zone_messages_concurrently( - world: &mut CucumberWorld, - step: &Step, - rows: Vec, -) -> StepResult { - let grouped = group_zone_messages_by_sequencer(&rows); - let handles = grouped - .keys() - .map(|sequencer_alias| { - log_step_error(step, world.zone.sequencer_client(sequencer_alias)) - .map(|handle| (sequencer_alias.clone(), handle.clone())) - }) - .collect::, _>>()?; - - join_all(handles.into_iter().map(|(sequencer_alias, handle)| { - let payloads = grouped[&sequencer_alias] - .iter() - .map(|message| message.payload.clone()) - .collect::>(); - - async move { - for payload in payloads { - handle.publish(payload).await.map_err(|error| { - StepError::LogicalError { - message: format!( - "Zone concurrent publish failed for sequencer '{sequencer_alias}': {error}" - ), - } - })?; - } - - Ok::<(), StepError>(()) - } - })) - .await - .into_iter() - .collect::, _>>()?; - - for row in rows { - world - .zone - .remember_zone_message(row.message_alias, row.payload, None, None, None); - } - - if world.zone.indexer().is_err() { - initialize_zone_indexer(world, step, DEFAULT_ZONE_SEQUENCER)?; - } - - Ok(()) -} - -pub(super) async fn start_named_sequencer( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: impl AsRef, - checkpoint: Option, - mode: DriveMode, -) -> StepResult { - let funding = log_step_error(step, sequencer_funding(world, sequencer_alias.as_ref()))?; - start_named_sequencer_with_config( - world, - step, - sequencer_alias, - checkpoint, - mode, - sequencer_config(funding), - ) - .await -} - -/// Fund sequencer transactions from the node's own funding wallet. -fn sequencer_funding( - world: &CucumberWorld, - sequencer_alias: &str, -) -> Result { - let node_name = world.zone.sequencer_node_name(sequencer_alias)?; - let funding_pk = world.funding_wallet(node_name)?.public_key()?; - Ok(FundingConfig { - funding_pk, - change_pk: None, - max_tx_fee: GasCost::new(u64::MAX), - priority_fee_percent: ZONE_TEST_PRIORITY_FEE_PERCENT, - }) -} - -pub(super) async fn start_named_sequencer_with_pending_submit_depth( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: impl AsRef, - checkpoint: Option, - mode: DriveMode, - max_pending_publish_depth: usize, -) -> StepResult { - let funding = log_step_error(step, sequencer_funding(world, sequencer_alias.as_ref()))?; - let config = sequencer_config_with_pending_submit_depth(max_pending_publish_depth, funding); - - start_named_sequencer_with_config(world, step, sequencer_alias, checkpoint, mode, config).await -} - -async fn start_named_sequencer_with_config( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: impl AsRef, - checkpoint: Option, - mode: DriveMode, - config: lb_zone_sdk::sequencer::SequencerConfig, -) -> StepResult { - let sequencer_alias = sequencer_alias.as_ref().to_owned(); - let signing_key = - log_step_error(step, world.zone.sequencer_signing_key(&sequencer_alias))?.clone(); - let node_client = log_step_error( - step, - world.zone_node_http_client_for_sequencer(&sequencer_alias), - )?; - let node_url = log_step_error(step, world.zone_node_url_for_sequencer(&sequencer_alias))?; - let sequencer = ZoneSequencer::init_with_config( - world.zone.sequencer_channel_id(&sequencer_alias)?, - signing_key, - ZoneNodeHttpClient::new(CommonHttpClient::new(None), node_url), - config, - checkpoint, - ); - - let runtime = start_sequencer_runtime(sequencer, mode); - let mut ready_rx = runtime.ready_rx.clone(); - - if let Err(error) = - wait_for_sequencer_ready(&sequencer_alias, &node_client, &mut ready_rx).await - { - runtime.task.abort(); - return Err(error); - } - - world.zone.set_sequencer_runtime( - sequencer_alias, - runtime.client, - runtime.task, - runtime.events, - runtime.checkpoint_rx, - runtime.channel_view_rx, - runtime.turn_to_write_rx, - runtime.tx_status_rx, - runtime.discarded_payloads, - ); - - Ok(()) -} - -async fn wait_for_sequencer_ready( - sequencer_alias: &str, - node_client: &NodeHttpClient, - ready_rx: &mut tokio::sync::watch::Receiver, -) -> StepResult { - timeout(SEQUENCER_READY_TIMEOUT, async { - let mut last_height = node_client.consensus_info().await?.cryptarchia_info.height; - - loop { - let poll = timeout(SEQUENCER_READY_POLL_TIMEOUT, async { - loop { - if ready_rx.changed().await.is_err() { - return Err(()); - } - if *ready_rx.borrow() { - return Ok(()); - } - } - }) - .await; - if matches!(poll, Ok(Ok(()))) { - return Ok(()); - } - - let _ = wait_for_height( - node_client, - last_height.saturating_add(1), - SEQUENCER_READY_HEIGHT_ADVANCE_TIMEOUT, - ) - .await; - - last_height = node_client - .consensus_info() - .await? - .cryptarchia_info - .height - .max(last_height); - } - }) - .await - .map_err(|_: Elapsed| StepError::Timeout { - message: format!( - "Sequencer `{sequencer_alias}` did not become ready within {} seconds", - SEQUENCER_READY_TIMEOUT.as_secs() - ), - })? -} - -fn from_policy_runtime( - rt: super::support::PolicyRuntime, - discarded_payloads: Option, -) -> StartedSequencerRuntime { - StartedSequencerRuntime { - task: rt.task, - client: rt.client, - events: rt.events, - checkpoint_rx: rt.checkpoint_rx, - ready_rx: rt.ready_rx, - channel_view_rx: rt.channel_view_rx, - turn_to_write_rx: rt.turn_to_write_rx, - tx_status_rx: rt.tx_status_rx, - discarded_payloads, - } -} - -fn start_sequencer_runtime( - sequencer: ZoneSequencer, - mode: DriveMode, -) -> StartedSequencerRuntime { - match mode { - DriveMode::Passive { republish_orphans } => from_policy_runtime( - start_sequencer_event_loop(sequencer, republish_orphans), - None, - ), - DriveMode::RepublishLineage { planned } => { - from_policy_runtime(start_republish_lineage_policy(sequencer, planned), None) - } - DriveMode::Sorted { discarded } => from_policy_runtime( - start_sorted_conflict_policy(sequencer, &discarded), - Some(discarded), - ), - DriveMode::BalanceAware { - initial_balances, - planned_payloads, - } => from_policy_runtime( - start_balance_aware_policy(sequencer, initial_balances, planned_payloads), - None, - ), - DriveMode::CustomRepublish { deps } => { - from_policy_runtime(start_custom_republish_policy(sequencer, *deps), None) - } - } -} diff --git a/tests/src/cucumber/steps/manual_zone/mod.rs b/tests/src/cucumber/steps/manual_zone/mod.rs deleted file mode 100644 index 73dd71c6d..000000000 --- a/tests/src/cucumber/steps/manual_zone/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -mod actions; -mod assertions; -mod errors; -pub mod runner; -pub mod steps; -mod support; -mod tables; diff --git a/tests/src/cucumber/steps/manual_zone/steps.rs b/tests/src/cucumber/steps/manual_zone/steps.rs deleted file mode 100644 index e56031bb7..000000000 --- a/tests/src/cucumber/steps/manual_zone/steps.rs +++ /dev/null @@ -1,1921 +0,0 @@ -use std::{ - collections::{HashMap, HashSet, VecDeque}, - sync::Arc, - time::Duration, -}; - -use cucumber::{gherkin::Step, given, when}; -use lb_core::mantle::ops::channel::inscribe::Inscription; - -use super::{ - actions::{ - DriveMode, initialize_zone_indexer, publish_atomic_zone_withdraw_transaction, - publish_zone_messages, publish_zone_messages_concurrently, - register_zone_sequencers_with_shared_key, remember_published_zone_message, - save_zone_checkpoint, start_named_sequencer, - start_named_sequencer_with_pending_submit_depth, start_nodes_with_zone_resources, - stop_zone_sequencer, submit_atomic_zone_deposit_transaction, submit_zone_channel_config, - submit_zone_channel_split_transaction, submit_zone_deposit_transaction, - submit_zone_multi_deposit_transaction, submit_zone_withdraw_transaction, - }, - assertions::{ - assert_sorted_outcome, scan_indexer_for_payloads, wait_for_indexer_unordered, - wait_until_sorted_conflict_settles, - }, - errors::{log_step_error, zone_step_error}, - runner::{TxSource, TxStatus}, - support::{ - CustomRepublishDeps, PublishDeadline, balance_update_payload, collect_indexed_messages, - collect_indexed_messages_exactly_once, ensure_zone_transactions_included, - parse_balance_payload, publish_message_with_retry, wait_for_channel_transfer_input_count, - wait_for_channel_view, wait_for_channel_wallet_counts, wait_for_channel_wallet_note, - wait_for_deposit, wait_for_exact_indexed_payload_count, - wait_for_finalized_deposit_via_sequencer_and_collect_mempool_pending, - wait_for_finalized_withdraw_via_sequencer_and_collect_mempool_pending, - wait_for_lib_advance, wait_for_on_chain_statuses_and_collect_mempool_pending, - wait_for_transactions_finalized, wait_for_turn_to_write, wait_for_tx_status_lifecycle, - wait_for_withdraw, - }, - tables::{ - ConcurrentZoneMessageRow, GeneratedZoneMessageBatch, concurrent_zone_message_rows, - custom_tx_rows, generated_zone_message_batches, generated_zone_message_sequencers, - group_zone_messages_by_sequencer, zone_account_balances, zone_atomic_withdraw_rows, - zone_balance_rows, zone_config_row, zone_message_rows, zone_node_resource_rows, - zone_sequencer_start_rows, zone_sequencing_state_row, - }, -}; -use crate::{ - common::mantle_inscription::make_inscription, - cucumber::{ - error::{StepError, StepResult}, - steps::parse_steps::single_column_table, - world::{CucumberWorld, ZoneSequencerStartup}, - }, -}; - -pub(super) const DEFAULT_ZONE_SEQUENCER: &str = "SEQ_A"; - -fn parse_submit_depth(step: &Step, value: &str) -> Result { - let value = value.trim(); - if matches!(value.to_lowercase().as_str(), "unlimited" | "none") { - return Ok(usize::MAX); - } - - let inner = value - .strip_prefix("Some(") - .and_then(|rest| rest.strip_suffix(')')) - .unwrap_or(value); - - let limit = inner - .parse::() - .map_err(|error| StepError::LogicalError { - message: format!( - "Invalid pending submit depth '{value}' in step '{}': {error}", - step.value - ), - })?; - - Ok(limit) -} - -fn parse_optional_submit_depth(step: &Step, value: &str) -> Result, StepError> { - let value = value.trim(); - if value.eq_ignore_ascii_case("default") { - return Ok(None); - } - - parse_submit_depth(step, value).map(Some) -} - -const fn passive_mode_for_startup(startup: ZoneSequencerStartup) -> DriveMode { - if startup.passive_republish_orphans { - DriveMode::passive_republish_orphans() - } else { - DriveMode::passive() - } -} - -async fn start_named_sequencer_with_startup( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: &str, - checkpoint: Option, - startup: ZoneSequencerStartup, -) -> StepResult { - let mode = passive_mode_for_startup(startup); - if let Some(submit_depth) = startup.pending_submit_depth { - start_named_sequencer_with_pending_submit_depth( - world, - step, - sequencer_alias, - checkpoint, - mode, - submit_depth, - ) - .await - } else { - start_named_sequencer(world, step, sequencer_alias, checkpoint, mode).await - } -} -const CONCURRENT_DUPLICATE_SETTLE_SECS: u64 = 30; - -#[given("I start nodes with wallet and sequencer resources:")] -#[when("I start nodes with wallet and sequencer resources:")] -async fn step_start_nodes_with_wallet_and_sequencer_resources( - world: &mut CucumberWorld, - step: &Step, -) -> StepResult { - let rows = zone_node_resource_rows(step)?; - - start_nodes_with_zone_resources(world, step, rows).await -} - -#[given(expr = "the following zone sequencers share the signing key of {string}:")] -#[expect( - clippy::needless_pass_by_value, - reason = "Cucumber string captures are provided as owned `String`s" -)] -fn step_zone_sequencers_share_signing_key( - world: &mut CucumberWorld, - step: &Step, - source_alias: String, -) -> StepResult { - let aliases = single_column_table(step, "alias", "zone sequencer aliases")?; - register_zone_sequencers_with_shared_key(world, &source_alias, aliases) -} - -#[given("the following zone account balances exist:")] -fn step_zone_account_balances(world: &mut CucumberWorld, step: &Step) -> StepResult { - let balances = zone_account_balances(step)? - .into_iter() - .map(|row| (row.account, row.balance)) - .collect(); - - world.zone.set_zone_account_balances(balances); - - Ok(()) -} - -#[when(expr = "I start zone sequencer {string}")] -async fn step_start_zone_sequencer( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, -) -> StepResult { - start_named_sequencer(world, step, sequencer_alias, None, DriveMode::passive()).await -} - -#[when(expr = "I start zone sequencer {string} with indexer")] -async fn step_start_zone_sequencer_with_indexer( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, -) -> StepResult { - start_sequencer_with_indexer(world, step, &sequencer_alias).await -} - -async fn start_sequencer_with_indexer( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: &str, -) -> StepResult { - start_named_sequencer(world, step, sequencer_alias, None, DriveMode::passive()).await?; - initialize_zone_indexer(world, step, sequencer_alias) -} - -#[when("I start zone sequencers:")] -async fn step_start_zone_sequencers(world: &mut CucumberWorld, step: &Step) -> StepResult { - for row in zone_sequencer_start_rows(step)? { - let alias = row.alias; - let startup = ZoneSequencerStartup { - pending_submit_depth: parse_optional_submit_depth(step, &row.pending_submit_depth)?, - passive_republish_orphans: row.passive_republish_orphans, - }; - world.zone.set_sequencer_startup(&alias, startup); - - start_named_sequencer_with_startup(world, step, &alias, None, startup).await?; - - if row.indexer { - initialize_zone_indexer(world, step, &alias)?; - } - } - - Ok(()) -} - -#[when(expr = "I stop zone sequencer {string}")] -fn step_stop_zone_sequencer( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, -) -> StepResult { - let _ = step; - stop_zone_sequencer(world, sequencer_alias) -} - -#[cucumber::when(expr = "the zone LIB advances in {int} seconds")] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require `&mut World` as the first parameter" -)] -async fn step_zone_lib_advances( - world: &mut CucumberWorld, - step: &Step, - timeout_seconds: u64, -) -> StepResult { - let client = log_step_error(step, world.zone_node_http_client())?; - let initial_lib_slot = client - .consensus_info() - .await - .map_err(|error| StepError::LogicalError { - message: format!("Failed to fetch zone consensus info: {error}"), - })? - .cryptarchia_info - .lib_slot; - - wait_for_lib_advance( - &client, - initial_lib_slot, - Duration::from_secs(timeout_seconds), - ) - .await - .map_err(|error| zone_step_error(step, &error)) -} - -#[when("I publish the following zone messages:")] -async fn step_publish_zone_messages(world: &mut CucumberWorld, step: &Step) -> StepResult { - publish_zone_messages( - world, - step, - DEFAULT_ZONE_SEQUENCER, - zone_message_rows(step)?, - ) - .await -} - -#[when(expr = "sequencer {string} publishes the following zone messages:")] -async fn step_publish_zone_messages_for_sequencer( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, -) -> StepResult { - publish_zone_messages(world, step, sequencer_alias, zone_message_rows(step)?).await -} - -/// Publishing while the sequencer's node is down must be rejected: with -/// funding configured, building a transaction needs the node's wallet, so -/// the sequencer fails fast with `Unavailable` once the stream drop is -/// noticed (or surfaces the funding error in the brief window before). A -/// fresh `Ready` event fires once the node is back and a live block -/// confirms the reconnect. -#[cucumber::then( - expr = "publishing zone message with data {string} via sequencer {string} fails while the node is down" -)] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require the world as the first `&mut` argument" -)] -async fn step_publish_fails_while_node_down( - world: &mut CucumberWorld, - step: &Step, - data: String, - sequencer_alias: String, -) -> StepResult { - let _ = step; - let payload = make_inscription(&data); - let client = world.zone.sequencer_client(&sequencer_alias)?.clone(); - match client.publish(payload).await { - Ok(_) => Err(StepError::LogicalError { - message: format!( - "Zone publish unexpectedly succeeded for sequencer '{sequencer_alias}' while its node is down" - ), - }), - Err(_expected) => Ok(()), - } -} - -#[when( - expr = "I submit zone message {string} to sequencer {string} with data {string} immediately" -)] -async fn step_publish_single_zone_message_for_sequencer( - world: &mut CucumberWorld, - step: &Step, - message_alias: String, - sequencer_alias: String, - data: String, -) -> StepResult { - let _ = step; - let payload = make_inscription(&data); - let handle = world.zone.sequencer_client(&sequencer_alias)?.clone(); - - let (published, _checkpoint) = handle - .publish(payload.clone()) - .await - .map_err(|error| StepError::LogicalError { - message: format!( - "Zone publish failed for sequencer '{sequencer_alias}' and message '{message_alias}': {error}" - ), - })?; - - remember_published_zone_message(world, &sequencer_alias, message_alias, payload, &published); - - Ok(()) -} - -#[when( - expr = "sequencer {string} submits zone message {string} with data {string} to queue immediately" -)] -async fn step_publish_single_zone_message_to_queue_for_sequencer( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, - message_alias: String, - data: String, -) -> StepResult { - step_publish_single_zone_message_for_sequencer( - world, - step, - message_alias, - sequencer_alias, - data, - ) - .await -} - -#[when( - "the following custom transactions are published concurrently with custom republish policy:" -)] -async fn step_publish_custom_txs_concurrently_with_policy( - world: &mut CucumberWorld, - step: &Step, -) -> StepResult { - let rows = custom_tx_rows(step)?; - let mut expected_payloads = Vec::new(); - - for row in &rows { - let batches: VecDeque> = (0..row.transactions) - .map(|tx_index| { - (0..row.inscriptions) - .map(|entry_index| { - make_inscription(&format!( - "custom-{}-{tx_index}-{entry_index}", - row.sequencer_alias - )) - }) - .collect() - }) - .collect(); - expected_payloads.extend(batches.iter().flatten().cloned()); - - let node_client = log_step_error( - step, - world.zone_node_http_client_for_sequencer(&row.sequencer_alias), - )?; - let node_name = world - .zone - .sequencer_node_name(&row.sequencer_alias)? - .to_owned(); - let funding_pk = world.funding_wallet(&node_name)?.public_key()?; - let deps = CustomRepublishDeps { - node_client, - channel_id: world.zone.sequencer_channel_id(&row.sequencer_alias)?, - signing_key: world - .zone - .sequencer_signing_key(&row.sequencer_alias)? - .clone(), - funding_pk, - batches, - }; - - start_named_sequencer( - world, - step, - &row.sequencer_alias, - None, - DriveMode::CustomRepublish { - deps: Box::new(deps), - }, - ) - .await?; - } - - world - .zone - .remember_expected_custom_payloads(expected_payloads); - Ok(()) -} - -#[cucumber::then(expr = "the zone indexer returns all custom payloads in {int} seconds")] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require `&mut World` as the first parameter" -)] -async fn step_zone_indexer_returns_all_custom_payloads( - world: &mut CucumberWorld, - step: &Step, - timeout_seconds: u64, -) -> StepResult { - let expected: HashSet = world - .zone - .expected_custom_payloads() - .iter() - .cloned() - .collect(); - if expected.is_empty() { - return Err(StepError::LogicalError { - message: "no custom transactions were planned".to_owned(), - }); - } - let indexer = log_step_error(step, world.zone.indexer())?; - - wait_for_indexer_unordered(indexer, &expected, Duration::from_secs(timeout_seconds)) - .await - .map_err(|error| zone_step_error(step, &error))?; - Ok(()) -} - -#[when( - expr = "I submit zone message {string} to sequencer {string} with data {string} on its turn" -)] -async fn step_publish_single_zone_message_for_sequencer_on_turn( - world: &mut CucumberWorld, - step: &Step, - message_alias: String, - sequencer_alias: String, - data: String, -) -> StepResult { - let payload = make_inscription(&data); - let handle = world.zone.sequencer_client(&sequencer_alias)?.clone(); - let mut view_rx = world.zone.sequencer_channel_view_rx(&sequencer_alias)?; - - wait_for_channel_view(&mut view_rx, Duration::from_mins(3), |view| { - view.our_turn_to_write - && view.authorized_key_index.is_some() - && view.authorized_key_index == view.own_key_index - }) - .await - .map_err(|error| zone_step_error(step, &error))?; - - let published = publish_message_with_retry( - &handle, - &payload, - PublishDeadline::from_now(Duration::from_mins(3)), - ) - .await - .map_err(|error| zone_step_error(step, &error))?; - - remember_published_zone_message(world, &sequencer_alias, message_alias, payload, &published); - - Ok(()) -} - -#[when(expr = "sequencer {string} submits the following zone messages to queue immediately:")] -async fn step_publish_zone_messages_to_queue_for_sequencer( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, -) -> StepResult { - let rows = zone_message_rows(step)?; - let handle = world.zone.sequencer_client(&sequencer_alias)?.clone(); - - for (message_alias, payload) in rows { - let (published, _checkpoint) = handle - .publish(payload.clone()) - .await - .map_err(|error| StepError::LogicalError { - message: format!( - "Zone publish failed for sequencer '{sequencer_alias}' and message '{message_alias}': {error}" - ), - })?; - - remember_published_zone_message( - world, - &sequencer_alias, - message_alias, - payload, - &published, - ); - } - - Ok(()) -} - -/// Publish via [`SequencerClient`] and record each inscription id, without -/// waiting for on-chain inclusion. -/// -/// Unlike `publishes the following zone messages` (which polls the node to -/// confirm inclusion), this performs no node HTTP calls, so it can be issued -/// while the node is down: each publish is accepted locally and posted on -/// reconnect. Recording the ids (the publish returns them locally) lets later -/// `... are finalized` / indexer assertions track the messages once the node is -/// back. -#[when( - expr = "sequencer {string} submits the following zone messages without waiting for inclusion:" -)] -async fn step_publish_zone_messages_without_inclusion_for_sequencer( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, -) -> StepResult { - let rows = zone_message_rows(step)?; - let handle = world.zone.sequencer_client(&sequencer_alias)?.clone(); - - for (message_alias, payload) in rows { - let (published, _checkpoint) = handle - .publish(payload.clone()) - .await - .map_err(|error| StepError::LogicalError { - message: format!( - "Zone publish failed for sequencer '{sequencer_alias}' and message '{message_alias}': {error}" - ), - })?; - remember_published_zone_message( - world, - &sequencer_alias, - message_alias, - payload, - &published, - ); - } - - Ok(()) -} - -#[when(expr = "I save current checkpoint of sequencer {string} as {string}")] -fn step_save_zone_checkpoint( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, - checkpoint_alias: String, -) -> StepResult { - save_zone_checkpoint(world, step, sequencer_alias, checkpoint_alias) -} - -#[when(expr = "I restart zone sequencer {string} from checkpoint {string}")] -async fn step_restart_zone_sequencer_from_checkpoint( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, - checkpoint_alias: String, -) -> StepResult { - let checkpoint = world.zone.resolve_checkpoint(checkpoint_alias)?; - let startup = world.zone.sequencer_startup_for(&sequencer_alias); - - start_named_sequencer_with_startup(world, step, &sequencer_alias, Some(checkpoint), startup) - .await -} - -#[when(expr = "I restart zone sequencer {string} fresh")] -async fn step_restart_zone_sequencer_fresh( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, -) -> StepResult { - let startup = world.zone.sequencer_startup_for(&sequencer_alias); - start_named_sequencer_with_startup(world, step, &sequencer_alias, None, startup).await -} - -#[when(expr = "sequencer {string} submits zone config transaction {string} authorizing:")] -async fn step_submit_zone_channel_config_transaction( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, - transaction_alias: String, -) -> StepResult { - let authorized_aliases = - single_column_table(step, "alias", "authorized zone sequencer aliases")?; - - submit_zone_channel_config( - world, - step, - &sequencer_alias, - transaction_alias, - authorized_aliases, - 0, - 0, - ) - .await -} - -#[when( - expr = "sequencer {string} submits zone config transaction {string} with posting timeframe {int} and timeout {int} authorizing:" -)] -async fn step_submit_zone_channel_config_transaction_with_posting_window( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, - transaction_alias: String, - posting_timeframe: u32, - posting_timeout: u32, -) -> StepResult { - let authorized_aliases = - single_column_table(step, "alias", "authorized zone sequencer aliases")?; - - submit_zone_channel_config( - world, - step, - &sequencer_alias, - transaction_alias, - authorized_aliases, - posting_timeframe, - posting_timeout, - ) - .await -} - -#[when(expr = "sequencer {string} submits zone config transaction:")] -async fn step_submit_zone_channel_config_transaction_from_table( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, -) -> StepResult { - let row = zone_config_row(step)?; - - submit_zone_channel_config( - world, - step, - &sequencer_alias, - row.config_name, - row.authorized_sequencers, - row.posting_timeframe, - row.posting_timeout, - ) - .await -} - -#[when( - expr = "I submit zone deposit transaction {string} into channel of {string} of {int} with metadata {string}" -)] -async fn step_submit_zone_deposit_transaction( - world: &mut CucumberWorld, - step: &Step, - transaction_alias: String, - channel_alias: String, - amount: u64, - metadata: String, -) -> StepResult { - submit_zone_deposit_transaction( - world, - step, - transaction_alias, - channel_alias, - amount, - metadata - .into_bytes() - .try_into() - .expect("Metadata too large for deposit op."), - ) - .await -} - -#[when( - expr = "I submit zone deposit transaction {string} into channel of {string} consuming notes valued {string} with metadata {string}" -)] -async fn step_submit_zone_multi_deposit_transaction( - world: &mut CucumberWorld, - step: &Step, - transaction_alias: String, - channel_alias: String, - values: String, - metadata: String, -) -> StepResult { - let input_values = values - .split(',') - .map(|part| part.trim().parse::()) - .collect::, _>>() - .expect("deposit note values must be a comma-separated list of integers"); - submit_zone_multi_deposit_transaction( - world, - step, - transaction_alias, - channel_alias, - input_values, - metadata - .into_bytes() - .try_into() - .expect("Metadata too large for deposit op."), - ) - .await -} - -#[when( - expr = "I submit zone deposit transaction {string} into channel of {string} consuming {int} notes of value {int} with metadata {string}" -)] -async fn step_submit_zone_bulk_deposit_transaction( - world: &mut CucumberWorld, - step: &Step, - transaction_alias: String, - channel_alias: String, - count: usize, - value: u64, - metadata: String, -) -> StepResult { - let input_values = vec![value; count]; - submit_zone_multi_deposit_transaction( - world, - step, - transaction_alias, - channel_alias, - input_values, - metadata - .into_bytes() - .try_into() - .expect("Metadata too large for deposit op."), - ) - .await -} - -#[when(expr = "sequencer {string} splits deposit {string} into {int} dust notes as {string}")] -async fn step_submit_zone_channel_split_transaction( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, - deposit_alias: String, - dust_count: usize, - transaction_alias: String, -) -> StepResult { - submit_zone_channel_split_transaction( - world, - step, - &sequencer_alias, - &deposit_alias, - dust_count, - transaction_alias, - ) - .await -} - -#[when( - expr = "sequencer {string} submits atomic zone deposit transaction {string} with inscription {string} of {int} with metadata {string}" -)] -async fn step_submit_atomic_zone_deposit_transaction( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, - transaction_alias: String, - message_alias: String, - amount: u64, - metadata: String, -) -> StepResult { - submit_atomic_zone_deposit_transaction( - world, - step, - &sequencer_alias, - transaction_alias, - message_alias, - amount, - metadata - .into_bytes() - .try_into() - .expect("Metadata too large for deposit op."), - ) - .await -} - -#[when( - expr = "sequencer {string} submits zone withdraw transaction {string} with inscription {string} of {int}" -)] -async fn step_submit_zone_withdraw_transaction( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, - transaction_alias: String, - message_alias: String, - amount: u64, -) -> StepResult { - submit_zone_withdraw_transaction( - world, - step, - &sequencer_alias, - transaction_alias, - message_alias, - amount, - ) - .await -} - -#[when(expr = "sequencer {string} publishes atomic withdraw {string} with inscription {string}:")] -async fn step_publish_atomic_zone_withdraw_transaction( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, - bundle_alias: String, - message_alias: String, -) -> StepResult { - let withdraw_rows = zone_atomic_withdraw_rows(step)?; - publish_atomic_zone_withdraw_transaction( - world, - step, - &sequencer_alias, - bundle_alias, - message_alias, - withdraw_rows, - ) - .await -} - -#[when("the following zone messages are published concurrently with republish policy:")] -async fn step_publish_zone_messages_concurrently_with_republish_policy( - world: &mut CucumberWorld, - step: &Step, -) -> StepResult { - let rows = concurrent_zone_message_rows(step)?; - publish_zone_messages_with_republish_policy(world, step, rows).await -} - -#[when( - expr = "each listed zone sequencer publishes {int} generated zone messages concurrently with republish policy:" -)] -async fn step_publish_generated_zone_messages_concurrently_with_republish_policy( - world: &mut CucumberWorld, - step: &Step, - messages_per_sequencer: usize, -) -> StepResult { - let rows = build_generated_zone_message_rows( - generated_zone_message_batches(step)?, - messages_per_sequencer, - ); - - publish_zone_messages_with_republish_policy(world, step, rows).await -} - -#[when( - expr = "each listed zone sequencer publishes {int} copies of zone message {string} concurrently with republish policy:" -)] -async fn step_publish_repeated_zone_messages_concurrently_with_republish_policy( - world: &mut CucumberWorld, - step: &Step, - copies_per_sequencer: usize, - payload: String, -) -> StepResult { - let rows = build_repeated_zone_message_rows( - generated_zone_message_sequencers(step)?, - copies_per_sequencer, - &payload, - ); - - publish_zone_messages_with_republish_policy(world, step, rows).await -} - -fn build_generated_zone_message_rows( - batches: Vec, - messages_per_sequencer: usize, -) -> Vec { - let mut builder = GeneratedZoneMessages::default(); - - for batch in batches { - builder.append_numbered_payloads(batch, messages_per_sequencer); - } - - builder.finish() -} - -fn build_repeated_zone_message_rows( - sequencer_aliases: Vec, - copies_per_sequencer: usize, - payload: &str, -) -> Vec { - let mut builder = GeneratedZoneMessages::default(); - let payload = make_inscription(payload); - - for sequencer_alias in sequencer_aliases { - builder.append_repeated_payloads(sequencer_alias, copies_per_sequencer, &payload); - } - - builder.finish() -} - -#[derive(Default)] -struct GeneratedZoneMessages { - next_message_number: usize, - rows: Vec, -} - -impl GeneratedZoneMessages { - fn append_numbered_payloads(&mut self, batch: GeneratedZoneMessageBatch, count: usize) { - let GeneratedZoneMessageBatch { - sequencer_alias, - data_prefix, - } = batch; - - for payload_number in 1..=count { - self.push( - sequencer_alias.clone(), - make_inscription(&format!("{data_prefix}{payload_number}")), - ); - } - } - - fn append_repeated_payloads( - &mut self, - sequencer_alias: String, - count: usize, - payload: &Inscription, - ) { - for _ in 1..count { - self.push(sequencer_alias.clone(), payload.clone()); - } - - if count > 0 { - self.push(sequencer_alias, payload.clone()); - } - } - - fn push(&mut self, sequencer_alias: String, payload: Inscription) { - self.next_message_number += 1; - - self.rows.push(ConcurrentZoneMessageRow { - sequencer_alias, - message_alias: format!("MSG_{}", self.next_message_number), - payload, - }); - } - - fn finish(self) -> Vec { - self.rows - } -} - -async fn publish_zone_messages_with_republish_policy( - world: &mut CucumberWorld, - step: &Step, - rows: Vec, -) -> StepResult { - let grouped = group_zone_messages_by_sequencer(&rows); - - for (sequencer_alias, messages) in &grouped { - let planned = messages - .iter() - .map(|message| message.payload.clone()) - .collect(); - start_named_sequencer_with_pending_submit_depth( - world, - step, - sequencer_alias, - None, - DriveMode::RepublishLineage { planned }, - usize::MAX, - ) - .await?; - } - - // Each sequencer's lineage policy owns publishing its own copies; the step - // only records the messages so the indexer assertions can find them. - for row in rows { - world - .zone - .remember_zone_message(row.message_alias, row.payload, None, None, None); - } - if world.zone.indexer().is_err() { - initialize_zone_indexer(world, step, DEFAULT_ZONE_SEQUENCER)?; - } - Ok(()) -} - -#[when("the following zone messages are published concurrently with sorted conflict policy:")] -async fn step_publish_zone_messages_concurrently_with_sorted_policy( - world: &mut CucumberWorld, - step: &Step, -) -> StepResult { - let rows = concurrent_zone_message_rows(step)?; - let grouped = group_zone_messages_by_sequencer(&rows); - let discarded = Arc::new(tokio::sync::Mutex::new(HashSet::new())); - - for sequencer_alias in grouped.keys() { - start_named_sequencer( - world, - step, - sequencer_alias, - None, - DriveMode::Sorted { - discarded: Arc::clone(&discarded), - }, - ) - .await?; - } - - world.zone.set_sorted_total_payloads(rows.len()); - world.zone.set_sorted_expected_by_sequencer( - grouped - .iter() - .map(|(sequencer_alias, messages)| { - ( - sequencer_alias.clone(), - messages - .iter() - .map(|message| message.payload.clone()) - .collect(), - ) - }) - .collect(), - ); - - publish_zone_messages_concurrently(world, step, rows).await -} - -#[when("the following zone balance updates are published concurrently with balance-aware policy:")] -async fn step_publish_zone_balance_updates_with_balance_policy( - world: &mut CucumberWorld, - step: &Step, -) -> StepResult { - let rows = zone_balance_rows(step)?; - let initial_balances = world.zone.zone_account_balances()?; - let grouped = rows.iter().fold( - HashMap::>::new(), - |mut grouped, row| { - let payload = balance_update_payload(&row.message_alias, &row.account, row.delta); - grouped - .entry(row.sequencer_alias.clone()) - .or_default() - .push((row.message_alias.clone(), payload)); - grouped - }, - ); - - for (sequencer_alias, planned) in &grouped { - start_named_sequencer( - world, - step, - sequencer_alias, - None, - DriveMode::BalanceAware { - initial_balances: initial_balances.clone(), - planned_payloads: planned.iter().map(|(_, payload)| payload.clone()).collect(), - }, - ) - .await?; - } - - for messages in grouped.values() { - for (message_alias, payload) in messages { - world.zone.remember_zone_message( - message_alias.clone(), - payload.clone(), - None, - None, - None, - ); - } - } - - if world.zone.indexer().is_err() { - initialize_zone_indexer(world, step, DEFAULT_ZONE_SEQUENCER)?; - } - - Ok(()) -} - -#[cucumber::then( - expr = "sequencer {string} reaches sequencing state OWN_KEY_INDEX {int} NOT_OUR_TURN with {int} pending publish txs in {int} seconds" -)] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require `&mut World` as the first parameter" -)] -async fn step_sequencer_reaches_sequencing_state_not_our_turn( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, - own_key_index: usize, - pending_publish_txs: usize, - timeout_seconds: u64, -) -> StepResult { - wait_for_sequencing_state( - world, - step, - &sequencer_alias, - own_key_index, - false, - pending_publish_txs, - timeout_seconds, - ) - .await -} - -#[cucumber::then( - expr = "sequencer {string} reaches sequencing state OWN_KEY_INDEX {int} OUR_TURN with {int} pending publish txs in {int} seconds" -)] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require `&mut World` as the first parameter" -)] -async fn step_sequencer_reaches_sequencing_state_our_turn( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, - own_key_index: usize, - pending_publish_txs: usize, - timeout_seconds: u64, -) -> StepResult { - wait_for_sequencing_state( - world, - step, - &sequencer_alias, - own_key_index, - true, - pending_publish_txs, - timeout_seconds, - ) - .await -} - -#[cucumber::then(expr = "sequencer {string} reaches sequencing state:")] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require `&mut World` as the first parameter" -)] -async fn step_sequencer_reaches_sequencing_state_from_table( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, -) -> StepResult { - let row = zone_sequencing_state_row(step)?; - - wait_for_sequencing_state( - world, - step, - &sequencer_alias, - row.own_key_index, - row.is_our_turn, - row.pending_transactions, - row.timeout_seconds, - ) - .await -} - -async fn wait_for_sequencing_state( - world: &CucumberWorld, - step: &Step, - sequencer_alias: &str, - own_key_index: usize, - is_our_turn: bool, - pending_publish_txs: usize, - timeout_seconds: u64, -) -> StepResult { - let _handle = log_step_error(step, world.zone.sequencer_client(sequencer_alias))?.clone(); - let mut view_rx = log_step_error(step, world.zone.sequencer_channel_view_rx(sequencer_alias))?; - - wait_for_channel_view( - &mut view_rx, - Duration::from_secs(timeout_seconds), - move |view| { - view.own_key_index == Some(own_key_index as u16) - && view.authorized_key_index.is_some() - && view.our_turn_to_write == is_our_turn - && (is_our_turn || view.authorized_key_index != view.own_key_index) - && (!is_our_turn || view.authorized_key_index == Some(own_key_index as u16)) - && view.pending_publish_txs == pending_publish_txs - }, - ) - .await - .map_err(|error| zone_step_error(step, &error))?; - - Ok(()) -} - -#[cucumber::then( - expr = "sequencer {string} is notified it is their turn to write in {int} seconds" -)] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require `&mut World` as the first parameter" -)] -async fn step_sequencer_notified_turn_to_write( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, - timeout_seconds: u64, -) -> StepResult { - let mut turn_rx = log_step_error( - step, - world.zone.sequencer_turn_to_write_rx(&sequencer_alias), - )?; - wait_for_turn_to_write(&mut turn_rx, Duration::from_secs(timeout_seconds)) - .await - .map_err(|error| zone_step_error(step, &error))?; - - Ok(()) -} - -#[cucumber::then( - expr = "sequencer {string} emits published events for queued zone messages on its turn in {int} seconds:" -)] -async fn step_sequencer_emits_published_events_for_queued_zone_messages_on_turn( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, - timeout_seconds: u64, -) -> StepResult { - let aliases = single_column_table(step, "alias", "zone message aliases")?; - let tx_hashes = log_step_error(step, world.zone.message_tx_hashes_for_aliases(&aliases))?; - let mut view_rx = log_step_error(step, world.zone.sequencer_channel_view_rx(&sequencer_alias))?; - wait_for_channel_view(&mut view_rx, Duration::from_secs(timeout_seconds), |view| { - view.our_turn_to_write - }) - .await - .map_err(|error| zone_step_error(step, &error))?; - - // The messages were remembered when queued; wait until they're mined - // (`OnChain`, not yet finalized) via the per-tx status stream. - let mut statuses = log_step_error( - step, - world.zone.take_sequencer_tx_status_rx(&sequencer_alias), - )?; - let mempool_pending = wait_for_on_chain_statuses_and_collect_mempool_pending( - &mut statuses, - &tx_hashes, - Duration::from_secs(timeout_seconds), - ) - .await - .map_err(|error| zone_step_error(step, &error))?; - - world - .zone - .record_mempool_pending(sequencer_alias.clone(), mempool_pending); - - Ok(()) -} - -#[cucumber::then(expr = "sequencer {string} observed mempool pending events for zone messages:")] -#[expect( - clippy::unused_async, - reason = "Cucumber step functions are async even when assertion is synchronous" -)] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require `&mut World` as the first parameter" -)] -async fn step_sequencer_emitted_mempool_pending_events_for_zone_messages( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, -) -> StepResult { - let aliases = single_column_table(step, "alias", "zone message aliases")?; - let tx_hashes = log_step_error(step, world.zone.message_tx_hashes_for_aliases(&aliases))?; - - for (alias, tx_hash) in aliases.iter().zip(tx_hashes.iter()) { - if !world - .zone - .has_observed_mempool_pending(&sequencer_alias, tx_hash) - { - return Err(StepError::LogicalError { - message: format!( - "Sequencer '{sequencer_alias}' did not emit mempool pending event for zone message '{alias}'" - ), - }); - } - } - - Ok(()) -} - -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require `&mut World` as the first parameter" -)] -#[cucumber::then(expr = "sequencer {string} has {int} pending publish txs in {int} seconds")] -async fn step_sequencer_has_pending_publish_txs( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, - pending_publish_txs: usize, - timeout_seconds: u64, -) -> StepResult { - let mut view_rx = log_step_error(step, world.zone.sequencer_channel_view_rx(&sequencer_alias))?; - - wait_for_channel_view( - &mut view_rx, - Duration::from_secs(timeout_seconds), - move |view| view.pending_publish_txs == pending_publish_txs, - ) - .await - .map_err(|error| zone_step_error(step, &error))?; - - Ok(()) -} - -#[cucumber::then( - expr = "sequencer {string} publishes {string} immediately while in turn in {int} seconds" -)] -async fn step_sequencer_publishes_immediately_while_in_turn( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, - message_alias: String, - timeout_seconds: u64, -) -> StepResult { - // The message was remembered when submitted; wait until it's mined - // (`OnChain`, not yet finalized) via the per-tx status stream. - let tx_hashes = log_step_error( - step, - world - .zone - .message_tx_hashes_for_aliases(std::slice::from_ref(&message_alias)), - )?; - let mut statuses = log_step_error( - step, - world.zone.take_sequencer_tx_status_rx(&sequencer_alias), - )?; - let mempool_pending = wait_for_on_chain_statuses_and_collect_mempool_pending( - &mut statuses, - &tx_hashes, - Duration::from_secs(timeout_seconds), - ) - .await - .map_err(|error| zone_step_error(step, &error))?; - - world - .zone - .record_mempool_pending(sequencer_alias.clone(), mempool_pending); - - Ok(()) -} - -#[cucumber::then(expr = "all zone messages are safe in {int} seconds")] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require `&mut World` as the first parameter" -)] -async fn step_all_zone_messages_are_safe( - world: &mut CucumberWorld, - step: &Step, - timeout_seconds: u64, -) -> StepResult { - let inscription_ids = log_step_error(step, world.zone.ordered_inscription_ids())?; - - if !world.zone.has_published_messages() { - return Err(StepError::LogicalError { - message: "No zone messages have been published".to_owned(), - }); - } - - let node = log_step_error(step, world.zone_node_http_client())?; - - ensure_zone_transactions_included( - &node, - &inscription_ids, - Duration::from_secs(timeout_seconds), - ) - .await - .map_err(|error| zone_step_error(step, &error)) -} - -#[cucumber::then(expr = "all zone messages are finalized in {int} seconds")] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require `&mut World` as the first parameter" -)] -async fn step_all_zone_messages_are_finalized( - world: &mut CucumberWorld, - step: &Step, - timeout_seconds: u64, -) -> StepResult { - let inscription_ids = log_step_error(step, world.zone.ordered_inscription_ids())?; - - if !world.zone.has_published_messages() { - return Err(StepError::LogicalError { - message: "No zone messages have been published".to_owned(), - }); - } - - let node_url = log_step_error(step, world.zone_node_url())?; - - wait_for_transactions_finalized( - node_url, - &inscription_ids, - Duration::from_secs(timeout_seconds), - ) - .await - .map_err(|error| zone_step_error(step, &error)) -} - -#[cucumber::then( - expr = "sequencer {string} emits the full transaction lifecycle for zone messages in {int} seconds:" -)] -#[cucumber::when( - expr = "sequencer {string} emits the full transaction lifecycle for zone messages in {int} seconds:" -)] -async fn step_sequencer_emits_full_transaction_lifecycle( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, - timeout_seconds: u64, -) -> StepResult { - let aliases = single_column_table(step, "alias", "zone message aliases")?; - let tx_hashes = log_step_error(step, world.zone.message_tx_hashes_for_aliases(&aliases))?; - let mut tx_status_rx = log_step_error( - step, - world.zone.take_sequencer_tx_status_rx(&sequencer_alias), - )?; - - wait_for_tx_status_lifecycle( - &mut tx_status_rx, - &tx_hashes, - &[ - TxStatus::AcceptedLocally, - TxStatus::PendingMempool, - TxStatus::OnChain(TxSource::Local), - TxStatus::Finalized(TxSource::Local), - ], - Duration::from_secs(timeout_seconds), - ) - .await - .map_err(|error| zone_step_error(step, &error)) -} - -#[cucumber::then("the zone indexer returns messages in this order:")] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require `&mut World` as the first parameter" -)] -async fn step_zone_indexer_returns_messages_in_order( - world: &mut CucumberWorld, - step: &Step, -) -> StepResult { - let aliases = single_column_table(step, "alias", "zone message aliases")?; - let expected = log_step_error(step, world.zone.message_payloads_for_aliases(&aliases))?; - let indexer = log_step_error(step, world.zone.indexer())?; - - let actual = collect_indexed_messages(indexer, &expected, Duration::from_mins(3)) - .await - .map_err(|error| zone_step_error(step, &error))?; - - if actual == expected { - return Ok(()); - } - - Err(StepError::LogicalError { - message: format!( - "Zone indexer returned messages in unexpected order: expected {} messages, got {}", - expected.len(), - actual.len() - ), - }) -} - -#[cucumber::then(expr = "the zone indexer returns messages in any order in {int} seconds:")] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require `&mut World` as the first parameter" -)] -async fn step_zone_indexer_returns_messages_in_any_order( - world: &mut CucumberWorld, - step: &Step, - timeout_seconds: u64, -) -> StepResult { - let aliases = single_column_table(step, "alias", "zone message aliases")?; - let expected = log_step_error(step, world.zone.message_payloads_for_aliases(&aliases))?; - let expected = expected.into_iter().collect::>(); - let indexer = log_step_error(step, world.zone.indexer())?; - - wait_for_indexer_unordered(indexer, &expected, Duration::from_secs(timeout_seconds)) - .await - .map_err(|error| zone_step_error(step, &error))?; - - Ok(()) -} - -#[cucumber::then("the zone indexer returns each of these messages exactly once in this order:")] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require `&mut World` as the first parameter" -)] -async fn step_zone_indexer_returns_messages_exactly_once_in_order( - world: &mut CucumberWorld, - step: &Step, -) -> StepResult { - let aliases = single_column_table(step, "alias", "zone message aliases")?; - let expected = log_step_error(step, world.zone.message_payloads_for_aliases(&aliases))?; - let indexer = log_step_error(step, world.zone.indexer())?; - - let actual = collect_indexed_messages_exactly_once(indexer, &expected, Duration::from_mins(3)) - .await - .map_err(|error| zone_step_error(step, &error))?; - - if actual == expected { - return Ok(()); - } - - Err(StepError::LogicalError { - message: format!( - "Zone indexer returned duplicate or out-of-order messages: expected {} messages, got {}", - expected.len(), - actual.len() - ), - }) -} - -#[cucumber::then(expr = "zone transaction {string} is included in {int} seconds")] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require `&mut World` as the first parameter" -)] -async fn step_zone_transaction_is_included( - world: &mut CucumberWorld, - step: &Step, - transaction_alias: String, - timeout_seconds: u64, -) -> StepResult { - let tx_hash = world.resolve_submitted_transaction(&transaction_alias)?; - let node = log_step_error(step, world.zone_node_http_client())?; - - ensure_zone_transactions_included(&node, &[tx_hash], Duration::from_secs(timeout_seconds)) - .await - .map_err(|error| zone_step_error(step, &error)) -} - -#[cucumber::then(expr = "zone transaction {string} is finalized in {int} seconds")] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require `&mut World` as the first parameter" -)] -async fn step_zone_transaction_is_finalized( - world: &mut CucumberWorld, - step: &Step, - transaction_alias: String, - timeout_seconds: u64, -) -> StepResult { - let tx_hash = world.resolve_submitted_transaction(&transaction_alias)?; - let node_url = log_step_error(step, world.zone_node_url())?; - - wait_for_transactions_finalized(node_url, &[tx_hash], Duration::from_secs(timeout_seconds)) - .await - .map_err(|error| zone_step_error(step, &error)) -} - -#[cucumber::then(expr = "the zone indexer returns finalized deposit {string} in {int} seconds")] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require `&mut World` as the first parameter" -)] -async fn step_zone_indexer_returns_finalized_deposit( - world: &mut CucumberWorld, - step: &Step, - deposit_alias: String, - timeout_seconds: u64, -) -> StepResult { - let (deposit, amount) = world - .zone - .resolve_submitted_deposit(&deposit_alias)? - .clone(); - let indexer = log_step_error(step, world.zone.indexer())?; - - wait_for_deposit( - indexer, - &deposit, - amount, - Duration::from_secs(timeout_seconds), - ) - .await - .map_err(|error| zone_step_error(step, &error)) -} - -#[cucumber::then(expr = "the zone indexer returns finalized withdraw {string} in {int} seconds")] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require `&mut World` as the first parameter" -)] -async fn step_zone_indexer_returns_finalized_withdraw( - world: &mut CucumberWorld, - step: &Step, - withdraw_alias: String, - timeout_seconds: u64, -) -> StepResult { - let withdraw = world - .zone - .resolve_submitted_withdraw(&withdraw_alias)? - .clone(); - let indexer = log_step_error(step, world.zone.indexer())?; - - wait_for_withdraw(indexer, &withdraw, Duration::from_secs(timeout_seconds)) - .await - .map_err(|error| zone_step_error(step, &error)) -} - -#[cucumber::then( - expr = "the zone indexer returns a finalized channel transfer consuming {int} inputs in {int} seconds" -)] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require `&mut World` as the first parameter" -)] -async fn step_zone_indexer_returns_finalized_channel_transfer_input_count( - world: &mut CucumberWorld, - step: &Step, - expected_inputs: usize, - timeout_seconds: u64, -) -> StepResult { - let indexer = log_step_error(step, world.zone.indexer())?; - - wait_for_channel_transfer_input_count( - indexer, - expected_inputs, - Duration::from_secs(timeout_seconds), - ) - .await - .map_err(|error| zone_step_error(step, &error)) -} - -#[cucumber::then( - expr = "the channel wallet of {string} contains a note of value {int} in {int} seconds" -)] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require `&mut World` as the first parameter" -)] -async fn step_channel_wallet_contains_note( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, - value: u64, - timeout_seconds: u64, -) -> StepResult { - let client = log_step_error(step, world.zone.sequencer_client(&sequencer_alias))?; - wait_for_channel_wallet_note(client, value, false, Duration::from_secs(timeout_seconds)) - .await - .map_err(|error| zone_step_error(step, &error)) -} - -#[cucumber::then( - expr = "the channel wallet of {string} contains a finalized note of value {int} in {int} seconds" -)] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require `&mut World` as the first parameter" -)] -async fn step_channel_wallet_contains_finalized_note( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, - value: u64, - timeout_seconds: u64, -) -> StepResult { - let client = log_step_error(step, world.zone.sequencer_client(&sequencer_alias))?; - wait_for_channel_wallet_note(client, value, true, Duration::from_secs(timeout_seconds)) - .await - .map_err(|error| zone_step_error(step, &error)) -} - -#[cucumber::then( - expr = "the channel wallet of {string} has exactly {int} finalized and {int} unfinalized notes in {int} seconds" -)] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require `&mut World` as the first parameter" -)] -async fn step_channel_wallet_has_exact_counts( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, - finalized: usize, - unfinalized: usize, - timeout_seconds: u64, -) -> StepResult { - let client = log_step_error(step, world.zone.sequencer_client(&sequencer_alias))?; - wait_for_channel_wallet_counts( - client, - finalized, - unfinalized, - Duration::from_secs(timeout_seconds), - ) - .await - .map_err(|error| zone_step_error(step, &error)) -} - -#[cucumber::then(expr = "sequencer {string} finalizes deposit {string} in {int} seconds")] -async fn step_zone_sequencer_finalizes_deposit( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, - deposit_alias: String, - timeout_seconds: u64, -) -> StepResult { - let (deposit, amount) = world - .zone - .resolve_submitted_deposit(&deposit_alias)? - .clone(); - let events = log_step_error(step, world.zone.sequencer_events_mut(&sequencer_alias))?; - - let mempool_pending = wait_for_finalized_deposit_via_sequencer_and_collect_mempool_pending( - events, - &deposit, - amount, - Duration::from_secs(timeout_seconds), - ) - .await - .map_err(|error| zone_step_error(step, &error))?; - world - .zone - .record_mempool_pending(sequencer_alias.clone(), mempool_pending); - Ok(()) -} - -#[cucumber::then(expr = "sequencer {string} finalizes withdraw {string} in {int} seconds")] -async fn step_zone_sequencer_finalizes_withdraw( - world: &mut CucumberWorld, - step: &Step, - sequencer_alias: String, - withdraw_alias: String, - timeout_seconds: u64, -) -> StepResult { - let withdraw = world - .zone - .resolve_submitted_withdraw(&withdraw_alias)? - .clone(); - let events = log_step_error(step, world.zone.sequencer_events_mut(&sequencer_alias))?; - - let mempool_pending = wait_for_finalized_withdraw_via_sequencer_and_collect_mempool_pending( - events, - &withdraw, - Duration::from_secs(timeout_seconds), - ) - .await - .map_err(|error| zone_step_error(step, &error))?; - world - .zone - .record_mempool_pending(sequencer_alias.clone(), mempool_pending); - Ok(()) -} - -#[cucumber::then( - expr = "the zone indexer returns all zone messages exactly once in any order in {int} seconds" -)] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require `&mut World` as the first parameter" -)] -async fn step_zone_indexer_returns_all_messages_exactly_once_any_order( - world: &mut CucumberWorld, - step: &Step, - timeout_seconds: u64, -) -> StepResult { - let expected_set = published_payload_set(world, step)?; - let indexer = log_step_error(step, world.zone.indexer())?; - - let seen = - wait_for_indexer_unordered(indexer, &expected_set, Duration::from_secs(timeout_seconds)) - .await - .map_err(|error| zone_step_error(step, &error))?; - - tokio::time::sleep(Duration::from_secs(CONCURRENT_DUPLICATE_SETTLE_SECS)).await; - - let all_payloads = scan_indexer_for_payloads(indexer, &expected_set) - .await - .map_err(|error| zone_step_error(step, &error))?; - - ensure_indexed_payloads_match_once(&expected_set, &seen, &all_payloads) -} - -fn published_payload_set( - world: &CucumberWorld, - step: &Step, -) -> Result, StepError> { - let expected_payloads = log_step_error(step, world.zone.published_message_payloads())?; - - Ok(expected_payloads.into_iter().collect()) -} - -fn ensure_indexed_payloads_match_once( - expected: &HashSet, - seen: &HashSet, - all_payloads: &[Inscription], -) -> StepResult { - let unique: HashSet<&Inscription> = all_payloads.iter().collect(); - - if unique.len() != all_payloads.len() { - return Err(StepError::LogicalError { - message: format!( - "Duplicate inscriptions detected on chain: expected {} unique, got {} total", - unique.len(), - all_payloads.len() - ), - }); - } - - if unique.len() != expected.len() || seen != expected { - return Err(StepError::LogicalError { - message: format!( - "Zone indexer did not return the expected message set: expected {}, got {}", - expected.len(), - unique.len() - ), - }); - } - - Ok(()) -} - -#[cucumber::then( - expr = "the zone indexer returns {int} copies of zone message {string} in {int} seconds" -)] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require `&mut World` as the first parameter" -)] -async fn step_zone_indexer_returns_payload_count( - world: &mut CucumberWorld, - step: &Step, - expected_count: usize, - payload: String, - timeout_seconds: u64, -) -> StepResult { - let indexer = log_step_error(step, world.zone.indexer())?; - wait_for_exact_indexed_payload_count( - indexer, - make_inscription(&payload), - expected_count, - Duration::from_secs(timeout_seconds), - ) - .await - .map_err(|error| zone_step_error(step, &error)) -} - -#[cucumber::then(expr = "zone balance updates keep all accounts non-negative after {int} seconds")] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require `&mut World` as the first parameter" -)] -async fn step_zone_balance_updates_keep_accounts_non_negative( - world: &mut CucumberWorld, - step: &Step, - settle_seconds: u64, -) -> StepResult { - tokio::time::sleep(Duration::from_secs(settle_seconds)).await; - - let mut balances = world.zone.zone_account_balances()?; - let expected_set = published_payload_set(world, step)?; - let indexer = log_step_error(step, world.zone.indexer())?; - let on_chain = scan_indexer_for_payloads(indexer, &expected_set) - .await - .map_err(|error| zone_step_error(step, &error))?; - - apply_indexed_balance_updates(&mut balances, &on_chain); - - ensure_balances_non_negative(&balances) -} - -#[cucumber::then( - expr = "the zone indexer preserves per-sequencer order and converges without duplicates in {int} seconds" -)] -#[expect( - clippy::needless_pass_by_ref_mut, - reason = "Cucumber step functions require `&mut World` as the first parameter" -)] -async fn step_zone_indexer_preserves_per_sequencer_order_without_duplicates( - world: &mut CucumberWorld, - step: &Step, - timeout_seconds: u64, -) -> StepResult { - let expected_set = published_payload_set(world, step)?; - let total = world.zone.sorted_total_payloads()?; - let expected_by_sequencer = world.zone.sorted_expected_by_sequencer()?; - let discarded = log_step_error(step, world.zone.discarded_payloads(DEFAULT_ZONE_SEQUENCER))?; - let indexer = log_step_error(step, world.zone.indexer())?; - - let on_chain = wait_until_sorted_conflict_settles( - indexer, - &expected_set, - &discarded, - total, - Duration::from_secs(timeout_seconds), - ) - .await - .map_err(|error| zone_step_error(step, &error))?; - - let discarded_snapshot = discarded.lock().await.clone(); - assert_sorted_outcome( - &on_chain, - &discarded_snapshot, - total, - &expected_by_sequencer, - ) -} - -fn apply_indexed_balance_updates(balances: &mut HashMap, payloads: &[Inscription]) { - for payload in payloads { - let Some((_, account, delta)) = parse_balance_payload(payload) else { - continue; - }; - - *balances.entry(account).or_default() += delta; - } -} - -fn ensure_balances_non_negative(balances: &HashMap) -> StepResult { - let negative = balances - .iter() - .filter(|(_, balance)| **balance < 0) - .map(|(account, balance)| format!("{account}={balance}")) - .collect::>(); - - if negative.is_empty() { - return Ok(()); - } - - Err(StepError::LogicalError { - message: format!( - "Zone account balances went negative: {}", - negative.join(", ") - ), - }) -} diff --git a/tests/src/cucumber/steps/manual_zone/support.rs b/tests/src/cucumber/steps/manual_zone/support.rs deleted file mode 100644 index 1d73d0652..000000000 --- a/tests/src/cucumber/steps/manual_zone/support.rs +++ /dev/null @@ -1,2147 +0,0 @@ -//! Zone SDK test helpers shared by Cucumber steps. -//! -//! The helpers in this module keep the feature steps focused on scenario -//! intent: start a zone-backed node, run sequencers, publish messages, observe -//! the indexer, and submit the channel operations that the zone layer relies -//! on. - -use std::{ - collections::{BTreeSet, HashMap, HashSet, VecDeque}, - sync::{Arc, LazyLock}, - time::{Duration, Instant}, -}; - -use lb_common_http_client::{CommonHttpClient, Slot}; -use lb_core::{ - mantle::{ - Note, Op, OpProof, RawMantleTx, Utxo, Value, - gas::GasCost, - ledger::{Inputs, Outputs, OutputsError}, - ops::{ - OpId as _, - channel::{ - ChannelId, MsgId, - channel_transfer::ChannelTransferOp, - deposit::{DepositOp, Metadata}, - inscribe::{Inscription, InscriptionOp}, - withdraw::ChannelWithdrawOp, - }, - transfer::TransferOp, - }, - traits::Hashable as _, - transactions::{OpsProofs, builder::MantleTxBuilder, states::Unverified}, - }, - proofs::channel_multi_sig_proof::{ChannelMultiSigProof, IndexedSignature}, -}; -use lb_http_api_common::bodies::{ - channel::{ChannelDepositRequestBody, ChannelDepositResponseBody}, - wallet::{ - fund::WalletFundRequestBody, - sign::{WalletSignTxZkRequestBody, WalletSignTxZkResponseBody}, - }, -}; -use lb_key_management_system_service::keys::{Ed25519Key, ZkPublicKey, ZkPublicKeys, ZkSignature}; -use lb_node::SignedMantleTx; -use lb_testing_framework::NodeHttpClient; -use lb_zone_sdk::{ - adapter::NodeHttpClient as ZoneNodeHttpClient, - sequencer::{ZoneSequencer, channel_inscriptions}, -}; -use rand::{Rng as _, thread_rng}; -use reqwest::Url; -use tokio::{ - task::JoinHandle, - time::{sleep, timeout}, -}; -use tracing::warn; - -use super::runner::{ - self, ChannelUpdate, ChannelUpdateTx, Event, FinalizedOp, FinalizedTx, FundingConfig, - InscriptionId, InscriptionInfo, PendingTx, PublishResult, SequencerChannelView, - SequencerCheckpoint, SequencerClient, SequencerConfig, TurnNotification, TxStatus, - TxStatusUpdate, WithdrawArg, WithdrawInputs, -}; - -/// Inscriptions in the just-finalized txs — the permanent, settled part of the -/// channel. Once a payload finalizes it's on chain for good, so a policy pins -/// these and never re-homes a finalized payload when it later drops off a -/// non-canonical branch. -fn finalized_inscriptions(finalized: &[FinalizedTx]) -> impl Iterator { - finalized - .iter() - .flat_map(|tx| tx.ops.iter()) - .filter_map(|op| match op { - FinalizedOp::Inscription(info) => Some(info), - FinalizedOp::Deposit(_) - | FinalizedOp::Withdraw(_) - | FinalizedOp::Config(_) - | FinalizedOp::ChannelTransfer(_) => None, - }) -} -use crate::{ - common::{ - chain::wait_for_transactions_inclusion, mantle_inscription::make_inscription, - wallet::build_wallet_funded_transfer, - }, - cucumber::world::ZoneReaderConfig, -}; - -#[derive(Debug, thiserror::Error)] -pub enum ZoneTestError { - #[error("timed out waiting for zone sequencer to accept a publish request")] - PublishTimeout, - #[error("zone indexer request failed: {message}")] - Indexer { message: String }, - #[error("timed out waiting for zone indexer to return all messages")] - IndexerTimeout, - #[error("zone indexer returned {actual} copies of '{payload}', expected {expected}")] - IndexedPayloadCountMismatch { - payload: String, - expected: usize, - actual: usize, - }, - #[error("timed out waiting for zone transactions to appear on the canonical chain")] - InclusionTimeout, - #[error("failed to fetch consensus info while checking finalized transactions: {message}")] - Consensus { message: String }, - #[error("failed to fetch block while checking finalized transactions: {message}")] - Block { message: String }, - #[error("timed out waiting for zone transactions to finalize")] - FinalizationTimeout, - #[error("channel wallet request failed: {message}")] - ChannelWallet { message: String }, - #[error("timed out waiting for a channel wallet note")] - ChannelWalletTimeout, - #[error("timed out waiting for zone LIB to advance")] - LibAdvanceTimeout, - #[error("timed out waiting for zone sequencer channel view condition: {message}")] - ChannelViewTimeout { message: String }, - #[error("failed to find a funding note with exact value {value}")] - MissingExactFundingNote { value: Value }, - #[error("failed to submit zone deposit: {message}")] - SubmitDeposit { message: String }, - #[error("failed to submit zone channel split transfer: {message}")] - SplitTransfer { message: String }, - #[error("failed to sign zone transaction: {message}")] - SignTransaction { message: String }, - #[error("failed to build atomic zone deposit transaction: {message}")] - BuildAtomicDeposit { message: String }, - #[error("failed to submit atomic zone deposit transaction: {message}")] - SubmitAtomicDeposit { message: String }, - #[error("failed to submit zone withdraw transaction: {message}")] - SubmitWithdraw { message: String }, - #[error("timed out waiting for zone withdraw to appear in the indexer")] - WithdrawTimeout, - #[error("failed to build custom zone transaction: {message}")] - BuildCustomTx { message: String }, - #[error("failed to submit custom zone transaction: {message}")] - SubmitCustomTx { message: String }, - #[error("zone sequencer event stream stopped before observing the expected event")] - SequencerStopped, - #[error(transparent)] - BoundedError(#[from] lb_utils::bounded::BoundedError), - #[error(transparent)] - OutputsError(#[from] OutputsError), -} - -/// Result of an atomic deposit scenario where a deposit and zone inscription -/// are submitted as one Mantle transaction. -pub struct AtomicZoneDepositSubmission { - pub deposit: DepositOp, - pub publish: PublishResult, - pub reserved_inputs: Vec, -} - -pub struct AtomicZoneDepositRequest { - pub channel_id: ChannelId, - pub funding_public_key: ZkPublicKey, - pub available_utxos: Vec, - pub amount: Value, - pub inscription_data: Inscription, - pub metadata: Metadata, -} - -/// Result of a withdraw scenario where the zone sequencer signs the channel -/// withdraw and publishes the accompanying inscription. -pub struct ZoneWithdrawSubmission { - pub withdraw: ChannelWithdrawOp, - pub publish: PublishResult, -} - -pub struct ZoneDeposit { - pub deposit: DepositOp, - pub reserved_inputs: Vec, - /// The channel notes the deposit re-creates (1:1 with its inputs). Computed - /// deterministically from the deposit's `op_id` and its input notes so a - /// later channel transfer can spend them without waiting on the indexer. - pub channel_notes: Vec, -} - -/// The channel notes a deposit re-creates: one per input, same note (value + -/// pk), re-homed under the deposit's `op_id`. Matches the ledger's deposit -/// execution (`DepositOp::outputs` re-creates inputs 1:1) and the zone-sdk's -/// channel-note derivation. -fn recreated_channel_notes(deposit: &DepositOp, reserved_inputs: &[Utxo]) -> Vec { - let op_id = deposit.op_id(); - reserved_inputs - .iter() - .enumerate() - .map(|(index, utxo)| Utxo::new(op_id, index, utxo.note)) - .collect() -} - -pub type DiscardedPayloads = Arc>>; -pub type ZoneAccountBalances = HashMap; - -/// Shared deadline for a publish attempt and the matching event wait so the -/// whole operation has one timeout budget. -#[derive(Clone, Copy)] -pub struct PublishDeadline { - started_at: Instant, - timeout: Duration, -} - -impl PublishDeadline { - #[must_use] - pub fn from_now(timeout: Duration) -> Self { - Self { - started_at: Instant::now(), - timeout, - } - } - - fn is_expired(self) -> bool { - self.started_at.elapsed() > self.timeout - } -} - -/// Bundle returned from policy starters so callers can wire the cucumber -/// world. Wraps [`runner::Runtime`] — events and checkpoints are exposed -/// uniformly across all policies because the policy runs inline on the -/// drive task; the event mpsc is purely for test observation. -pub struct PolicyRuntime { - pub task: JoinHandle<()>, - pub client: SequencerClient, - pub events: tokio::sync::broadcast::Receiver, - pub checkpoint_rx: tokio::sync::watch::Receiver>, - pub ready_rx: tokio::sync::watch::Receiver, - pub channel_view_rx: tokio::sync::watch::Receiver, - pub turn_to_write_rx: tokio::sync::watch::Receiver, - pub tx_status_rx: tokio::sync::broadcast::Receiver, -} - -fn to_policy_runtime(rt: runner::Runtime) -> PolicyRuntime { - PolicyRuntime { - task: rt.task, - client: rt.client, - events: rt.event_rx, - checkpoint_rx: rt.checkpoint_rx, - ready_rx: rt.ready_rx, - channel_view_rx: rt.channel_view_rx, - turn_to_write_rx: rt.turn_to_write_rx, - tx_status_rx: rt.tx_status_rx, - } -} - -/// Spawn a sequencer drive task with a no-op policy. Step bodies drive -/// publishes via [`SequencerClient`]; events flow to `PolicyRuntime.events`. -/// If `republish_orphans` is set, the [`OrphanRepublishPolicy`] runs inline -/// inside the drive loop. -pub fn start_sequencer_event_loop( - sequencer: ZoneSequencer, - republish_orphans: bool, -) -> PolicyRuntime { - if republish_orphans { - to_policy_runtime(runner::spawn(sequencer, OrphanRepublishPolicy::default())) - } else { - to_policy_runtime(runner::spawn(sequencer, runner::PassivePolicy)) - } -} - -/// Drives a competing-sequencer policy that publishes `planned` once ready and -/// re-publishes its own orphans (tracked by intent lineage) until they land — -/// correct even when payloads repeat. -pub fn start_republish_lineage_policy( - sequencer: ZoneSequencer, - planned: Vec, -) -> PolicyRuntime { - let policy = RepublishLineagePolicy { - planned, - published_initial: false, - lineage: LineageTracker::default(), - }; - to_policy_runtime(runner::spawn(sequencer, policy)) -} - -/// Drives a policy that republishes orphaned balance updates only when the -/// local canonical view can still apply the update without going negative, -/// and lays planned balance updates whenever it's our turn to write. -pub fn start_balance_aware_policy( - sequencer: ZoneSequencer, - initial_balances: ZoneAccountBalances, - planned_payloads: Vec, -) -> PolicyRuntime { - let view_rx = sequencer.subscribe_channel_view(); - let policy = BalanceAwarePolicy { - balances: BalanceAwareState::new(initial_balances), - planned: VecDeque::from(planned_payloads), - view_rx, - }; - to_policy_runtime(runner::spawn(sequencer, policy)) -} - -/// Drives a deterministic conflict policy used by tests that expect the final -/// zone chain to converge to sorted payload order. -pub fn start_sorted_conflict_policy( - sequencer: ZoneSequencer, - discarded: &DiscardedPayloads, -) -> PolicyRuntime { - let policy = SortedConflictPolicy { - state: SortedConflictState::new(Arc::clone(discarded)), - }; - to_policy_runtime(runner::spawn(sequencer, policy)) -} - -/// Inline policy: republish orphaned inscriptions not already back on the -/// canonical chain. Plain inscriptions only — bundles re-prepare themselves. -/// Assumes unique payloads; for repeating payloads see -/// [`RepublishLineagePolicy`]. -/// -/// Tracks canonical on-chain state keyed by id, decided by payload (see -/// `on_event`): a dead twin's id leaves while a live twin keeps the payload -/// covered, so a payload still on chain is never re-homed. -#[derive(Default)] -struct OrphanRepublishPolicy { - /// Canonical on-chain inscriptions by id (adopted-unfinalized + finalized). - /// Finalized entries are added and never removed — they can't be orphaned — - /// so this one set is the whole on-chain view. - canonical: HashMap, -} - -impl runner::Policy for OrphanRepublishPolicy -where - Node: lb_zone_sdk::adapter::Node + Clone + Send + Sync + 'static, -{ - async fn on_event(&mut self, sequencer: &mut ZoneSequencer, event: &Event) { - let Event::BlocksProcessed { - channel_update, - finalized, - .. - } = event - else { - return; - }; - // 1. Remove orphaned by id — a dead twin leaves, a live twin stays. - for entry in &channel_update.orphaned { - if let Some(info) = entry.inscription() { - self.canonical.remove(&info.this_msg); - } - } - // 2. Add finalized by id (permanent — finalized can't be orphaned). - for info in finalized_inscriptions(finalized) { - self.canonical.insert(info.this_msg, info.payload.clone()); - } - // 3. Add adopted by id (this block's new canonical). - for info in channel_update - .adopted - .iter() - .filter_map(ChannelUpdateTx::inscription) - { - self.canonical.insert(info.this_msg, info.payload.clone()); - } - // 4. Republish orphaned whose payload no canonical id still carries. - for entry in &channel_update.orphaned { - let ChannelUpdateTx::Inscription(info) = entry else { - continue; - }; - if self - .canonical - .values() - .any(|payload| *payload == info.payload) - { - continue; - } - if let Err(error) = sequencer.handle().publish(info.payload.clone()).await { - warn!(%error, "Failed to re-publish orphaned zone payload"); - } - } - } -} - -/// Tracks our published inscriptions by intent lineage, so republishing works -/// even when payloads repeat (identical bytes published as distinct messages). -/// -/// Each original publish is its own intent, rooted at its `this_msg`; every -/// republish we issue for an orphaned member is recorded under the same root. -/// An intent is "live" while any of its `this_msg`s is on the channel -/// (`adopted`) or in flight as a publish/republish we issued. Identical -/// payloads form distinct intents (distinct `this_msg`s), so each lands once, -/// and other sequencers' inscriptions are never in our map, so we never -/// republish theirs. -#[derive(Default)] -struct LineageTracker { - /// Every `this_msg` we've published (originals + republishes) → intent - /// root. - intent_root: HashMap, - /// Per intent root, the `this_msg`s currently pending (in the - /// non-finalized channel view). - pending: HashMap>, - /// Intent roots that have finalized — permanently landed, so the intent is - /// considered live forever and never re-homed again. - finalized_roots: HashSet, -} - -impl LineageTracker { - /// Record an original publish as its own intent, in flight. - fn record_publish(&mut self, this_msg: MsgId) { - self.intent_root.insert(this_msg, this_msg); - self.pending.entry(this_msg).or_default().insert(this_msg); - } - - /// Record a republish of `orphan` as a new live member of its intent. - fn record_republish(&mut self, orphan: MsgId, republished: MsgId) { - let root = self.intent_root.get(&orphan).copied().unwrap_or(orphan); - self.intent_root.insert(republished, root); - self.pending.entry(root).or_default().insert(republished); - } - - /// Fold a delta into per-intent liveness — only our `msg_id`s are relevant. - /// Adopted members become live; orphaned members stop being live. - fn observe(&mut self, channel_update: &ChannelUpdate) { - for info in channel_update - .adopted - .iter() - .filter_map(ChannelUpdateTx::inscription) - { - if let Some(&root) = self.intent_root.get(&info.this_msg) { - self.pending.entry(root).or_default().insert(info.this_msg); - } - } - for entry in &channel_update.orphaned { - if let ChannelUpdateTx::Inscription(info) = entry - && let Some(&root) = self.intent_root.get(&info.this_msg) - && let Some(members) = self.pending.get_mut(&root) - { - members.remove(&info.this_msg); - } - } - } - - /// Pin the intents of any finalized `this_msg`s of ours as permanently - /// live — once a member finalizes the payload is on chain for good. - fn observe_finalized(&mut self, finalized: impl Iterator) { - for this_msg in finalized { - if let Some(&root) = self.intent_root.get(&this_msg) { - self.finalized_roots.insert(root); - } - } - } - - /// True if `this_msg` is one of ours. - fn is_ours(&self, this_msg: &MsgId) -> bool { - self.intent_root.contains_key(this_msg) - } - - /// True if the intent of `this_msg` has finalized, or still has a live - /// member. - fn intent_live(&self, this_msg: &MsgId) -> bool { - let root = self.intent_root.get(this_msg).copied().unwrap_or(*this_msg); - self.finalized_roots.contains(&root) - || self - .pending - .get(&root) - .is_some_and(|members| !members.is_empty()) - } -} - -/// Inline republish policy for channels whose payloads can repeat. Publishes -/// its own `planned` payloads once the sequencer is ready, then republishes any -/// of *our* orphans whose intent has no live member, tracking msg-id lineage -/// (the payload can't identify the message when it repeats). Owning the -/// publishes is what gives the policy its outbox: every `this_msg` it sends is -/// recorded. -struct RepublishLineagePolicy { - planned: Vec, - published_initial: bool, - lineage: LineageTracker, -} - -impl runner::Policy for RepublishLineagePolicy -where - Node: lb_zone_sdk::adapter::Node + Clone + Send + Sync + 'static, -{ - async fn on_event(&mut self, sequencer: &mut ZoneSequencer, event: &Event) { - match event { - Event::Ready if !self.published_initial => { - self.published_initial = true; - for payload in self.planned.clone() { - match sequencer.handle().publish(payload).await { - Ok((result, _checkpoint)) => { - self.lineage - .record_publish(result.tx.inscription().this_msg); - } - Err(error) => warn!(%error, "Failed to publish planned zone payload"), - } - } - } - Event::BlocksProcessed { - channel_update, - finalized, - .. - } => { - self.lineage - .observe_finalized(finalized_inscriptions(finalized).map(|i| i.this_msg)); - self.lineage.observe(channel_update); - for entry in &channel_update.orphaned { - let ChannelUpdateTx::Inscription(info) = entry else { - continue; - }; - if !self.lineage.is_ours(&info.this_msg) - || self.lineage.intent_live(&info.this_msg) - { - continue; - } - match sequencer.handle().publish(info.payload.clone()).await { - Ok((result, _checkpoint)) => { - self.lineage - .record_republish(info.this_msg, result.tx.inscription().this_msg); - } - Err(error) => warn!(%error, "Failed to re-publish orphaned zone payload"), - } - } - } - _ => {} - } - } -} - -/// Inline policy: republish orphans only when the local balance view still -/// allows it; publish planned payloads as soon as it's our turn to write. -/// -/// The balance view is rebuilt from the full delta — every orphaned op is -/// removed and every adopted op applied — so affordability reflects all -/// inscriptions on the channel. Removing an orphan we never applied (never- -/// landed pending) is a no-op, and an already-adopted op is skipped because its -/// id is already in the applied set after `record_adopted_payloads`. -struct BalanceAwarePolicy { - balances: BalanceAwareState, - planned: VecDeque, - view_rx: tokio::sync::watch::Receiver, -} - -impl runner::Policy for BalanceAwarePolicy -where - Node: lb_zone_sdk::adapter::Node + Clone + Send + Sync + 'static, -{ - async fn on_event(&mut self, sequencer: &mut ZoneSequencer, event: &Event) { - if let Event::BlocksProcessed { - channel_update, - finalized, - .. - } = event - { - self.balances.record_finalized_payloads(finalized); - let ChannelUpdate { orphaned, adopted } = channel_update; - let orphaned_inscriptions: Vec = orphaned - .iter() - .filter_map(|o| match o { - ChannelUpdateTx::Inscription(i) => Some(i.clone()), - ChannelUpdateTx::AtomicWithdraw(_) - | ChannelUpdateTx::Custom(_) - | ChannelUpdateTx::Config(_) => None, - }) - .collect(); - self.balances - .remove_orphaned_payloads(&orphaned_inscriptions); - self.balances.record_adopted_payloads(adopted); - for info in orphaned_inscriptions { - if !self.balances.should_republish(&info.payload) { - continue; - } - if let Err(error) = sequencer.handle().publish(info.payload.clone()).await { - warn!(%error, "Failed to re-publish balance-aware zone payload"); - continue; - } - self.balances.record_republished_payload(&info.payload); - } - } - - if !self.view_rx.borrow().our_turn_to_write { - return; - } - while let Some(payload) = self.planned.pop_front() { - if !self.balances.should_republish(&payload) { - continue; - } - if let Err(error) = sequencer.handle().publish(payload.clone()).await { - warn!(%error, "Failed to publish planned balance-aware zone payload"); - self.planned.push_front(payload); - break; - } - self.balances.record_republished_payload(&payload); - } - } -} - -/// Inline policy: republish orphans only when they preserve sorted-payload -/// order; otherwise mark them as discarded. -/// -/// The full delta lets us rebuild the on-chain payload set each update (drop -/// orphaned, add adopted), so the order floor we gate republishing on falls -/// back correctly when the highest payload is orphaned. -struct SortedConflictPolicy { - state: SortedConflictState, -} - -impl runner::Policy for SortedConflictPolicy -where - Node: lb_zone_sdk::adapter::Node + Clone + Send + Sync + 'static, -{ - async fn on_event(&mut self, sequencer: &mut ZoneSequencer, event: &Event) { - let Event::BlocksProcessed { - channel_update, - finalized, - .. - } = event - else { - return; - }; - // Pin finalized payloads first. - self.state.record_finalized(finalized); - let ChannelUpdate { orphaned, adopted } = channel_update; - let orphaned_inscriptions: Vec<&InscriptionInfo> = orphaned - .iter() - .filter_map(|o| match o { - ChannelUpdateTx::Inscription(i) => Some(i), - ChannelUpdateTx::AtomicWithdraw(_) - | ChannelUpdateTx::Custom(_) - | ChannelUpdateTx::Config(_) => None, - }) - .collect(); - - // Rebuild on-chain state from this delta before deciding anything. - self.state.revert_orphaned(&orphaned_inscriptions); - self.state.record_adoptions(adopted).await; - - let readopted: HashSet<&Inscription> = adopted - .iter() - .filter_map(|tx| tx.inscription().map(|i| &i.payload)) - .collect(); - - // Consider this round's fresh orphans together with everything parked, - // in sorted order (a `BTreeSet` iterates ascending). A payload parked - // under a higher floor on another branch then slots in ahead of a higher - // fresh orphan instead of being locked out, and the chain stays sorted. - // Finalized payloads are excluded — they're already permanently landed. - let mut candidates: BTreeSet = orphaned_inscriptions - .iter() - .map(|i| i.payload.clone()) - .filter(|payload| !readopted.contains(payload) && !self.state.is_finalized(payload)) - .collect(); - candidates.extend(self.state.discarded_snapshot().await); - - for payload in candidates { - if self.state.is_finalized(&payload) { - continue; - } - if self.state.preserves_order(&payload) { - if let Err(error) = sequencer.handle().publish(payload.clone()).await { - warn!(%error, "Failed to re-publish sorted zone payload"); - continue; - } - self.state.record_published_payload(payload).await; - } else { - self.state.discard(payload).await; - } - } - } -} - -struct BalanceAwareState { - initial_balances: ZoneAccountBalances, - applied: HashMap>, - finalized: HashSet, -} - -impl BalanceAwareState { - fn new(initial_balances: ZoneAccountBalances) -> Self { - Self { - initial_balances, - applied: HashMap::new(), - finalized: HashSet::new(), - } - } - - /// Pin finalized payloads. - fn record_finalized_payloads(&mut self, finalized: &[FinalizedTx]) { - for inscription in finalized_inscriptions(finalized) { - if let Some((uuid, _, _)) = parse_balance_payload(&inscription.payload) { - self.finalized.insert(uuid); - } - self.record_applied_payload(&inscription.payload); - } - } - - fn record_applied_payload(&mut self, payload: &Inscription) { - let Some((uuid, account, delta)) = parse_balance_payload(payload) else { - return; - }; - - self.applied.entry(account).or_default().insert(uuid, delta); - } - - fn remove_orphaned_payloads(&mut self, orphaned: &[InscriptionInfo]) { - for inscription in orphaned { - let Some((uuid, account, _)) = parse_balance_payload(&inscription.payload) else { - continue; - }; - - // A finalized delta is permanent — never drop it on an orphan. - if self.finalized.contains(&uuid) { - continue; - } - - if let Some(account_updates) = self.applied.get_mut(&account) { - account_updates.remove(&uuid); - } - } - } - - fn record_adopted_payloads(&mut self, adopted: &[ChannelUpdateTx]) { - for info in adopted.iter().filter_map(ChannelUpdateTx::inscription) { - self.record_applied_payload(&info.payload); - } - } - - fn should_republish(&self, payload: &Inscription) -> bool { - let Some((uuid, account, delta)) = parse_balance_payload(payload) else { - return false; - }; - - if self.finalized.contains(&uuid) || self.account_updates(&account).contains_key(&uuid) { - return false; - } - - self.available_balance(&account) + delta >= 0 - } - - fn record_republished_payload(&mut self, payload: &Inscription) { - self.record_applied_payload(payload); - } - - fn available_balance(&self, account: &str) -> i64 { - self.initial_balances.get(account).copied().unwrap_or(0) - + self.account_updates(account).values().sum::() - } - - fn account_updates(&self, account: &str) -> &HashMap { - self.applied.get(account).unwrap_or(&EMPTY_BALANCE_UPDATES) - } -} - -static EMPTY_BALANCE_UPDATES: LazyLock> = LazyLock::new(HashMap::new); - -struct SortedConflictState { - /// The local channel view: pending (non-finalized) payloads plus the - /// pinned finalized base, kept as the ordering floor. - channel_view: BTreeSet, - discarded: DiscardedPayloads, - finalized: HashSet, -} - -impl SortedConflictState { - fn new(discarded: DiscardedPayloads) -> Self { - Self { - channel_view: BTreeSet::new(), - discarded, - finalized: HashSet::new(), - } - } - - /// Pin finalized payloads into the channel view permanently. - fn record_finalized(&mut self, finalized: &[FinalizedTx]) { - for inscription in finalized_inscriptions(finalized) { - self.finalized.insert(inscription.payload.clone()); - self.channel_view.insert(inscription.payload.clone()); - } - } - - fn is_finalized(&self, payload: &Inscription) -> bool { - self.finalized.contains(payload) - } - - /// Drop orphaned payloads from the channel view — the order floor falls - /// back to the max of whatever remains. Finalized payloads stay put. - fn revert_orphaned(&mut self, orphaned: &[&InscriptionInfo]) { - for inscription in orphaned { - if self.finalized.contains(&inscription.payload) { - continue; - } - self.channel_view.remove(&inscription.payload); - } - } - - async fn record_adoptions(&mut self, adopted: &[ChannelUpdateTx]) { - for info in adopted.iter().filter_map(ChannelUpdateTx::inscription) { - self.discarded.lock().await.remove(&info.payload); - self.channel_view.insert(info.payload.clone()); - } - } - - async fn record_published_payload(&mut self, payload: Inscription) { - self.discarded.lock().await.remove(&payload); - self.channel_view.insert(payload); - } - - fn preserves_order(&self, payload: &Inscription) -> bool { - self.channel_view.last().is_none_or(|max| payload >= max) - } - - async fn discard(&self, payload: Inscription) { - self.discarded.lock().await.insert(payload); - } - - async fn discarded_snapshot(&self) -> Vec { - self.discarded.lock().await.iter().cloned().collect() - } -} - -/// Creates a scenario-local sequencer key. -#[must_use] -pub fn keygen() -> Ed25519Key { - let mut key_bytes = [0u8; 32]; - thread_rng().fill(&mut key_bytes); - Ed25519Key::from_bytes(&key_bytes) -} - -/// Encodes a balance-affecting zone payload used by balance-aware sequencer -/// scenarios. -#[must_use] -pub fn balance_update_payload(uuid: &str, account: &str, delta: i64) -> Inscription { - make_inscription(&format!("{uuid}:{account}:{delta}")) -} - -/// Parses a balance-affecting payload in the same format produced by -/// [`balance_update_payload`]. -pub fn parse_balance_payload(payload: &Inscription) -> Option<(String, String, i64)> { - let payload = std::str::from_utf8(payload.as_slice()).ok()?; - let parts = payload.splitn(3, ':').collect::>(); - let [uuid, account, delta] = parts.as_slice() else { - return None; - }; - - Some(( - (*uuid).to_owned(), - (*account).to_owned(), - delta.parse().ok()?, - )) -} - -/// Uses a short resubmit interval so retry-sensitive zone scenarios settle -/// quickly enough for CI. -#[must_use] -pub const fn sequencer_config(funding: FundingConfig) -> SequencerConfig { - SequencerConfig { - resubmit_interval: Duration::from_secs(3), - min_slots_remaining_in_turn: 2, - ..SequencerConfig::new(funding) - } -} - -/// Uses the same retry profile while overriding pending publish submit depth. -#[must_use] -pub const fn sequencer_config_with_pending_submit_depth( - max_pending_publish_depth: usize, - funding: FundingConfig, -) -> SequencerConfig { - SequencerConfig { - max_pending_publish_depth, - ..sequencer_config(funding) - } -} - -/// Publishes a zone payload through the runner and returns the SDK's -/// [`PublishResult`] inline. Retries transient publish errors until the -/// deadline elapses. No "wait for event" — the SDK accepts the publish -/// inline (funding it via the node when configured) and the runner forwards -/// the call through the drive task. -pub async fn publish_message_with_retry( - client: &SequencerClient, - data: &Inscription, - deadline: PublishDeadline, -) -> Result { - loop { - if deadline.is_expired() { - return Err(ZoneTestError::PublishTimeout); - } - match client.publish(data.clone()).await { - Ok((result, _cp)) => return Ok(result), - Err(error) => { - warn!(error = %error, "Zone sequencer publish failed, retrying"); - sleep(Duration::from_millis(500)).await; - } - } - } -} - -/// Waits until every tx in `tx_hashes` reports [`TxStatus::OnChain`] on the -/// sequencer's status stream, collecting the tx hashes seen as -/// [`TxStatus::PendingMempool`] along the way. Own publishes don't echo in -/// [`ChannelUpdate::adopted`] on chain extension (the sequencer already -/// tracks them), so the per-tx status stream is where "landed on chain, not -/// yet finalized" is observable. -pub async fn wait_for_on_chain_statuses_and_collect_mempool_pending( - statuses: &mut tokio::sync::broadcast::Receiver, - tx_hashes: &[InscriptionId], - duration: Duration, -) -> Result, ZoneTestError> { - timeout(duration, async { - let mut on_chain: HashSet = HashSet::new(); - let mut mempool_pending = HashSet::new(); - - while on_chain.len() < tx_hashes.len() { - let update = match statuses.recv().await { - Ok(update) => update, - Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => { - warn!("status subscriber lagged by {n}, recovering"); - continue; - } - Err(tokio::sync::broadcast::error::RecvError::Closed) => { - return Err(ZoneTestError::SequencerStopped); - } - }; - match update.status { - TxStatus::PendingMempool => { - mempool_pending.insert(update.tx_hash); - } - TxStatus::OnChain(_) if tx_hashes.contains(&update.tx_hash) => { - on_chain.insert(update.tx_hash); - } - _ => {} - } - } - - Ok(mempool_pending) - }) - .await - .map_err(|_| ZoneTestError::PublishTimeout)? -} - -pub async fn wait_for_tx_status_lifecycle( - tx_status_rx: &mut tokio::sync::broadcast::Receiver, - tx_hashes: &[InscriptionId], - statuses: &[TxStatus], - duration: Duration, -) -> Result<(), ZoneTestError> { - let mut remaining: HashSet<(InscriptionId, TxStatus)> = tx_hashes - .iter() - .flat_map(|tx_hash| statuses.iter().map(move |status| (*tx_hash, *status))) - .collect(); - - timeout(duration, async { - while !remaining.is_empty() { - let update = match tx_status_rx.recv().await { - Ok(update) => update, - Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => { - warn!("tx-status subscriber lagged by {n}, recovering"); - continue; - } - Err(tokio::sync::broadcast::error::RecvError::Closed) => { - return Err(ZoneTestError::SequencerStopped); - } - }; - remaining.remove(&(update.tx_hash, update.status)); - if remaining.is_empty() { - return Ok(()); - } - } - Ok(()) - }) - .await - .map_err(|_| ZoneTestError::IndexerTimeout)? -} - -/// Waits until the subscribed channel view satisfies the supplied predicate. -pub async fn wait_for_channel_view( - view_rx: &mut tokio::sync::watch::Receiver, - duration: Duration, - predicate: impl Fn(&SequencerChannelView) -> bool + Send + Sync, -) -> Result { - timeout(duration, async { - loop { - let current = view_rx.borrow().clone(); - if predicate(¤t) { - return Ok(current); - } - - view_rx - .changed() - .await - .map_err(|error| ZoneTestError::Indexer { - message: format!("channel view sender closed: {error}"), - })?; - } - }) - .await - .map_err(|_| ZoneTestError::ChannelViewTimeout { - message: format!( - "condition not reached within {} seconds", - duration.as_secs() - ), - })? -} - -/// Waits until the sequencer emits a turn-to-write notification. -pub async fn wait_for_turn_to_write( - turn_rx: &mut tokio::sync::watch::Receiver, - duration: Duration, -) -> Result { - timeout(duration, async { - loop { - let current = turn_rx.borrow().clone(); - if current.our_turn_to_write { - return Ok(current); - } - - turn_rx - .changed() - .await - .map_err(|error| ZoneTestError::Indexer { - message: format!("turn-to-write sender closed: {error}"), - })?; - } - }) - .await - .map_err(|_| ZoneTestError::ChannelViewTimeout { - message: format!( - "turn to write not reached within {} seconds", - duration.as_secs() - ), - })? -} - -/// Replays the channel's finalized history by cold-starting a fresh -/// read-only sequencer: a random signing key that is not part of the channel -/// rotation, so the instance can never publish or repost anything — -/// inscription posting is turn-gated. Finalized txs are collected from the -/// backfill events until the sequencer reports `Ready`, then the instance is -/// dropped; each call observes a fresh snapshot up to the LIB at connect -/// time. -pub async fn replay_finalized_history( - reader: &ZoneReaderConfig, -) -> Result, ZoneTestError> { - let node = ZoneNodeHttpClient::new(CommonHttpClient::new(None), reader.node_url.clone()); - // Placeholder funding: the reader never publishes (random key, posting is - // turn-gated), so the funding wallet is never exercised. - let funding = FundingConfig { - funding_pk: lb_groth16::Fr::from(1u64).into(), - change_pk: None, - max_tx_fee: GasCost::new(u64::MAX), - priority_fee_percent: FundingConfig::DEFAULT_PRIORITY_FEE_PERCENT, - }; - let mut sequencer = ZoneSequencer::init(reader.channel_id, keygen(), node, funding, None); - - timeout(Duration::from_mins(3), async { - let mut finalized = Vec::new(); - loop { - match sequencer.next_event().await { - Event::BlocksProcessed { - finalized: batch, .. - } => finalized.extend(batch), - Event::Ready => return finalized, - Event::MempoolPending(_) | Event::TurnNotification { .. } => {} - } - } - }) - .await - .map_err(|_| ZoneTestError::IndexerTimeout) -} - -/// Ordered inscription payloads within a finalized-history replay. -pub fn replayed_inscription_payloads(history: &[FinalizedTx]) -> Vec { - finalized_inscriptions(history) - .map(|info| info.payload.clone()) - .collect() -} - -/// Collects indexed block payloads until all expected messages have appeared. -/// -/// The returned order is the finalized on-chain order, which lets assertions -/// decide whether ordering matters for the scenario. -pub async fn collect_indexed_messages( - reader: &ZoneReaderConfig, - expected_messages: &[Inscription], - duration: Duration, -) -> Result, ZoneTestError> { - let expected: HashSet = expected_messages.iter().cloned().collect(); - - timeout(duration, async { - loop { - let payloads = replayed_inscription_payloads(&replay_finalized_history(reader).await?); - let mut seen: HashSet = HashSet::new(); - let mut ordered: Vec = Vec::new(); - for payload in payloads { - if expected.contains(&payload) && seen.insert(payload.clone()) { - ordered.push(payload); - } - } - - if seen == expected { - return Ok(ordered); - } - - sleep(Duration::from_millis(500)).await; - } - }) - .await - .map_err(|_| ZoneTestError::IndexerTimeout)? -} - -/// Replays the finalized history until it exactly matches the expected -/// message sequence without duplicates. -pub async fn collect_indexed_messages_exactly_once( - reader: &ZoneReaderConfig, - expected_messages: &[Inscription], - duration: Duration, -) -> Result, ZoneTestError> { - let expected: HashSet = expected_messages.iter().cloned().collect(); - - timeout(duration, async { - loop { - let ordered: Vec = - replayed_inscription_payloads(&replay_finalized_history(reader).await?) - .into_iter() - .filter(|payload| expected.contains(payload)) - .collect(); - - if ordered == expected_messages { - return Ok(ordered); - } - - sleep(Duration::from_millis(500)).await; - } - }) - .await - .map_err(|_| ZoneTestError::IndexerTimeout)? -} - -/// Waits until the finalized history contains exactly `expected_count` copies -/// of one payload after a short settle period. -/// -/// This intentionally counts duplicate payload bytes, which is required for -/// shared-payload zone tests where each inscription has the same data but a -/// distinct transaction lineage. -pub async fn wait_for_exact_indexed_payload_count( - reader: &ZoneReaderConfig, - expected_payload: Inscription, - expected_count: usize, - duration: Duration, -) -> Result<(), ZoneTestError> { - timeout(duration, async { - loop { - let count = count_indexed_payload(reader, &expected_payload).await?; - - if count >= expected_count { - sleep(Duration::from_secs(30)).await; - - let final_count = count_indexed_payload(reader, &expected_payload).await?; - if final_count == expected_count { - return Ok(()); - } - - return Err(ZoneTestError::IndexedPayloadCountMismatch { - payload: String::from_utf8_lossy(expected_payload.as_slice()).to_string(), - expected: expected_count, - actual: final_count, - }); - } - - sleep(Duration::from_millis(500)).await; - } - }) - .await - .map_err(|_| ZoneTestError::IndexerTimeout)? -} - -async fn count_indexed_payload( - reader: &ZoneReaderConfig, - expected_payload: &Inscription, -) -> Result { - Ok( - replayed_inscription_payloads(&replay_finalized_history(reader).await?) - .iter() - .filter(|payload| *payload == expected_payload) - .count(), - ) -} - -/// Polls until the wallet holds exactly the given number of finalized and -/// unfinalized notes — an exact-count check that catches double-counting. -pub async fn wait_for_channel_wallet_counts( - client: &SequencerClient, - finalized: usize, - unfinalized: usize, - duration: Duration, -) -> Result<(), ZoneTestError> { - timeout(duration, async { - loop { - let view = - client - .channel_wallet() - .await - .map_err(|error| ZoneTestError::ChannelWallet { - message: error.to_string(), - })?; - if view.finalized.len() == finalized && view.unfinalized.len() == unfinalized { - return Ok(()); - } - sleep(Duration::from_millis(500)).await; - } - }) - .await - .map_err(|_| ZoneTestError::ChannelWalletTimeout)? -} - -/// Polls the sequencer's channel wallet until a note of `value` is present. -/// With `finalized_only`, only the finalized layer counts. -pub async fn wait_for_channel_wallet_note( - client: &SequencerClient, - value: Value, - finalized_only: bool, - duration: Duration, -) -> Result<(), ZoneTestError> { - timeout(duration, async { - loop { - let view = - client - .channel_wallet() - .await - .map_err(|error| ZoneTestError::ChannelWallet { - message: error.to_string(), - })?; - let unfinalized = (!finalized_only) - .then_some(view.unfinalized.iter()) - .into_iter() - .flatten(); - if view - .finalized - .iter() - .chain(unfinalized) - .any(|note| note.value == value) - { - return Ok(()); - } - sleep(Duration::from_millis(500)).await; - } - }) - .await - .map_err(|_| ZoneTestError::ChannelWalletTimeout)? -} - -/// Waits until the finalized channel history contains the expected channel -/// deposit, including its amount. -pub async fn wait_for_deposit( - reader: &ZoneReaderConfig, - expected: &DepositOp, - expected_amount: Value, - duration: Duration, -) -> Result<(), ZoneTestError> { - poll_replayed_history_until(reader, duration, ZoneTestError::IndexerTimeout, |op| { - matches!(op, FinalizedOp::Deposit(deposit) - if deposit.inputs == expected.inputs - && deposit.amount == expected_amount - && deposit.metadata == expected.metadata) - }) - .await -} - -/// Waits until the finalized channel history contains the expected withdraw. -pub async fn wait_for_withdraw( - reader: &ZoneReaderConfig, - expected: &ChannelWithdrawOp, - timeout_duration: Duration, -) -> Result<(), ZoneTestError> { - poll_replayed_history_until( - reader, - timeout_duration, - ZoneTestError::WithdrawTimeout, - |op| matches!(op, FinalizedOp::Withdraw(withdraw) if withdraw.op.inputs == expected.inputs), - ) - .await -} - -/// Waits until the finalized channel history contains a channel transfer whose -/// input set has exactly `expected_inputs` notes. -/// -/// This is the on-chain record of the withdrawal's note selection: in the -/// dust-flood scenario it proves best-fit largest-first consumed a single -/// covering note rather than sweeping the >255-note dust flood into a transfer -/// the ledger would reject. -pub async fn wait_for_channel_transfer_input_count( - reader: &ZoneReaderConfig, - expected_inputs: usize, - timeout_duration: Duration, -) -> Result<(), ZoneTestError> { - poll_replayed_history_until( - reader, - timeout_duration, - ZoneTestError::IndexerTimeout, - move |op| { - matches!(op, FinalizedOp::ChannelTransfer(transfer) - if transfer.op.inputs.len() == expected_inputs) - }, - ) - .await -} - -async fn poll_replayed_history_until( - reader: &ZoneReaderConfig, - duration: Duration, - timeout_error: ZoneTestError, - mut predicate: impl FnMut(&FinalizedOp) -> bool, -) -> Result<(), ZoneTestError> { - timeout(duration, async { - loop { - let history = replay_finalized_history(reader).await?; - if history - .iter() - .flat_map(|tx| tx.ops.iter()) - .any(&mut predicate) - { - return Ok(()); - } - - sleep(Duration::from_millis(500)).await; - } - }) - .await - .map_err(|_| timeout_error)? -} - -/// Waits until the sequencer's event stream surfaces the expected deposit -/// in [`Event::BlocksProcessed::finalized`] (matched by `inputs`, `amount`, -/// and `metadata`) while collecting any mempool-pending events. Drains the -/// events channel as it goes — call this after any earlier event consumers in -/// the scenario have moved past the relevant publish events. -pub async fn wait_for_finalized_deposit_via_sequencer_and_collect_mempool_pending( - events: &mut tokio::sync::broadcast::Receiver, - expected: &DepositOp, - expected_amount: Value, - duration: Duration, -) -> Result, ZoneTestError> { - poll_sequencer_finalized_until_and_collect_mempool_pending( - events, - duration, - ZoneTestError::IndexerTimeout, - |op| { - matches!(op, FinalizedOp::Deposit(d) - if d.inputs == expected.inputs - && d.amount == expected_amount - && d.metadata == expected.metadata) - }, - ) - .await -} - -/// Waits until the sequencer's event stream surfaces the expected withdraw -/// (matched by `outputs`) while collecting any mempool-pending events. Drains -/// the events channel as it goes. -pub async fn wait_for_finalized_withdraw_via_sequencer_and_collect_mempool_pending( - events: &mut tokio::sync::broadcast::Receiver, - expected: &ChannelWithdrawOp, - duration: Duration, -) -> Result, ZoneTestError> { - poll_sequencer_finalized_until_and_collect_mempool_pending( - events, - duration, - ZoneTestError::WithdrawTimeout, - |op| matches!(op, FinalizedOp::Withdraw(w) if w.op.inputs == expected.inputs), - ) - .await -} - -async fn poll_sequencer_finalized_until_and_collect_mempool_pending( - events: &mut tokio::sync::broadcast::Receiver, - duration: Duration, - timeout_error: ZoneTestError, - mut predicate: impl FnMut(&FinalizedOp) -> bool, -) -> Result, ZoneTestError> { - timeout(duration, async { - let mut mempool_pending = HashSet::new(); - loop { - let event = match events.recv().await { - Ok(event) => event, - Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => { - warn!("event subscriber lagged by {n}, recovering"); - continue; - } - Err(tokio::sync::broadcast::error::RecvError::Closed) => { - return Err(ZoneTestError::SequencerStopped); - } - }; - if let Event::MempoolPending(tx_hash) = event { - mempool_pending.insert(tx_hash); - continue; - } - let Event::BlocksProcessed { finalized, .. } = event else { - continue; - }; - for tx in finalized { - if tx.ops.iter().any(&mut predicate) { - return Ok(mempool_pending); - } - } - } - }) - .await - .map_err(|_| timeout_error)? -} - -/// Waits until node mempool/chain observation confirms the submitted zone -/// transactions reached the canonical chain. -pub async fn ensure_zone_transactions_included( - client: &NodeHttpClient, - tx_hashes: &[InscriptionId], - duration: Duration, -) -> Result<(), ZoneTestError> { - let included = wait_for_transactions_inclusion(client, tx_hashes, duration).await; - - if included { - return Ok(()); - } - - Err(ZoneTestError::InclusionTimeout) -} - -/// Walks back from LIB until every expected zone transaction is found in the -/// finalized chain. -pub async fn wait_for_transactions_finalized( - node_url: Url, - tx_hashes: &[InscriptionId], - duration: Duration, -) -> Result<(), ZoneTestError> { - let client = CommonHttpClient::new(None); - let expected: HashSet<_> = tx_hashes.iter().copied().collect(); - - timeout(duration, async { - loop { - let info = client - .consensus_info(node_url.clone()) - .await - .map_err(|error| ZoneTestError::Consensus { - message: error.to_string(), - })?; - - let mut found = HashSet::new(); - let mut current = info.cryptarchia_info.lib; - - while let Some(block) = client - .get_block_by_id(node_url.clone(), current) - .await - .map_err(|error| ZoneTestError::Block { - message: error.to_string(), - })? - { - for tx in &block.transactions { - let hash = tx.mantle_tx().hash(); - if expected.contains(&hash) { - found.insert(hash); - } - } - - current = block.header.parent_block; - } - - if found == expected { - return Ok(()); - } - - sleep(Duration::from_millis(500)).await; - } - }) - .await - .map_err(|_| ZoneTestError::FinalizationTimeout)? -} - -/// Waits for LIB movement after a restart so stale-checkpoint scenarios can -/// distinguish old local state from new canonical chain progress. -pub async fn wait_for_lib_advance( - client: &NodeHttpClient, - initial_lib_slot: Slot, - duration: Duration, -) -> Result<(), ZoneTestError> { - timeout(duration, async { - loop { - let info = client - .consensus_info() - .await - .map_err(|error| ZoneTestError::Consensus { - message: error.to_string(), - })?; - - if info.cryptarchia_info.lib_slot > initial_lib_slot { - return Ok(()); - } - - sleep(Duration::from_millis(500)).await; - } - }) - .await - .map_err(|_| ZoneTestError::LibAdvanceTimeout)? -} - -/// Builds a regular channel deposit for an existing funding note with the -/// exact deposit value. -pub fn build_zone_deposit( - available_utxos: Vec, - channel_id: ChannelId, - amount: Value, - metadata: Metadata, -) -> Result { - let note = available_utxos - .into_iter() - .find(|utxo| utxo.note.value == amount) - .ok_or(ZoneTestError::MissingExactFundingNote { value: amount })?; - - let deposit = DepositOp { - channel_id, - inputs: Inputs::new([note.id()]), - metadata, - }; - let reserved_inputs = vec![note]; - let channel_notes = recreated_channel_notes(&deposit, &reserved_inputs); - Ok(ZoneDeposit { - deposit, - reserved_inputs, - channel_notes, - }) -} - -/// Build a deposit that consumes one wallet note per listed value, in order. -/// A deposit re-creates its inputs 1:1 as channel notes, so this yields a -/// multi-input deposit whose recreated notes carry those exact per-note values -/// — used to cover the channel wallet's per-note tracking. -pub fn build_zone_deposit_from_values( - available_utxos: Vec, - channel_id: ChannelId, - input_values: &[Value], - metadata: Metadata, -) -> Result { - let mut remaining = available_utxos; - let mut reserved_inputs = Vec::new(); - for &value in input_values { - let index = remaining - .iter() - .position(|utxo| utxo.note.value == value) - .ok_or(ZoneTestError::MissingExactFundingNote { value })?; - reserved_inputs.push(remaining.remove(index)); - } - - let input_ids: Vec<_> = reserved_inputs.iter().map(Utxo::id).collect(); - let deposit = DepositOp { - channel_id, - inputs: Inputs::try_new(input_ids).map_err(|error| ZoneTestError::SubmitDeposit { - message: format!("deposit input set exceeds bound: {error:?}"), - })?, - metadata, - }; - let channel_notes = recreated_channel_notes(&deposit, &reserved_inputs); - Ok(ZoneDeposit { - deposit, - reserved_inputs, - channel_notes, - }) -} - -/// Generous cap on channel transaction fees at genesis gas prices; actual -/// fees are a few hundred gas units for these small transactions. -const MAX_ZONE_DEPOSIT_TX_FEE: u64 = 10_000; - -/// Submits a regular channel deposit through the node wallet API. -pub async fn submit_zone_deposit( - node_url: &Url, - deposit: &DepositOp, - funding_public_key: ZkPublicKey, -) -> Result { - let body = ChannelDepositRequestBody { - tip: None, - deposit: deposit.clone(), - change_public_key: funding_public_key, - funding_public_keys: vec![funding_public_key], - max_tx_fee: MAX_ZONE_DEPOSIT_TX_FEE.into(), - }; - - let request_url = - node_url - .join("/channel/deposit") - .map_err(|error| ZoneTestError::SubmitDeposit { - message: error.to_string(), - })?; - - let response: ChannelDepositResponseBody = CommonHttpClient::new(None) - .post(request_url, &body) - .await - .map_err(|error| ZoneTestError::SubmitDeposit { - message: error.to_string(), - })?; - - Ok(response.hash) -} - -/// Splits one channel note into `dust_count` value-1 dust notes via a raw, -/// node-funded `ChannelTransfer`, submitted straight to the node mempool (no -/// zone-sdk involvement — mirrors how deposits are submitted). -/// -/// The transfer is value-preserving, so the input note's value must equal -/// `dust_count` (every output is value 1). The channel authorizes the transfer -/// with the sequencer's accredited key at index 0 (single-signer channel, -/// `transfer_threshold == 1`); the node appends and proves the fee transfer. -pub async fn submit_zone_channel_split( - node_url: &Url, - channel_id: ChannelId, - signing_key: &Ed25519Key, - funding_pk: ZkPublicKey, - input_note: Utxo, - dust_count: usize, -) -> Result { - let input_value = input_note.note.value; - if input_value != dust_count as u64 { - return Err(ZoneTestError::SplitTransfer { - message: format!( - "channel note value {input_value} must equal dust count {dust_count} \ - (each dust note is value 1)" - ), - }); - } - - let outputs = - Outputs::try_new(vec![Note::new(1, funding_pk); dust_count]).map_err(|error| { - ZoneTestError::SplitTransfer { - message: format!("dust outputs exceed bound: {error:?}"), - } - })?; - let transfer = ChannelTransferOp { - channel_id, - inputs: Inputs::new([input_note.id()]), - outputs, - }; - - let tx_builder = MantleTxBuilder::new() - .push_op(Op::ChannelTransfer(transfer)) - .map_err(|error| ZoneTestError::SplitTransfer { - message: format!("too many ops: {error}"), - })?; - - let node = NodeHttpClient::from_url(node_url.clone()); - let response = node - .fund_tx(WalletFundRequestBody { - tip: None, - priority_fee_percent: 0, - tx_builder, - change_public_key: funding_pk, - funding_public_keys: vec![funding_pk], - max_tx_fee: GasCost::new(u64::MAX), - }) - .await - .map_err(|error| ZoneTestError::SplitTransfer { - message: format!("funding failed: {error}"), - })?; - - // The channel multi-sig proves the transfer over the funded tx hash; the - // funding appends its own fee transfer proof as the last op. - let funded_tx = response.funded_tx; - let tx_hash = funded_tx.hash(); - let signature = signing_key.sign_payload(tx_hash.as_signing_bytes().as_ref()); - let proof = ChannelMultiSigProof::try_new([IndexedSignature::new(0, signature)].into()) - .map_err(|error| ZoneTestError::SplitTransfer { - message: format!("multi-sig proof assembly failed: {error:?}"), - })?; - let mut ops_proofs = OpsProofs::new_unchecked(vec![OpProof::ChannelMultiSigProof(proof)]); - if let Some(transfer_proof) = response.transfer_proof { - ops_proofs - .try_push(transfer_proof) - .map_err(|error| ZoneTestError::SplitTransfer { - message: format!("too many operation proofs: {error:?}"), - })?; - } - - let signed_tx = SignedMantleTx::new(funded_tx, ops_proofs); - node.submit_transaction(&signed_tx) - .await - .map_err(|error| ZoneTestError::SplitTransfer { - message: format!("submit failed: {error}"), - })?; - - Ok(tx_hash) -} - -/// Builds and submits a single transaction that both creates the deposit note -/// and publishes the zone inscription that consumes it. -pub async fn submit_atomic_zone_deposit( - node_url: &Url, - client: &SequencerClient, - request: AtomicZoneDepositRequest, -) -> Result { - let AtomicZoneDepositRequest { - channel_id, - funding_public_key, - available_utxos, - amount, - metadata, - inscription_data, - } = request; - let (transfer, reserved_inputs) = - build_atomic_deposit_transfer(available_utxos, funding_public_key, amount)?; - let deposit = build_atomic_deposit_op(channel_id, metadata, &transfer)?; - - let (tx, msg_id, sequencer_sig) = client - .prepare_tx( - [Op::Transfer(transfer), Op::ChannelDeposit(deposit.clone())].into(), - inscription_data, - ) - .await - .map_err(|error| ZoneTestError::BuildAtomicDeposit { - message: error.to_string(), - })?; - - let user_sig = sign_tx_zk(node_url, &tx, vec![funding_public_key]).await?; - let signed_tx = SignedMantleTx::new( - tx, - [ - OpProof::ZkSig(user_sig.clone()), - OpProof::ZkSig(user_sig), - OpProof::Ed25519Sig(sequencer_sig), - ] - .into(), - ); - - let (result, _cp) = client - .submit_signed_tx(signed_tx, msg_id) - .await - .map_err(|error| ZoneTestError::SubmitAtomicDeposit { - message: error.to_string(), - })?; - - Ok(AtomicZoneDepositSubmission { - deposit, - publish: result, - reserved_inputs, - }) -} - -async fn build_funded_custom_tx( - node_client: &NodeHttpClient, - channel_id: ChannelId, - signing_key: &Ed25519Key, - funding_pk: ZkPublicKey, - payloads: &[Inscription], - mut parent: MsgId, -) -> Result<(SignedMantleTx, MsgId), ZoneTestError> { - let signer = signing_key.public_key(); - let mut tx_builder = MantleTxBuilder::new(); - for payload in payloads { - let op = InscriptionOp { - channel_id, - inscription: payload.clone(), - parent, - signer, - }; - parent = op.id(); - tx_builder = tx_builder - .push_op(Op::ChannelInscribe(op)) - .map_err(|error| ZoneTestError::BuildCustomTx { - message: format!("too many ops: {error}"), - })?; - } - - let response = node_client - .fund_tx(WalletFundRequestBody { - tip: None, - priority_fee_percent: 0, - tx_builder, - change_public_key: funding_pk, - funding_public_keys: vec![funding_pk], - max_tx_fee: GasCost::new(u64::MAX), - }) - .await - .map_err(|error| ZoneTestError::SubmitCustomTx { - message: format!("funding failed: {error}"), - })?; - - // Funding appends the fee transfer as the last op; every inscription is - // proven by the sequencer key over the funded tx hash. - let funded_tx = response.funded_tx; - let signature = signing_key.sign_payload(funded_tx.hash().as_signing_bytes().as_ref()); - let mut ops_proofs = - OpsProofs::new_unchecked(vec![OpProof::Ed25519Sig(signature); payloads.len()]); - if let Some(proof) = response.transfer_proof { - ops_proofs - .try_push(proof) - .map_err(|error| ZoneTestError::BuildCustomTx { - message: format!("too many operation proofs: {error:?}"), - })?; - } - let signed_tx = SignedMantleTx::new(funded_tx, ops_proofs); - - Ok((signed_tx, parent)) -} - -pub struct CustomRepublishDeps { - pub node_client: NodeHttpClient, - pub channel_id: ChannelId, - pub signing_key: Ed25519Key, - pub funding_pk: ZkPublicKey, - pub batches: VecDeque>, -} - -pub fn start_custom_republish_policy( - sequencer: ZoneSequencer, - deps: CustomRepublishDeps, -) -> PolicyRuntime { - let view_rx = sequencer.subscribe_channel_view(); - let policy = CustomRepublishPolicy { - deps, - view_rx, - pending: HashSet::new(), - finalized: HashSet::new(), - chain_tip: None, - ready: false, - }; - to_policy_runtime(runner::spawn(sequencer, policy)) -} - -/// [`OrphanRepublishPolicy`] for the custom-tx flow: orphans that are -/// neither in `pending` nor finalized are rebuilt and re-submitted. -struct CustomRepublishPolicy { - deps: CustomRepublishDeps, - view_rx: tokio::sync::watch::Receiver, - pending: HashSet, - finalized: HashSet, - /// Where our own submitted chain ends; reset on orphans so rebuilds - /// chain from the channel tip instead. - chain_tip: Option, - /// No submissions until ready — a fail-fast submit would leak its - /// funding reservation. - ready: bool, -} - -impl CustomRepublishPolicy { - async fn submit( - &mut self, - sequencer: &mut ZoneSequencer, - payloads: Vec, - ) -> bool - where - Node: lb_zone_sdk::adapter::Node + Clone + Send + Sync + 'static, - { - let parent = self - .chain_tip - .unwrap_or_else(|| self.view_rx.borrow().tip_message); - let built = build_funded_custom_tx( - &self.deps.node_client, - self.deps.channel_id, - &self.deps.signing_key, - self.deps.funding_pk, - &payloads, - parent, - ) - .await; - let (signed_tx, msg_id) = match built { - Ok(built) => built, - Err(error) => { - warn!(%error, "Failed to build custom zone tx"); - return false; - } - }; - match sequencer.handle().submit_signed_tx(signed_tx, msg_id) { - Ok((_result, _checkpoint)) => { - self.pending.extend(payloads); - self.chain_tip = Some(msg_id); - true - } - Err(error) => { - warn!(%error, "Failed to submit custom zone tx"); - false - } - } - } - - fn entry_payloads(&self, entry: &ChannelUpdateTx) -> Vec { - match entry { - ChannelUpdateTx::Custom(tx) => channel_inscriptions(tx, self.deps.channel_id) - .into_iter() - .map(|info| info.payload) - .collect(), - typed => typed - .inscription() - .map(|info| info.payload.clone()) - .into_iter() - .collect(), - } - } -} - -impl runner::Policy for CustomRepublishPolicy -where - Node: lb_zone_sdk::adapter::Node + Clone + Send + Sync + 'static, -{ - async fn on_event(&mut self, sequencer: &mut ZoneSequencer, event: &Event) { - let (channel_update, finalized) = match event { - Event::Ready => { - self.ready = true; - (None, None) - } - Event::BlocksProcessed { - channel_update, - finalized, - .. - } => (Some(channel_update), Some(finalized)), - _ => return, - }; - - if let Some(finalized) = finalized { - self.finalized - .extend(finalized_inscriptions(finalized).map(|info| info.payload.clone())); - } - - if let Some(channel_update) = channel_update { - let orphaned: HashSet = channel_update - .orphaned - .iter() - .flat_map(|entry| self.entry_payloads(entry)) - .collect(); - let adopted: Vec = channel_update - .adopted - .iter() - .flat_map(|entry| self.entry_payloads(entry)) - .collect(); - for payload in &orphaned { - self.pending.remove(payload); - } - self.pending.extend(adopted); - - let republish: Vec = orphaned - .into_iter() - .filter(|payload| { - !self.pending.contains(payload) && !self.finalized.contains(payload) - }) - .collect(); - if self.ready && !republish.is_empty() { - self.chain_tip = None; - if !self.submit(sequencer, republish.clone()).await { - self.deps.batches.push_back(republish); - } - } - } - - // One attempt per batch per event: a failed submission stops the - // drain and is retried on the next event. - while self.ready { - let Some(batch) = self.deps.batches.pop_front() else { - break; - }; - if !self.submit(sequencer, batch.clone()).await { - self.deps.batches.push_front(batch); - break; - } - } - } -} - -/// Builds the funding transfer that creates the note consumed by an atomic -/// zone deposit. -/// Generous fee margin for the atomic `[Transfer, Deposit, Inscribe]` -/// transaction. The mandatory fee (execution + size-based storage gas) is -/// roughly 2k and varies with input count and change-note presence, so a -/// tight margin intermittently underfunds the tx — which is permanently -/// invalid and silently evicted at block assembly. Matches -/// `MAX_ZONE_DEPOSIT_TX_FEE`; the excess above the mandatory fee is a tip. -const ATOMIC_DEPOSIT_FEE_MARGIN: u64 = 10_000; - -fn build_atomic_deposit_transfer( - available_utxos: Vec, - funding_public_key: ZkPublicKey, - amount: Value, -) -> Result<(TransferOp, Vec), ZoneTestError> { - let deposit_note = Note::new(amount, funding_public_key); - let funded_transfer = build_wallet_funded_transfer( - available_utxos, - vec![deposit_note], - funding_public_key, - ATOMIC_DEPOSIT_FEE_MARGIN, - ) - .map_err(|error| ZoneTestError::BuildAtomicDeposit { - message: error.to_string(), - })?; - - Ok(funded_transfer.into_parts()) -} - -/// Points the channel deposit at the note created by the atomic funding -/// transfer, keeping both operations in the same transaction. -fn build_atomic_deposit_op( - channel_id: ChannelId, - metadata: Metadata, - transfer: &TransferOp, -) -> Result { - let deposit_note_id = transfer - .outputs - .utxo_by_index(0, transfer) - .ok_or_else(|| ZoneTestError::BuildAtomicDeposit { - message: "transfer did not produce the deposit note".to_owned(), - })? - .id(); - - Ok(DepositOp { - channel_id, - inputs: Inputs::new([deposit_note_id]), - metadata, - }) -} - -/// Submits a channel withdraw signed by the active zone sequencer and publishes -/// the withdraw inscription as part of the same SDK flow. -/// -/// The withdraw pays a single note of `amount` back to `funding_public_key` -/// (self-withdraw). Inputs are selected automatically by the SDK -/// (`WithdrawInputs::Auto`, best-fit largest-first, capped at 255 inputs). -pub async fn submit_zone_withdraw( - client: &SequencerClient, - _channel_id: ChannelId, - funding_public_key: ZkPublicKey, - amount: Value, - inscription_data: Inscription, -) -> Result { - let (result, _cp) = client - .publish_atomic_withdraw( - inscription_data, - vec![WithdrawArg { - outputs: Outputs::new([Note::new(amount, funding_public_key)]), - }], - WithdrawInputs::Auto, - ) - .await - .map_err(|error| ZoneTestError::SubmitWithdraw { - message: error.to_string(), - })?; - - let PendingTx::AtomicWithdraw(info) = result.tx else { - return Err(ZoneTestError::SubmitWithdraw { - message: "publish_atomic_withdraw returned a non-AtomicWithdraw publish result" - .to_owned(), - }); - }; - let withdraw = info - .withdraws - .first() - .ok_or_else(|| ZoneTestError::SubmitWithdraw { - message: "atomic withdraw bundle had no withdraw ops".to_owned(), - })? - .op - .clone(); - - Ok(ZoneWithdrawSubmission { - withdraw, - publish: PublishResult { - tx: PendingTx::AtomicWithdraw(info), - }, - }) -} - -/// Result of publishing an atomic inscription+withdraw bundle. Carries every -/// withdraw op produced by the SDK (one per `WithdrawArg`, in submission -/// order) so a multi-withdraw scenario can match each by its outputs. -pub struct ZoneAtomicWithdrawSubmission { - pub withdraws: Vec, - pub publish: PublishResult, -} - -/// Publishes an atomic inscription+withdraw bundle through the runner. -/// Returns every withdraw op (with the nonce filled by the SDK) from the -/// publish call's return value, so downstream cucumber assertions can -/// match each withdraw by its outputs. -/// -/// `outputs_per_arg` carries one entry per `WithdrawArg`; each inner `Vec` -/// becomes that arg's `Outputs` (one `Note::new(amount, funding_pk)` per -/// listed amount). Exercises the SDK API at full width: multiple args, with -/// any arg able to carry multiple output notes. -pub async fn publish_atomic_zone_withdraw( - client: &SequencerClient, - funding_public_key: ZkPublicKey, - outputs_per_arg: Vec>, - inscription_data: Inscription, - _deadline: PublishDeadline, -) -> Result { - if outputs_per_arg.is_empty() { - return Err(ZoneTestError::SubmitWithdraw { - message: "publish_atomic_zone_withdraw requires at least one withdraw arg".to_owned(), - }); - } - let withdraw_args: Vec = outputs_per_arg - .iter() - .map(|amounts| { - Ok::(WithdrawArg { - outputs: Outputs::try_new( - amounts - .iter() - .map(|amount| Note::new(*amount, funding_public_key)) - .collect::>(), - )?, - }) - }) - .collect::, ZoneTestError>>()?; - - let (result, _cp) = client - .publish_atomic_withdraw(inscription_data, withdraw_args, WithdrawInputs::Auto) - .await - .map_err(|error| ZoneTestError::SubmitWithdraw { - message: error.to_string(), - })?; - - let PendingTx::AtomicWithdraw(info) = result.tx else { - return Err(ZoneTestError::SubmitWithdraw { - message: "publish_atomic_withdraw returned a non-AtomicWithdraw publish result" - .to_owned(), - }); - }; - if info.withdraws.is_empty() { - return Err(ZoneTestError::SubmitWithdraw { - message: "atomic withdraw bundle had no withdraw ops".to_owned(), - }); - } - Ok(ZoneAtomicWithdrawSubmission { - withdraws: info.withdraws.iter().map(|w| w.op.clone()).collect(), - publish: PublishResult { - tx: PendingTx::AtomicWithdraw(info), - }, - }) -} - -/// Asks the node wallet service to sign a Mantle transaction for the requested -/// ZK keys. -async fn sign_tx_zk( - node_url: &Url, - tx: &RawMantleTx, - public_keys: Vec, -) -> Result { - let request_url = - node_url - .join("wallet/sign/zk") - .map_err(|error| ZoneTestError::SignTransaction { - message: error.to_string(), - })?; - let response: WalletSignTxZkResponseBody = CommonHttpClient::new(None) - .post( - request_url, - &WalletSignTxZkRequestBody { - tx_hash: tx.hash(), - pks: ZkPublicKeys::try_from(public_keys)?, - }, - ) - .await - .map_err(|error| ZoneTestError::SignTransaction { - message: error.to_string(), - })?; - - Ok(response.sig) -} diff --git a/tests/src/cucumber/steps/manual_mempool/actions.rs b/tests/src/cucumber/steps/mempool/actions.rs similarity index 98% rename from tests/src/cucumber/steps/manual_mempool/actions.rs rename to tests/src/cucumber/steps/mempool/actions.rs index aa992cd4c..e62beb694 100644 --- a/tests/src/cucumber/steps/manual_mempool/actions.rs +++ b/tests/src/cucumber/steps/mempool/actions.rs @@ -25,9 +25,7 @@ use crate::{ common::wallet::{WalletTransactionError, WalletTransactionIntent}, cucumber::{ error::StepError, - steps::{ - TARGET, manual_transactions::tracked_transactions::create_stateless_invalid_transaction, - }, + steps::{TARGET, transactions::tracked_transactions::create_stateless_invalid_transaction}, utils::{tx_hash_to_hex, user_config_from_node_yaml}, wallet::submissions::{ SignedUserWalletSubmission, prepare_user_wallet_transaction_submission, diff --git a/tests/src/cucumber/steps/manual_mempool/assertions.rs b/tests/src/cucumber/steps/mempool/assertions.rs similarity index 100% rename from tests/src/cucumber/steps/manual_mempool/assertions.rs rename to tests/src/cucumber/steps/mempool/assertions.rs diff --git a/tests/src/cucumber/steps/manual_mempool/mod.rs b/tests/src/cucumber/steps/mempool/mod.rs similarity index 100% rename from tests/src/cucumber/steps/manual_mempool/mod.rs rename to tests/src/cucumber/steps/mempool/mod.rs diff --git a/tests/src/cucumber/steps/manual_mempool/steps.rs b/tests/src/cucumber/steps/mempool/steps.rs similarity index 99% rename from tests/src/cucumber/steps/manual_mempool/steps.rs rename to tests/src/cucumber/steps/mempool/steps.rs index 39a303d6e..eb03fc6fd 100644 --- a/tests/src/cucumber/steps/manual_mempool/steps.rs +++ b/tests/src/cucumber/steps/mempool/steps.rs @@ -2,7 +2,7 @@ use cucumber::{gherkin::Step, then, when}; use crate::cucumber::{ error::{StepError, StepResult}, - steps::manual_mempool::{ + steps::mempool::{ actions::{ prepare_transfer_transaction, submit_prepared_transaction_through_blend, submit_prepared_transaction_to_nodes, try_submit_invalid_transaction, diff --git a/tests/src/cucumber/steps/mod.rs b/tests/src/cucumber/steps/mod.rs index a043a4192..8a4c86a7c 100644 --- a/tests/src/cucumber/steps/mod.rs +++ b/tests/src/cucumber/steps/mod.rs @@ -2,16 +2,16 @@ pub mod run; pub mod scenario; pub mod workloads; +pub mod cluster; pub mod fees; -pub mod manual_cluster; -pub mod manual_k8s; -pub mod manual_mempool; -pub mod manual_nodes; -pub mod manual_transactions; -pub mod manual_zone; +pub mod k8s; +pub mod mempool; +pub mod nodes; pub mod parse_steps; pub mod pow; pub mod tokio_console; +pub mod transactions; pub mod wallet_fund; +pub mod zone; const TARGET: &str = "cucumber_steps"; diff --git a/tests/src/cucumber/steps/manual_nodes/config_override.rs b/tests/src/cucumber/steps/nodes/config_override.rs similarity index 100% rename from tests/src/cucumber/steps/manual_nodes/config_override.rs rename to tests/src/cucumber/steps/nodes/config_override.rs diff --git a/tests/src/cucumber/steps/manual_nodes/diagnostics.rs b/tests/src/cucumber/steps/nodes/diagnostics.rs similarity index 99% rename from tests/src/cucumber/steps/manual_nodes/diagnostics.rs rename to tests/src/cucumber/steps/nodes/diagnostics.rs index b4cd9c50d..68cd013ad 100644 --- a/tests/src/cucumber/steps/manual_nodes/diagnostics.rs +++ b/tests/src/cucumber/steps/nodes/diagnostics.rs @@ -12,7 +12,7 @@ use tracing::{info, warn}; use crate::cucumber::{ TARGET, error::{StepError, StepResult}, - steps::manual_nodes::config_override::set_deployment_config_override, + steps::nodes::config_override::set_deployment_config_override, utils::{peer_id_from_node_yaml, user_config_from_node_yaml}, world::{BlendDiagnosticPhase, CucumberWorld}, }; diff --git a/tests/src/cucumber/steps/manual_nodes/lib_assertions.rs b/tests/src/cucumber/steps/nodes/lib_assertions.rs similarity index 100% rename from tests/src/cucumber/steps/manual_nodes/lib_assertions.rs rename to tests/src/cucumber/steps/nodes/lib_assertions.rs diff --git a/tests/src/cucumber/steps/nodes/mod.rs b/tests/src/cucumber/steps/nodes/mod.rs new file mode 100644 index 000000000..b699dfaa1 --- /dev/null +++ b/tests/src/cucumber/steps/nodes/mod.rs @@ -0,0 +1,22 @@ +pub(crate) mod config_override; +pub(crate) mod diagnostics; +pub mod lib_assertions; +pub mod parameters; +pub(crate) mod snapshots; +pub mod steps; + +mod operations; + +pub(crate) use operations::{ + NodesToStartUnordered, WalletStartInfo, create_snapshot_all_nodes_with_wallet_state, + create_snapshot_node_with_wallet_state, create_snapshots_all_nodes, + ensure_all_nodes_agree_on_lib, ensure_fee_sponsorship_and_fork_groups_are_not_mixed, + fetch_public_peer_consensus, genesis_block_utxos, get_cryptarchia_info_all_nodes, + nodes_converged, parse_genesis_wallet_tokens_row, parse_mining_wallet_resources_table_row, + parse_url, parse_wallet_resources_table_row, poll_all_nodes_and_update_consensus_cache, + restart_node, start_node, start_nodes_order_respecting_dependencies, stop_node, + verify_genesis_wallet_resources_table_indexes, + verify_mining_node_wallet_resources_table_indexes, verify_node_wallet_resources_table_indexes, + verify_reponsive_and_network_ready_with_timeout, wait_all_nodes_responive, + wait_for_all_nodes_to_be_synced_to_chain, +}; diff --git a/tests/src/cucumber/steps/nodes/operations/consensus.rs b/tests/src/cucumber/steps/nodes/operations/consensus.rs new file mode 100644 index 000000000..485b6dbe8 --- /dev/null +++ b/tests/src/cucumber/steps/nodes/operations/consensus.rs @@ -0,0 +1,376 @@ +use super::*; + +fn tips_aligned_at_min_difference( + nodes_chain_info: &HashMap, + all_nodes_min: u64, +) -> (AlignmentStatus, Vec) { + // Always return per-node view at min_height for logging + let mut anchor_hashes: Vec = Vec::with_capacity(nodes_chain_info.len()); + + for node_name in nodes_chain_info.keys() { + let peer_chain = nodes_chain_info + .get(node_name) + .expect("nodes_chain_info must be pre-initialized"); + anchor_hashes.push(MaybeSnapshot { + height: all_nodes_min, + header_hash: peer_chain.get(&all_nodes_min).cloned(), + }); + } + + let all_have = anchor_hashes.iter().all(|snap| snap.header_hash.is_some()); + if !all_have { + return (AlignmentStatus::MissingChainInfo, anchor_hashes); + } + + let compare_hash = anchor_hashes[0].header_hash.as_ref().unwrap(); + let all_same = anchor_hashes + .iter() + .all(|snap| snap.header_hash.as_ref().unwrap() == compare_hash); + + if all_same { + (AlignmentStatus::Aligned, anchor_hashes) + } else { + (AlignmentStatus::Fork, anchor_hashes) + } +} + +async fn fetch_and_update_chain_info( + step: &str, + nodes_info: &mut HashMap, + nodes_chain_info: &mut HashMap, +) -> Result<(u64, u64, Vec), StepError> { + poll_all_nodes_and_update_consensus_cache(step, nodes_info).await?; + + let mut best_node_heights: Vec = Vec::with_capacity(nodes_info.len()); + + for node_info in nodes_info.values() { + let max_height = node_info.best_height().unwrap_or_default(); + best_node_heights.push(max_height); + + let started_node_name = node_info.started_node.name.clone(); + let chain = + nodes_chain_info + .get_mut(&started_node_name) + .ok_or(StepError::LogicalError { + message: format!( + "Started node '{started_node_name}' not found in chain info map", + ), + })?; + let chain_info = node_info.chain_info(); + for (height, hash) in chain_info { + chain.insert(*height, hash.clone()); + } + } + + let all_nodes_min = *best_node_heights.iter().min().unwrap_or(&0); + let all_nodes_max = *best_node_heights.iter().max().unwrap_or(&0); + let diff = all_nodes_max - all_nodes_min; + + Ok((all_nodes_min, diff, best_node_heights)) +} + +fn log_waiting_status( + status: &AlignmentStatus, + min_height: Option, + diff: u64, + peer_heights: &[u64], + peer_min: u64, + anchor_hashes: &[MaybeSnapshot], + start: Instant, +) { + match status { + AlignmentStatus::Aligned => { + let converge = min_height.map_or_else( + || "Waiting for all nodes to converge".to_owned(), + |min_height| format!("Waiting for at least {min_height} blocks converged"), + ); + info!( + target: TARGET, + "{converge} - elapsed: {:.2?}, diff: {diff}, heights: {peer_heights:?}", + start.elapsed() + ); + } + AlignmentStatus::MissingChainInfo => { + info!( + target: TARGET, + "Waiting for all node's hashes at height {peer_min} - elapsed: {:.2?}, diff: \ + {diff}, heights: {peer_heights:?}, anchors: {:?}", + start.elapsed(), + anchor_hashes.iter().map(|snap| &snap.header_hash).collect::>() + ); + } + AlignmentStatus::Fork => { + let fork_hashes: HashSet<_> = anchor_hashes + .iter() + .filter_map(|snap| snap.header_hash.as_ref()) + .collect(); + info!( + target: TARGET, + "{} fork chains detected!!! Elapsed: {:.2?}, diff: {diff}, heights: {peer_heights:?}, \ + fork hashes at height {}: {:?}", + fork_hashes.len(), start.elapsed(), anchor_hashes[0].height, fork_hashes + ); + } + } +} + +pub async fn nodes_converged( + world: &mut CucumberWorld, + step: &str, + min_height: Option, + max_diff_height: u64, + time_out_seconds: u64, +) -> StepResult { + let nodes_info = &world.nodes_info.values().collect::>(); + let start = Instant::now(); + let time_out = Duration::from_secs(time_out_seconds); + + // node_name -> (height -> header_id) (overwrites on reorg) + let mut nodes_chain_info: HashMap = + HashMap::with_capacity(nodes_info.len()); + + // Pre-initialize so lookups are deterministic + for node_info in nodes_info { + nodes_chain_info + .entry(node_info.started_node.name.clone()) + .or_default(); + } + + let mut count = 0usize; + loop { + let (all_nodes_min, diff, peer_heights) = + fetch_and_update_chain_info(step, &mut world.nodes_info, &mut nodes_chain_info).await?; + let (status, anchor_hashes) = + tips_aligned_at_min_difference(&nodes_chain_info, all_nodes_min); + + if diff <= max_diff_height + && matches!(status, AlignmentStatus::Aligned) + && all_nodes_min >= min_height.unwrap_or_default() + { + if let Some(min_height) = min_height { + info!( + target: TARGET, + "All nodes have at least {min_height} blocks, converged in {:.2?} - max diff: \ + {diff}, heights: {peer_heights:?}", + start.elapsed() + ); + } else { + info!( + target: TARGET, + "All nodes converged in {:.2?} - max diff: {diff}, heights: {peer_heights:?}", + start.elapsed() + ); + } + return Ok(()); + } + + if count.is_multiple_of(50) { + log_waiting_status( + &status, + min_height, + diff, + &peer_heights, + all_nodes_min, + &anchor_hashes, + start, + ); + } + + if start.elapsed() >= time_out { + let err = min_height.map_or_else(|| StepError::StepFail { + message: format!( + "Step `{step}` error: Nodes did not converge to {max_diff_height} blocks at in \ + {time_out_seconds} s" + ), + }, |min_height| StepError::StepFail { + message: format!( + "Step `{step}` error: Nodes did not converge to {max_diff_height} blocks at minimum height \ + {min_height} in {time_out_seconds} s" + ), + }); + return Err(err); + } + + sleep(Duration::from_millis(100)).await; + count += 1; + } +} + +pub async fn ensure_all_nodes_agree_on_lib( + world: &CucumberWorld, + step: &str, + time_out_seconds: u64, +) -> StepResult { + let start = Instant::now(); + let time_out = Duration::from_secs(time_out_seconds); + let mut count = 0usize; + + loop { + let snapshots = try_join_all(world.nodes_info.values().map(async |node| { + let consensus = node.started_node.client.consensus_info().await?; + Ok::<_, StepError>(( + node.name.clone(), + consensus.cryptarchia_info.height, + consensus.cryptarchia_info.lib.encode_hex::(), + )) + })) + .await?; + + let libs = snapshots + .iter() + .map(|(_, _, lib)| lib.clone()) + .collect::>(); + + if libs.len() == 1 { + info!( + target: TARGET, + "All nodes agree on LIB in {:.2?}", + start.elapsed() + ); + return Ok(()); + } + + if count.is_multiple_of(50) { + let status = format_lib_agreement_status(&snapshots); + + info!( + target: TARGET, + "Waiting for all nodes to agree on LIB - elapsed {:.2?}, {status}", + start.elapsed() + ); + } + + if start.elapsed() >= time_out { + let status = format_lib_agreement_status(&snapshots); + + return Err(StepError::StepFail { + message: format!( + "Step `{step}` error: Nodes did not agree on LIB in {time_out_seconds} s ({status})" + ), + }); + } + + sleep(Duration::from_millis(100)).await; + count += 1; + } +} + +fn format_lib_agreement_status(snapshots: &[(String, u64, String)]) -> String { + snapshots + .iter() + .map(|(node_name, height, lib)| format!("{node_name}: {height}/{}", truncate_hash(lib, 16))) + .collect::>() + .join(", ") +} + +pub async fn poll_all_nodes_and_update_consensus_cache( + step: &str, + nodes_info: &mut HashMap, +) -> Result<(), StepError> { + use futures_util::future::join_all; + + let nodes = nodes_info.values().collect::>(); + + // Query every node, but do not fail-fast on the first error. + let info_futures = nodes.iter().map(async |node| { + let node_name = node.name.clone(); + let result = node.started_node.client.consensus_info().await; + (node_name, result) + }); + + let results = join_all(info_futures).await; + + let mut snapshots = Vec::::new(); + let mut failed_nodes = Vec::::new(); + + for (node_name, result) in results { + match result { + Ok(info) => snapshots.push(ConsensusSnapshot { + node_name, + height: info.cryptarchia_info.height, + header_hash: info.cryptarchia_info.tip.encode_hex(), + }), + Err(e) => { + // If both `consensus_info` and `network_info` fail, assume the node is no + // longer responsive. + if let Err(e2) = poll_network_info( + nodes_info.get_mut(&node_name).expect("Failed to get node"), + &node_name, + 5, + ) + .await + { + return Err(StepError::StepFail { + message: format!( + "Step `{step}` error: {node_name} is not responsive anymore: {e} / {e2}" + ), + }); + } + warn!( + target: TARGET, + "Step `{step}` error: node `{node_name}` did not respond with consensus_info: {e}", + ); + failed_nodes.push(node_name); + } + } + } + + // If all nodes failed in this poll, surface a hard error. + // If at least one succeeded, update cache for those and let caller keep + // polling. + if snapshots.is_empty() { + let failed = if failed_nodes.is_empty() { + "none".to_owned() + } else { + failed_nodes.join(", ") + }; + return Err(StepError::StepFail { + message: format!( + "Step `{step}` error: all nodes failed to respond with consensus_info in this poll \ + (failed: [{failed}])" + ), + }); + } + + for snap in &snapshots { + let node = nodes_info + .get_mut(&snap.node_name) + .ok_or(StepError::LogicalError { + message: format!( + "Step `{step}` error: Runtime node '{}' not found in world.nodes_info", + snap.node_name + ), + })?; + node.upsert_tip(snap.height, snap.header_hash.clone()); + } + + if !failed_nodes.is_empty() { + warn!( + target: TARGET, + "Step `{step}` warning: partial consensus poll failure; updated {}/{} node(s), failed: [{}]", + snapshots.len(), + snapshots.len() + failed_nodes.len(), + failed_nodes.join(", "), + ); + } + + Ok(()) +} + +async fn poll_network_info( + node_info: &NodeInfo, + node_name: &str, + time_out_seconds: u64, +) -> Result<(), String> { + let start = TokioInstant::now(); + let time_out = Duration::from_secs(time_out_seconds); + while start.elapsed() <= time_out { + if node_info.started_node.client.network_info().await.is_ok() { + return Ok(()); + } + sleep(Duration::from_millis(250)).await; + } + Err(format!( + "Node `{node_name}` did not respond to network_info after {time_out_seconds:.2?}" + )) +} diff --git a/tests/src/cucumber/steps/nodes/operations/lifecycle.rs b/tests/src/cucumber/steps/nodes/operations/lifecycle.rs new file mode 100644 index 000000000..6d00ffa46 --- /dev/null +++ b/tests/src/cucumber/steps/nodes/operations/lifecycle.rs @@ -0,0 +1,995 @@ +use super::*; + +// Sort nodes_to_start with empty peers first to ensure standalone nodes start +// before connected nodes, then by dependency order to ensure all peers of a +// node are started before the node itself is started. If there is a circular +// dependency, return an error. +pub fn start_nodes_order_respecting_dependencies( + nodes_to_start: NodesToStartUnordered, + already_started: HashSet, +) -> Result { + let mut remaining = nodes_to_start; + // Peers that are already running (started by earlier steps) count as + // satisfied dependencies, so a node in this batch may connect to them. + let mut started = already_started; + let mut ordered = Vec::new(); + + // Step 1: Find all nodes whose peer dependencies are already satisfied + // (no in-batch peers, or all peers already running). + let nodes_without_peers: Vec = remaining + .iter() + .filter(|&(_, (_, peers))| peers.iter().all(|peer| started.contains(peer))) + .map(|(node_name, (_, _))| node_name.clone()) + .collect(); + + if nodes_without_peers.is_empty() && !remaining.is_empty() { + return Err(StepError::InvalidArgument { + message: "No nodes without peer dependencies found. Possible circular dependency." + .to_owned(), + }); + } + + // Update start list with all nodes without peers + for node_name in nodes_without_peers { + if let Some((wallet_infos, initial_peers)) = remaining.remove(&node_name) { + ordered.push((node_name.clone(), wallet_infos, initial_peers)); + started.insert(node_name); + } + } + + // Step 2: Iteratively find nodes whose peer dependencies are already included + // in the start list + while !remaining.is_empty() { + let mut made_progress = false; + + let ready_nodes: Vec = remaining + .iter() + .filter_map(|(node_name, (_, peers))| { + let all_peers_started = peers.iter().all(|peer| started.contains(peer)); + all_peers_started.then(|| node_name.clone()) + }) + .collect(); + + for node_name in ready_nodes { + if let Some((wallet_infos, mut peers)) = remaining.remove(&node_name) { + peers.sort(); + peers.dedup(); + ordered.push((node_name.clone(), wallet_infos, peers)); + started.insert(node_name); + made_progress = true; + } + } + + if !made_progress { + let remaining_nodes: Vec = remaining.keys().cloned().collect(); + return Err(StepError::InvalidArgument { + message: format!("Circular dependency detected among nodes: {remaining_nodes:?}"), + }); + } + } + + Ok(ordered) +} + +#[expect( + clippy::too_many_lines, + reason = "Covers startup, optional snapshot seeding, wallet wiring, and readiness in one path" +)] +#[expect( + clippy::cognitive_complexity, + reason = "Singular fn with multiple branches to handle different events and futures." +)] +pub async fn start_node( + world: &mut CucumberWorld, + step: &str, + node_name: &str, + wallet_start_info: &[WalletStartInfo], + initial_peers: &[String], + immediate_start: bool, + extra_user_overrides: &[ConfigOverride], +) -> StepResult { + if world.cluster.local_cluster.is_none() { + return Err(StepError::LogicalError { + message: "No local cluster available".into(), + }); + } + let mut startup_settings = + get_startup_settings(world, initial_peers, node_name).inspect_err(|e| { + warn!(target: TARGET, "Step `{step}` error: {e}"); + })?; + // Merge per-node user config overrides (e.g. a mining node's derived + // `pow.claim_address`) on top of the scenario-wide ones, upserting by path. + for extra in extra_user_overrides { + if let Some(existing) = startup_settings + .user_config_overrides + .iter_mut() + .find(|item| item.path == extra.path) + { + existing.value = extra.value.clone(); + } else { + startup_settings.user_config_overrides.push(extra.clone()); + } + } + let is_bootstrap_node = startup_settings.is_bootstrap_node; + let join_external_network = startup_settings.join_external_network; + let persist_dir = world.lifecycle.scenario_base_dir.join(node_name); + let runtime_dir_prefix = format!("{node_name}_"); + let final_dir_ignore_list = matching_child_dirs(&persist_dir, &runtime_dir_prefix); + let tokio_console_node = startup_settings.tokio_console_node.clone(); + let scenario_wallet_key_ids = world + .wallet_registry + .wallet_accounts + .values() + .map(wallet_account_key_id) + .chain( + world + .wallet_registry + .fee_state + .wallet_account + .iter() + .map(wallet_account_key_id), + ) + .collect(); + let start_options = StartNodeOptions::default() + .with_peers(startup_settings.peer_selection) + .with_persist_dir(persist_dir) + .create_patch(move |mut config: RunConfig| { + prepare_config_patch( + &mut config, + startup_settings.join_external_network, + startup_settings.deployment_settings_override.as_ref(), + &startup_settings.manual_node_config_overrides, + startup_settings.initial_peers_override.as_ref(), + &startup_settings.ibd_peers, + &startup_settings.user_config_overrides, + &startup_settings.deployment_config_overrides, + startup_settings.tokio_console_node.as_ref(), + &scenario_wallet_key_ids, + )?; + Ok(config) + }); + + let started_node = { + let cluster = world + .cluster + .local_cluster + .as_ref() + .expect("local cluster checked"); + Box::pin(cluster.start_node_with(node_name, start_options)) + .await + .inspect_err(|e| { + warn!(target: TARGET, "Step `{step}` error: {e}"); + })? + }; + + let node_final_dir = extract_child_dir_name( + &world.lifecycle.scenario_base_dir, + &runtime_dir_prefix, + &final_dir_ignore_list, + ) + .inspect_err(|e| { + warn!(target: TARGET, "Step `{step}` error: {e}"); + })?; + let node_runtime_dir = world + .lifecycle + .scenario_base_dir + .join(node_final_dir.clone()); + populate_slots_per_epoch_from_deployment(world, &node_runtime_dir)?; + let started_node_name = started_node.name.clone(); + info!( + target: TARGET, + "Starting node `{node_name}` with runtime_dir='{}'", + display_last_path_components(&node_runtime_dir, 4) + ); + + // `StartNodeOptions::with_persist_dir` currently creates a fresh runtime + // directory for each launch. Seed that runtime directory and restart once + // to effectively initialize from a named snapshot. + let restored_node_snapshot = if let Some(node_snapshot) = + world.snapshots.node_snapshot_on_startup.clone() + { + let stop_result = { + let cluster = world + .cluster + .local_cluster + .as_ref() + .expect("local cluster checked"); + cluster + .stop_node(&started_node_name) + .await + .inspect_err(|e| { + warn!(target: TARGET, "Step `{step}` error: {e}"); + }) + }; + stop_result?; + + restore_node_state_from_snapshot(&node_snapshot, &node_runtime_dir).inspect_err(|e| { + warn!(target: TARGET, "Step `{step}` error: {e}"); + })?; + populate_slots_per_epoch_from_deployment(world, &node_runtime_dir)?; + + let restart_result = { + let cluster = world + .cluster + .local_cluster + .as_ref() + .expect("local cluster checked"); + cluster + .restart_node(&started_node_name) + .await + .inspect_err(|e| { + warn!(target: TARGET, "Step `{step}` error: {e}"); + }) + }; + restart_result?; + info!( + target: TARGET, + "Node {node_name} started from snapshot {}/{}", + node_snapshot.name, node_snapshot.node + ); + Some(node_snapshot) + } else { + None + }; + + // Scrape the final node directory name to get the correct path to the node's + // YAML file for extracting the peer ID, since the actual directory name has + // a random suffix added by the deployer. + world.cluster.node_peer_ids.insert( + node_name.to_owned(), + peer_id_from_node_yaml(&node_runtime_dir.join(USER_CONFIG_FILE))?, + ); + + let wallet_info = add_wallets( + world, + step, + node_name, + wallet_start_info, + &started_node, + &node_runtime_dir, + join_external_network, + ) + .inspect_err(|e| { + warn!(target: TARGET, "Step `{step}` error: {e}"); + })?; + + world + .wallet_registry + .wallet_info + .extend(wallet_info.iter().map(|(k, v)| (k.clone(), v.clone()))); + + let client = started_node.client.clone(); + // Move `started_node` into the world's NodeInfo (no clone required) + world.nodes_info.insert( + node_name.to_owned(), + NodeInfo { + name: node_name.to_owned(), + started_node, + run_config: None, + chain_info: HashMap::default(), + wallet_info, + runtime_dir: node_runtime_dir, + immediate_start, + }, + ); + + if let Some(node_snapshot) = restored_node_snapshot + && let Some(snapshot_name) = world.snapshots.restore.extensions.clone() + { + restore_wallet_snapshot_if_present( + &snapshot_name, + &node_snapshot.node, + node_name, + &client, + world, + ) + .await + .inspect_err(|e| { + warn!(target: TARGET, "Step `{step}` error: {e}"); + })?; + } + + // All nodes are required to be network ready responsive, and bootstrap nodes + // must be `Mode::OnLine` for IBD of other peers to succeed + if !immediate_start { + let cluster = world + .cluster + .local_cluster + .as_ref() + .expect("local cluster checked"); + ensure_node_ready( + cluster, + &client, + node_name, + &started_node_name, + is_bootstrap_node, + world.startup.require_all_peers_mode_online_at_startup, + startup_settings.join_external_network, + ) + .await + .inspect_err(|e| { + warn!(target: TARGET, "Step `{step}` error: {e}"); + })?; + } + + if world.snapshots.node_snapshot_on_startup.is_some() { + match client.consensus_info().await { + Ok(info) => { + info!( + target: TARGET, + "Node `{node_name}` snapshot state - height: {}/{}, tip: {}, lib: {}", + info.cryptarchia_info.height, + info.cryptarchia_info.slot.into_inner(), + truncate_hash(&info.cryptarchia_info.tip.encode_hex::(), 16), + truncate_hash(&info.cryptarchia_info.lib.encode_hex::(), 16) + ); + } + Err(e) => { + warn!( + target: TARGET, + "Node `{node_name}` failed to fetch post-start consensus after snapshot init: {e}" + ); + } + } + } + + if let Some(tokio_console) = tokio_console_node { + check_tokio_console_port(node_name, tokio_console.port); + } + + Ok(()) +} + +fn check_tokio_console_port(node_name: &str, port: u16) { + let addr = SocketAddr::from((Ipv4Addr::LOCALHOST, port)); + + match TcpStream::connect_timeout(&addr, Duration::from_secs(2)) { + Ok(_) => info!( + target: TARGET, + "Tokio console endpoint for `{node_name}` is listening at port `{port}`, connect with \ + `tokio-console http://127.0.0.1:{port}`" + ), + Err(error) => warn!( + target: TARGET, + "Tokio console endpoint for `{node_name}` is not reachable at \ + `http://127.0.0.1:{port}`: {error}. Refer to the repo root `README.md -> Tokio task \ + profiling` for general instructions." + ), + } +} + +/// Stop a node and leave it down. +/// +/// Unlike [`restart_node`], which brings it back up and waits for readiness, +/// this leaves the node down, useful to exercise reconnect behavior while the +/// node is down. +pub async fn stop_node(world: &CucumberWorld, step: &str, node_name: &str) -> StepResult { + let cluster = world + .cluster + .local_cluster + .as_ref() + .ok_or(StepError::LogicalError { + message: "No local cluster available".into(), + })?; + let started_node_name = world + .resolve_node_runtime_name(node_name) + .inspect_err(|e| { + warn!(target: TARGET, "Step `{step}` error: {e}"); + })?; + + log_node_lifecycle_marker(world, "node_stop", node_name, "before").await; + + cluster + .stop_node(&started_node_name) + .await + .inspect_err(|e| { + warn!(target: TARGET, "Step `{step}` error: {e}"); + })?; + + log_node_lifecycle_marker(world, "node_stop", node_name, "after").await; + + info!( + target: TARGET, + "Stopped node `{node_name}` (runtime name `{started_node_name}`)" + ); + Ok(()) +} + +pub async fn restart_node(world: &CucumberWorld, step: &str, node_name: &str) -> StepResult { + let cluster = world + .cluster + .local_cluster + .as_ref() + .ok_or(StepError::LogicalError { + message: "No local cluster available".into(), + })?; + let started_node_name = world + .resolve_node_runtime_name(node_name) + .inspect_err(|e| { + warn!(target: TARGET, "Step `{step}` error: {e}"); + })?; + + log_node_lifecycle_marker(world, "node_restart", node_name, "before").await; + + cluster + .restart_node(&started_node_name) + .await + .inspect_err(|e| { + warn!(target: TARGET, "Step `{step}` error: {e}"); + })?; + + log_node_lifecycle_marker(world, "node_restart", node_name, "after").await; + let client = world.resolve_node_http_client(node_name).inspect_err(|e| { + warn!(target: TARGET, "Step `{step}` error: {e}"); + })?; + ensure_node_ready( + cluster, + &client, + node_name, + &started_node_name, + // TODO: Add `is_bootstrap_node` to world + false, + None, + world.startup.join_external_network.unwrap_or_default(), + ) + .await + .inspect_err(|e| { + warn!(target: TARGET, "Step `{step}` error: {e}"); + })?; + + info!( + target: TARGET, + "Restarted node `{node_name}` (runtime name `{started_node_name}`)" + ); + + Ok(()) +} + +fn add_wallets( + world: &CucumberWorld, + step: &str, + node_name: &str, + wallet_start_info: &[WalletStartInfo], + started_node: &StartedNode, + node_runtime_dir: &Path, + join_external_network: bool, +) -> Result { + let wallet_info = compile_wallet_in_map( + wallet_start_info, + node_name, + world, + step, + node_runtime_dir, + join_external_network, + )?; + for (wallet_name, info) in &wallet_info { + let wallet_type = match info.wallet_type.clone() { + WalletType::User { .. } => "User", + WalletType::Funding { .. } => "Funding", + }; + info!(target: TARGET, "{wallet_type} wallet `{}/{node_name}` created: {}", + wallet_name, + format!("{}wallet/{}/balance", started_node.client.base_url(), info.public_key_hex()) + ); + } + + Ok(wallet_info) +} + +struct StartupSettings { + peer_selection: PeerSelection, + ibd_peers: HashSet, + is_bootstrap_node: bool, + initial_peers_override: Option>, + join_external_network: bool, + user_config_overrides: Vec, + deployment_config_overrides: Vec, + deployment_settings_override: Option, + manual_node_config_overrides: ManualNodeConfigOverrides, + tokio_console_node: Option, +} + +fn get_startup_settings( + world: &CucumberWorld, + initial_peers: &[String], + node_name: &str, +) -> Result { + let peer_selection = if initial_peers.is_empty() { + PeerSelection::None + } else { + let named = initial_peers + .iter() + .map(|peer| world.resolve_node_runtime_name(peer)) + .collect::, StepError>>()?; + PeerSelection::Named(named) + }; + let mut ibd_peers = world.startup.ibd_peers_override.clone().unwrap_or_default(); + let populate_ibd_peers_from_initial_peers = world + .startup + .populate_ibd_peers_from_initial_peers + .unwrap_or_default(); + if populate_ibd_peers_from_initial_peers { + for peer in initial_peers { + if let Some(peer_id) = world.cluster.node_peer_ids.get(peer) { + ibd_peers.insert(*peer_id); + } + } + } + let is_bootstrap_node = initial_peers.is_empty(); + let initial_peers_override = world.startup.initial_peers_override.clone(); + let join_external_network = world.startup.join_external_network.unwrap_or_default(); + let deployment_settings_override = world + .startup + .deployment_config_override_path + .clone() + .map(|path| load_run_config(&path)) + .transpose()?; + let user_config_overrides = world.startup.user_config_overrides.clone(); + let deployment_config_overrides = world.startup.deployment_config_overrides.clone(); + let tokio_console_node = world.tokio_console_profile.node(node_name).cloned(); + + Ok(StartupSettings { + peer_selection, + ibd_peers, + is_bootstrap_node, + initial_peers_override, + join_external_network, + deployment_settings_override, + manual_node_config_overrides: world.startup.manual_node_config_overrides.clone(), + user_config_overrides, + deployment_config_overrides, + tokio_console_node, + }) +} + +#[expect(clippy::too_many_arguments, reason = "all needed")] +fn prepare_config_patch( + config: &mut RunConfig, + join_external_network: bool, + deployment_override: Option<&DeploymentSettings>, + config_overrides: &ManualNodeConfigOverrides, + initial_peers_override: Option<&Vec>, + ibd_peers: &HashSet, + user_config_overrides: &[ConfigOverride], + deployment_config_overrides: &[ConfigOverride], + tokio_console_node: Option<&TokioConsoleProfileNode>, + scenario_wallet_key_ids: &HashSet, +) -> Result<(), StepError> { + if join_external_network { + config.deployment = deployment_override + .cloned() + .unwrap_or_else(DeploymentSettings::default); + } else if let Some(deployment_override) = deployment_override { + config.deployment = deployment_override.clone(); + } + + config_overrides.apply_to(config); + + if let Some(initial_peers) = &initial_peers_override { + config + .user + .network + .backend + .initial_peers + .clone_from(initial_peers); + } + config + .user + .cryptarchia + .network + .bootstrap + .ibd + .peers + .clone_from(ibd_peers); + if let Some(node) = &tokio_console_node { + config.user.tracing.console = ConsoleLayer::Console(TokioConfig { + bind_address: IpAddr::V4(Ipv4Addr::LOCALHOST), + port: node.port, + recording_path: node + .record_raw + .then(|| PathBuf::from("tokio-console-raw.jsonl")), + }); + } + + apply_user_config_overrides(config, user_config_overrides)?; + apply_deployment_config_overrides(config, deployment_config_overrides)?; + if join_external_network { + remove_external_scenario_wallet_keys(config, scenario_wallet_key_ids); + } + Ok(()) +} + +pub(super) fn wallet_account_key_id(account: &WalletAccount) -> KeyId { + let key: Key = account.secret_key.clone().into(); + key_id_for_preload_backend(&key) +} + +fn remove_external_scenario_wallet_keys( + config: &mut RunConfig, + scenario_wallet_key_ids: &HashSet, +) { + remove_external_scenario_wallet_keys_from_maps( + &mut config.user.wallet.known_keys, + &mut config.user.kms.backend.keys, + scenario_wallet_key_ids, + ); +} + +pub(super) fn remove_external_scenario_wallet_keys_from_maps( + known_keys: &mut HashMap, + kms_keys: &mut HashMap, + scenario_wallet_key_ids: &HashSet, +) { + for key_id in scenario_wallet_key_ids { + known_keys.remove(key_id); + kms_keys.remove(key_id); + } +} + +fn load_run_config(path: &Path) -> Result { + let text = fs::read_to_string(path).map_err(|e| StepError::LogicalError { + message: format!("Failed to read '{}': {e}", path.display()), + })?; + serde_yaml::from_str::(&text).map_err(|e| StepError::LogicalError { + message: format!("Failed to parse '{}': {e}", path.display()), + }) +} + +fn populate_slots_per_epoch_from_deployment( + world: &mut CucumberWorld, + node_runtime_dir: &Path, +) -> Result<(), StepError> { + let path = node_runtime_dir.join("deployment.yaml"); + let text = fs::read_to_string(&path).map_err(|source| StepError::LogicalError { + message: format!( + "failed to read effective deployment config '{}': {source}", + path.display() + ), + })?; + let deployment = serde_yaml::from_str::(&text).map_err(|source| { + StepError::LogicalError { + message: format!( + "failed to parse effective deployment config '{}': {source}", + path.display() + ), + } + })?; + let slots_per_epoch = deployment.cryptarchia.slots_per_epoch(); + let slots_per_epoch = NonZero::new(slots_per_epoch).ok_or_else(|| StepError::LogicalError { + message: format!( + "effective deployment config '{}' has zero slots per epoch", + path.display() + ), + })?; + world.chain.slots_per_epoch = slots_per_epoch; + info!( + target: TARGET, + "Loaded effective epoch configuration from '{}': slots_per_epoch={slots_per_epoch}", + path.display() + ); + Ok(()) +} + +// Ensure this node is ready, and achieved `Mode::OnLine` if it is a bootstrap +// node. +async fn ensure_node_ready( + cluster: &LbcManualCluster, + client: &NodeHttpClient, + node_name: &str, + started_node_name: &str, + is_bootstrap_node: bool, + require_all_peers_mode_online_at_startup: Option, + join_external_network: bool, +) -> StepResult { + // General readiness check to ensure the node is responsive. + let operation = format!("node '{started_node_name}' readiness"); + track_progress(&operation, Duration::from_secs(5), async { + cluster + .wait_node_ready(started_node_name) + .await + .map_err(|source| StepError::StepFail { + message: format!( + "node '{started_node_name}' did not become ready after start: {source}" + ), + }) + }) + .await?; + + verify_reponsive_and_network_ready(client, node_name, started_node_name).await?; + + if !is_bootstrap_node && require_all_peers_mode_online_at_startup.is_none() + || join_external_network + { + return Ok(()); + } + + verify_online( + client, + node_name, + started_node_name, + require_all_peers_mode_online_at_startup, + ) + .await?; + Ok(()) +} + +async fn verify_online( + client: &NodeHttpClient, + node_name: &str, + started_node_name: &str, + time_out: Option, +) -> StepResult { + let time_out = time_out.unwrap_or_else(|| Duration::from_mins(1)); + let start = Instant::now(); + let mut count = 0usize; + loop { + let mut mode_online = false; + match client.consensus_info().await { + Ok(val) => { + if matches!(val.phase, PhaseTag::Following) { + mode_online = true; + } + } + Err(e) if start.elapsed() < time_out => { + if count.is_multiple_of(20) { + info!( + target: TARGET, + "Waiting for node `{node_name}/{started_node_name}` to be `Mode::OnLine` - \ + elapsed: {:.2?} ({e})", + start.elapsed() + ); + } + } + Err(e) => { + return Err(StepError::StepFail { + message: format!( + "Node `{node_name}/{started_node_name}` failed `Mode::OnLine` - elapsed \ + {:.2?}: {e}", + start.elapsed() + ), + }); + } + } + if mode_online { + info!( + target: TARGET, + "Node `{node_name}/{started_node_name}` achieved `Mode::OnLine` and listen \ + addresses in {:.2?}", + start.elapsed() + ); + return Ok(()); + } + sleep(Duration::from_millis(100)).await; + count += 1; + } +} + +/// Wait for all nodes to become responsive +pub async fn wait_all_nodes_responive( + cluster: &LbcManualCluster, + time_out: Duration, +) -> StepResult { + timeout(time_out, cluster.wait_network_ready()) + .await + .map_err(|_| StepError::StepFail { + message: format!("Not all nodes became responsive after {time_out:?}"), + })? + .map_err(|e| StepError::StepFail { + message: format!("Failed to check all nodes ready: {e}"), + }) +} + +async fn verify_reponsive_and_network_ready( + client: &NodeHttpClient, + node_name: &str, + started_node_name: &str, +) -> StepResult { + verify_reponsive_and_network_ready_with_timeout( + client, + node_name, + started_node_name, + Duration::from_mins(1), + ) + .await +} + +/// Wait for the node to be responsive and network ready, with a timeout. +#[expect( + clippy::cognitive_complexity, + reason = "Singular fn with multiple branches to handle different events and futures." +)] +pub async fn verify_reponsive_and_network_ready_with_timeout( + client: &NodeHttpClient, + node_name: &str, + started_node_name: &str, + time_out: Duration, +) -> StepResult { + let start = Instant::now(); + let mut count = 0usize; + let mut can_provide_consensus_info; + let mut is_network_ready; + + loop { + can_provide_consensus_info = false; + match client.consensus_info().await { + Ok(_) => { + can_provide_consensus_info = true; + } + Err(e) if start.elapsed() < time_out => { + if count.is_multiple_of(20) { + info!( + target: TARGET, + "Waiting for node `{node_name}/{started_node_name}` to be responsive - \ + elapsed: {:.2?} ({e})", + start.elapsed() + ); + } + } + Err(e) => { + return Err(StepError::StepFail { + message: format!( + "Node `{node_name}/{started_node_name}` failed to be responsive - elapsed \ + {:.2?}: {e}", + start.elapsed() + ), + }); + } + } + is_network_ready = false; + match client.network_info().await { + Ok(val) => { + is_network_ready = !val.listen_addresses.is_empty(); + } + Err(e) if start.elapsed() < time_out => { + if count.is_multiple_of(20) { + info!( + target: TARGET, + "Waiting for node `{node_name}/{started_node_name}` to be network ready - \ + elapsed: {:.2?} ({e})", + start.elapsed() + ); + } + } + Err(e) => { + return Err(StepError::StepFail { + message: format!( + "Node `{node_name}/{started_node_name}` failed to be network ready - elapsed \ + {:.2?}: {e}", + start.elapsed() + ), + }); + } + } + if can_provide_consensus_info && is_network_ready { + info!( + target: TARGET, + "Node `{node_name}/{started_node_name}` is responsive and network ready in {:.2?}", + start.elapsed() + ); + return Ok(()); + } + sleep(Duration::from_millis(100)).await; + count += 1; + } +} + +fn compile_wallet_in_map( + wallet_start_info: &[WalletStartInfo], + node_name: &str, + world: &CucumberWorld, + step: &str, + node_runtime_dir: &Path, + join_external_network: bool, +) -> Result { + let mut wallet_info: WalletInfoMap = HashMap::new(); + for wallet in wallet_start_info { + let wallet_account = match world + .wallet_registry + .wallet_accounts + .get(&wallet.account_index) + { + Some(wallet_account) => wallet_account.clone(), + None => { + if join_external_network { + WalletAccount::random() + .map_err(|source| StepError::LogicalError { + message: format!( + "Step `{step}` error: Failed to derive random wallet account for index {}: {source}", + wallet.account_index + ), + })? + } else { + WalletAccount::deterministic( + wallet.account_index as u64, + 0, + true, + ) + .map_err(|source| StepError::LogicalError { + message: format!( + "Step `{step}` error: Failed to derive deterministic wallet account for index {}: {source}", + wallet.account_index + ), + })? + } + } + }; + + wallet_info.insert( + wallet.wallet_name.clone(), + WalletInfo { + wallet_name: wallet.wallet_name.clone(), + node_name: node_name.to_owned(), + wallet_type: WalletType::User { wallet_account }, + }, + ); + } + + let node_wallet_keys = + node_wallet_keys_from_node_yaml(&node_runtime_dir.join(USER_CONFIG_FILE))?; + let user_wallets_by_pk = world + .wallet_registry + .wallet_accounts + .values() + .map(|account| (account.public_key_hex(), account.label.clone())) + .chain( + world + .wallet_registry + .fee_state + .wallet_account + .iter() + .map(|account| (account.public_key_hex(), account.label.clone())), + ) + .collect::>(); + let mut generic_key_index = 0usize; + + for node_wallet_key in node_wallet_keys { + if let Some(user_wallet_name) = user_wallets_by_pk.get(&node_wallet_key.wallet_pk) { + if node_wallet_key.role != NodeWalletKeyRole::General { + return Err(StepError::LogicalError { + message: format!( + "Scenario wallet `{user_wallet_name}` public key conflicts with the \ + {role:?} key owned by `{node_name}`", + role = node_wallet_key.role, + ), + }); + } + info!( + target: TARGET, + "Scenario wallet `{user_wallet_name}` is registered with `{node_name}`; \ + excluding its public key from node-wallet aliases" + ); + continue; + } + + let wallet_name = node_wallet_name(node_name, &node_wallet_key, &mut generic_key_index); + wallet_info.insert( + wallet_name.clone(), + WalletInfo { + wallet_name, + node_name: node_name.to_owned(), + wallet_type: WalletType::Funding { + key: node_wallet_key, + }, + }, + ); + } + + Ok(wallet_info) +} + +pub(super) fn node_wallet_name( + node_name: &str, + key: &NodeWalletKey, + generic_key_index: &mut usize, +) -> String { + let role = match key.role { + NodeWalletKeyRole::Funding => "FUNDING".to_owned(), + NodeWalletKeyRole::VoucherMaster => "VOUCHER_MASTER".to_owned(), + NodeWalletKeyRole::BlendZk => "BLEND_ZK".to_owned(), + NodeWalletKeyRole::General => { + *generic_key_index += 1; + format!("GENERAL_{}", *generic_key_index) + } + }; + format!("{node_name}_WALLET_{role}") +} diff --git a/tests/src/cucumber/steps/nodes/operations/mod.rs b/tests/src/cucumber/steps/nodes/operations/mod.rs new file mode 100644 index 000000000..0d6fa4eb1 --- /dev/null +++ b/tests/src/cucumber/steps/nodes/operations/mod.rs @@ -0,0 +1,145 @@ +use std::{ + collections::{HashMap, HashSet}, + fs, + net::{IpAddr, Ipv4Addr, SocketAddr, TcpStream}, + num::NonZero, + path::{Path, PathBuf}, + time::{Duration, Instant}, +}; + +use cucumber::gherkin::Table; +use futures::future::try_join_all; +use hex::ToHex as _; +use lb_chain_service::{ChainServiceInfo, CryptarchiaInfo, PhaseTag}; +use lb_config::kms::key_id_for_preload_backend; +use lb_core::mantle::{Utxo, ops::OpId as _, traits::GenesisTx as _}; +use lb_http_api_common::paths::CRYPTARCHIA_INFO; +use lb_key_management_system_service::{backend::preload::KeyId, keys::Key}; +use lb_libp2p::PeerId; +use lb_node::config::{ + DeploymentSettings, RunConfig, + tracing::serde::console::{Layer as ConsoleLayer, TokioConfig}, +}; +use lb_testing_framework::{ + LbcEnv, LbcManualCluster, NodeHttpClient, USER_CONFIG_FILE, configs::wallet::WalletAccount, +}; +use libp2p::Multiaddr; +use reqwest::{Client, Url}; +use testing_framework_core::scenario::{PeerSelection, StartNodeOptions, StartedNode}; +use tokio::time::{Instant as TokioInstant, sleep, timeout}; +use tracing::{info, warn}; + +use crate::cucumber::{ + error::{StepError, StepResult}, + steps::{ + TARGET, + nodes::{ + config_override::{apply_deployment_config_overrides, apply_user_config_overrides}, + diagnostics::log_node_lifecycle_marker, + snapshots::{ + reset_named_snapshot, restore_node_state_from_snapshot, + save_named_node_state_snapshot, validate_snapshot_path_component, + }, + }, + tokio_console::profile::TokioConsoleProfileNode, + }, + utils::{ + display_last_path_components, extract_child_dir_name, matching_child_dirs, + node_wallet_keys_from_node_yaml, peer_id_from_node_yaml, track_progress, truncate_hash, + }, + wallet::snapshot::{create_and_save_all_wallets_snapshot, restore_wallet_snapshot_if_present}, + world::{ + ChainInfoMap, ConfigOverride, CucumberWorld, ManualNodeConfigOverrides, NodeInfo, + NodeWalletKey, NodeWalletKeyRole, PublicCryptarchiaEndpointPeer, WalletInfo, WalletInfoMap, + WalletType, + }, +}; + +pub type NodesToStartUnordered = HashMap, Vec)>; +type NodesToStartOrdered = Vec<(String, Vec, Vec)>; + +const CHAIN_SYNC_POLL_INTERVAL: Duration = Duration::from_secs(5); +const CHAIN_SYNC_STATUS_LOG_INTERVAL: Duration = Duration::from_mins(2); + +// Returns the root directory for a named snapshot. + +enum AlignmentStatus { + MissingChainInfo, + Fork, + Aligned, +} + +#[derive(Debug, Clone)] +struct ConsensusSnapshot { + node_name: String, + height: u64, + header_hash: String, +} + +#[derive(Debug, Clone)] +struct MaybeSnapshot { + height: u64, + header_hash: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct SyncTargetStats { + lib: String, + tip: String, + slot: u64, + height: u64, +} + +#[derive(Debug, Clone)] +struct PublicPeerConsensusSnapshot { + peer_url: String, + stats: SyncTargetStats, +} + +mod consensus; +mod lifecycle; +mod resources; +mod snapshots; +mod synchronization; + +pub use consensus::{ + ensure_all_nodes_agree_on_lib, nodes_converged, poll_all_nodes_and_update_consensus_cache, +}; +pub use lifecycle::{ + restart_node, start_node, start_nodes_order_respecting_dependencies, stop_node, + verify_reponsive_and_network_ready_with_timeout, wait_all_nodes_responive, +}; +pub use resources::{ + ensure_fee_sponsorship_and_fork_groups_are_not_mixed, genesis_block_utxos, + parse_genesis_wallet_tokens_row, parse_mining_wallet_resources_table_row, + parse_wallet_resources_table_row, verify_genesis_wallet_resources_table_indexes, + verify_mining_node_wallet_resources_table_indexes, verify_node_wallet_resources_table_indexes, +}; +pub use snapshots::{ + WalletStartInfo, create_snapshot_all_nodes_with_wallet_state, + create_snapshot_node_with_wallet_state, create_snapshots_all_nodes, + get_cryptarchia_info_all_nodes, +}; +pub use synchronization::{ + fetch_public_peer_consensus, parse_url, wait_for_all_nodes_to_be_synced_to_chain, +}; + +#[cfg(test)] +mod tests; + +#[derive(Debug, Clone)] +struct MajorityPublicSyncTarget { + peer_urls: Vec, + stats: SyncTargetStats, +} + +impl SyncTargetStats { + fn from_cryptarchia_info(info: &CryptarchiaInfo) -> Self { + Self { + lib: info.lib.encode_hex::(), + tip: info.tip.encode_hex::(), + slot: info.slot.into_inner(), + height: info.height, + } + } +} diff --git a/tests/src/cucumber/steps/nodes/operations/resources.rs b/tests/src/cucumber/steps/nodes/operations/resources.rs new file mode 100644 index 000000000..e26e39dea --- /dev/null +++ b/tests/src/cucumber/steps/nodes/operations/resources.rs @@ -0,0 +1,313 @@ +use super::*; + +#[must_use] +pub fn genesis_block_utxos(genesis_tx: &lb_core::mantle::transactions::GenesisTx) -> Vec { + let transfer_op = genesis_tx.genesis_transfer().clone(); + let transfer_id = transfer_op.op_id(); + + transfer_op + .outputs + .iter() + .enumerate() + .map(|(idx, note)| Utxo::new(transfer_id, idx, *note)) + .collect() +} + +const ACCOUNT_INDEX: &str = "account_index"; +const ACCOUNT_INDEX_IDX_T1: usize = 0; +const TOKEN_COUNT: &str = "token_count"; +const TOKEN_COUNT_IDX: usize = 1; +const TOKEN_AMOUNT: &str = "token_amount"; +const TOKEN_AMOUNT_IDX: usize = 2; + +pub fn verify_genesis_wallet_resources_table_indexes( + table: &Table, + step: &str, +) -> Result<(), StepError> { + if table.rows.is_empty() + || table.rows[0].len() != 3 + || table.rows[0][ACCOUNT_INDEX_IDX_T1] != ACCOUNT_INDEX + || table.rows[0][TOKEN_COUNT_IDX] != TOKEN_COUNT + || table.rows[0][TOKEN_AMOUNT_IDX] != TOKEN_AMOUNT + { + return Err(StepError::InvalidArgument { + message: format!( + "Step `{step}` error: Wallet resources table must have a header row with columns: \ + {ACCOUNT_INDEX}, {TOKEN_COUNT}, {TOKEN_AMOUNT}" + ), + }); + } + // All wallet account indexes must be unique + let wallet_accounts: HashSet<_> = table + .rows + .iter() + .map(|row| &row[ACCOUNT_INDEX_IDX_T1]) + .collect(); + if wallet_accounts.len() != table.rows.len() { + return Err(StepError::InvalidArgument { + message: format!( + "Step `{step}` error: Duplicate {ACCOUNT_INDEX} indexes found in the table" + ), + }); + } + + Ok(()) +} + +pub fn parse_genesis_wallet_tokens_row( + step: &str, + row: &[String], +) -> Result<(usize, usize, u64), StepError> { + let account_index = + row[ACCOUNT_INDEX_IDX_T1] + .parse::() + .map_err(|_| StepError::InvalidArgument { + message: format!("Step `{step}` error: {ACCOUNT_INDEX} must be a valid number"), + })?; + let token_count = + row[TOKEN_COUNT_IDX] + .parse::() + .map_err(|_| StepError::InvalidArgument { + message: format!("Step `{step}` error: {TOKEN_COUNT} must be a valid number"), + })?; + let token_amount = + row[TOKEN_AMOUNT_IDX] + .parse::() + .map_err(|_| StepError::InvalidArgument { + message: format!("Step `{step}` error: {TOKEN_AMOUNT} must be a valid number"), + })?; + Ok((account_index, token_count, token_amount)) +} + +const NODE_NAME: &str = "node_name"; +const NODE_NAME_IDX: usize = 0; +const ACCOUNT_INDEX_IDX_T2: usize = 1; +const WALLET_NAME: &str = "wallet_name"; +const WALLET_NAME_IDX: usize = 2; +const CONNECTED_TO: &str = "connected_to"; +const CONNECTED_TO_IDX: usize = 3; + +// Mining-node wallet-resources table adds an `is_mining_wallet` column between +// `wallet_name` and `connected_to`. +const IS_MINING_WALLET: &str = "is_mining_wallet"; +const IS_MINING_WALLET_IDX: usize = 3; +const MINING_CONNECTED_TO_IDX: usize = 4; + +pub fn verify_node_wallet_resources_table_indexes( + table: &Table, + step: &str, +) -> Result<(), StepError> { + if table.rows.is_empty() + || table.rows[0].len() != 4 + || table.rows[0][NODE_NAME_IDX] != NODE_NAME + || table.rows[0][ACCOUNT_INDEX_IDX_T2] != ACCOUNT_INDEX + || table.rows[0][WALLET_NAME_IDX] != WALLET_NAME + || table.rows[0][CONNECTED_TO_IDX] != CONNECTED_TO + { + return Err(StepError::InvalidArgument { + message: format!( + "Step `{step}` error: Wallet resources table must have a header row with columns: {NODE_NAME}, {ACCOUNT_INDEX}, {WALLET_NAME}, {CONNECTED_TO}" + ), + }); + } + // All wallet indexes must be unique + let account_indexes: HashSet<_> = table + .rows + .iter() + .map(|row| &row[ACCOUNT_INDEX_IDX_T2]) + .collect(); + if account_indexes.len() != table.rows.len() { + return Err(StepError::InvalidArgument { + message: format!( + "Step `{step}` error: Duplicate {ACCOUNT_INDEX} indexes found in the table" + ), + }); + } + // All wallet names must be unique + let wallet_names: HashSet<_> = table.rows.iter().map(|row| &row[WALLET_NAME_IDX]).collect(); + if wallet_names.len() != table.rows.len() { + return Err(StepError::InvalidArgument { + message: format!( + "Step `{step}` error: Duplicate {WALLET_NAME} indexes found in the table" + ), + }); + } + // node_name and connected_to must be different + for row in table.rows.iter().skip(1) { + let node_name = row[NODE_NAME_IDX].trim(); + let connected_to = row + .get(CONNECTED_TO_IDX) + .map(|s| s.trim()) + .filter(|s| !s.is_empty()); + if let Some(peer) = connected_to + && peer == node_name + { + return Err(StepError::InvalidArgument { + message: format!( + "Step `{step}` error: {NODE_NAME} and {CONNECTED_TO} cannot be the same" + ), + }); + } + } + + Ok(()) +} + +pub fn parse_wallet_resources_table_row( + step: &str, + row: &[String], +) -> Result<(String, WalletStartInfo, Option), StepError> { + let node_name = row[NODE_NAME_IDX].trim().to_owned(); + if node_name.is_empty() { + return Err(StepError::InvalidArgument { + message: format!("Step `{step}` error: {NODE_NAME} cannot be empty"), + }); + } + let account_index = row[ACCOUNT_INDEX_IDX_T2] + .trim() + .parse::() + .map_err(|_| StepError::InvalidArgument { + message: format!( + "Step `{step}` error: {ACCOUNT_INDEX} '{}' must be a valid number", + row[ACCOUNT_INDEX_IDX_T2] + ), + })?; + let wallet_name = row[WALLET_NAME_IDX].trim().to_owned(); + if wallet_name.is_empty() { + return Err(StepError::InvalidArgument { + message: format!("Step `{step}` error: {WALLET_NAME} cannot be empty"), + }); + } + let connected_to = row + .get(CONNECTED_TO_IDX) + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .map(str::to_owned); + + Ok(( + node_name, + WalletStartInfo { + wallet_name, + account_index, + }, + connected_to, + )) +} + +pub fn verify_mining_node_wallet_resources_table_indexes( + table: &Table, + step: &str, +) -> Result<(), StepError> { + if table.rows.is_empty() + || table.rows[0].len() != 5 + || table.rows[0][NODE_NAME_IDX] != NODE_NAME + || table.rows[0][ACCOUNT_INDEX_IDX_T2] != ACCOUNT_INDEX + || table.rows[0][WALLET_NAME_IDX] != WALLET_NAME + || table.rows[0][IS_MINING_WALLET_IDX] != IS_MINING_WALLET + || table.rows[0][MINING_CONNECTED_TO_IDX] != CONNECTED_TO + { + return Err(StepError::InvalidArgument { + message: format!( + "Step `{step}` error: Mining wallet resources table must have a header row with columns: {NODE_NAME}, {ACCOUNT_INDEX}, {WALLET_NAME}, {IS_MINING_WALLET}, {CONNECTED_TO}" + ), + }); + } + // All wallet indexes must be unique. + let account_indexes: HashSet<_> = table + .rows + .iter() + .map(|row| &row[ACCOUNT_INDEX_IDX_T2]) + .collect(); + if account_indexes.len() != table.rows.len() { + return Err(StepError::InvalidArgument { + message: format!( + "Step `{step}` error: Duplicate {ACCOUNT_INDEX} indexes found in the table" + ), + }); + } + // All wallet names must be unique. + let wallet_names: HashSet<_> = table.rows.iter().map(|row| &row[WALLET_NAME_IDX]).collect(); + if wallet_names.len() != table.rows.len() { + return Err(StepError::InvalidArgument { + message: format!( + "Step `{step}` error: Duplicate {WALLET_NAME} indexes found in the table" + ), + }); + } + + Ok(()) +} + +pub fn parse_mining_wallet_resources_table_row( + step: &str, + row: &[String], +) -> Result<(String, WalletStartInfo, bool, Option), StepError> { + let node_name = row[NODE_NAME_IDX].trim().to_owned(); + if node_name.is_empty() { + return Err(StepError::InvalidArgument { + message: format!("Step `{step}` error: {NODE_NAME} cannot be empty"), + }); + } + let account_index = row[ACCOUNT_INDEX_IDX_T2] + .trim() + .parse::() + .map_err(|_| StepError::InvalidArgument { + message: format!( + "Step `{step}` error: {ACCOUNT_INDEX} '{}' must be a valid number", + row[ACCOUNT_INDEX_IDX_T2] + ), + })?; + let wallet_name = row[WALLET_NAME_IDX].trim().to_owned(); + if wallet_name.is_empty() { + return Err(StepError::InvalidArgument { + message: format!("Step `{step}` error: {WALLET_NAME} cannot be empty"), + }); + } + let is_mining_wallet = match row[IS_MINING_WALLET_IDX].trim() { + "true" => true, + "false" => false, + other => { + return Err(StepError::InvalidArgument { + message: format!( + "Step `{step}` error: {IS_MINING_WALLET} '{other}' must be 'true' or 'false'" + ), + }); + } + }; + let connected_to = row + .get(MINING_CONNECTED_TO_IDX) + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .map(str::to_owned); + + Ok(( + node_name, + WalletStartInfo { + wallet_name, + account_index, + }, + is_mining_wallet, + connected_to, + )) +} + +pub fn ensure_fee_sponsorship_and_fork_groups_are_not_mixed( + world: &CucumberWorld, + step_value: &str, +) -> StepResult { + if world + .wallet_registry + .fee_state + .sponsored_genesis_account + .is_some() + && !world.fork_groups.groups().is_empty() + { + return Err(StepError::InvalidArgument { + message: format!( + "Step `{step_value}` error: sponsored fee accounts cannot be combined with distinct node groups in the same scenario" + ), + }); + } + + Ok(()) +} diff --git a/tests/src/cucumber/steps/nodes/operations/snapshots.rs b/tests/src/cucumber/steps/nodes/operations/snapshots.rs new file mode 100644 index 000000000..4cb8ef551 --- /dev/null +++ b/tests/src/cucumber/steps/nodes/operations/snapshots.rs @@ -0,0 +1,132 @@ +use super::*; + +/// This struct represents the wallet resources to be associated with a node at +/// startup. +pub struct WalletStartInfo { + // Logical name of the wallet resource, used for referencing in steps. + pub wallet_name: String, + // The account index in the genesis tokens that this resource corresponds to. + pub account_index: usize, +} + +/// Saves the current node state of all nodes into a named snapshot location for +/// later use. +pub fn create_snapshots_all_nodes( + world: &CucumberWorld, + snapshot_name: &str, +) -> Result<(), StepError> { + validate_snapshot_path_component(snapshot_name, "Snapshot name")?; + reset_named_snapshot(snapshot_name)?; + + let runtime_dir_by_node_name: Vec<(String, PathBuf)> = world + .nodes_info + .iter() + .map(|(node_name, info)| (node_name.clone(), info.runtime_dir.clone())) + .collect(); + + for (node_name, node_runtime_dir) in &runtime_dir_by_node_name { + save_named_node_state_snapshot(snapshot_name, node_name, node_runtime_dir)?; + info!( + target: TARGET, + "Saved snapshot `{snapshot_name}` for node `{node_name}`", + ); + } + Ok(()) +} + +pub async fn create_snapshot_all_nodes_with_wallet_state( + world: &mut CucumberWorld, + snapshot_name: &str, +) -> StepResult { + if world.nodes_info.is_empty() { + return Err(StepError::InvalidArgument { + message: "cannot create snapshot: no running nodes".to_owned(), + }); + } + + create_and_save_all_wallets_snapshot(snapshot_name, world).await?; + create_snapshots_all_nodes(world, snapshot_name) +} + +pub async fn create_snapshot_node_with_wallet_state( + world: &mut CucumberWorld, + snapshot_name: &str, + node_name: &str, +) -> StepResult { + if world.nodes_info.is_empty() { + return Err(StepError::InvalidArgument { + message: "cannot create snapshot: no running nodes".to_owned(), + }); + } + + if let Some(runtime_dir) = world + .nodes_info + .get(node_name) + .map(|info| info.runtime_dir.clone()) + { + reset_named_snapshot(snapshot_name)?; + create_and_save_all_wallets_snapshot(snapshot_name, world).await?; + save_named_node_state_snapshot(snapshot_name, node_name, &runtime_dir)?; + info!( + target: TARGET, + "Saved snapshot `{snapshot_name}` for node {}", + runtime_dir.display() + ); + Ok(()) + } else { + Err(StepError::InvalidArgument { + message: format!("Node {node_name} does not exist"), + }) + } +} + +/// Fetches and logs the consensus info of all nodes, for debugging purposes. +/// Does not require the nodes to be aligned or have any specific state, and is +/// resilient to some nodes being offline or unresponsive. +#[expect( + clippy::cognitive_complexity, + reason = "Singular fn with multiple branches to handle different events and futures." +)] +pub async fn get_cryptarchia_info_all_nodes(world: &CucumberWorld, step: &str) { + let mut node_names = world.nodes_info.keys().cloned().collect::>(); + node_names.sort(); + + if node_names.is_empty() { + warn!( + target: TARGET, + "Step `{step}` no nodes found for CRYPTARCHIA_INFO_ALL_NODES" + ); + return; + } + + for node_name in node_names { + let Some(node_info) = world.nodes_info.get(&node_name) else { + continue; + }; + match node_info.started_node.client.consensus_info().await { + Ok(consensus) => { + let mode = if matches!(consensus.phase, PhaseTag::Following) { + "Online" + } else { + "Bootstrapping" + }; + info!( + target: TARGET, + "cryptarchia/info - '{}', '{}', {}/{}, tip '{} ...', lib '{} ...'", + node_name, + mode, + consensus.cryptarchia_info.height, + consensus.cryptarchia_info.slot.into_inner(), + truncate_hash(&consensus.cryptarchia_info.tip.encode_hex::(), 16), + truncate_hash(&consensus.cryptarchia_info.lib.encode_hex::(), 16), + ); + } + Err(e) => { + warn!( + target: TARGET, + "Step `{step}` CRYPTARCHIA_INFO failed for node `{node_name}`: {e}", + ); + } + } + } +} diff --git a/tests/src/cucumber/steps/nodes/operations/synchronization.rs b/tests/src/cucumber/steps/nodes/operations/synchronization.rs new file mode 100644 index 000000000..ca2ddded4 --- /dev/null +++ b/tests/src/cucumber/steps/nodes/operations/synchronization.rs @@ -0,0 +1,244 @@ +use super::{ + CHAIN_SYNC_POLL_INTERVAL, CHAIN_SYNC_STATUS_LOG_INTERVAL, CRYPTARCHIA_INFO, ChainServiceInfo, + Client, CryptarchiaInfo, CucumberWorld, Duration, HashMap, Instant, MajorityPublicSyncTarget, + PublicCryptarchiaEndpointPeer, PublicPeerConsensusSnapshot, StepError, StepResult, + SyncTargetStats, TARGET, Url, get_cryptarchia_info_all_nodes, info, sleep, truncate_hash, warn, +}; + +pub async fn wait_for_all_nodes_to_be_synced_to_chain( + world: &mut CucumberWorld, + step: &str, +) -> StepResult { + let public_cryptarchia_endpoint_peers = world + .startup + .public_cryptarchia_endpoint_peers + .clone() + .unwrap_or_default(); + if public_cryptarchia_endpoint_peers.is_empty() { + return Err(StepError::InvalidArgument { + message: format!( + "Step `{step}` error: no public cryptarchia endpoint peers configured" + ), + }); + } + if world.nodes_info.is_empty() { + return Err(StepError::InvalidArgument { + message: format!("Step `{step}` error: no local nodes are available to check sync"), + }); + } + + let client = Client::new(); + let start = Instant::now(); + let mut last_status_log_at = None; + + loop { + let public_snapshots = + fetch_public_peer_consensus_snapshots(&client, &public_cryptarchia_endpoint_peers) + .await; + let majority_target = select_majority_public_sync_target(&public_snapshots); + + if let Some(target) = majority_target.as_ref() + && all_local_nodes_match_sync_target(world, target).await + { + get_cryptarchia_info_all_nodes(world, step).await; + info!( + target: TARGET, + "All nodes synced to the chain in {:.2?}", + start.elapsed() + ); + + catch_up_known_wallet_tracking_after_chain_sync(world, step).await?; + + return Ok(()); + } + + if should_log_chain_sync_status(last_status_log_at) { + log_chain_sync_progress( + start.elapsed(), + public_cryptarchia_endpoint_peers.len(), + &public_snapshots, + majority_target.as_ref(), + ); + get_cryptarchia_info_all_nodes(world, step).await; + last_status_log_at = Some(Instant::now()); + } + + sleep(CHAIN_SYNC_POLL_INTERVAL).await; + } +} + +async fn catch_up_known_wallet_tracking_after_chain_sync( + world: &mut CucumberWorld, + step: &str, +) -> StepResult { + if world.wallet_registry.wallet_info.is_empty() { + return Ok(()); + } + + let started_at = Instant::now(); + // The wallet scanner timeout is moderate here because the contract is that the + // majority nodes have been synced prior just to this step. + world + .wait_for_wallet_scanner_catch_up(Duration::from_secs(30)) + .await?; + + info!( + target: TARGET, + "Wallet scanner caught up after chain sync for step `{step}` in {:.2?}", + started_at.elapsed() + ); + + Ok(()) +} + +pub fn parse_url(raw: &str) -> Result { + let mut trimmed = raw.trim(); + trimmed = trimmed.trim_end_matches('/'); + if trimmed.is_empty() { + return Err("url cannot be empty".to_owned()); + } + + Url::parse(trimmed).map_err(|e| format!("invalid url '{trimmed}': {e}"))?; + + Ok(trimmed.to_owned()) +} + +async fn fetch_public_peer_consensus_snapshots( + client: &Client, + peers: &[PublicCryptarchiaEndpointPeer], +) -> Vec { + let mut snapshots = Vec::new(); + + for peer in peers { + match fetch_public_peer_consensus(client, peer).await { + Ok(info) => snapshots.push(PublicPeerConsensusSnapshot { + peer_url: peer.base_url.clone(), + stats: SyncTargetStats::from_cryptarchia_info(&info), + }), + Err(e) => warn!( + target: TARGET, + "Failed to fetch public cryptarchia info from '{}': {e}", + peer.base_url + ), + } + } + + snapshots +} + +/// Fetch the current consensus info from a public cryptarchia endpoint peer. +/// Returns an error if the request fails or the response is invalid. +pub async fn fetch_public_peer_consensus( + client: &Client, + peer: &PublicCryptarchiaEndpointPeer, +) -> Result { + let request_url = Url::parse(&format!( + "{peer_url}/{path}", + peer_url = peer.base_url.as_str(), + path = CRYPTARCHIA_INFO.trim_start_matches('/') + )) + .map_err(|e| StepError::InvalidArgument { + message: format!( + "Invalid public cryptarchia info URL for '{}': {e}", + peer.base_url.as_str() + ), + })?; + + Ok(client + .get(request_url) + .basic_auth(&peer.username, Some(&peer.password)) + .send() + .await? + .error_for_status()? + .json::() + .await + .map_err(StepError::from)? + .cryptarchia_info) +} + +fn select_majority_public_sync_target( + snapshots: &[PublicPeerConsensusSnapshot], +) -> Option { + let mut groups = HashMap::>::new(); + for snapshot in snapshots { + groups + .entry(snapshot.stats.clone()) + .or_default() + .push(snapshot.peer_url.clone()); + } + + let best = groups + .into_iter() + .max_by(|(left_stats, left_peers), (right_stats, right_peers)| { + left_peers + .len() + .cmp(&right_peers.len()) + .then_with(|| left_stats.height.cmp(&right_stats.height)) + .then_with(|| left_stats.slot.cmp(&right_stats.slot)) + }) + .map(|(stats, peer_urls)| MajorityPublicSyncTarget { peer_urls, stats })?; + + if best.peer_urls.len() * 2 <= snapshots.len() { + return None; + } + + Some(best) +} + +async fn all_local_nodes_match_sync_target( + world: &CucumberWorld, + target: &MajorityPublicSyncTarget, +) -> bool { + let mut node_names = world.nodes_info.keys().cloned().collect::>(); + node_names.sort(); + + for node_name in node_names { + let Some(node_info) = world.nodes_info.get(&node_name) else { + return false; + }; + + let Ok(consensus) = node_info.started_node.client.consensus_info().await else { + return false; + }; + if SyncTargetStats::from_cryptarchia_info(&consensus.cryptarchia_info) != target.stats { + return false; + } + } + + true +} + +fn should_log_chain_sync_status(last_status_log_at: Option) -> bool { + last_status_log_at.is_none_or(|last| last.elapsed() >= CHAIN_SYNC_STATUS_LOG_INTERVAL) +} + +fn log_chain_sync_progress( + elapsed: Duration, + total_public_peers: usize, + public_snapshots: &[PublicPeerConsensusSnapshot], + majority_target: Option<&MajorityPublicSyncTarget>, +) { + if let Some(target) = majority_target { + info!( + target: TARGET, + "Waiting to be synced - elapsed {:.2?}, height {}/{}, public peers {}/{}, majority {}/{}, tip '{} ...', lib '{} ...'", + elapsed, + target.stats.height, + target.stats.slot, + public_snapshots.len(), + total_public_peers, + target.peer_urls.len(), + public_snapshots.len(), + truncate_hash(&target.stats.tip, 16), + truncate_hash(&target.stats.lib, 16), + ); + } else { + info!( + target: TARGET, + "Waiting to be synced - elapsed {:.2?}, no majority public peer consensus ({}/{} reachable)", + elapsed, + public_snapshots.len(), + total_public_peers, + ); + } +} diff --git a/tests/src/cucumber/steps/nodes/operations/tests.rs b/tests/src/cucumber/steps/nodes/operations/tests.rs new file mode 100644 index 000000000..1750f66c3 --- /dev/null +++ b/tests/src/cucumber/steps/nodes/operations/tests.rs @@ -0,0 +1,178 @@ +#[cfg(test)] +mod wallet_name_tests { + use super::{NodeWalletKey, NodeWalletKeyRole, node_wallet_name}; + + fn node_wallet_key(role: NodeWalletKeyRole) -> NodeWalletKey { + NodeWalletKey { + wallet_pk: "00".repeat(32), + role, + } + } + + #[test] + fn node_wallet_names_use_semantic_roles_before_numbered_fallbacks() { + let mut generic_key_index = 0; + + assert_eq!( + node_wallet_name( + "NODE_1", + &node_wallet_key(NodeWalletKeyRole::Funding), + &mut generic_key_index + ), + "NODE_1_WALLET_FUNDING" + ); + assert_eq!( + node_wallet_name( + "NODE_1", + &node_wallet_key(NodeWalletKeyRole::VoucherMaster), + &mut generic_key_index + ), + "NODE_1_WALLET_VOUCHER_MASTER" + ); + assert_eq!( + node_wallet_name( + "NODE_1", + &node_wallet_key(NodeWalletKeyRole::BlendZk), + &mut generic_key_index + ), + "NODE_1_WALLET_BLEND_ZK" + ); + assert_eq!( + node_wallet_name( + "NODE_1", + &node_wallet_key(NodeWalletKeyRole::General), + &mut generic_key_index + ), + "NODE_1_WALLET_GENERAL_1" + ); + assert_eq!( + node_wallet_name( + "NODE_1", + &node_wallet_key(NodeWalletKeyRole::General), + &mut generic_key_index + ), + "NODE_1_WALLET_GENERAL_2" + ); + } + + #[test] + fn semantic_wallet_names_do_not_consume_general_indexes() { + let mut generic_key_index = 0; + + assert_eq!( + node_wallet_name( + "NODE_1", + &node_wallet_key(NodeWalletKeyRole::Funding), + &mut generic_key_index + ), + "NODE_1_WALLET_FUNDING" + ); + assert_eq!(generic_key_index, 0); + } +} + +#[cfg(test)] +mod scenario_wallet_key_tests { + use lb_key_management_system_service::keys::{Ed25519Key, secured_key::SecuredKey as _}; + + use super::*; + + #[test] + fn removes_only_external_scenario_wallet_keys() { + let mut kms_keys = HashMap::new(); + let mut known_keys = HashMap::new(); + let scenario_accounts = [ + WalletAccount::deterministic(100, 0, true).expect("account"), + WalletAccount::deterministic(101, 0, true).expect("account"), + ]; + let sponsored_fee_account = + WalletAccount::deterministic(103, 0, true).expect("fee account"); + let scenario_key_ids = scenario_accounts + .iter() + .map(wallet_account_key_id) + .chain(std::iter::once(wallet_account_key_id( + &sponsored_fee_account, + ))) + .collect::>(); + + for account in scenario_accounts + .iter() + .chain(std::iter::once(&sponsored_fee_account)) + { + let key: Key = account.secret_key.clone().into(); + let key_id = key_id_for_preload_backend(&key); + kms_keys.insert(key_id.clone(), key); + known_keys.insert(key_id, account.secret_key.as_public_key()); + } + + let unrelated = WalletAccount::deterministic(102, 0, true).expect("account"); + let unrelated_key: Key = unrelated.secret_key.clone().into(); + let unrelated_key_id = key_id_for_preload_backend(&unrelated_key); + kms_keys.insert(unrelated_key_id.clone(), unrelated_key); + known_keys.insert( + unrelated_key_id.clone(), + unrelated.secret_key.as_public_key(), + ); + + let ed25519_key: Key = Ed25519Key::from_bytes(&[9; 32]).into(); + let ed25519_key_id = key_id_for_preload_backend(&ed25519_key); + kms_keys.insert(ed25519_key_id.clone(), ed25519_key); + let voucher_master = WalletAccount::deterministic(104, 0, true).expect("account"); + let voucher_key: Key = voucher_master.secret_key.clone().into(); + let voucher_master_key_id = key_id_for_preload_backend(&voucher_key); + kms_keys.insert(voucher_master_key_id.clone(), voucher_key); + known_keys.insert( + voucher_master_key_id.clone(), + voucher_master.secret_key.as_public_key(), + ); + + remove_external_scenario_wallet_keys_from_maps( + &mut known_keys, + &mut kms_keys, + &scenario_key_ids, + ); + remove_external_scenario_wallet_keys_from_maps( + &mut known_keys, + &mut kms_keys, + &scenario_key_ids, + ); + + for key_id in &scenario_key_ids { + assert!(!kms_keys.contains_key(key_id)); + assert!(!known_keys.contains_key(key_id)); + } + assert!(kms_keys.contains_key(&unrelated_key_id)); + assert!(known_keys.contains_key(&unrelated_key_id)); + assert!(kms_keys.contains_key(&ed25519_key_id)); + assert!(kms_keys.contains_key(&voucher_master_key_id)); + assert!(known_keys.contains_key(&voucher_master_key_id)); + } + + #[test] + fn empty_scenario_wallet_set_changes_nothing() { + let mut kms_keys = HashMap::new(); + let mut known_keys = HashMap::new(); + let account = WalletAccount::deterministic(105, 0, true).expect("account"); + let key: Key = account.secret_key.clone().into(); + let key_id = key_id_for_preload_backend(&key); + kms_keys.insert(key_id.clone(), key); + known_keys.insert(key_id, account.secret_key.as_public_key()); + let before_kms = kms_keys.clone(); + let before_known_keys = known_keys.clone(); + + remove_external_scenario_wallet_keys_from_maps( + &mut known_keys, + &mut kms_keys, + &HashSet::new(), + ); + + assert_eq!(kms_keys, before_kms); + assert_eq!(known_keys, before_known_keys); + } +} +use super::{ + lifecycle::{ + node_wallet_name, remove_external_scenario_wallet_keys_from_maps, wallet_account_key_id, + }, + *, +}; diff --git a/tests/src/cucumber/steps/manual_nodes/parameters.rs b/tests/src/cucumber/steps/nodes/parameters.rs similarity index 100% rename from tests/src/cucumber/steps/manual_nodes/parameters.rs rename to tests/src/cucumber/steps/nodes/parameters.rs diff --git a/tests/src/cucumber/steps/manual_nodes/snapshots.rs b/tests/src/cucumber/steps/nodes/snapshots.rs similarity index 100% rename from tests/src/cucumber/steps/manual_nodes/snapshots.rs rename to tests/src/cucumber/steps/nodes/snapshots.rs diff --git a/tests/src/cucumber/steps/nodes/steps/blend.rs b/tests/src/cucumber/steps/nodes/steps/blend.rs new file mode 100644 index 000000000..cab5907a6 --- /dev/null +++ b/tests/src/cucumber/steps/nodes/steps/blend.rs @@ -0,0 +1,416 @@ +use super::*; + +#[when( + expr = "all nodes have at least {int} blocks and converged to within {int} blocks in {int} seconds" +)] +#[then( + expr = "all nodes have at least {int} blocks and converged to within {int} blocks in {int} seconds" +)] +async fn step_all_nodes_reached_min_height_and_converged( + world: &mut CucumberWorld, + step: &Step, + min_height: u64, + max_diff_height: u64, + time_out_seconds: u64, +) -> StepResult { + nodes_converged( + world, + &step.value, + Some(min_height), + max_diff_height, + time_out_seconds, + ) + .await +} + +#[when(expr = "all nodes agree on LIB in {int} seconds")] +#[then(expr = "all nodes agree on LIB in {int} seconds")] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require the world as the first `&mut` argument" +)] +async fn step_all_nodes_agree_on_lib( + world: &mut CucumberWorld, + step: &Step, + time_out_seconds: u64, +) -> StepResult { + ensure_all_nodes_agree_on_lib(world, &step.value, time_out_seconds).await +} + +#[when("I wait for all nodes to be synced to the chain")] +#[then("I wait for all nodes to be synced to the chain")] +async fn step_wait_for_all_nodes_to_be_synced_to_the_chain( + world: &mut CucumberWorld, + step: &Step, +) -> StepResult { + wait_for_all_nodes_to_be_synced_to_chain(world, &step.value).await +} + +#[when("I query cryptarchia info for all nodes")] +#[then("I query cryptarchia info for all nodes")] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require the world as the first `&mut` argument" +)] +async fn step_query_cryptarchia_info_all_nodes(world: &mut CucumberWorld, step: &Step) { + get_cryptarchia_info_all_nodes(world, &step.value).await; +} + +#[then(expr = "I stop all nodes")] +async fn step_stop_all_nodes(world: &mut CucumberWorld) -> StepResult { + let runtime_dir_by_node_name: Vec<(String, String)> = world + .nodes_info + .iter() + .map(|(node_name, info)| (node_name.clone(), info.started_node.name.clone())) + .collect(); + + if world.snapshots.save.extensions.is_some() { + prepare_all_wallets_snapshot(world).await?; + } + + world.reset_wallet_scanner_after_current_iteration().await; + world.zone.clear(); + stop_active_manual_cluster(world)?; + + if let Some(snapshot_name) = world.snapshots.save.node_state.take() { + create_snapshots_all_nodes(world, &snapshot_name)?; + } + + if let Some(snapshot_name) = world.snapshots.save.extensions.take() { + save_prepared_all_wallets_snapshot(&snapshot_name, world)?; + } + + for (node_name, _) in &runtime_dir_by_node_name { + info!(target: TARGET, "Stopping node '{node_name}'"); + } + world.nodes_info.clear(); + + Ok(()) +} + +#[when( + expr = "I send {int} transactions of {int} LGO each from wallet {string} to blend core zk key of node {string}" +)] +async fn step_send_multiple_transactions_to_blend_core_zk_key( + world: &mut CucumberWorld, + step: &Step, + number_of_transactions: usize, + output_value: u64, + sender_wallet_name: String, + receiver_node_name: String, +) -> StepResult { + let receiver_blend_zk_pk = blend_zk_pk_for_node(world, &receiver_node_name)?; + let sender_node_name = world.resolve_wallet(&sender_wallet_name)?.node_name; + let sender_node_client = world + .nodes_info + .get(&sender_node_name) + .ok_or_else(|| StepError::LogicalError { + message: format!("Node '{sender_node_name}' not found in world state"), + })? + .started_node + .client + .clone(); + + let mut available_utxos = WalletUtxos::new(); + let best_node_info = wait_wallet_send_ready( + world, + &step.value, + &sender_wallet_name, + 180, + number_of_transactions as u64 * output_value, + WalletSendReadiness::TotalValueOnly, + &mut available_utxos, + &HashSet::new(), + ) + .await?; + + for _ in 0..number_of_transactions { + let tx_hashes = create_and_submit_transaction_hashes_with_utxo_cache( + world, + &step.value, + &sender_wallet_name, + &[(receiver_blend_zk_pk, output_value)], + Some(&best_node_info), + Some(&mut available_utxos), + ) + .await + .inspect_err(|error| { + warn!(target: TARGET, "Step `{}` error: {error}", step.value); + })?; + + wait_for_transactions_inclusion(&sender_node_client, &tx_hashes, Duration::from_mins(2)) + .await + .inspect_err(|error| { + warn!(target: TARGET, "Step `{}` error: {error}", step.value); + })?; + + info!( + target: TARGET, + "Sent and included normal transaction from `{sender_wallet_name}` to blend zk key of {receiver_node_name}, value: {output_value}, tx count: {}", + tx_hashes.len(), + ); + } + + Ok(()) +} + +fn blend_zk_pk_for_node(world: &CucumberWorld, node_name: &str) -> Result { + let node_info = world + .nodes_info + .get(node_name) + .ok_or_else(|| StepError::LogicalError { + message: format!("Node '{node_name}' not found in world state"), + })?; + + let user_config_path = node_info.runtime_dir.join(USER_CONFIG_FILE); + let blend_zk_pk_hex = blend_core_zk_pk_from_node_yaml(&user_config_path)?; + let blend_zk_pk = ZkPublicKey::from_bytes(&hex::decode(blend_zk_pk_hex)?)?; + + Ok(blend_zk_pk) +} + +/// Wait for the node-local wallet API to expose a funded note for a Blend key. +/// +/// Blend ZK keys are read from node configuration and are not scenario wallets, +/// so the wallet scanner does not currently track them. Keep this exception +/// node-local because the returned note is immediately consumed by that node's +/// SDP declaration endpoint. +async fn wait_for_blend_funded_note( + world: &CucumberWorld, + node_name: &str, + blend_zk_pk: ZkPublicKey, +) -> Result { + let base_url = world + .nodes_info + .get(node_name) + .ok_or_else(|| StepError::LogicalError { + message: format!("Node '{node_name}' not found in world state"), + })? + .started_node + .client + .base_url() + .clone(); + let timeout = Duration::from_secs(30); + let started = Instant::now(); + let client = CommonHttpClient::new(None); + + loop { + let last_error = match client + .get_wallet_balance(base_url.clone(), blend_zk_pk, None) + .await + { + Ok(wallet_balance) => { + if let Some(note_id) = wallet_balance.notes.keys().next().copied() { + return Ok(note_id); + } + "wallet has no notes yet".to_owned() + } + Err(error) => error.to_string(), + }; + + if started.elapsed() >= timeout { + return Err(StepError::Timeout { + message: format!( + "Timed out waiting for a funded note on Blend ZK key of '{node_name}' via \ + '{base_url}' (last error: {last_error})" + ), + }); + } + + sleep(Duration::from_millis(250)).await; + } +} + +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Required to be mutable by cucumber step function signature" +)] +#[expect(unused_variables, reason = "Cucumber step function signature")] +#[then(expr = "I declare node {string} as blend core node via the CLI binary")] +async fn step_run_blend_sdp_declaration_cli( + world: &mut CucumberWorld, + step: &Step, + declarer_node_name: String, +) -> StepResult { + let user_config_path = node_user_config_path(world, &declarer_node_name)?; + let locator = blend_core_locator_from_node_yaml(&user_config_path)?; + let blend_zk_pk = blend_zk_pk_for_node(world, &declarer_node_name)?; + let service_note_id = + wait_for_blend_funded_note(world, &declarer_node_name, blend_zk_pk).await?; + let service_note_id_json = + serde_json::to_string(&service_note_id).map_err(|error| StepError::LogicalError { + message: format!("Failed to serialize service note ID: {error}"), + })?; + let service_note_id_hex = service_note_id_json.trim_matches('"').to_owned(); + + let declarer_api_base_url = world + .nodes_info + .get(&declarer_node_name) + .ok_or_else(|| StepError::LogicalError { + message: format!("Node '{declarer_node_name}' not found in world state"), + })? + .started_node + .client + .base_url() + .clone(); + + let workspace_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(".."); + let output = tokio::process::Command::new("cargo") + .current_dir(workspace_root) + .arg("run") + .arg("-p") + .arg("logos-blockchain-tools") + .arg("--bin") + .arg("logos-blockchain-tools-api") + .arg("--") + .arg("sdp") + .arg("post-blend-declaration") + .arg("--user-config-path") + .arg(user_config_path) + .arg("--blend-addr") + .arg(format!("{locator}")) + .arg("--service-note-id") + .arg(service_note_id_hex) + .arg("--node-address") + .arg(declarer_api_base_url.to_string()) + .output() + .await?; + + if !output.status.success() { + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(StepError::StepFail { + message: format!( + "Blend declaration CLI failed for node '{declarer_node_name}'\nstdout:\n{stdout}\nstderr:\n{stderr}" + ), + }); + } + + Ok(()) +} + +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Required to be mutable by cucumber step function signature" +)] +#[then(expr = "I declare node {string} as blend core node via the API")] +async fn step_run_blend_sdp_declaration_api( + world: &mut CucumberWorld, + step: &Step, + declarer_node_name: String, +) -> StepResult { + let user_config_path = node_user_config_path(world, &declarer_node_name)?; + let locator = blend_core_locator_from_node_yaml(&user_config_path)?; + let blend_zk_pk = blend_zk_pk_for_node(world, &declarer_node_name)?; + let service_note_id = + wait_for_blend_funded_note(world, &declarer_node_name, blend_zk_pk).await?; + + let declarer_node_client = world + .nodes_info + .get(&declarer_node_name) + .ok_or_else(|| StepError::LogicalError { + message: format!("Node '{declarer_node_name}' not found in world state"), + })? + .started_node + .client + .clone(); + + let declaration_id = declarer_node_client + .join_blend_network(locator, service_note_id) + .await + .inspect_err(|error| { + warn!(target: TARGET, "Step `{}` error: {error}", step.value); + })?; + + info!( + target: TARGET, + "Node '{declarer_node_name}' joined blend core via API, declaration id: {declaration_id}" + ); + + Ok(()) +} + +fn node_user_config_path(world: &CucumberWorld, node_name: &str) -> Result { + let node_info = world + .nodes_info + .get(node_name) + .ok_or_else(|| StepError::LogicalError { + message: format!("Node '{node_name}' not found in world state"), + })?; + + Ok(node_info.runtime_dir.join(USER_CONFIG_FILE)) +} + +#[expect( + clippy::cognitive_complexity, + reason = "TODO: Address this at some point." +)] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Required to be mutable by cucumber step function signature" +)] +#[then(expr = "blend core SDP declaration for node {string} is included on node {string}")] +async fn step_verify_blend_sdp_declaration_included( + world: &mut CucumberWorld, + step: &Step, + declarer_node_name: String, + api_node_name: String, +) -> StepResult { + let blend_zk_pk = blend_zk_pk_for_node(world, &declarer_node_name)?; + let service_note_id = + wait_for_blend_funded_note(world, &declarer_node_name, blend_zk_pk).await?; + + let step_timeout = Duration::from_secs(30); + let start_time = Instant::now(); + loop { + let declarations_result = world + .nodes_info + .get(&api_node_name) + .ok_or_else(|| StepError::LogicalError { + message: format!("Node '{api_node_name}' not found in world state"), + })? + .started_node + .client + .get_sdp_declarations() + .await; + + let declarations = match declarations_result { + Ok(declarations) => declarations, + Err(error) => { + let error_message = error.to_string(); + if error_message.contains("404 Not Found") { + info!( + target: TARGET, + "Skipping declaration visibility assertion on '{api_node_name}' because testing SDP endpoint is unavailable: {error_message}", + ); + return Ok(()); + } + + warn!(target: TARGET, "Step `{}` error: {error}", step.value); + return Err(error.into()); + } + }; + + if declarations.values().any(|declaration| { + declaration.service_note_id == service_note_id && declaration.zk_id == blend_zk_pk + }) { + info!( + target: TARGET, + "Blend declaration observed for node '{declarer_node_name}'" + ); + break; + } + + if start_time.elapsed() >= step_timeout { + return Err(StepError::Timeout { + message: format!( + "Timed out waiting for declaration submitted by '{declarer_node_name}' to appear on node '{api_node_name}'" + ), + }); + } + + sleep(Duration::from_millis(250)).await; + } + + Ok(()) +} diff --git a/tests/src/cucumber/steps/nodes/steps/configuration.rs b/tests/src/cucumber/steps/nodes/steps/configuration.rs new file mode 100644 index 000000000..f7d65d7d4 --- /dev/null +++ b/tests/src/cucumber/steps/nodes/steps/configuration.rs @@ -0,0 +1,135 @@ +use super::{ + CucumberWorld, SdpFundingConfig, Step, StepError, StepResult, + ensure_fee_sponsorship_and_fork_groups_are_not_mixed, given, + rebuild_pending_local_manual_cluster, set_blend_diagnostic_parameter_set, + set_deployment_config_override, set_user_config_override, when, +}; + +#[given(expr = "the cluster uses Blend diagnostic parameter set {string}")] +#[when(expr = "the cluster uses Blend diagnostic parameter set {string}")] +#[expect( + clippy::needless_pass_by_value, + reason = "Cucumber step arguments must use owned types" +)] +fn step_set_blend_diagnostic_parameter_set( + world: &mut CucumberWorld, + step: &Step, + parameter_set_name: String, +) -> StepResult { + set_blend_diagnostic_parameter_set(world, &step.value, ¶meter_set_name) +} + +#[given(expr = "we use IBD peers")] +#[when(expr = "we use IBD peers")] +const fn step_we_use_ibd_peers(world: &mut CucumberWorld) { + world.startup.populate_ibd_peers_from_initial_peers = Some(true); +} + +#[given(expr = "we join an external network")] +#[when(expr = "we join an external network")] +const fn step_we_join_external_network(world: &mut CucumberWorld) { + world.startup.join_external_network = Some(true); +} + +#[given(expr = "we will have distinct node groups to query wallet balances:")] +#[when(expr = "we will have distinct node groups to query wallet balances:")] +fn step_define_node_groups(world: &mut CucumberWorld, step: &Step) -> Result<(), StepError> { + ensure_fee_sponsorship_and_fork_groups_are_not_mixed(world, &step.value)?; + + let table = step.table.as_ref().ok_or(StepError::LogicalError { + message: "Expected a data table".to_owned(), + })?; + + if table.rows.is_empty() || table.rows[0].len() != 2 { + return Err(StepError::LogicalError { + message: "Expected table columns: | group_name | node_name |".to_owned(), + }); + } + + if table.rows[0][0].trim() != "group_name" || table.rows[0][1].trim() != "node_name" { + return Err(StepError::LogicalError { + message: "Expected table columns: | group_name | node_name |".to_owned(), + }); + } + + let assignments = table + .rows + .iter() + .skip(1) + .map(|row| { + if row.len() != 2 { + return Err(StepError::LogicalError { + message: "Each node-group row must have exactly two columns".to_owned(), + }); + } + + Ok((row[0].trim().to_owned(), row[1].trim().to_owned())) + }) + .collect::, _>>()?; + + world.fork_groups.replace_all(assignments) +} + +#[given(expr = "I have user config override {string} as {string}")] +#[when(expr = "I have user config override {string} as {string}")] +#[expect( + clippy::needless_pass_by_value, + reason = "Required by cucumber expression" +)] +fn step_set_user_config_setting( + world: &mut CucumberWorld, + step: &Step, + setting_path: String, + setting_value: String, +) -> StepResult { + set_user_config_override(world, &step.value, &setting_path, &setting_value) +} + +#[given(expr = "I have deployment config override {string} as {string}")] +#[when(expr = "I have deployment config override {string} as {string}")] +#[expect( + clippy::needless_pass_by_value, + reason = "Required by cucumber expression" +)] +fn step_set_deployment_config_setting( + world: &mut CucumberWorld, + step: &Step, + setting_path: String, + setting_value: String, +) -> StepResult { + set_deployment_config_override(world, &step.value, &setting_path, &setting_value) +} + +#[given(expr = "the first {int} nodes are declared as blend providers")] +#[when(expr = "the first {int} nodes are declared as blend providers")] +fn step_blend_provider_count(world: &mut CucumberWorld, provider_count: usize) -> StepResult { + world.cluster.blend_core_nodes = Some(provider_count); + rebuild_pending_local_manual_cluster(world) +} + +#[given(expr = "the cluster uses SDP funding of {int} per provider split across {int} notes")] +#[when(expr = "the cluster uses SDP funding of {int} per provider split across {int} notes")] +fn step_set_sdp_funding( + world: &mut CucumberWorld, + total_value_per_node: u64, + target_notes_per_node: usize, +) -> StepResult { + if target_notes_per_node == 0 { + return Err(StepError::InvalidArgument { + message: "SDP funding note count must be greater than zero".to_owned(), + }); + } + + world.set_sdp_funding_config(SdpFundingConfig::new( + total_value_per_node, + target_notes_per_node, + )); + rebuild_pending_local_manual_cluster(world) +} + +#[given(expr = "no nodes are declared as blend providers")] +#[when(expr = "no nodes are declared as blend providers")] +fn step_no_blend_providers(world: &mut CucumberWorld) -> StepResult { + world.cluster.blend_core_nodes = Some(0); + rebuild_pending_local_manual_cluster(world) +} diff --git a/tests/src/cucumber/steps/nodes/steps/genesis.rs b/tests/src/cucumber/steps/nodes/steps/genesis.rs new file mode 100644 index 000000000..8afd27397 --- /dev/null +++ b/tests/src/cucumber/steps/nodes/steps/genesis.rs @@ -0,0 +1,119 @@ +use super::{ + CucumberWorld, Duration, GenesisTime, Instant, NodeBinaryProfile, OffsetDateTime, Step, + StepError, StepResult, TARGET, TimeDuration, ensure_node_binary_built, given, info, + rebuild_pending_local_manual_cluster, sleep, then, when, +}; + +pub(super) fn resolve_step_genesis_time( + step_value: &str, + now: OffsetDateTime, + seconds: i64, +) -> Result { + if seconds < 0 { + return Err(StepError::InvalidArgument { + message: format!("step `{step_value}` requires a non-negative offset"), + }); + } + + let genesis_datetime = now + .checked_add(TimeDuration::seconds(seconds)) + .ok_or_else(|| StepError::InvalidArgument { + message: format!( + "step `{step_value}` has an invalid genesis time: offset is out of range" + ), + })?; + + GenesisTime::try_from(genesis_datetime).map_err(|error| StepError::InvalidArgument { + message: format!("step `{step_value}` has an invalid genesis time: {error}"), + }) +} + +pub(super) fn validate_genesis_time_change( + existing_genesis_time: Option, + nodes_started: bool, + requested_genesis_time: GenesisTime, +) -> StepResult { + if nodes_started && existing_genesis_time != Some(requested_genesis_time) { + return Err(StepError::LogicalError { + message: "cannot change genesis time after nodes have started".to_owned(), + }); + } + + Ok(()) +} + +#[given(expr = "the chain starts {int} seconds from now")] +#[when(expr = "the chain starts {int} seconds from now")] +async fn step_chain_starts_from_now( + world: &mut CucumberWorld, + step: &Step, + seconds: i64, +) -> StepResult { + let node_binary_profile = if world.tokio_console_profile_enabled() { + NodeBinaryProfile::TokioConsole + } else { + NodeBinaryProfile::default() + }; + ensure_node_binary_built(&node_binary_profile) + .await + .map_err(|error| StepError::Preflight { + message: format!("failed to resolve/build node binary: {error}"), + })?; + + let genesis_time = resolve_step_genesis_time(&step.value, OffsetDateTime::now_utc(), seconds)?; + validate_genesis_time_change( + world.lifecycle.genesis_time, + !world.nodes_info.is_empty(), + genesis_time, + )?; + + world.set_genesis_time(genesis_time); + if world.nodes_info.is_empty() && world.cluster.manual_cluster_spec.is_some() { + rebuild_pending_local_manual_cluster(world)?; + } + + Ok(()) +} + +#[then(expr = "the configured genesis time has not passed for {int} seconds")] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require the world as the first `&mut` argument" +)] +async fn step_genesis_time_has_not_passed( + world: &mut CucumberWorld, + step: &Step, + seconds: u64, +) -> StepResult { + let genesis_time = world + .lifecycle + .genesis_time + .ok_or_else(|| StepError::LogicalError { + message: "the scenario has no configured genesis time".to_owned(), + })?; + let genesis_datetime = OffsetDateTime::from(genesis_time); + let timeout = Duration::from_secs(seconds); + let started_waiting = Instant::now(); + + loop { + let now = OffsetDateTime::now_utc(); + if now >= genesis_datetime { + return Err(StepError::StepFail { + message: format!( + "Step `{}` failed: configured genesis time {genesis_datetime} had passed at {now}", + step.value + ), + }); + } + + if started_waiting.elapsed() >= timeout { + info!( + target: TARGET, + "Configured genesis time {genesis_datetime} remained in the future for {seconds} seconds" + ); + return Ok(()); + } + + sleep(Duration::from_millis(250)).await; + } +} diff --git a/tests/src/cucumber/steps/nodes/steps/lifecycle.rs b/tests/src/cucumber/steps/nodes/steps/lifecycle.rs new file mode 100644 index 000000000..d9c00a53b --- /dev/null +++ b/tests/src/cucumber/steps/nodes/steps/lifecycle.rs @@ -0,0 +1,445 @@ +use super::{ + ConfigOverride, CucumberWorld, Duration, GenesisTokens, HashMap, Instant, ManualClusterKind, + ManualClusterSpec, NodesToStartUnordered, Step, StepError, StepResult, TARGET, WalletAccount, + assert_manual_node_has_peers, connect_manual_node_to_node, + ensure_fee_sponsorship_and_fork_groups_are_not_mixed, given, install_local_manual_cluster, + non_zero, parse_genesis_wallet_tokens_row, parse_mining_wallet_resources_table_row, + parse_wallet_resources_table_row, restart_node, start_node, + start_nodes_order_respecting_dependencies, stop_node, then, + verify_genesis_wallet_resources_table_indexes, + verify_mining_node_wallet_resources_table_indexes, verify_node_wallet_resources_table_indexes, + warn, when, +}; + +#[given(expr = "I have a cluster with capacity of {int} nodes")] +#[when(expr = "I have a cluster with capacity of {int} nodes")] +fn step_manual_cluster(world: &mut CucumberWorld, step: &Step, nodes_count: usize) -> StepResult { + install_local_manual_cluster( + world, + ManualClusterSpec { + kind: ManualClusterKind::Generated, + capacity: nodes_count, + }, + ) + .inspect_err(|e| { + warn!(target: TARGET, "Step '{step}' error: {e}"); + }) +} + +#[given(expr = "I have a devnet cluster with capacity of {int} nodes")] +#[when(expr = "I have a devnet cluster with capacity of {int} nodes")] +fn step_manual_devnet_cluster( + world: &mut CucumberWorld, + step: &Step, + nodes_count: usize, +) -> StepResult { + install_local_manual_cluster( + world, + ManualClusterSpec { + kind: ManualClusterKind::Devnet, + capacity: nodes_count, + }, + ) + .inspect_err(|e| { + warn!(target: TARGET, "Step '{step}' error: {e}"); + }) +} + +#[given("the genesis block has the following wallet resources:")] +#[when("the genesis block has the following wallet resources:")] +fn step_cluster_has_wallet_resources(world: &mut CucumberWorld, step: &Step) -> StepResult { + let table = step + .table + .as_ref() + .ok_or(StepError::MissingTable) + .inspect_err(|e| { + warn!(target: TARGET, "Step `{}` error: {e}", step.value); + })?; + + verify_genesis_wallet_resources_table_indexes(table, &step.value)?; + world.chain.genesis_tokens.clear(); + for row in table.rows.iter().skip(1) { + let (account_index, token_count, token_amount) = + parse_genesis_wallet_tokens_row(&step.value, row)?; + + world.chain.genesis_tokens.push(GenesisTokens { + account_index, + token_count, + token_amount, + }); + } + + Ok(()) +} + +#[given(expr = "we have a sponsored genesis fee account with {int} tokens of {int} value each")] +#[when(expr = "we have a sponsored genesis fee account with {int} tokens of {int} value each")] +fn step_sponsored_genesis_fee_account( + world: &mut CucumberWorld, + step: &Step, + token_count: usize, + token_value: u64, +) -> StepResult { + ensure_fee_sponsorship_and_fork_groups_are_not_mixed(world, step.value.as_str())?; + + let token_count = non_zero!("genesis fee token count", token_count)?; + let token_value = non_zero!("genesis fee token value", token_value)?; + + world + .wallet_registry + .fee_state + .set_sponsored_genesis_account(token_count, token_value); + Ok(()) +} + +#[given("I start nodes with wallet resources:")] +#[when("I start nodes with wallet resources:")] +async fn step_start_nodes_with_wallet_resources( + world: &mut CucumberWorld, + step: &Step, +) -> StepResult { + let table = step + .table + .as_ref() + .ok_or(StepError::MissingTable) + .inspect_err(|e| { + warn!(target: TARGET, "Step `{}` error: {e}", step.value); + })?; + + // Map wallet start info and connected peers to node name + verify_node_wallet_resources_table_indexes(table, &step.value)?; + let mut nodes_to_start: NodesToStartUnordered = HashMap::new(); + for row in table.rows.iter().skip(1) { + let (node_name, wallet_start_info, connected_to) = + parse_wallet_resources_table_row(&step.value, row)?; + let entry = nodes_to_start + .entry(node_name) + .or_insert_with(|| (Vec::new(), Vec::new())); + entry.0.push(wallet_start_info); + if let Some(peer) = connected_to { + entry.1.push(peer); + } + } + + let nodes_to_start_ordered = start_nodes_order_respecting_dependencies( + nodes_to_start, + world.nodes_info.keys().cloned().collect(), + ) + .inspect_err(|e| { + warn!(target: TARGET, "Step `{}` error: {e}", step.value); + })?; + for (node_name, wallet_start_info, mut initial_peers) in nodes_to_start_ordered { + initial_peers.sort(); + initial_peers.dedup(); + start_node( + world, + &step.value, + &node_name, + &wallet_start_info, + &initial_peers, + false, + &[], + ) + .await?; + } + + world.ensure_wallet_scanner_started().await?; + + Ok(()) +} + +/// Starts mining nodes, each carrying one or more wallet resources of which +/// exactly one is flagged `is_mining_wallet`. That wallet's public key is +/// derived and applied as the node's `pow.claim_address`, so mined rewards are +/// paid to a wallet the test tracks. Multiple mining nodes are supported; each +/// gets its own single mining wallet / claim address. +#[given("I start mining nodes with wallet resources:")] +#[when("I start mining nodes with wallet resources:")] +async fn step_start_mining_nodes_with_wallet_resources( + world: &mut CucumberWorld, + step: &Step, +) -> StepResult { + let table = step + .table + .as_ref() + .ok_or(StepError::MissingTable) + .inspect_err(|e| { + warn!(target: TARGET, "Step `{}` error: {e}", step.value); + })?; + + verify_mining_node_wallet_resources_table_indexes(table, &step.value)?; + + let mut nodes_to_start: NodesToStartUnordered = HashMap::new(); + let mut node_claim_overrides: HashMap = HashMap::new(); + let mut node_mining_wallet_count: HashMap = HashMap::new(); + for row in table.rows.iter().skip(1) { + let (node_name, wallet_start_info, is_mining_wallet, connected_to) = + parse_mining_wallet_resources_table_row(&step.value, row)?; + + if is_mining_wallet { + *node_mining_wallet_count + .entry(node_name.clone()) + .or_insert(0) += 1; + let account = + WalletAccount::deterministic(wallet_start_info.account_index as u64, 0, true) + .map_err(|source| StepError::InvalidArgument { + message: format!( + "Step `{}` error: failed to derive mining wallet public key for \ + `{}`: {source}", + step.value, wallet_start_info.wallet_name + ), + })?; + let value = serde_yaml::to_value(account.public_key()).map_err(|source| { + StepError::InvalidArgument { + message: format!( + "Step `{}` error: failed to serialize mining wallet public key for \ + `{}`: {source}", + step.value, wallet_start_info.wallet_name + ), + } + })?; + node_claim_overrides.insert( + node_name.clone(), + ConfigOverride { + path: "pow.claim_address".to_owned(), + value, + }, + ); + } + + let entry = nodes_to_start + .entry(node_name) + .or_insert_with(|| (Vec::new(), Vec::new())); + entry.0.push(wallet_start_info); + if let Some(peer) = connected_to { + entry.1.push(peer); + } + } + + // Every mining node must configure exactly one mining wallet. + for node_name in nodes_to_start.keys() { + let count = node_mining_wallet_count + .get(node_name) + .copied() + .unwrap_or(0); + if count != 1 { + return Err(StepError::InvalidArgument { + message: format!( + "Step `{}` error: mining node `{node_name}` must have exactly one \ + is_mining_wallet row, found {count}", + step.value + ), + }); + } + } + + let nodes_to_start_ordered = start_nodes_order_respecting_dependencies( + nodes_to_start, + world.nodes_info.keys().cloned().collect(), + ) + .inspect_err(|e| { + warn!(target: TARGET, "Step `{}` error: {e}", step.value); + })?; + for (node_name, wallet_start_info, mut initial_peers) in nodes_to_start_ordered { + initial_peers.sort(); + initial_peers.dedup(); + let extra_user_overrides = node_claim_overrides + .get(&node_name) + .map_or(&[][..], std::slice::from_ref); + start_node( + world, + &step.value, + &node_name, + &wallet_start_info, + &initial_peers, + false, + extra_user_overrides, + ) + .await?; + } + + world.ensure_wallet_scanner_started().await?; + + Ok(()) +} + +#[given(expr = "I start node {string}")] +#[when(expr = "I start node {string}")] +async fn step_start_manual_stand_alone_node( + world: &mut CucumberWorld, + step: &Step, + node_name: String, +) -> StepResult { + start_node( + world, + &step.value, + &node_name, + &Vec::new(), + &Vec::new(), + false, + &[], + ) + .await +} + +#[given(expr = "I immediate start node {string}")] +#[when(expr = "I immediate start node {string}")] +async fn step_start_manual_network_ready_only_stand_alone_node( + world: &mut CucumberWorld, + step: &Step, + node_name: String, +) -> StepResult { + start_node( + world, + &step.value, + &node_name, + &Vec::new(), + &Vec::new(), + true, + &[], + ) + .await +} + +#[when(expr = "I start node {string} to be ready between {int} and {int} seconds")] +async fn step_start_manual_stand_alone_node_not_ready_before( + world: &mut CucumberWorld, + step: &Step, + node_name: String, + min_wait_seconds: u64, + max_wait_seconds: u64, +) -> StepResult { + let start = Instant::now(); + start_node( + world, + &step.value, + &node_name, + &Vec::new(), + &Vec::new(), + false, + &[], + ) + .await?; + + let elapsed = start.elapsed(); + if elapsed < Duration::from_secs(min_wait_seconds) { + return Err(StepError::StepFail { + message: format!( + "Step `{}` error: Node '{node_name}' became ready too early: elapsed {:.2?}, \ + expected at least {min_wait_seconds}s", + step.value, elapsed, + ), + }); + } + if elapsed > Duration::from_secs(max_wait_seconds) { + return Err(StepError::StepFail { + message: format!( + "Step `{}` error: Node '{node_name}' took too long to become ready: elapsed {:.2?}, \ + expected at most {max_wait_seconds}s", + step.value, elapsed, + ), + }); + } + + Ok(()) +} + +#[when( + expr = "I start peer node {string} connected to node {string} to be ready between {int} and {int} seconds" +)] +async fn step_start_manual_peer_node_not_ready_before( + world: &mut CucumberWorld, + step: &Step, + node_name: String, + peer_name: String, + min_wait_seconds: u64, + max_wait_seconds: u64, +) -> StepResult { + let start = Instant::now(); + start_node( + world, + &step.value, + &node_name, + &Vec::new(), + &[peer_name], + false, + &[], + ) + .await?; + + let elapsed = start.elapsed(); + if elapsed < Duration::from_secs(min_wait_seconds) { + return Err(StepError::StepFail { + message: format!( + "Step `{}` error: Node '{node_name}' became ready too early: elapsed {:.2?}, \ + expected at least {min_wait_seconds}s", + step.value, elapsed, + ), + }); + } + if elapsed > Duration::from_secs(max_wait_seconds) { + return Err(StepError::StepFail { + message: format!( + "Step `{}` error: Node '{node_name}' took too long to become ready: elapsed {:.2?}, \ + expected at most {max_wait_seconds}s", + step.value, elapsed, + ), + }); + } + + Ok(()) +} + +#[when(expr = "I connect node {string} to node {string} at runtime")] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step entrypoints must take `&mut World`" +)] +async fn step_connect_nodes_at_runtime( + world: &mut CucumberWorld, + source_node_name: String, + target_node_name: String, +) -> StepResult { + connect_manual_node_to_node(world, &source_node_name, &target_node_name).await +} + +#[then(expr = "node {string} has at least {int} peers within {int} seconds")] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step entrypoints must take `&mut World`" +)] +async fn step_node_has_peers( + world: &mut CucumberWorld, + node_name: String, + min_peers: usize, + timeout_secs: u64, +) -> StepResult { + assert_manual_node_has_peers(world, &node_name, min_peers, timeout_secs).await +} + +#[when(expr = "I restart node {string}")] +async fn step_restart_node( + world: &mut CucumberWorld, + step: &Step, + node_name: String, +) -> StepResult { + restart_node(world, &step.value, &node_name).await?; + if world.blend_diagnostics.observation_count > 0 { + world.blend_diagnostics.stopped_nodes.remove(&node_name); + world.blend_diagnostics.phase = + Some(crate::cucumber::world::BlendDiagnosticPhase::Recovery); + } + Ok(()) +} + +#[when(expr = "I stop node {string}")] +async fn step_stop_node(world: &mut CucumberWorld, step: &Step, node_name: String) -> StepResult { + if world.blend_diagnostics.observation_count > 0 { + world.blend_diagnostics.phase = Some(crate::cucumber::world::BlendDiagnosticPhase::Outage); + } + stop_node(world, &step.value, &node_name).await?; + if world.blend_diagnostics.observation_count > 0 { + world.blend_diagnostics.stopped_nodes.insert(node_name); + } + Ok(()) +} diff --git a/tests/src/cucumber/steps/nodes/steps/mod.rs b/tests/src/cucumber/steps/nodes/steps/mod.rs new file mode 100644 index 000000000..0b3427224 --- /dev/null +++ b/tests/src/cucumber/steps/nodes/steps/mod.rs @@ -0,0 +1,91 @@ +use std::{ + collections::{HashMap, HashSet}, + path::PathBuf, + time::Duration, +}; + +use cucumber::{gherkin::Step, given, then, when}; +use lb_common_http_client::CommonHttpClient; +use lb_core::{codec::DeserializeOp as _, mantle::GenesisTime}; +use lb_key_management_system_service::keys::ZkPublicKey; +use lb_libp2p::{Multiaddr, PeerId}; +use lb_testing_framework::{ + USER_CONFIG_FILE, + configs::{ + deployment::{NodeBinaryProfile, SdpFundingConfig}, + wallet::WalletAccount, + }, + ensure_node_binary_built, +}; +use time::{Duration as TimeDuration, OffsetDateTime}; +use tokio::time::{Instant, sleep}; +use tracing::{info, warn}; + +use crate::{ + common::wallet::WalletUtxos, + cucumber::{ + error::{StepError, StepResult}, + steps::{ + TARGET, + cluster::{ + assert_manual_node_has_peers, connect_manual_node_to_node, + install_local_manual_cluster, rebuild_pending_local_manual_cluster, + stop_active_manual_cluster, + }, + nodes::{ + NodesToStartUnordered, + config_override::{set_deployment_config_override, set_user_config_override}, + create_snapshot_all_nodes_with_wallet_state, + create_snapshot_node_with_wallet_state, create_snapshots_all_nodes, + diagnostics::set_blend_diagnostic_parameter_set, + ensure_all_nodes_agree_on_lib, + ensure_fee_sponsorship_and_fork_groups_are_not_mixed, + get_cryptarchia_info_all_nodes, nodes_converged, parse_genesis_wallet_tokens_row, + parse_mining_wallet_resources_table_row, parse_url, + parse_wallet_resources_table_row, poll_all_nodes_and_update_consensus_cache, + restart_node, + snapshots::validate_snapshot_path_component, + start_node, start_nodes_order_respecting_dependencies, stop_node, + verify_genesis_wallet_resources_table_indexes, + verify_mining_node_wallet_resources_table_indexes, + verify_node_wallet_resources_table_indexes, + verify_reponsive_and_network_ready_with_timeout, wait_all_nodes_responive, + wait_for_all_nodes_to_be_synced_to_chain, + }, + transactions::utils::{ + create_and_submit_transaction_hashes_with_utxo_cache, + wait_for_transactions_inclusion, + }, + }, + utils::{ + blend_core_locator_from_node_yaml, blend_core_zk_pk_from_node_yaml, + resolve_literal_or_env, + }, + wallet::{ + snapshot::{ + prepare_all_wallets_snapshot, prepare_wallet_snapshot_restore_if_present, + save_prepared_all_wallets_snapshot, + }, + sync::{WalletSendReadiness, wait_wallet_send_ready}, + }, + world::{ + ConfigOverride, CucumberWorld, GenesisTokens, ManualClusterKind, ManualClusterSpec, + NodeSnapshot, PublicCryptarchiaEndpointPeer, + }, + }, + non_zero, +}; + +const PUBLIC_CRYPTARCHIA_ENDPOINT: &str = "public_cryptarchia_endpoint"; +const PUBLIC_CRYPTARCHIA_ENDPOINT_USERNAME: &str = "username"; +const PUBLIC_CRYPTARCHIA_ENDPOINT_PASSWORD: &str = "password"; + +mod blend; +mod configuration; +mod genesis; +mod lifecycle; +mod network; +mod snapshots; + +#[cfg(test)] +mod tests; diff --git a/tests/src/cucumber/steps/nodes/steps/network.rs b/tests/src/cucumber/steps/nodes/steps/network.rs new file mode 100644 index 000000000..19dd1a922 --- /dev/null +++ b/tests/src/cucumber/steps/nodes/steps/network.rs @@ -0,0 +1,383 @@ +use super::{ + CucumberWorld, Duration, HashSet, Instant, Multiaddr, PUBLIC_CRYPTARCHIA_ENDPOINT, + PUBLIC_CRYPTARCHIA_ENDPOINT_PASSWORD, PUBLIC_CRYPTARCHIA_ENDPOINT_USERNAME, PeerId, + PublicCryptarchiaEndpointPeer, Step, StepError, StepResult, TARGET, given, info, + nodes_converged, parse_url, poll_all_nodes_and_update_consensus_cache, resolve_literal_or_env, + sleep, start_node, then, verify_reponsive_and_network_ready_with_timeout, + wait_all_nodes_responive, when, +}; + +#[given("I have public cryptarchia endpoint peers:")] +#[when("I have public cryptarchia endpoint peers:")] +fn step_set_public_cryptarchia_endpoint_peers( + world: &mut CucumberWorld, + step: &Step, +) -> StepResult { + let table = step.table.as_ref().ok_or(StepError::MissingTable)?; + + if table.rows.is_empty() { + return Err(StepError::InvalidArgument { + message: format!( + "Step `{step}` error: public cryptarchia endpoint peers table cannot be empty" + ), + }); + } + if table.rows.iter().any(|row| row.len() != 3) { + return Err(StepError::InvalidArgument { + message: format!( + "Step `{step}` error: public cryptarchia endpoint peers table must have exactly three columns" + ), + }); + } + if !matches!(table.rows[0][0].trim(), PUBLIC_CRYPTARCHIA_ENDPOINT) + || !matches!( + table.rows[0][1].trim(), + PUBLIC_CRYPTARCHIA_ENDPOINT_USERNAME + ) + || !matches!( + table.rows[0][2].trim(), + PUBLIC_CRYPTARCHIA_ENDPOINT_PASSWORD + ) + { + return Err(StepError::InvalidArgument { + message: format!( + "Step `{step}` error: public cryptarchia endpoint peers table header row must be \ + '{PUBLIC_CRYPTARCHIA_ENDPOINT}', '{PUBLIC_CRYPTARCHIA_ENDPOINT_USERNAME}', \ + '{PUBLIC_CRYPTARCHIA_ENDPOINT_PASSWORD}'" + ), + }); + } + + let mut endpoint_peers = Vec::with_capacity(table.rows.len().saturating_sub(1)); + for row in table.rows.iter().skip(1) { + let url = parse_url(&row[0]).map_err(|e| StepError::InvalidArgument { + message: format!( + "Step `{}` error: invalid public cryptarchia endpoint '{}': {e}", + step.value, row[0] + ), + })?; + + let username = + resolve_literal_or_env(row[1].trim(), "public cryptarchia endpoint username").map_err( + |e| StepError::InvalidArgument { + message: format!("Step `{}` error: {e}", step.value), + }, + )?; + if username.is_empty() { + return Err(StepError::InvalidArgument { + message: format!( + "Step `{}` error: username cannot be empty for public cryptarchia endpoint '{}'", + step.value, url + ), + }); + } + + let password = + resolve_literal_or_env(row[2].trim(), "public cryptarchia endpoint password").map_err( + |e| StepError::InvalidArgument { + message: format!("Step `{}` error: {e}", step.value), + }, + )?; + if password.is_empty() { + return Err(StepError::InvalidArgument { + message: format!( + "Step `{}` error: password cannot be empty for public cryptarchia endpoint '{}'", + step.value, url + ), + }); + } + + endpoint_peers.push(PublicCryptarchiaEndpointPeer { + base_url: url, + username, + password, + }); + } + + if endpoint_peers.is_empty() { + return Err(StepError::InvalidArgument { + message: format!( + "Step `{step}` error: at least one public cryptarchia endpoint peer is required" + ), + }); + } + world.startup.public_cryptarchia_endpoint_peers = Some(endpoint_peers); + + Ok(()) +} + +#[given(expr = "all peers must be mode online after startup in {int} seconds")] +#[when(expr = "all peers must be mode online after startup in {int} seconds")] +const fn step_all_nodes_to_be_mode_online(world: &mut CucumberWorld, on_line_time_out: u64) { + world.startup.require_all_peers_mode_online_at_startup = + Some(Duration::from_secs(on_line_time_out)); +} + +#[given("I have initial peers:")] +#[when("I have initial peers:")] +fn step_set_initial_peers(world: &mut CucumberWorld, step: &Step) -> StepResult { + let table = step.table.as_ref().ok_or(StepError::MissingTable)?; + if table.rows.is_empty() || table.rows[0].len() != 1 || table.rows[0][0] != "initial_peer" { + return Err(StepError::InvalidArgument { + message: format!( + "Step `{}` error: initial peers table header must be `initial_peer`", + step.value + ), + }); + } + + let mut peers = Vec::with_capacity(table.rows.len().saturating_sub(1)); + for row in table.rows.iter().skip(1) { + let peer = row[0] + .trim() + .parse::() + .map_err(|e| StepError::InvalidArgument { + message: format!( + "Step `{}` error: invalid initial peer '{}': {e}", + step.value, row[0] + ), + })?; + peers.push(peer); + } + + world.startup.initial_peers_override = Some(peers); + Ok(()) +} + +#[given("I have IBD peers:")] +#[when("I have IBD peers:")] +fn step_set_ibd_peers(world: &mut CucumberWorld, step: &Step) -> StepResult { + let table = step.table.as_ref().ok_or(StepError::MissingTable)?; + if table.rows.is_empty() || table.rows[0].len() != 1 || table.rows[0][0] != "ibd_peer" { + return Err(StepError::InvalidArgument { + message: format!( + "Step `{}` error: IBD peers table header must be `ibd_peer`", + step.value + ), + }); + } + + let mut peers = HashSet::with_capacity(table.rows.len().saturating_sub(1)); + for row in table.rows.iter().skip(1) { + let peer = row[0] + .trim() + .parse::() + .map_err(|e| StepError::InvalidArgument { + message: format!( + "Step `{}` error: invalid IBD peer '{}': {e}", + step.value, row[0] + ), + })?; + peers.insert(peer); + } + + world.startup.ibd_peers_override = Some(peers); + world.startup.populate_ibd_peers_from_initial_peers = Some(true); + Ok(()) +} + +#[given(expr = "I start peer node {string} connected to node {string}")] +#[when(expr = "I start peer node {string} connected to node {string}")] +async fn step_start_manual_connected_node( + world: &mut CucumberWorld, + step: &Step, + node_name: String, + peer_name: String, +) -> StepResult { + start_node( + world, + &step.value, + &node_name, + &Vec::new(), + &[peer_name], + false, + &[], + ) + .await +} + +#[given(expr = "I immediate start peer node {string} connected to node {string}")] +#[when(expr = "I immediate start peer node {string} connected to node {string}")] +async fn step_immediate_start_manual_connected_node( + world: &mut CucumberWorld, + step: &Step, + node_name: String, + peer_name: String, +) -> StepResult { + start_node( + world, + &step.value, + &node_name, + &Vec::new(), + &[peer_name], + true, + &[], + ) + .await +} + +#[given(expr = "I start peer node {string} connected to node {string} and node {string}")] +#[when(expr = "I start peer node {string} connected to node {string} and node {string}")] +async fn step_start_manual_two_connected_nodes( + world: &mut CucumberWorld, + step: &Step, + node_name: String, + peer_name1: String, + peer_name2: String, +) -> StepResult { + start_node( + world, + &step.value, + &node_name, + &Vec::new(), + &[peer_name1, peer_name2], + false, + &[], + ) + .await +} + +#[given(expr = "I immediate start peer node {string} connected to node {string} and node {string}")] +#[when(expr = "I immediate start peer node {string} connected to node {string} and node {string}")] +async fn step_immediate_start_manual_two_connected_nodes( + world: &mut CucumberWorld, + step: &Step, + node_name: String, + peer_name1: String, + peer_name2: String, +) -> StepResult { + start_node( + world, + &step.value, + &node_name, + &Vec::new(), + &[peer_name1, peer_name2], + true, + &[], + ) + .await +} + +#[when(expr = "I wait for all nodes to be responsive in {int} seconds")] +#[then(expr = "I wait for all nodes to be responsive in {int} seconds")] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require the world as the first `&mut` argument" +)] +async fn step_wait_all_nodes_responsive( + world: &mut CucumberWorld, + step: &Step, + time_out_seconds: u64, +) -> StepResult { + let cluster = world + .cluster + .local_cluster + .as_ref() + .ok_or(StepError::LogicalError { + message: "No local cluster available".into(), + })?; + if let Err(e) = wait_all_nodes_responive(cluster, Duration::from_secs(time_out_seconds)).await { + return Err(StepError::StepFail { + message: format!("Step `{}` error: {e}", step.value), + }); + } + + let wait_tasks: Vec<_> = world + .nodes_info + .values() + .map(|node| { + let fut = verify_reponsive_and_network_ready_with_timeout( + &node.started_node.client, + &node.name, + &node.started_node.name, + Duration::from_secs(time_out_seconds), + ); + let step_value = step.value.clone(); + async move { + fut.await.map_err(|e| StepError::StepFail { + message: format!("Step `{step_value}` error: {e}"), + }) + } + }) + .collect(); + + futures::future::try_join_all(wait_tasks).await?; + + Ok(()) +} + +#[when(expr = "node {string} is at height {int} in {int} seconds")] +#[then(expr = "node {string} is at height {int} in {int} seconds")] +async fn step_node_is_at_height( + world: &mut CucumberWorld, + step: &Step, + node_name: String, + height: u64, + time_out_seconds: u64, +) -> StepResult { + let start = Instant::now(); + let time_out = Duration::from_secs(time_out_seconds); + + let mut count = 0usize; + loop { + poll_all_nodes_and_update_consensus_cache(&step.value, &mut world.nodes_info).await?; + let best_height = world.node_best_height(&node_name)?.unwrap_or_default(); + if best_height >= height { + info!( + target: TARGET, + "Node '{node_name}' reached height {height} in {:.2?}", + start.elapsed() + ); + return Ok(()); + } else if count.is_multiple_of(50) { + info!( + target: TARGET, + "Waiting for '{node_name}' to reach height {height} - elapsed: {:.2?}, current \ + height: {}", start.elapsed(), best_height + ); + } + + if start.elapsed() >= time_out { + return Err(StepError::StepFail { + message: format!( + "Step `{}` error: Node '{node_name}' did not reach height {height} in {time_out_seconds} s", + step.value + ), + }); + } + sleep(Duration::from_millis(100)).await; + count += 1; + } +} + +#[when(expr = "node {string} is exactly at height")] +#[then(expr = "node {string} is exactly at height")] +async fn step_node_is_exactly_at_height( + world: &mut CucumberWorld, + step: &Step, + node_name: String, + height: u64, +) -> StepResult { + poll_all_nodes_and_update_consensus_cache(&step.value, &mut world.nodes_info).await?; + let node_height = world.node_best_height(&node_name)?.unwrap_or_default(); + if node_height != height { + return Err(StepError::StepFail { + message: format!( + "Step `{}` error: Node '{node_name}' is at height {node_height}, required {height}", + step.value + ), + }); + } + Ok(()) +} + +#[when(expr = "all nodes converged to within {int} blocks in {int} seconds")] +#[then(expr = "all nodes converged to within {int} blocks in {int} seconds")] +async fn step_all_nodes_converged( + world: &mut CucumberWorld, + step: &Step, + max_diff_height: u64, + time_out_seconds: u64, +) -> StepResult { + nodes_converged(world, &step.value, None, max_diff_height, time_out_seconds).await +} diff --git a/tests/src/cucumber/steps/nodes/steps/snapshots.rs b/tests/src/cucumber/steps/nodes/steps/snapshots.rs new file mode 100644 index 000000000..00464cc92 --- /dev/null +++ b/tests/src/cucumber/steps/nodes/steps/snapshots.rs @@ -0,0 +1,73 @@ +use super::{ + CucumberWorld, NodeSnapshot, StepError, StepResult, + create_snapshot_all_nodes_with_wallet_state, create_snapshot_node_with_wallet_state, given, + prepare_wallet_snapshot_restore_if_present, then, validate_snapshot_path_component, when, +}; + +#[given(expr = "I will create a snapshot {string} of all nodes when stopping")] +#[when(expr = "I will create a snapshot {string} of all nodes when stopping")] +#[expect( + clippy::needless_pass_by_value, + reason = "Required by cucumber expression" +)] +fn step_set_snapshot_all_nodes_on_stop( + world: &mut CucumberWorld, + snapshot_name: String, +) -> StepResult { + if snapshot_name.trim().is_empty() { + return Err(StepError::InvalidArgument { + message: "Snapshot name cannot be empty".to_owned(), + }); + } + validate_snapshot_path_component(&snapshot_name, "Snapshot name")?; + let snapshot_name = snapshot_name.trim().to_owned(); + world.snapshots.save.node_state = Some(snapshot_name.clone()); + world.snapshots.save.extensions = Some(snapshot_name); + Ok(()) +} + +#[given(expr = "I will initialize started nodes from snapshot {string} source node {string}")] +#[when(expr = "I will initialize started nodes from snapshot {string} source node {string}")] +#[expect( + clippy::needless_pass_by_value, + reason = "Required by cucumber expression" +)] +fn step_set_node_snapshot_on_startup( + world: &mut CucumberWorld, + snapshot_name: String, + node_name: String, +) -> StepResult { + validate_snapshot_path_component(&snapshot_name, "Snapshot name")?; + validate_snapshot_path_component(&node_name, "Node name")?; + + let snapshot_name = snapshot_name.trim().to_owned(); + world.snapshots.node_snapshot_on_startup = Some(NodeSnapshot { + name: snapshot_name.clone(), + node: node_name.trim().to_owned(), + }); + world.snapshots.restore.extensions = Some(snapshot_name); + if let Some(snapshot_name) = world.snapshots.restore.extensions.clone() { + prepare_wallet_snapshot_restore_if_present(&snapshot_name, world)?; + } + Ok(()) +} + +#[given(expr = "I create a snapshot {string} of all nodes")] +#[when(expr = "I create a snapshot {string} of all nodes")] +async fn step_create_snapshot_all_nodes_now( + world: &mut CucumberWorld, + snapshot_name: String, +) -> StepResult { + create_snapshot_all_nodes_with_wallet_state(world, &snapshot_name).await +} + +#[given(expr = "I create a snapshot {string} of node {string}")] +#[when(expr = "I create a snapshot {string} of node {string}")] +#[then(expr = "I create a snapshot {string} of node {string}")] +async fn step_create_snapshot_node_now( + world: &mut CucumberWorld, + snapshot_name: String, + node_name: String, +) -> StepResult { + create_snapshot_node_with_wallet_state(world, &snapshot_name, &node_name).await +} diff --git a/tests/src/cucumber/steps/nodes/steps/tests.rs b/tests/src/cucumber/steps/nodes/steps/tests.rs new file mode 100644 index 000000000..03e12e44d --- /dev/null +++ b/tests/src/cucumber/steps/nodes/steps/tests.rs @@ -0,0 +1,35 @@ +use super::{ + GenesisTime, OffsetDateTime, StepError, + genesis::{resolve_step_genesis_time, validate_genesis_time_change}, +}; + +#[test] +fn step_relative_genesis_time_uses_now_plus_offset() { + let now = OffsetDateTime::from_unix_timestamp(1_000).expect("valid timestamp"); + + let genesis_time = resolve_step_genesis_time("the chain starts 60 seconds from now", now, 60) + .expect("offset should produce a valid genesis time"); + + assert_eq!(genesis_time, GenesisTime::new(1_060)); +} + +#[test] +fn overflowing_step_relative_genesis_time_is_invalid_argument() { + let now = time::Date::MAX.midnight().assume_utc(); + + let error = resolve_step_genesis_time("the chain starts 1 seconds from now", now, 1) + .expect_err("overflowing offset should fail"); + + assert!(matches!(error, StepError::InvalidArgument { .. })); +} + +#[test] +fn genesis_time_change_after_nodes_started_is_rejected() { + let error = + validate_genesis_time_change(Some(GenesisTime::new(1_000)), true, GenesisTime::new(1_001)) + .expect_err("genesis time should not change after nodes start"); + + assert!( + matches!(error, StepError::LogicalError { message } if message == "cannot change genesis time after nodes have started") + ); +} diff --git a/tests/src/cucumber/steps/manual_transactions/drain_wallets.rs b/tests/src/cucumber/steps/transactions/drain_wallets.rs similarity index 100% rename from tests/src/cucumber/steps/manual_transactions/drain_wallets.rs rename to tests/src/cucumber/steps/transactions/drain_wallets.rs diff --git a/tests/src/cucumber/steps/manual_transactions/faucet.rs b/tests/src/cucumber/steps/transactions/faucet.rs similarity index 99% rename from tests/src/cucumber/steps/manual_transactions/faucet.rs rename to tests/src/cucumber/steps/transactions/faucet.rs index 422abe537..2187ed34e 100644 --- a/tests/src/cucumber/steps/manual_transactions/faucet.rs +++ b/tests/src/cucumber/steps/transactions/faucet.rs @@ -9,7 +9,7 @@ use tracing::{info, warn}; use crate::cucumber::{ error::StepError, - steps::{TARGET, manual_nodes::utils::fetch_public_peer_consensus}, + steps::{TARGET, nodes::fetch_public_peer_consensus}, utils::truncate_hash, world::PublicCryptarchiaEndpointPeer, }; diff --git a/tests/src/cucumber/steps/manual_transactions/inscriptions.rs b/tests/src/cucumber/steps/transactions/inscriptions.rs similarity index 99% rename from tests/src/cucumber/steps/manual_transactions/inscriptions.rs rename to tests/src/cucumber/steps/transactions/inscriptions.rs index 0b1a59fd6..3bc7cdd2c 100644 --- a/tests/src/cucumber/steps/manual_transactions/inscriptions.rs +++ b/tests/src/cucumber/steps/transactions/inscriptions.rs @@ -17,7 +17,7 @@ use crate::{ error::{StepError, StepResult}, steps::{ TARGET, - manual_transactions::utils::{ + transactions::utils::{ prepare_user_wallet_transaction_submission, submit_prepared_user_wallet_transaction, }, }, diff --git a/tests/src/cucumber/steps/transactions/manual_control/execution/control.rs b/tests/src/cucumber/steps/transactions/manual_control/execution/control.rs new file mode 100644 index 000000000..05654e36b --- /dev/null +++ b/tests/src/cucumber/steps/transactions/manual_control/execution/control.rs @@ -0,0 +1,54 @@ +use super::{ + CucumberWorld, Duration, Instant, MANUAL_COMMAND_FILE_ENV, MANUAL_COMMAND_POLL_INTERVAL_ENV, + Path, StepError, TARGET, env, execute_manual_command, info, sleep, take_next_command, +}; + +#[expect( + clippy::cognitive_complexity, + reason = "Singular fn with multiple branches to handle different events and futures." +)] +pub async fn perform_manual_step_control( + world: &mut CucumberWorld, + step: &str, + timeout_seconds: u64, +) -> Result<(), StepError> { + let command_file = + env::var(MANUAL_COMMAND_FILE_ENV).map_err(|_| StepError::InvalidArgument { + message: format!( + "Step `{step}` requires environment variable '{MANUAL_COMMAND_FILE_ENV}' to be set", + ), + })?; + let poll_interval_ms = env::var(MANUAL_COMMAND_POLL_INTERVAL_ENV) + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(300); + + info!( + target: TARGET, + "Manual control step started. Monitoring command file: `{command_file}`" + ); + + let time_out = Duration::from_secs(timeout_seconds); + let start = Instant::now(); + while start.elapsed() < time_out { + if let Some(command) = take_next_command(Path::new(&command_file))? { + info!(target: TARGET, "====> manual command: {command:?}"); + if matches!( + execute_manual_command(world, step, &command).await, + Ok(true) + ) { + info!( + target: TARGET, + "Manual command loop stopped by STOP command after {:.2?}", + start.elapsed() + ); + return Ok(()); + } + } else { + sleep(Duration::from_millis(poll_interval_ms)).await; + } + } + info!(target: TARGET, "Manual command loop stopped by tine-out after {:.2?}", start.elapsed()); + + Ok(()) +} diff --git a/tests/src/cucumber/steps/transactions/manual_control/execution/dispatch.rs b/tests/src/cucumber/steps/transactions/manual_control/execution/dispatch.rs new file mode 100644 index 000000000..34c5bf7d2 --- /dev/null +++ b/tests/src/cucumber/steps/transactions/manual_control/execution/dispatch.rs @@ -0,0 +1,493 @@ +use super::{ + BTreeMap, BuildHasher, CucumberWorld, Duration, HashSet, ManualCommand, NoteId, StepError, + TARGET, TxHash, WalletOutputState, WalletSendReadiness, WalletUtxos, all_user_wallets, + build_cycle_fee_policy, clear_all_wallet_encumbrances, clear_wallet_encumbrances, + create_snapshot_all_nodes_with_wallet_state, create_snapshot_node_with_wallet_state, + current_available_utxos_for_user_wallets, drain_all_node_wallets, execute_coin_split, + execute_coin_split_with_utxo_cache, execute_continuous_round_robin, execute_drain, + execute_send, export_funds, extend_note_id_set, extend_tx_hash_set, handle_verify_command, + info, log_wallet_balance, log_wallet_balances, nodes, + prepare_ring_send_round_send_with_utxo_cache, request_faucet_funds_all_funding_wallets, + request_faucet_funds_all_user_wallets, restart_node, sync, utils, + validate_fee_horizon_after_wallet_batch, verify_no_duplicate_transactions, + wait_for_all_nodes_to_be_synced_to_chain, wait_for_observed_transaction_hashes, +}; + +pub async fn execute_manual_command( + world: &mut CucumberWorld, + step: &str, + command: &ManualCommand, +) -> Result { + if matches!(command, ManualCommand::Stop) { + return Ok(true); + } + + execute_non_stop_manual_command(world, step, command).await?; + Ok(false) +} + +#[expect( + clippy::too_many_arguments, + reason = "Cucumber transaction shape and fee policy inputs" +)] +pub async fn execute_continuous_round_robin_user_wallets( + world: &mut CucumberWorld, + step: &str, + coin_split_outputs: usize, + coin_split_value: u64, + num_transactions: usize, + value: u64, + cycles: usize, + epochs_headroom: u32, +) -> Result<(), StepError> { + let command = ManualCommand::ContinuousRoundRobinUserWallets { + coin_split_outputs, + coin_split_value, + num_transactions, + value, + cycles, + epochs_headroom, + }; + execute_non_stop_manual_command(world, step, &command).await +} + +pub async fn execute_coin_splits_all_user_wallets( + world: &mut CucumberWorld, + step: &str, + splits_per_wallet: usize, + outputs: usize, + value: u64, +) -> Result<(), StepError> { + let mut wallet_names: Vec<_> = world + .all_user_wallets() + .iter() + .map(|w| w.wallet_name.clone()) + .collect(); + if wallet_names.len() < 2 { + return Err(StepError::InvalidArgument { + message: "coin split for all user wallets requires at least two wallets".to_owned(), + }); + } + wallet_names.sort(); + let mut available_utxos = current_available_utxos_for_user_wallets(world, step).await?; + + for wallet_name in &wallet_names { + let best_node_info = sync::wait_wallet_send_ready( + world, + step, + wallet_name, + 180, + splits_per_wallet as u64 * outputs as u64 * value, + WalletSendReadiness::TotalValueOnly, + &mut available_utxos, + &HashSet::new(), + ) + .await?; + + for _ in 0..splits_per_wallet { + execute_coin_split_with_utxo_cache( + world, + step, + wallet_name, + outputs, + value, + Some(&best_node_info), + &mut available_utxos, + ) + .await?; + } + } + + Ok(()) +} + +pub async fn verify_min_outputs_all_user_wallets( + world: &mut CucumberWorld, + step: &str, + min_outputs: usize, + timeout_seconds: u64, + wallet_state_type: WalletOutputState, +) -> Result<(), StepError> { + let mut wallet_names: Vec<_> = world + .all_user_wallets() + .iter() + .map(|w| w.wallet_name.clone()) + .collect(); + wallet_names.sort(); + + for wallet_name in &wallet_names { + utils::wait_for_wallet_output_state( + world, + step, + wallet_name.clone(), + Some(&min_outputs), + None, + None, + None, + timeout_seconds, + wallet_state_type, + ) + .await?; + } + + Ok(()) +} + +fn destructure_next_wallet_command( + command: &ManualCommand, +) -> Result<(usize, usize, u64, u32), StepError> { + let ManualCommand::ContinuousNextWalletUserWallets { + cycles, + num_transactions, + value, + epochs_headroom, + } = command + else { + return Err(StepError::LogicalError { + message: "expected ContinuousNextWalletUserWallets command".to_owned(), + }); + }; + Ok((*cycles, *num_transactions, *value, *epochs_headroom)) +} + +pub async fn execute_continuous_next_wallet_user_wallet( + world: &mut CucumberWorld, + step: &str, + command: &ManualCommand, +) -> Result<(), StepError> { + execute_continuous_next_wallet_user_wallet_inner(world, step, command).await +} + +async fn execute_continuous_next_wallet_user_wallet_inner( + world: &mut CucumberWorld, + step: &str, + command: &ManualCommand, +) -> Result<(), StepError> { + let (cycles, transactions_per_wallet, value, epochs_headroom) = + destructure_next_wallet_command(command)?; + let wallet_names = all_user_wallets(world)?; + + let mut used_input_note_ids: HashSet = HashSet::new(); + let mut all_next_wallet_tx_hashes = HashSet::new(); + for cycle in 0..cycles { + let mut available_utxos = current_available_utxos_for_user_wallets(world, step).await?; + + let (cycle_tx_hashes, cycle_used_input_note_ids) = execute_ring_send_round_with_utxo_cache( + world, + step, + &wallet_names, + transactions_per_wallet, + value, + cycle, + epochs_headroom, + &mut available_utxos, + &used_input_note_ids, + ) + .await?; + verify_no_duplicate_transactions( + &cycle_tx_hashes, + &all_next_wallet_tx_hashes, + cycle, + "CONTINUOUS NEXT WALLET", + )?; + extend_note_id_set(&mut used_input_note_ids, &cycle_used_input_note_ids); + + verify_transactions_mined( + world, + step, + &cycle_tx_hashes, + wallet_names.len() * transactions_per_wallet, + Some(cycle + 1), + "CONTINUOUS NEXT WALLET", + "D", + ) + .await?; + extend_tx_hash_set(&mut all_next_wallet_tx_hashes, &cycle_tx_hashes); + } + + let expected_total = cycles * wallet_names.len() * transactions_per_wallet; + if all_next_wallet_tx_hashes.len() != expected_total { + return Err(StepError::StepFail { + message: format!( + "CONTINUOUS NEXT WALLET submitted {} unique transaction hash(es), expected \ + {expected_total}", + all_next_wallet_tx_hashes.len(), + ), + }); + } + + info!( + target: TARGET, + "CONTINUOUS NEXT WALLET scenario complete: {} unique submitted transaction(s) verified \ + across {} cycle(s)", + all_next_wallet_tx_hashes.len(), + cycles, + ); + + Ok(()) +} + +pub(super) async fn verify_transactions_mined( + world: &mut CucumberWorld, + step: &str, + tx_hashes: &HashSet, + expected_tx_count: usize, + cycle: Option, + tag: &str, + phase: &str, +) -> Result<(), StepError> { + if tx_hashes.len() != expected_tx_count { + return Err(StepError::StepFail { + message: format!( + "{tag}{} submitted {} transaction hash(es), expected {expected_tx_count}", + cycle.map_or_else(String::new, |cycle| format!(" cycle {cycle}")), + tx_hashes.len(), + ), + }); + } + + info!( + target: TARGET, + "{tag}{} {phase}: Wait for {} submitted transaction hashes to be observed in chain blocks", + cycle.map_or_else(String::new, |cycle| format!(" cycle {cycle}")), + tx_hashes.len(), + ); + + wait_for_observed_transaction_hashes(world, step, tx_hashes, Duration::from_mins(10)).await +} + +pub(super) fn log_phase_counts( + tag: &str, + cycle: usize, + phase: &str, + kind: &str, + counts: &BTreeMap, +) { + let counts = counts + .iter() + .map(|(wallet, count)| format!("{wallet}={count}")) + .collect::>() + .join(", "); + info!( + target: TARGET, + "{tag} cycle {} {phase}: {kind} tx counts by sender wallet: {counts}", + cycle + 1, + ); +} + +#[expect(clippy::too_many_arguments, reason = "Need all args")] +async fn execute_ring_send_round_with_utxo_cache( + world: &mut CucumberWorld, + step: &str, + wallet_names: &[String], + transactions_per_wallet: usize, + value: u64, + cycle: usize, + epochs_headroom: u32, + available_utxos: &mut WalletUtxos, + used_input_note_ids: &HashSet, +) -> Result<(HashSet, HashSet), StepError> { + let policy = build_cycle_fee_policy(world, step, &wallet_names[0], epochs_headroom).await?; + + let mut signed_submissions = Vec::with_capacity(wallet_names.len() * transactions_per_wallet); + let mut prepared_counts = BTreeMap::new(); + + for i in 0..wallet_names.len() { + info!( + target: TARGET, + "CONTINUOUS NEXT WALLET cycle {} A: Await funds", + cycle + 1 + ); + let from = &wallet_names[i]; + let to = &wallet_names[(i + 1) % wallet_names.len()]; + + let required_available = transactions_per_wallet as u64 * value; + sync::wait_wallet_send_ready( + world, + step, + from, + 180, + required_available, + WalletSendReadiness::EligibleUtxoBatch { + min_required_outputs: transactions_per_wallet, + min_value_per_transaction: value, + }, + available_utxos, + used_input_note_ids, + ) + .await?; + + info!( + target: TARGET, + "CONTINUOUS NEXT WALLET cycle {} B: Prepare transactions to next wallet concurrently", + cycle + 1 + ); + let mut prepared = prepare_ring_send_round_send_with_utxo_cache( + world, + step, + transactions_per_wallet, + value, + from, + to, + available_utxos, + Some(policy.horizon.ceiling_prices.clone()), + policy.priority_fee_percent, + ) + .await?; + prepared_counts.insert(from.clone(), prepared.len()); + validate_fee_horizon_after_wallet_batch(world, &policy, from, prepared.len()).await?; + signed_submissions.append(&mut prepared); + } + + let mut cycle_used_input_note_ids: HashSet = HashSet::new(); + for submission in &signed_submissions { + extend_note_id_set( + &mut cycle_used_input_note_ids, + &submission.reserved_inputs().input_note_ids_list(), + ); + } + + log_phase_counts( + "CONTINUOUS NEXT WALLET", + cycle, + "C", + "prepared", + &prepared_counts, + ); + + let submitted_hashes = utils::submit_signed_user_wallet_submissions_concurrently( + world, + signed_submissions, + Some(&policy), + ) + .await?; + let mut submitted_counts = BTreeMap::new(); + for (sender, _) in &submitted_hashes { + *submitted_counts.entry(sender.clone()).or_insert(0usize) += 1; + } + log_phase_counts( + "CONTINUOUS NEXT WALLET", + cycle, + "D", + "submitted", + &submitted_counts, + ); + let cycle_tx_hashes = submitted_hashes + .into_iter() + .map(|(_, tx_hash)| tx_hash) + .collect::>(); + + Ok((cycle_tx_hashes, cycle_used_input_note_ids)) +} + +#[expect(clippy::too_many_lines, reason = "Test function.")] +async fn execute_non_stop_manual_command( + world: &mut CucumberWorld, + step: &str, + command: &ManualCommand, +) -> Result<(), StepError> { + match command { + ManualCommand::CreateSnapshotAllNodes { snapshot_name } => { + create_snapshot_all_nodes_with_wallet_state(world, snapshot_name).await + } + ManualCommand::CreateSnapshotNode { + snapshot_name, + node_name, + } => create_snapshot_node_with_wallet_state(world, snapshot_name, node_name).await, + ManualCommand::CoinSplit { + wallet, + outputs, + value, + } => execute_coin_split(world, step, wallet, *outputs, *value) + .await + .map(|_| ()), + ManualCommand::Verify { .. } => handle_verify_command(world, step, command).await, + ManualCommand::WalletBalance { wallet_name } => { + log_wallet_balance(world, step, wallet_name).await + } + ManualCommand::WalletBalanceAllUserWallets => { + log_wallet_balances(world, step, world.all_user_wallets()).await + } + ManualCommand::WalletBalanceAllFundingWallets => { + log_wallet_balances(world, step, world.all_node_wallets()).await + } + ManualCommand::WalletBalanceAllWallets => { + let mut wallets = world.all_user_wallets(); + wallets.extend(world.all_node_wallets()); + + log_wallet_balances(world, step, wallets).await + } + ManualCommand::ExportFunds { + wallet_name, + value, + output_path, + include_secret, + } => { + export_funds( + world, + step, + wallet_name, + *value, + output_path, + *include_secret, + ) + .await + } + ManualCommand::ClearEncumbrances { wallet_name } => { + clear_wallet_encumbrances(world, step, wallet_name) + } + ManualCommand::ClearEncumbrancesAllWallets => clear_all_wallet_encumbrances(world, step), + ManualCommand::Send { + num_transactions, + value, + from, + to, + } => execute_send(world, step, *num_transactions, *value, from, to).await, + ManualCommand::Drain { from, to } => execute_drain(world, step, from, to).await, + ManualCommand::DrainAllNodeWallets { node_name, to } => { + drain_all_node_wallets(world, node_name, to).await + } + ManualCommand::ContinuousRoundRobinUserWallets { .. } => { + execute_continuous_round_robin(world, step, command).await + } + ManualCommand::FaucetFundsAllUserWallets { rounds } => { + request_faucet_funds_all_user_wallets(world, step, *rounds) + } + ManualCommand::FaucetFundsAllFundingWallets { rounds } => { + request_faucet_funds_all_funding_wallets(world, step, *rounds) + } + ManualCommand::RestartNode { node_name } => restart_node(world, step, node_name).await, + ManualCommand::CryptarchiaInfoAllNodes => { + nodes::get_cryptarchia_info_all_nodes(world, step).await; + Ok(()) + } + ManualCommand::WaitAllNodesSyncedToChain => { + wait_for_all_nodes_to_be_synced_to_chain(world, step).await + } + ManualCommand::CoinSplitAllUserWallets { + splits_per_wallet, + outputs, + value, + } => { + execute_coin_splits_all_user_wallets(world, step, *splits_per_wallet, *outputs, *value) + .await + } + ManualCommand::VerifyMinAvailableOutputsAllUserWallets { + min_outputs, + timeout_seconds, + } => { + verify_min_outputs_all_user_wallets( + world, + step, + *min_outputs, + *timeout_seconds, + WalletOutputState::Available, + ) + .await + } + ManualCommand::ContinuousNextWalletUserWallets { .. } => { + execute_continuous_next_wallet_user_wallet(world, step, command).await + } + ManualCommand::Stop => Ok(()), + } +} diff --git a/tests/src/cucumber/steps/transactions/manual_control/execution/fee_policy.rs b/tests/src/cucumber/steps/transactions/manual_control/execution/fee_policy.rs new file mode 100644 index 000000000..1bb8f3b40 --- /dev/null +++ b/tests/src/cucumber/steps/transactions/manual_control/execution/fee_policy.rs @@ -0,0 +1,73 @@ +use super::{ + CucumberWorld, GasPrices, MAX_TEST_EPOCH_HEADROOM, StepError, TARGET, TransactionFeePolicy, + build_fee_horizon, get_best_node_info, info, +}; + +pub async fn build_cycle_fee_policy( + world: &CucumberWorld, + step: &str, + representative_wallet: &str, + epochs_headroom: u32, +) -> Result { + if epochs_headroom > MAX_TEST_EPOCH_HEADROOM { + return Err(StepError::InvalidArgument { + message: format!("epochs headroom per cycle must be at most {MAX_TEST_EPOCH_HEADROOM}"), + }); + } + + let best = get_best_node_info(world, representative_wallet, None).await?; + let best_node = best.best_node_for_wallet(world, representative_wallet)?; + let client = world.resolve_node_http_client(&best_node)?; + let consensus = client + .consensus_info() + .await + .map_err(|source| StepError::StepFail { + message: format!("Step `{step}` error: consensus query failed: {source}"), + })?; + let tip = consensus.cryptarchia_info.tip; + let prices = client + .gas_prices(Some(tip)) + .await + .map_err(|source| StepError::StepFail { + message: format!("Step `{step}` error: gas prices query failed: {source}"), + })?; + if prices.tip != tip { + return Err(StepError::StepFail { + message: format!( + "Step `{step}` error: gas prices response referenced tip {:?}, requested {:?}", + prices.tip, tip + ), + }); + } + let live_prices = GasPrices { + execution_base_gas_price: prices.execution_base_gas_price, + storage_gas_price: prices.storage_gas_price, + }; + let horizon = build_fee_horizon( + tip, + u64::from(consensus.cryptarchia_info.slot), + world.chain.slots_per_epoch, + epochs_headroom, + live_prices, + ) + .map_err(|source| StepError::LogicalError { + message: format!("failed to build transaction fee horizon: {source}"), + })?; + let policy = TransactionFeePolicy::new(horizon).map_err(|source| StepError::LogicalError { + message: format!("failed to build transaction fee policy: {source}"), + })?; + info!( + target: TARGET, + "Cycle fee horizon: prepared tip {:?}, slot {:?}, epoch {:?}, valid through {:?}, execution {}->{}, storage {}->{}, priority reserve {}%, representative wallet `{representative_wallet}`", + policy.horizon.prepared_at_tip, + consensus.cryptarchia_info.slot, + policy.horizon.prepared_at_epoch, + policy.horizon.valid_through_epoch, + policy.horizon.live_prices.execution_base_gas_price.into_inner(), + policy.horizon.ceiling_prices.execution_base_gas_price.into_inner(), + policy.horizon.live_prices.storage_gas_price.into_inner(), + policy.horizon.ceiling_prices.storage_gas_price.into_inner(), + policy.priority_fee_percent, + ); + Ok(policy) +} diff --git a/tests/src/cucumber/steps/transactions/manual_control/execution/mod.rs b/tests/src/cucumber/steps/transactions/manual_control/execution/mod.rs new file mode 100644 index 000000000..52b78b65e --- /dev/null +++ b/tests/src/cucumber/steps/transactions/manual_control/execution/mod.rs @@ -0,0 +1,119 @@ +//! This module executes manual commands for Cucumber scenarios. +//! +//! External command controller: +//! - Set `CUCUMBER_MANUAL_COMMAND_FILE=/tmp/cucumber-manual-commands.txt`. +//! - Start the scenario. +//! - Prepare the command file beforehand, or append commands while the test +//! runs. +//! +//! Supported commands (one per line): +//! +//! ```text +//! COIN_SPLIT, wallet '', outputs , value +//! VERIFY, wallet '', outputs , time_out +//! BALANCE, wallet '' +//! EXPORT_FUNDS, wallet '', value , output '', include_secret true|false +//! BALANCE_ALL_WALLETS +//! BALANCE_ALL_USER_WALLETS +//! BALANCE_ALL_FUNDING_WALLETS +//! CLEAR_ENCUMBRANCES, wallet '' +//! CLEAR_ENCUMBRANCES_ALL_WALLETS +//! SEND, num_transactions , value , from '', to '' +//! DRAIN, from '', to '' +//! DRAIN_ALL_NODE_WALLETS, node_name '', to '' +//! VERIFY_MAX, wallet '', wallet_state_type 'on-chain'/'encumbered'/'available', outputs , value 14000, time_out +//! VERIFY_MIN, wallet '', wallet_state_type 'on-chain'/'encumbered'/'available', outputs , value 14000, time_out +//! CONTINUOUS_ROUND_ROBIN_USER_WALLETS, coin_split_outputs , coin_split_value , num_transactions , value , cycles , epochs_headroom +//! COIN_SPLIT_ALL_USER_WALLETS, splits_per_wallet , outputs , value +//! VERIFY_MIN_AVAILABLE_OUTPUTS_ALL_USER_WALLETS, min_outputs , timeout_seconds +//! CONTINUOUS_NEXT_WALLET_USER_WALLETS, cycles , num_transactions , value , epochs_headroom +//! FAUCET_ALL_USER_WALLETS, rounds +//! FAUCET_ALL_FUNDING_WALLETS, rounds +//! CREATE_SNAPSHOT_ALL_NODES, snapshot_name '' +//! CREATE_SNAPSHOT_NODE, snapshot_name '', node_name '' +//! RESTART_NODE, node_name '' +//! CRYPTARCHIA_INFO_ALL_NODES +//! WAIT_ALL_NODES_SYNCED_TO_CHAIN +//! STOP +//! ``` + +use std::{ + collections::{BTreeMap, HashSet}, + env, fs, + hash::BuildHasher, + num::NonZero, + path::Path, + time::Duration, +}; + +use lb_core::mantle::{ + NoteId, Utxo, + transactions::{GasPrices, hash::TxHash}, +}; +use lb_key_management_system_service::keys::ZkPublicKey; +use lb_wallet::WalletError; +use serde::Serialize; +use tokio::time::{Instant, sleep}; +use tracing::{info, warn}; + +use crate::{ + common::wallet::{TransactionFeePolicy, WalletStateView, WalletUtxos, build_fee_horizon}, + cucumber::{ + error::{StepError, StepResult}, + steps::{ + TARGET, nodes, + nodes::{ + create_snapshot_all_nodes_with_wallet_state, + create_snapshot_node_with_wallet_state, restart_node, + wait_for_all_nodes_to_be_synced_to_chain, + }, + transactions::{ + drain_wallets::{drain_all_node_wallets, drain_node_wallet, drain_user_wallet}, + manual_control::parsing::{ManualCommand, take_next_command}, + utils, + utils::{BestNodeInfo, WalletOutputState, extend_note_id_set, extend_tx_hash_set}, + }, + }, + wallet::{ + best_node::get_best_node_info, + checks::wait_for_observed_transaction_hashes, + submissions::{SignedUserWalletSubmission, validate_fee_horizon_after_wallet_batch}, + sync, + sync::{WalletSendReadiness, current_available_utxos_for_user_wallets}, + }, + world::{CucumberWorld, WalletInfo, WalletType}, + }, +}; + +const MANUAL_COMMAND_FILE_ENV: &str = "CUCUMBER_MANUAL_COMMAND_FILE"; +const MANUAL_COMMAND_POLL_INTERVAL_ENV: &str = "CUCUMBER_MANUAL_COMMAND_POLL_INTERVAL_MS"; +const MAX_TEST_EPOCH_HEADROOM: u32 = 16; + +mod control; +mod dispatch; +mod fee_policy; +mod round_robin; +mod transactions; +mod wallet_state; + +pub use control::perform_manual_step_control; +pub use dispatch::{ + execute_coin_splits_all_user_wallets, execute_continuous_next_wallet_user_wallet, + execute_continuous_round_robin_user_wallets, execute_manual_command, + verify_min_outputs_all_user_wallets, +}; +use dispatch::{log_phase_counts, verify_transactions_mined}; +pub use fee_policy::build_cycle_fee_policy; +use round_robin::{ + all_user_wallets, execute_continuous_round_robin, verify_no_duplicate_transactions, +}; +use transactions::{ + execute_coin_split, execute_coin_split_with_utxo_cache, execute_send, handle_verify_command, + prepare_coin_splits_all_wallets_with_utxo_cache, prepare_ring_send_round_send_with_utxo_cache, + request_faucet_funds_all_funding_wallets, request_faucet_funds_all_user_wallets, +}; +pub use wallet_state::log_wallet_balances; +use wallet_state::{ + clear_all_wallet_encumbrances, clear_wallet_encumbrances, execute_drain, export_funds, + log_wallet_balance, +}; diff --git a/tests/src/cucumber/steps/transactions/manual_control/execution/round_robin.rs b/tests/src/cucumber/steps/transactions/manual_control/execution/round_robin.rs new file mode 100644 index 000000000..8a9d8b8aa --- /dev/null +++ b/tests/src/cucumber/steps/transactions/manual_control/execution/round_robin.rs @@ -0,0 +1,537 @@ +use super::{ + BTreeMap, BuildHasher, CucumberWorld, Duration, GasPrices, HashSet, Instant, ManualCommand, + NoteId, SignedUserWalletSubmission, StepError, TARGET, TransactionFeePolicy, TxHash, + WalletSendReadiness, WalletUtxos, build_cycle_fee_policy, extend_note_id_set, + extend_tx_hash_set, get_best_node_info, info, log_phase_counts, + prepare_coin_splits_all_wallets_with_utxo_cache, sleep, sync, utils, + validate_fee_horizon_after_wallet_batch, verify_transactions_mined, + wait_for_observed_transaction_hashes, warn, +}; + +fn destructure_round_robin_command( + command: &ManualCommand, +) -> Result<(usize, u64, usize, u64, usize, u32), StepError> { + let ManualCommand::ContinuousRoundRobinUserWallets { + coin_split_outputs, + coin_split_value, + num_transactions, + value, + cycles, + epochs_headroom, + } = command + else { + return Err(StepError::LogicalError { + message: "expected ContinuousRoundRobinUserWallets command".to_owned(), + }); + }; + Ok(( + *coin_split_outputs, + *coin_split_value, + *num_transactions, + *value, + *cycles, + *epochs_headroom, + )) +} + +pub(super) fn all_user_wallets(world: &CucumberWorld) -> Result, StepError> { + let mut wallet_names = world + .all_user_wallets() + .iter() + .map(|w| w.wallet_name.clone()) + .collect::>(); + if wallet_names.len() < 2 { + return Err(StepError::InvalidArgument { + message: "This command requires at least two user wallets".to_owned(), + }); + } + wallet_names.sort(); + Ok(wallet_names) +} + +#[expect(clippy::too_many_arguments, reason = "Transaction preparation inputs")] +async fn prepare_and_submit_round_robin_transactions( + world: &mut CucumberWorld, + step: &str, + cycle: usize, + wallet_names: &[String], + num_transactions: usize, + value: u64, + available_utxos: &mut WalletUtxos, + epochs_headroom: u32, +) -> Result<(HashSet, HashSet), StepError> { + let policy = build_cycle_fee_policy(world, step, &wallet_names[0], epochs_headroom).await?; + + let mut signed_submissions = Vec::with_capacity(wallet_names.len() * num_transactions); + let mut prepared_counts = BTreeMap::new(); + for sender in wallet_names { + let recipients = recipient_wallets(wallet_names, sender)?; + let mut prepared = prepare_round_robin_with_utxo_cache( + world, + step, + sender, + &recipients, + num_transactions, + value, + available_utxos, + Some(policy.horizon.ceiling_prices.clone()), + policy.priority_fee_percent, + ) + .await + .map_err(|e| StepError::StepFail { + message: format!( + "CONTINUOUS ROUND ROBIN cycle {} failed to prepare transactions for sender \ + '{sender}': {e}", + cycle + 1, + ), + })?; + + prepared_counts.insert(sender.clone(), prepared.len()); + validate_fee_horizon_after_wallet_batch(world, &policy, sender, prepared.len()).await?; + signed_submissions.append(&mut prepared); + } + + let mut cycle_used_input_note_ids: HashSet = HashSet::new(); + for submission in &signed_submissions { + extend_note_id_set( + &mut cycle_used_input_note_ids, + &submission.reserved_inputs().input_note_ids_list(), + ); + } + + log_phase_counts( + "CONTINUOUS ROUND ROBIN", + cycle, + "D", + "prepared", + &prepared_counts, + ); + + let submitted_hashes = utils::submit_signed_user_wallet_submissions_concurrently( + world, + signed_submissions, + Some(&policy), + ) + .await?; + let mut submitted_counts = BTreeMap::new(); + for (sender, _) in &submitted_hashes { + *submitted_counts.entry(sender.clone()).or_insert(0usize) += 1; + } + log_phase_counts( + "CONTINUOUS ROUND ROBIN", + cycle, + "D", + "submitted", + &submitted_counts, + ); + + let cycle_tx_hashes = submitted_hashes + .into_iter() + .map(|(_, tx_hash)| tx_hash) + .collect::>(); + + Ok((cycle_tx_hashes, cycle_used_input_note_ids)) +} + +/// Manages the coin split process for a round robin cycle, including performing +/// the coin splits, waiting for them to be mined, and verifying that the +/// transactions were successfully mined. Returns a set of used input note IDs +/// from the coin split transactions. Note: This function needs a readiness +/// prepared UTXO cache. +#[expect(clippy::too_many_arguments, reason = "Round-robin split inputs")] +async fn manage_round_robin_coin_splits_with_utxo_cache( + world: &mut CucumberWorld, + step: &str, + cycle: usize, + wallet_names: &[String], + coin_split_outputs: usize, + coin_split_value: u64, + epochs_headroom: u32, + available_utxos: &mut WalletUtxos, +) -> Result, StepError> { + let policy = build_cycle_fee_policy(world, step, &wallet_names[0], epochs_headroom).await?; + + let (split_tx_hashes, split_used_input_note_ids) = + perform_coin_splits_for_round_robin_with_utxo_cache( + world, + step, + wallet_names, + coin_split_outputs, + coin_split_value, + cycle, + available_utxos, + &policy, + ) + .await?; + + wait_for_n_blocks_or_warn(world, step, wallet_names, Duration::from_mins(3), 2, cycle).await?; + + verify_transactions_mined( + world, + step, + &split_tx_hashes, + wallet_names.len(), + Some(cycle + 1), + "CONTINUOUS ROUND ROBIN", + "B", + ) + .await?; + + Ok(split_used_input_note_ids) +} + +async fn refresh_round_robin_sender_cache_entries( + world: &mut CucumberWorld, + step: &str, + wallet_names: &[String], + required_available: u64, + readiness: WalletSendReadiness, + available_utxos: &mut WalletUtxos, + used_input_note_ids: &HashSet, +) -> Result<(), StepError> { + for sender in wallet_names { + sync::wait_wallet_send_ready( + world, + step, + sender, + 180, + required_available, + readiness, + available_utxos, + used_input_note_ids, + ) + .await?; + } + + Ok(()) +} + +pub(super) fn verify_no_duplicate_transactions( + cycle_tx_hashes: &HashSet, + all_tx_hashes: &HashSet, + cycle: usize, + scenario_tag: &str, +) -> Result<(), StepError> { + let duplicate_hashes = cycle_tx_hashes + .intersection(all_tx_hashes) + .copied() + .collect::>(); + if duplicate_hashes.is_empty() { + Ok(()) + } else { + Err(StepError::StepFail { + message: format!( + "{scenario_tag} cycle {} prepared/submitted {} duplicate transaction hash(es) from \ + previous cycles", + cycle + 1, + duplicate_hashes.len(), + ), + }) + } +} + +#[expect(clippy::too_many_lines, reason = "Test function.")] +#[expect( + clippy::cognitive_complexity, + reason = "This function has multiple steps that are logically distinct." +)] +pub(super) async fn execute_continuous_round_robin( + world: &mut CucumberWorld, + step: &str, + command: &ManualCommand, +) -> Result<(), StepError> { + let (coin_split_outputs, coin_split_value, num_transactions, value, cycles, epochs_headroom) = + destructure_round_robin_command(command)?; + let wallet_names = all_user_wallets(world)?; + + let mut used_input_note_ids: HashSet = HashSet::new(); + let mut all_round_robin_tx_hashes = HashSet::new(); + let mut available_utxos = WalletUtxos::new(); + for cycle in 0..cycles { + info!( + target: TARGET, + "CONTINUOUS ROUND ROBIN cycle {} A: Wait for available coin-split funds all wallets", + cycle + 1 + ); + + refresh_round_robin_sender_cache_entries( + world, + step, + &wallet_names, + coin_split_outputs as u64 * coin_split_value, + WalletSendReadiness::TotalValueOnly, + &mut available_utxos, + &used_input_note_ids, + ) + .await?; + + info!( + target: TARGET, + "CONTINUOUS ROUND ROBIN cycle {} B: Perform coin splits all wallets and wait mined", + cycle + 1 + ); + + let split_used_input_note_ids = manage_round_robin_coin_splits_with_utxo_cache( + world, + step, + cycle, + &wallet_names, + coin_split_outputs, + coin_split_value, + epochs_headroom, + &mut available_utxos, + ) + .await?; + extend_note_id_set(&mut used_input_note_ids, &split_used_input_note_ids); + + info!( + target: TARGET, + "CONTINUOUS ROUND ROBIN cycle {} C: Wait all wallets ready with coin-split outputs", + cycle + 1 + ); + + refresh_round_robin_sender_cache_entries( + world, + step, + &wallet_names, + num_transactions as u64 * value, + WalletSendReadiness::EligibleUtxoBatch { + min_required_outputs: num_transactions, + min_value_per_transaction: value, + }, + &mut available_utxos, + &used_input_note_ids, + ) + .await?; + + info!( + target: TARGET, + "CONTINUOUS ROUND ROBIN cycle {} D: Prepare and send all round robin transactions", + cycle + 1 + ); + + let (cycle_tx_hashes, cycle_used_input_note_ids) = + prepare_and_submit_round_robin_transactions( + world, + step, + cycle, + &wallet_names, + num_transactions, + value, + &mut available_utxos, + epochs_headroom, + ) + .await?; + verify_no_duplicate_transactions( + &cycle_tx_hashes, + &all_round_robin_tx_hashes, + cycle, + "CONTINUOUS ROUND ROBIN", + )?; + extend_note_id_set(&mut used_input_note_ids, &cycle_used_input_note_ids); + + // Assert transaction count + verify_transactions_mined( + world, + step, + &cycle_tx_hashes, + wallet_names.len() * num_transactions, + Some(cycle + 1), + "CONTINUOUS ROUND ROBIN", + "E", + ) + .await?; + + // Collect hashes for final drain verification + extend_tx_hash_set(&mut all_round_robin_tx_hashes, &cycle_tx_hashes); + } + + // Final drain: verify submitted D-phase transaction hashes are observed in + // chain blocks. + info!( + target: TARGET, + "CONTINUOUS ROUND ROBIN final: Verify {} submitted round-robin transaction(s) were observed \ + in chain blocks", + all_round_robin_tx_hashes.len(), + ); + + wait_for_observed_transaction_hashes( + world, + step, + &all_round_robin_tx_hashes, + Duration::from_mins(10), + ) + .await?; + + info!( + target: TARGET, + "CONTINUOUS ROUND ROBIN scenario complete: {} transaction(s) verified from observed chain block transaction hashes across {} cycle(s)", + all_round_robin_tx_hashes.len(), + cycles + ); + + Ok(()) +} + +async fn wait_for_n_blocks_or_warn( + world: &CucumberWorld, + step: &str, + wallet_names: &[String], + time_out: Duration, + blocks_to_wait: u64, + cycle: usize, +) -> Result<(), StepError> { + if wallet_names.is_empty() { + return Err(StepError::InvalidArgument { + message: "No wallet names provided for wait_for_n_blocks".to_owned(), + }); + } + let mut last_msg = String::new(); + let best_node_info = get_best_node_info(world, &wallet_names[0], Some(&mut last_msg)).await?; + let node = world + .resolve_node_http_client(&best_node_info.best_node_for_wallet(world, &wallet_names[0])?)?; + let start_height = node.consensus_info().await?.cryptarchia_info.height; + let start = Instant::now(); + loop { + sleep(Duration::from_secs(1)).await; + let best_node_info = + get_best_node_info(world, &wallet_names[0], Some(&mut last_msg)).await?; + let node = world.resolve_node_http_client( + &best_node_info.best_node_for_wallet(world, &wallet_names[0])?, + )?; + let height = node.consensus_info().await?.cryptarchia_info.height; + if height >= start_height + blocks_to_wait { + return Ok(()); + } + if start.elapsed() > time_out { + warn!( + target: TARGET, + "Step `{step}` cycle {}: Chain could not grow by {blocks_to_wait} blocks in {time_out:.2?}", + cycle + 1 + ); + return Ok(()); + } + } +} + +#[expect(clippy::too_many_arguments, reason = "Round-robin split inputs")] +async fn perform_coin_splits_for_round_robin_with_utxo_cache( + world: &mut CucumberWorld, + step: &str, + wallet_names: &[String], + coin_split_outputs: usize, + coin_split_value: u64, + cycle: usize, + available_utxos: &mut WalletUtxos, + policy: &TransactionFeePolicy, +) -> Result<(HashSet, HashSet), StepError> { + info!(target: TARGET, "CONTINUOUS ROUND ROBIN cycle {} B: Perform coin splits all wallets", cycle + 1); + + let (signed_submissions, prepared_split_counts) = + prepare_coin_splits_all_wallets_with_utxo_cache( + world, + step, + wallet_names, + coin_split_outputs, + coin_split_value, + available_utxos, + Some(policy.horizon.ceiling_prices.clone()), + policy.priority_fee_percent, + ) + .await?; + log_phase_counts( + "CONTINUOUS ROUND ROBIN", + cycle, + "B", + "split prepared", + &prepared_split_counts, + ); + + let mut split_used_input_note_ids: HashSet = HashSet::new(); + for submission in &signed_submissions { + extend_note_id_set( + &mut split_used_input_note_ids, + &submission.reserved_inputs().input_note_ids_list(), + ); + } + + let submitted_split_hashes = utils::submit_signed_user_wallet_submissions_concurrently( + world, + signed_submissions, + Some(policy), + ) + .await?; + let mut submitted_split_counts = BTreeMap::new(); + for (sender, _) in &submitted_split_hashes { + *submitted_split_counts + .entry(sender.clone()) + .or_insert(0usize) += 1; + } + log_phase_counts( + "CONTINUOUS ROUND ROBIN", + cycle, + "B", + "split submitted", + &submitted_split_counts, + ); + + let split_tx_hashes = submitted_split_hashes + .into_iter() + .map(|(_, tx_hash)| tx_hash) + .collect::>(); + + Ok((split_tx_hashes, split_used_input_note_ids)) +} + +fn recipient_wallets(wallet_names: &[String], sender: &str) -> Result, StepError> { + let recipients: Vec<_> = wallet_names + .iter() + .filter(|wallet| wallet.as_str() != sender) + .cloned() + .collect(); + if recipients.is_empty() { + return Err(StepError::InvalidArgument { + message: format!("No recipient wallets available for sender '{sender}'"), + }); + } + + Ok(recipients) +} + +#[expect(clippy::too_many_arguments, reason = "Transaction preparation inputs")] +async fn prepare_round_robin_with_utxo_cache( + world: &mut CucumberWorld, + step: &str, + sender: &str, + recipients: &[String], + transactions: usize, + value: u64, + available_utxos: &mut WalletUtxos, + gas_prices: Option, + priority_fee_percent: u64, +) -> Result, StepError> { + let mut reserved_submissions = Vec::with_capacity(transactions); + + for i in 0..transactions { + let receiver_name = &recipients[i % recipients.len()]; + let receiver = world.resolve_recipient(receiver_name)?; + let receiver_pk = receiver.public_key; + + let receivers = vec![(receiver_pk, value)]; + let reserved_submission = + utils::reserve_user_wallet_transaction_submission_with_utxo_cache( + world, + step, + sender, + &receivers, + available_utxos, + gas_prices.clone(), + priority_fee_percent, + ) + .await?; + + reserved_submissions.push(reserved_submission); + } + + utils::finalize_reserved_user_wallet_submissions_concurrently(step, reserved_submissions).await +} diff --git a/tests/src/cucumber/steps/transactions/manual_control/execution/transactions.rs b/tests/src/cucumber/steps/transactions/manual_control/execution/transactions.rs new file mode 100644 index 000000000..c9304c786 --- /dev/null +++ b/tests/src/cucumber/steps/transactions/manual_control/execution/transactions.rs @@ -0,0 +1,301 @@ +use super::{ + BTreeMap, BestNodeInfo, CucumberWorld, GasPrices, HashSet, ManualCommand, NonZero, + SignedUserWalletSubmission, StepError, TxHash, WalletError, WalletInfo, WalletSendReadiness, + WalletUtxos, ZkPublicKey, sync, utils, +}; + +pub(super) async fn handle_verify_command( + world: &mut CucumberWorld, + step: &str, + command: &ManualCommand, +) -> Result<(), StepError> { + let ManualCommand::Verify { + wallet, + outputs, + value, + time_out, + wallet_state_type, + verify_max, + } = command + else { + unreachable!("handle_verify_command must be called with ManualCommand::Verify") + }; + + let verify_min = !*verify_max; + utils::wait_for_wallet_output_state( + world, + step, + wallet.clone(), + if verify_min { outputs.as_ref() } else { None }, + if *verify_max { outputs.as_ref() } else { None }, + if verify_min { value.as_ref() } else { None }, + if *verify_max { value.as_ref() } else { None }, + *time_out, + *wallet_state_type, + ) + .await +} + +pub(super) fn request_faucet_funds_all_user_wallets( + world: &mut CucumberWorld, + step: &str, + rounds: usize, +) -> Result<(), StepError> { + let number_of_rounds = NonZero::new(rounds).ok_or_else(|| StepError::InvalidArgument { + message: "Invalid value for 'rounds': '0'".to_owned(), + })?; + let all_wallets_pk_hex = world + .wallet_registry + .wallet_info + .values() + .filter(|w| w.is_user_wallet()) + .map(WalletInfo::public_key_hex) + .collect::>(); + utils::request_faucet_funds(world, step, number_of_rounds, &all_wallets_pk_hex) +} + +pub(super) fn request_faucet_funds_all_funding_wallets( + world: &mut CucumberWorld, + step: &str, + rounds: usize, +) -> Result<(), StepError> { + let number_of_rounds = NonZero::new(rounds).ok_or_else(|| StepError::InvalidArgument { + message: "Invalid value for 'rounds': '0'".to_owned(), + })?; + let all_wallets_pk_hex = world + .wallet_registry + .wallet_info + .values() + .filter(|wallet| wallet.is_node_funding_wallet()) + .map(WalletInfo::public_key_hex) + .collect::>(); + utils::request_faucet_funds(world, step, number_of_rounds, &all_wallets_pk_hex) +} + +pub(super) async fn execute_coin_split( + world: &mut CucumberWorld, + step: &str, + wallet_name: &str, + outputs: usize, + value: u64, +) -> Result, StepError> { + let wallet = world.resolve_wallet(wallet_name)?; + let self_pk = wallet.public_key()?; + let receivers = vec![(self_pk, value); outputs]; + + let mut available_utxos = WalletUtxos::new(); + let best_node_info = sync::wait_wallet_send_ready( + world, + step, + wallet_name, + 180, + outputs as u64 * value, + WalletSendReadiness::TotalValueOnly, + &mut available_utxos, + &HashSet::new(), + ) + .await?; + + utils::create_and_submit_transaction_hashes_with_utxo_cache( + world, + step, + wallet_name, + &receivers, + Some(&best_node_info), + Some(&mut available_utxos), + ) + .await +} + +pub(super) async fn execute_coin_split_with_utxo_cache( + world: &mut CucumberWorld, + step: &str, + wallet_name: &str, + outputs: usize, + value: u64, + best_node_info: Option<&BestNodeInfo>, + available_utxos: &mut WalletUtxos, +) -> Result, StepError> { + let wallet = world.resolve_wallet(wallet_name)?; + let self_pk = wallet.public_key()?; + let receivers = vec![(self_pk, value); outputs]; + utils::create_and_submit_transaction_hashes_with_utxo_cache( + world, + step, + wallet_name, + &receivers, + best_node_info, + Some(available_utxos), + ) + .await +} + +async fn prepare_signed_submissions_with_utxo_cache( + world: &mut CucumberWorld, + step: &str, + requests: Vec<(String, Vec<(ZkPublicKey, u64)>)>, + available_utxos: &mut WalletUtxos, + gas_prices: Option, + priority_fee_percent: u64, +) -> Result, StepError> { + let mut reserved_submissions = Vec::with_capacity(requests.len()); + + for (sender, receivers) in requests { + let reserved_submission = + utils::reserve_user_wallet_transaction_submission_with_utxo_cache( + world, + step, + &sender, + &receivers, + available_utxos, + gas_prices.clone(), + priority_fee_percent, + ) + .await?; + reserved_submissions.push(reserved_submission); + } + + utils::finalize_reserved_user_wallet_submissions_concurrently(step, reserved_submissions).await +} + +#[expect(clippy::too_many_arguments, reason = "Coin-split preparation inputs")] +pub(super) async fn prepare_coin_splits_all_wallets_with_utxo_cache( + world: &mut CucumberWorld, + step: &str, + wallet_names: &[String], + outputs: usize, + value: u64, + available_utxos: &mut WalletUtxos, + gas_prices: Option, + priority_fee_percent: u64, +) -> Result<(Vec, BTreeMap), StepError> { + let mut requests = Vec::with_capacity(wallet_names.len()); + let mut prepared_counts = BTreeMap::new(); + + for wallet_name in wallet_names { + let wallet = world.resolve_wallet(wallet_name)?; + let self_pk = wallet.public_key()?; + let receivers = vec![(self_pk, value); outputs]; + *prepared_counts.entry(wallet_name.clone()).or_insert(0usize) += 1; + requests.push((wallet_name.clone(), receivers)); + } + + let signed_submissions = prepare_signed_submissions_with_utxo_cache( + world, + step, + requests, + available_utxos, + gas_prices, + priority_fee_percent, + ) + .await?; + Ok((signed_submissions, prepared_counts)) +} + +pub(super) async fn execute_send( + world: &mut CucumberWorld, + step: &str, + number_of_transactions: usize, + value: u64, + from: &str, + to: &str, +) -> Result<(), StepError> { + let receiver = world.resolve_recipient(to)?; + let receiver_pk = receiver.public_key; + + let mut available_utxos = WalletUtxos::new(); + let best_node_info = sync::wait_wallet_send_ready( + world, + step, + from, + 180, + number_of_transactions as u64 * value, + WalletSendReadiness::EligibleUtxoBatch { + min_required_outputs: number_of_transactions, + min_value_per_transaction: value, + }, + &mut available_utxos, + &HashSet::new(), + ) + .await?; + + for i in 0..number_of_transactions { + let result = utils::create_and_submit_transaction( + world, + step, + from, + &[(receiver_pk, value)], + Some(&best_node_info), + Some(&mut available_utxos), + ) + .await; + + if let Err(StepError::WalletError(WalletError::InsufficientFunds { available })) = result { + return Err(StepError::FundsDeficit { + available, + num_utxos_required: number_of_transactions - i, + value_per_utxos_required: value, + }); + } + result?; + } + Ok(()) +} + +#[expect(clippy::too_many_arguments, reason = "Transaction preparation inputs")] +pub(super) async fn prepare_ring_send_round_send_with_utxo_cache( + world: &mut CucumberWorld, + step: &str, + transactions: usize, + value: u64, + from: &str, + to: &str, + available_utxos: &mut WalletUtxos, + gas_prices: Option, + priority_fee_percent: u64, +) -> Result, StepError> { + let receiver = world.resolve_recipient(to)?; + let receiver_pk = receiver.public_key; + let mut reserved_submissions = Vec::with_capacity(transactions); + + for i in 0..transactions { + let sender_utxo_count_before = available_utxos.get(from).map_or(0usize, Vec::len); + + let receivers = vec![(receiver_pk, value)]; + let reserved_submission = + utils::reserve_user_wallet_transaction_submission_with_utxo_cache( + world, + step, + from, + &receivers, + available_utxos, + gas_prices.clone(), + priority_fee_percent, + ) + .await + .map_err(|error| match error { + StepError::WalletError(WalletError::InsufficientFunds { available }) => { + StepError::FundsDeficit { + available, + num_utxos_required: transactions - i, + value_per_utxos_required: value, + } + } + error => error, + })?; + let sender_utxo_count_after = available_utxos.get(from).map_or(0usize, Vec::len); + + if transactions > 1 && sender_utxo_count_after >= sender_utxo_count_before { + return Err(StepError::LogicalError { + message: format!( + "Batch cache accounting failed for '{from}': expected available input count to \ + decrease between submissions ({sender_utxo_count_before} -> {sender_utxo_count_after})" + ), + }); + } + + reserved_submissions.push(reserved_submission); + } + + utils::finalize_reserved_user_wallet_submissions_concurrently(step, reserved_submissions).await +} diff --git a/tests/src/cucumber/steps/transactions/manual_control/execution/wallet_state.rs b/tests/src/cucumber/steps/transactions/manual_control/execution/wallet_state.rs new file mode 100644 index 000000000..52bf9056a --- /dev/null +++ b/tests/src/cucumber/steps/transactions/manual_control/execution/wallet_state.rs @@ -0,0 +1,317 @@ +use super::{ + BTreeMap, CucumberWorld, Path, Serialize, StepError, StepResult, TARGET, Utxo, WalletError, + WalletInfo, WalletOutputState, WalletStateView, WalletType, + current_available_utxos_for_user_wallets, drain_node_wallet, drain_user_wallet, fs, info, + utils, warn, +}; + +pub(super) async fn execute_drain( + world: &mut CucumberWorld, + step: &str, + from: &str, + to: &str, +) -> StepResult { + let sender = world.resolve_wallet(from)?; + let receiver_pk = world.resolve_recipient(to)?.public_key; + + if sender.public_key()? == receiver_pk { + return Err(StepError::InvalidArgument { + message: format!("Cannot drain wallet `{from}` into itself"), + }); + } + + match sender.wallet_type { + WalletType::User { .. } => drain_user_wallet(world, step, &sender, receiver_pk).await, + WalletType::Funding { .. } => drain_node_wallet(world, &sender, receiver_pk).await, + } +} + +pub async fn log_wallet_balances( + world: &mut CucumberWorld, + step: &str, + wallets: Vec, +) -> StepResult { + let tracked_wallets = wallets + .iter() + .filter(|wallet| wallet.is_user_wallet()) + .cloned() + .collect::>(); + let states = if tracked_wallets.is_empty() { + BTreeMap::new() + } else { + utils::current_wallet_states_for_wallets(world, step, &tracked_wallets).await? + }; + + for wallet in &wallets { + if wallet.is_node_wallet() { + log_node_wallet_balance(world, wallet).await?; + continue; + } + let state = + states + .get(wallet.wallet_name.as_str()) + .ok_or_else(|| StepError::LogicalError { + message: format!( + "Wallet `{}` balance state is not tracked", + wallet.wallet_name + ), + })?; + log_wallet_state_balance(&wallet.wallet_name, &wallet.public_key_hex(), state); + } + Ok(()) +} + +async fn log_node_wallet_balance(world: &CucumberWorld, wallet: &WalletInfo) -> StepResult { + let node = world + .nodes_info + .get(&wallet.node_name) + .ok_or_else(|| StepError::LogicalError { + message: format!( + "Node '{}' for wallet '{}' not found", + wallet.node_name, wallet.wallet_name + ), + })?; + + let balance_response = node + .started_node + .client + .wallet_balance(wallet.public_key()?, None) + .await; + match balance_response { + Ok(balance) => { + info!( + target: TARGET, + "Wallet `{}` [On-chain] {}/{} LGO, {}", + wallet.wallet_name, + balance.notes.len(), + balance.balance, + wallet.public_key_hex(), + ); + } + Err(_) => { + info!( + target: TARGET, + "Wallet `{}` [On-chain] no funds yet, {}", + wallet.wallet_name, + wallet.public_key_hex(), + ); + } + } + + Ok(()) +} + +pub(super) async fn log_wallet_balance( + world: &mut CucumberWorld, + step: &str, + wallet_name: &str, +) -> StepResult { + let wallet = world.resolve_wallet(wallet_name)?; + log_wallet_balances(world, step, vec![wallet]).await +} + +fn log_wallet_state_balance(wallet_name: &str, public_key_hex: &str, state: &WalletStateView) { + let available = state.balance(WalletOutputState::Available); + let reserved = state.balance(WalletOutputState::Reserved); + let on_chain = state.balance(WalletOutputState::OnChain); + + info!( + target: TARGET, + "Wallet `{wallet_name}` [Available] {}/{} LGO, [Encumbered] {}/{} LGO, \ + [On-chain] {}/{} LGO, {}", + available.output_count, + available.value, + reserved.output_count, + reserved.value, + on_chain.output_count, + on_chain.value, + public_key_hex, + ); +} + +#[derive(Serialize)] +struct WalletFundsExport { + wallet: String, + node_url: String, + public_key: String, + #[serde(skip_serializing_if = "Option::is_none")] + secret_key: Option, + requested_value: u64, + selected_value: u64, + #[serde(skip_serializing_if = "Option::is_none")] + height: Option, + utxos: Vec, +} + +#[derive(Serialize)] +struct ExportedUtxo { + utxo_id: String, + value: u64, + encoded_utxo: String, +} + +pub(super) async fn export_funds( + world: &mut CucumberWorld, + step: &str, + wallet_name: &str, + value: u64, + output_path: &str, + include_secret: bool, +) -> Result<(), StepError> { + let wallet = world.resolve_wallet(wallet_name)?.clone(); + let available_utxos = current_available_utxos_for_user_wallets(world, step) + .await? + .get(wallet_name) + .cloned() + .ok_or(StepError::LogicalError { + message: format!("Wallet '{wallet_name}' not found in updated balances"), + })?; + let selected = select_utxos_covering(available_utxos.clone(), value)?; + let selected_value = selected.iter().map(|utxo| utxo.note.value).sum(); + let export = WalletFundsExport { + wallet: wallet.wallet_name.clone(), + node_url: format!( + "{}", + world + .resolve_node_http_client(&wallet.node_name)? + .base_url() + ), + public_key: wallet.public_key_hex(), + secret_key: exported_secret_key(&wallet, include_secret)?, + requested_value: value, + selected_value, + height: best_known_wallet_node_height(world, &wallet).await, + utxos: selected + .iter() + .map(exported_utxo) + .collect::, _>>()?, + }; + + let path = Path::new(output_path); + if let Some(parent) = path.parent() + && !parent.as_os_str().is_empty() + { + fs::create_dir_all(parent).map_err(|error| StepError::StepFail { + message: format!( + "Failed to create EXPORT_FUNDS output directory '{}': {error}", + parent.display() + ), + })?; + } + let json = serde_json::to_string_pretty(&export).map_err(|error| StepError::StepFail { + message: format!("Failed to serialize EXPORT_FUNDS JSON: {error}"), + })?; + fs::write(path, json).map_err(|error| StepError::StepFail { + message: format!( + "Failed to write EXPORT_FUNDS output '{}': {error}", + path.display() + ), + })?; + + info!( + target: TARGET, + "EXPORT_FUNDS wrote {} UTXO(s), selected value {}, requested value {}, output '{}'", + export.utxos.len(), + selected_value, + value, + path.display() + ); + Ok(()) +} + +fn select_utxos_covering(mut utxos: Vec, value: u64) -> Result, StepError> { + utxos.sort_by_key(|utxo| std::cmp::Reverse(utxo.note.value)); + let available = utxos.iter().map(|utxo| utxo.note.value).sum(); + let mut selected = Vec::new(); + let mut selected_value = 0u64; + + for utxo in utxos { + selected_value = selected_value.saturating_add(utxo.note.value); + selected.push(utxo); + if selected_value >= value { + return Ok(selected); + } + } + + Err(StepError::WalletError(WalletError::InsufficientFunds { + available, + })) +} + +fn exported_secret_key( + wallet: &WalletInfo, + include_secret: bool, +) -> Result, StepError> { + if !include_secret { + return Ok(None); + } + + let WalletType::User { wallet_account } = &wallet.wallet_type else { + return Err(StepError::InvalidArgument { + message: format!( + "EXPORT_FUNDS include_secret true requires a user wallet; '{}' is a funding wallet", + wallet.wallet_name + ), + }); + }; + + bincode::serialize(&wallet_account.secret_key) + .map(hex::encode) + .map(Some) + .map_err(|error| StepError::StepFail { + message: format!("Failed to encode wallet secret key for EXPORT_FUNDS: {error}"), + }) +} + +fn exported_utxo(utxo: &Utxo) -> Result { + Ok(ExportedUtxo { + utxo_id: hex::encode(utxo.id().as_bytes()), + value: utxo.note.value, + encoded_utxo: bincode::serialize(utxo).map(hex::encode).map_err(|error| { + StepError::StepFail { + message: format!("Failed to encode UTXO for EXPORT_FUNDS: {error}"), + } + })?, + }) +} + +async fn best_known_wallet_node_height(world: &CucumberWorld, wallet: &WalletInfo) -> Option { + let node = world.nodes_info.get(&wallet.node_name)?; + node.started_node + .client + .consensus_info() + .await + .ok() + .map(|info| info.cryptarchia_info.height) +} + +pub(super) fn clear_wallet_encumbrances( + world: &mut CucumberWorld, + step: &str, + wallet_name: &str, +) -> StepResult { + if world.resolve_wallet(wallet_name).is_err() { + warn!(target: TARGET, "Step `{}` error: wallet '{wallet_name}' not found in world state", step); + return Err(StepError::LogicalError { + message: format!("wallet '{wallet_name}' not found in world state"), + }); + } + + world.with_wallets_mut(|wallets| wallets.clear_encumbrances(wallet_name))?; + world + .wallet_registry + .fee_state + .clear_wallet_reservations(wallet_name); + info!(target: TARGET, "Cleared encumbrances for wallet '{wallet_name}'"); + Ok(()) +} + +pub(super) fn clear_all_wallet_encumbrances(world: &mut CucumberWorld, step: &str) -> StepResult { + let wallet_names: Vec = world.wallet_registry.wallet_info.keys().cloned().collect(); + + for wallet_name in wallet_names { + clear_wallet_encumbrances(world, step, &wallet_name)?; + } + info!(target: TARGET, "Cleared encumbrances for all wallets"); + Ok(()) +} diff --git a/tests/src/cucumber/steps/transactions/manual_control/mod.rs b/tests/src/cucumber/steps/transactions/manual_control/mod.rs new file mode 100644 index 000000000..513a260dc --- /dev/null +++ b/tests/src/cucumber/steps/transactions/manual_control/mod.rs @@ -0,0 +1,4 @@ +mod execution; +pub mod parsing; + +pub(crate) use execution::*; diff --git a/tests/src/cucumber/steps/manual_transactions/command_file_parsing.rs b/tests/src/cucumber/steps/transactions/manual_control/parsing.rs similarity index 99% rename from tests/src/cucumber/steps/manual_transactions/command_file_parsing.rs rename to tests/src/cucumber/steps/transactions/manual_control/parsing.rs index 0b04c514f..5943e2663 100644 --- a/tests/src/cucumber/steps/manual_transactions/command_file_parsing.rs +++ b/tests/src/cucumber/steps/transactions/manual_control/parsing.rs @@ -2,7 +2,7 @@ use std::{fs, path::Path}; use crate::cucumber::{ error::StepError, - steps::manual_transactions::utils::{WalletOutputState, parse_wallet_output_state}, + steps::transactions::utils::{WalletOutputState, parse_wallet_output_state}, }; #[cfg_attr(test, derive(strum_macros::EnumCount))] diff --git a/tests/src/cucumber/steps/manual_transactions/mod.rs b/tests/src/cucumber/steps/transactions/mod.rs similarity index 67% rename from tests/src/cucumber/steps/manual_transactions/mod.rs rename to tests/src/cucumber/steps/transactions/mod.rs index 56e26bb97..8577f09cc 100644 --- a/tests/src/cucumber/steps/manual_transactions/mod.rs +++ b/tests/src/cucumber/steps/transactions/mod.rs @@ -1,8 +1,7 @@ -pub mod command_file_parsing; -pub mod command_file_utils; mod drain_wallets; mod faucet; pub mod inscriptions; +pub mod manual_control; pub mod steps; pub(crate) mod tracked_transactions; pub mod utils; diff --git a/tests/src/cucumber/steps/transactions/steps/draining.rs b/tests/src/cucumber/steps/transactions/steps/draining.rs new file mode 100644 index 000000000..88a4ae1af --- /dev/null +++ b/tests/src/cucumber/steps/transactions/steps/draining.rs @@ -0,0 +1,81 @@ +use super::{ + CucumberWorld, Duration, Step, StepError, StepResult, TARGET, WalletType, + assert_tracked_wallet_fees_equal_sponsored_fee_account_spend, drain_all_node_wallets, + drain_node_wallet, drain_user_wallet, then, wait_for_wallet_submitted_transactions_inclusion, + warn, when, +}; + +#[when(expr = "I drain wallet {string} into {string}")] +#[then(expr = "I drain wallet {string} into {string}")] +async fn step_drain_wallet( + world: &mut CucumberWorld, + step: &Step, + sender_wallet_name: String, + receiver_wallet_name: String, +) -> StepResult { + let sender = world.resolve_wallet(&sender_wallet_name)?; + let receiver = world.resolve_recipient(&receiver_wallet_name)?; + let sender_pk = sender.public_key()?; + let receiver_pk = receiver.public_key; + + if sender_pk == receiver_pk { + return Err(StepError::InvalidArgument { + message: format!("Cannot drain wallet `{sender_wallet_name}` into itself"), + }); + } + + match sender.wallet_type { + WalletType::User { .. } => { + drain_user_wallet(world, &step.value, &sender, receiver_pk).await + } + WalletType::Funding { .. } => drain_node_wallet(world, &sender, receiver_pk).await, + } +} + +#[when(expr = "I drain all node {string} wallets into {string}")] +#[then(expr = "I drain all node {string} wallets into {string}")] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require a mutable World reference" +)] +async fn step_drain_all_node_wallets( + world: &mut CucumberWorld, + node_name: String, + receiver_wallet_name: String, +) -> StepResult { + drain_all_node_wallets(world, &node_name, &receiver_wallet_name).await +} + +#[when(expr = "wallet {string} has all submitted transactions settled in {int} seconds")] +#[then(expr = "wallet {string} has all submitted transactions settled in {int} seconds")] +#[when(expr = "wallet {string} has all submitted transactions included in {int} seconds")] +#[then(expr = "wallet {string} has all submitted transactions included in {int} seconds")] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require the world as the first `&mut` argument" +)] +async fn step_wallet_has_all_submitted_transactions_settled( + world: &mut CucumberWorld, + step: &Step, + wallet_name: String, + time_out_seconds: u64, +) -> StepResult { + wait_for_wallet_submitted_transactions_inclusion( + world, + &wallet_name, + Duration::from_secs(time_out_seconds), + ) + .await + .inspect_err(|e| { + warn!(target: TARGET, "Step `{}` error: {e}", step.value); + }) +} + +#[when(expr = "tracked wallet fees equal sponsored fee account spent fees")] +#[then(expr = "tracked wallet fees equal sponsored fee account spent fees")] +async fn step_tracked_wallet_fees_equal_sponsored_fee_account_spend( + world: &mut CucumberWorld, + step: &Step, +) -> StepResult { + assert_tracked_wallet_fees_equal_sponsored_fee_account_spend(world, &step.value).await +} diff --git a/tests/src/cucumber/steps/transactions/steps/faucet.rs b/tests/src/cucumber/steps/transactions/steps/faucet.rs new file mode 100644 index 000000000..b54fc82f9 --- /dev/null +++ b/tests/src/cucumber/steps/transactions/steps/faucet.rs @@ -0,0 +1,104 @@ +use super::{ + CucumberWorld, Step, StepError, StepResult, TARGET, WalletInfo, given, non_zero, utils, warn, + when, +}; + +#[given(expr = "I have a faucet with URL {string}")] +#[when(expr = "I have a faucet with URL {string}")] +fn step_faucet_details(world: &mut CucumberWorld, base_url: String) { + world.wallet_registry.faucet_base_url = Some(base_url); +} + +#[given(expr = "I request {int} rounds of faucet funds for wallet {string}")] +#[when(expr = "I request {int} rounds of faucet funds for wallet {string}")] +#[expect( + clippy::needless_pass_by_value, + reason = "Required by cucumber expression" +)] +fn step_request_faucet_funds_for_wallet( + world: &mut CucumberWorld, + step: &Step, + number_of_rounds: usize, + wallet_name: String, +) -> StepResult { + let wallet = world.resolve_wallet(&wallet_name).inspect_err(|error| { + warn!(target: TARGET, "Step `{}` error: {error}", step.value); + })?; + + let wallet_pk_hex = wallet.public_key_hex(); + + utils::request_faucet_funds( + world, + &step.value, + non_zero!("number of rounds", number_of_rounds)?, + &[wallet_pk_hex], + ) +} + +#[given(expr = "I request {int} rounds of faucet funds for all wallets")] +#[when(expr = "I request {int} rounds of faucet funds for all wallets")] +fn step_request_faucet_funds_for_all_wallets( + world: &mut CucumberWorld, + step: &Step, + number_of_rounds: usize, +) -> StepResult { + let all_wallets_pk_hex = world + .wallet_registry + .wallet_info + .values() + .map(WalletInfo::public_key_hex) + .collect::>(); + + utils::request_faucet_funds( + world, + &step.value, + non_zero!("number of rounds", number_of_rounds)?, + &all_wallets_pk_hex, + ) +} + +#[given(expr = "I request {int} rounds of faucet funds for all user wallets")] +#[when(expr = "I request {int} rounds of faucet funds for all user wallets")] +fn step_request_faucet_funds_for_all_user_wallets( + world: &mut CucumberWorld, + step: &Step, + number_of_rounds: usize, +) -> StepResult { + let all_wallets_pk_hex = world + .wallet_registry + .wallet_info + .values() + .filter(|w| w.is_user_wallet()) + .map(WalletInfo::public_key_hex) + .collect::>(); + + utils::request_faucet_funds( + world, + &step.value, + non_zero!("number of rounds", number_of_rounds)?, + &all_wallets_pk_hex, + ) +} + +#[given(expr = "I request {int} rounds of faucet funds for all funding wallets")] +#[when(expr = "I request {int} rounds of faucet funds for all funding wallets")] +fn step_request_faucet_funds_for_all_funding_wallets( + world: &mut CucumberWorld, + step: &Step, + number_of_rounds: usize, +) -> StepResult { + let all_wallets_pk_hex = world + .wallet_registry + .wallet_info + .values() + .filter(|wallet| wallet.is_node_funding_wallet()) + .map(WalletInfo::public_key_hex) + .collect::>(); + + utils::request_faucet_funds( + world, + &step.value, + non_zero!("number of rounds", number_of_rounds)?, + &all_wallets_pk_hex, + ) +} diff --git a/tests/src/cucumber/steps/transactions/steps/mod.rs b/tests/src/cucumber/steps/transactions/steps/mod.rs new file mode 100644 index 000000000..2a2b98070 --- /dev/null +++ b/tests/src/cucumber/steps/transactions/steps/mod.rs @@ -0,0 +1,51 @@ +use std::{collections::HashSet, time::Duration}; + +use cucumber::{gherkin::Step, given, then, when}; +use tokio::time::timeout; +use tracing::{info, warn}; + +use crate::{ + common::wallet::WalletUtxos, + cucumber::{ + error::{StepError, StepResult}, + steps::{ + TARGET, + transactions::{ + drain_wallets::{drain_all_node_wallets, drain_node_wallet, drain_user_wallet}, + manual_control::{ + execute_coin_splits_all_user_wallets, + execute_continuous_next_wallet_user_wallet, + execute_continuous_round_robin_user_wallets, log_wallet_balances, + parsing::ManualCommand, perform_manual_step_control, + verify_min_outputs_all_user_wallets, + }, + tracked_transactions::{ + submit_funded_transfer_transaction, submit_invalid_transfer_transaction, + submit_stateless_invalid_transfer_transaction, + transaction_is_not_included_in_seconds, + transaction_is_rejected_during_preverification, + }, + utils, + utils::{ + WalletOutputState, + assert_tracked_wallet_fees_equal_sponsored_fee_account_spend, + create_and_submit_transaction, parse_wallet_output_state, + wait_for_wallet_output_state, wait_for_wallet_submitted_transactions_inclusion, + }, + }, + }, + wallet::{ + submissions::create_and_submit_transaction_hashes_with_utxo_cache, + sync::{WalletSendReadiness, wait_wallet_send_ready}, + }, + world::{CucumberWorld, WalletInfo, WalletType}, + }, + non_zero, +}; + +mod draining; +mod faucet; +mod submissions; +mod transfers; +mod wallets; +mod workloads; diff --git a/tests/src/cucumber/steps/transactions/steps/submissions.rs b/tests/src/cucumber/steps/transactions/steps/submissions.rs new file mode 100644 index 000000000..d5d9ba69f --- /dev/null +++ b/tests/src/cucumber/steps/transactions/steps/submissions.rs @@ -0,0 +1,180 @@ +use super::{ + CucumberWorld, HashSet, Step, StepResult, TARGET, WalletSendReadiness, WalletUtxos, + create_and_submit_transaction, info, submit_funded_transfer_transaction, + submit_invalid_transfer_transaction, submit_stateless_invalid_transfer_transaction, then, + transaction_is_not_included_in_seconds, transaction_is_rejected_during_preverification, + wait_wallet_send_ready, warn, when, +}; + +#[when( + expr = "I send {int} transactions of {int} LGO each from wallet {string} to wallet {string}" +)] +async fn step_send_multiple_transactions_to_single_wallet( + world: &mut CucumberWorld, + step: &Step, + number_of_transactions: usize, + output_value: u64, + sender_wallet_name: String, + receiver_wallet_name: String, +) -> StepResult { + let sender_wallet = world.resolve_wallet(&sender_wallet_name).inspect_err(|e| { + warn!(target: TARGET, "Step `{}` error: {e}", step.value); + })?; + let receiver = world.resolve_recipient(&receiver_wallet_name)?; + let receiver_wallet_pk = receiver.public_key; + + let mut available_utxos = WalletUtxos::new(); + let best_node_info = wait_wallet_send_ready( + world, + &step.value, + &sender_wallet_name, + 180, + number_of_transactions as u64 * output_value, + WalletSendReadiness::TotalValueOnly, + &mut available_utxos, + &HashSet::new(), + ) + .await?; + + for _ in 0..number_of_transactions { + let tx_hash_hex = create_and_submit_transaction( + world, + &step.value, + &sender_wallet_name, + &[(receiver_wallet_pk, output_value)], + Some(&best_node_info), + Some(&mut available_utxos), + ) + .await + .inspect_err(|e| { + warn!(target: TARGET, "Step `{}` error: {e}", step.value); + })?; + + info!( + target: TARGET, + "Sent normal transaction from `{sender_wallet_name}/{}` to {}, \ + value: {output_value}, tx hash: {tx_hash_hex}", + sender_wallet.node_name, + receiver.label + ); + } + + Ok(()) +} + +#[when(expr = "I submit invalid transfer transaction {string} to node {string}")] +async fn step_submit_invalid_transfer_transaction( + world: &mut CucumberWorld, + step: &Step, + transaction_alias: String, + node_name: String, +) -> StepResult { + submit_invalid_transfer_transaction(world, &step.value, transaction_alias, node_name).await +} + +#[when(expr = "I submit a stateless-invalid transfer transaction {string} to node {string}")] +async fn step_submit_stateless_invalid_transfer_transaction( + world: &mut CucumberWorld, + step: &Step, + transaction_alias: String, + node_name: String, +) -> StepResult { + submit_stateless_invalid_transfer_transaction(world, &step.value, transaction_alias, node_name) + .await +} + +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require `&mut World` as the first parameter" +)] +#[expect( + clippy::needless_pass_by_value, + reason = "Cucumber step captures are always owned `String`s, even when the step only needs to borrow them" +)] +#[then(expr = "transaction {string} is rejected during preverification")] +fn step_transaction_is_rejected_during_preverification( + world: &mut CucumberWorld, + transaction_alias: String, +) -> StepResult { + transaction_is_rejected_during_preverification(world, &transaction_alias) +} + +#[when( + expr = "I submit funded transfer transaction {string} of {int} LGO from wallet {string} to wallet {string}" +)] +async fn step_submit_funded_transfer_transaction( + world: &mut CucumberWorld, + step: &Step, + transaction_alias: String, + amount: u64, + sender_wallet_name: String, + receiver_wallet_name: String, +) -> StepResult { + submit_funded_transfer_transaction( + world, + &step.value, + transaction_alias, + amount, + sender_wallet_name, + receiver_wallet_name, + ) + .await +} + +#[when(expr = "transaction {string} is not included in {int} seconds")] +#[then(expr = "transaction {string} is not included in {int} seconds")] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require `&mut World` as the first parameter" +)] +async fn step_transaction_is_not_included_in_seconds( + world: &mut CucumberWorld, + step: &Step, + transaction_alias: String, + timeout_seconds: u64, +) -> StepResult { + transaction_is_not_included_in_seconds(world, &step.value, transaction_alias, timeout_seconds) + .await +} + +#[when( + expr = "I send one transaction with {int} outputs of {int} LGO each from wallet {string} to wallet {string}" +)] +async fn step_send_single_transaction_multiple_outputs_to_single_wallet( + world: &mut CucumberWorld, + step: &Step, + number_of_outputs: usize, + output_value: u64, + sender_wallet_name: String, + receiver_wallet_name: String, +) -> StepResult { + let sender_wallet = world.resolve_wallet(&sender_wallet_name).inspect_err(|e| { + warn!(target: TARGET, "Step `{}` error: {e}", step.value); + })?; + let receiver = world.resolve_recipient(&receiver_wallet_name)?; + let receiver_wallet_pk = receiver.public_key; + + let receivers = vec![(receiver_wallet_pk, output_value); number_of_outputs]; + let tx_hash_hex = create_and_submit_transaction( + world, + &step.value, + &sender_wallet_name, + &receivers, + None, + None, + ) + .await + .inspect_err(|e| { + warn!(target: TARGET, "Step `{}` error: {e}", step.value); + })?; + + info!( + target: TARGET, + "Sent normal transaction from `{sender_wallet_name}/{}` to {}, \ + number_of_outputs: {number_of_outputs}, value: {output_value}, tx hash: {tx_hash_hex}", + sender_wallet.node_name, + receiver.label + ); + + Ok(()) +} diff --git a/tests/src/cucumber/steps/transactions/steps/transfers.rs b/tests/src/cucumber/steps/transactions/steps/transfers.rs new file mode 100644 index 000000000..4f347856e --- /dev/null +++ b/tests/src/cucumber/steps/transactions/steps/transfers.rs @@ -0,0 +1,129 @@ +use super::{ + CucumberWorld, HashSet, Step, StepError, StepResult, TARGET, WalletSendReadiness, WalletUtxos, + create_and_submit_transaction, create_and_submit_transaction_hashes_with_utxo_cache, info, + wait_wallet_send_ready, warn, when, +}; + +#[when(expr = "I do a coin split for {string} of {int} UTXOs valued at {int} LGO tokens each")] +async fn step_do_coin_split( + world: &mut CucumberWorld, + step: &Step, + wallet_name: String, + number_of_outputs: usize, + output_value: u64, +) -> StepResult { + let wallet = world.resolve_wallet(&wallet_name).inspect_err(|e| { + warn!(target: TARGET, "Step `{}` error: {e}", step.value); + })?; + + let mut available_utxos = WalletUtxos::new(); + let best_node_info = wait_wallet_send_ready( + world, + &step.value, + &wallet_name, + 180, + number_of_outputs as u64 * output_value, + WalletSendReadiness::TotalValueOnly, + &mut available_utxos, + &HashSet::new(), + ) + .await?; + + let self_pk = wallet.public_key().inspect_err(|e| { + warn!(target: TARGET, "Step `{}` error: {e}", step.value); + })?; + let receivers = vec![(self_pk, output_value); number_of_outputs]; + let tx_hash_hex = create_and_submit_transaction( + world, + &step.value, + &wallet_name, + &receivers, + Some(&best_node_info), + Some(&mut available_utxos), + ) + .await + .inspect_err(|e| { + warn!(target: TARGET, "Step `{}` error: {e}", step.value); + })?; + + info!( + target: TARGET, + "Submitted coin split transaction for `{wallet_name}/{}`, outputs: {number_of_outputs}, \ + value: {output_value}, tx hash: {tx_hash_hex}", + wallet.node_name + ); + + Ok(()) +} + +/// Tops up a node's funding wallet from a user wallet with `note_count` +/// notes of `note_value` LGO each, in a single transaction. +/// +/// Funding a transaction reserves one wallet note until the transaction is +/// mined, so a funding wallet supports at most `note count` concurrent +/// in-flight funded transactions — and its single `10_000` genesis note cannot +/// pay the fees of scenarios that publish many messages at non-zero gas +/// prices. This mirrors the workflow of a real node operator provisioning +/// their sequencer's funding wallet. +#[when( + expr = "wallet {string} sends {int} notes of {int} LGO to node {string} funding wallet as {string}" +)] +async fn step_topup_node_funding_wallet( + world: &mut CucumberWorld, + step: &Step, + wallet_name: String, + note_count: usize, + note_value: u64, + node_name: String, + transaction_alias: String, +) -> StepResult { + let funding_wallet = world.funding_wallet(&node_name).inspect_err(|e| { + warn!(target: TARGET, "Step `{}` error: {e}", step.value); + })?; + let funding_pk = funding_wallet.public_key().inspect_err(|e| { + warn!(target: TARGET, "Step `{}` error: {e}", step.value); + })?; + + let mut available_utxos = WalletUtxos::new(); + let best_node_info = wait_wallet_send_ready( + world, + &step.value, + &wallet_name, + 180, + note_count as u64 * note_value, + WalletSendReadiness::TotalValueOnly, + &mut available_utxos, + &HashSet::new(), + ) + .await?; + + let receivers = vec![(funding_pk, note_value); note_count]; + let tx_hashes = create_and_submit_transaction_hashes_with_utxo_cache( + world, + &step.value, + &wallet_name, + &receivers, + Some(&best_node_info), + Some(&mut available_utxos), + ) + .await + .inspect_err(|e| { + warn!(target: TARGET, "Step `{}` error: {e}", step.value); + })?; + let tx_hash = tx_hashes + .first() + .copied() + .ok_or_else(|| StepError::LogicalError { + message: "funding wallet top-up produced no transaction".to_owned(), + })?; + world.remember_submitted_transaction(transaction_alias.clone(), tx_hash); + + info!( + target: TARGET, + "Submitted funding wallet top-up `{transaction_alias}`: {note_count} notes of \ + {note_value} LGO from `{wallet_name}` to `{}`", + funding_wallet.wallet_name, + ); + + Ok(()) +} diff --git a/tests/src/cucumber/steps/transactions/steps/wallets.rs b/tests/src/cucumber/steps/transactions/steps/wallets.rs new file mode 100644 index 000000000..f89eeb475 --- /dev/null +++ b/tests/src/cucumber/steps/transactions/steps/wallets.rs @@ -0,0 +1,246 @@ +use super::{ + CucumberWorld, Step, StepResult, WalletOutputState, log_wallet_balances, then, + wait_for_wallet_output_state, when, +}; + +#[when(expr = "wallet {string} has {int} or more outputs in {int} seconds")] +#[then(expr = "wallet {string} has {int} or more outputs in {int} seconds")] +async fn step_wallet_has_at_least_coins( + world: &mut CucumberWorld, + step: &Step, + wallet_name: String, + min_coin_count: usize, + time_out_seconds: u64, +) -> StepResult { + wait_for_wallet_output_state( + world, + &step.value, + wallet_name, + Some(&min_coin_count), + None, + None, + None, + time_out_seconds, + WalletOutputState::OnChain, + ) + .await +} + +#[when(expr = "wallet {string} has {int} or less outputs in {int} seconds")] +#[then(expr = "wallet {string} has {int} or less outputs in {int} seconds")] +async fn step_wallet_has_at_most_coins( + world: &mut CucumberWorld, + step: &Step, + wallet_name: String, + max_coin_count: usize, + time_out_seconds: u64, +) -> StepResult { + wait_for_wallet_output_state( + world, + &step.value, + wallet_name, + None, + Some(&max_coin_count), + None, + None, + time_out_seconds, + WalletOutputState::OnChain, + ) + .await +} + +#[when(expr = "wallet {string} has exactly {int} outputs in {int} seconds")] +#[then(expr = "wallet {string} has exactly {int} outputs in {int} seconds")] +async fn step_wallet_has_exact_coins( + world: &mut CucumberWorld, + step: &Step, + wallet_name: String, + coin_count: usize, + time_out_seconds: u64, +) -> StepResult { + wait_for_wallet_output_state( + world, + &step.value, + wallet_name, + Some(&coin_count), + Some(&coin_count), + None, + None, + time_out_seconds, + WalletOutputState::OnChain, + ) + .await +} + +#[when(expr = "wallet {string} has {int} or less encumbered outputs in {int} seconds")] +#[then(expr = "wallet {string} has {int} or less encumbered outputs in {int} seconds")] +async fn step_wallet_has_at_most_encumbered_coins( + world: &mut CucumberWorld, + step: &Step, + wallet_name: String, + max_coin_count: usize, + time_out_seconds: u64, +) -> StepResult { + wait_for_wallet_output_state( + world, + &step.value, + wallet_name, + None, + Some(&max_coin_count), + None, + None, + time_out_seconds, + WalletOutputState::Reserved, + ) + .await +} + +#[when(expr = "wallet {string} has {int} or more LGO in {int} seconds")] +#[then(expr = "wallet {string} has {int} or more LGO in {int} seconds")] +async fn step_wallet_has_at_least_value( + world: &mut CucumberWorld, + step: &Step, + wallet_name: String, + min_token_value: u64, + time_out_seconds: u64, +) -> StepResult { + wait_for_wallet_output_state( + world, + &step.value, + wallet_name, + None, + None, + Some(&min_token_value), + None, + time_out_seconds, + WalletOutputState::OnChain, + ) + .await +} + +#[when(expr = "wallet {string} has exactly {int} LGO in {int} seconds")] +#[then(expr = "wallet {string} has exactly {int} LGO in {int} seconds")] +async fn step_wallet_has_exact_value( + world: &mut CucumberWorld, + step: &Step, + wallet_name: String, + token_value: u64, + time_out_seconds: u64, +) -> StepResult { + wait_for_wallet_output_state( + world, + &step.value, + wallet_name, + None, + None, + Some(&token_value), + Some(&token_value), + time_out_seconds, + WalletOutputState::OnChain, + ) + .await +} + +#[when(expr = "wallet {string} has {int} or less LGO in {int} seconds")] +#[then(expr = "wallet {string} has {int} or less LGO in {int} seconds")] +async fn step_wallet_has_at_most_value( + world: &mut CucumberWorld, + step: &Step, + wallet_name: String, + max_token_value: u64, + time_out_seconds: u64, +) -> StepResult { + wait_for_wallet_output_state( + world, + &step.value, + wallet_name, + None, + None, + None, + Some(&max_token_value), + time_out_seconds, + WalletOutputState::OnChain, + ) + .await +} + +#[when(expr = "wallet {string} has {int} or more outputs and {int} or more LGO in {int} seconds")] +#[then(expr = "wallet {string} has {int} or more outputs and {int} or more LGO in {int} seconds")] +async fn step_wallet_has_at_least_coins_and_value( + world: &mut CucumberWorld, + step: &Step, + wallet_name: String, + min_coin_count: usize, + min_token_value: u64, + time_out_seconds: u64, +) -> StepResult { + wait_for_wallet_output_state( + world, + &step.value, + wallet_name, + Some(&min_coin_count), + None, + Some(&min_token_value), + None, + time_out_seconds, + WalletOutputState::OnChain, + ) + .await +} + +#[when(expr = "wallet {string} has {int} or less outputs and {int} or less LGO in {int} seconds")] +#[then(expr = "wallet {string} has {int} or less outputs and {int} or less LGO in {int} seconds")] +async fn step_wallet_has_at_most_coins_and_value( + world: &mut CucumberWorld, + step: &Step, + wallet_name: String, + max_coin_count: usize, + max_token_value: u64, + time_out_seconds: u64, +) -> StepResult { + wait_for_wallet_output_state( + world, + &step.value, + wallet_name, + None, + Some(&max_coin_count), + None, + Some(&max_token_value), + time_out_seconds, + WalletOutputState::OnChain, + ) + .await +} + +#[when(expr = "wallet {string} has exactly {int} outputs and {int} LGO in {int} seconds")] +#[then(expr = "wallet {string} has exactly {int} outputs and {int} LGO in {int} seconds")] +async fn step_wallet_has_exact_coins_and_value( + world: &mut CucumberWorld, + step: &Step, + wallet_name: String, + coin_count: usize, + token_value: u64, + time_out_seconds: u64, +) -> StepResult { + wait_for_wallet_output_state( + world, + &step.value, + wallet_name, + Some(&coin_count), + Some(&coin_count), + Some(&token_value), + Some(&token_value), + time_out_seconds, + WalletOutputState::OnChain, + ) + .await +} + +#[when(expr = "I log wallet balances for all wallets")] +#[then(expr = "I log wallet balances for all wallets")] +async fn step_wallet_balance_all_wallets(world: &mut CucumberWorld, step: &Step) -> StepResult { + let mut wallets = world.all_user_wallets(); + wallets.extend(world.all_node_wallets()); + + log_wallet_balances(world, &step.value, wallets).await +} diff --git a/tests/src/cucumber/steps/transactions/steps/workloads.rs b/tests/src/cucumber/steps/transactions/steps/workloads.rs new file mode 100644 index 000000000..e39f36e1b --- /dev/null +++ b/tests/src/cucumber/steps/transactions/steps/workloads.rs @@ -0,0 +1,181 @@ +use super::{ + CucumberWorld, Duration, ManualCommand, Step, StepError, StepResult, TARGET, + execute_coin_splits_all_user_wallets, execute_continuous_next_wallet_user_wallet, + execute_continuous_round_robin_user_wallets, info, parse_wallet_output_state, + perform_manual_step_control, timeout, verify_min_outputs_all_user_wallets, warn, when, +}; + +#[when(expr = "I perform manual control of transactions for all wallets for {int} seconds")] +async fn step_manual_control_transactions( + world: &mut CucumberWorld, + step: &Step, + timeout_seconds: u64, +) -> StepResult { + perform_manual_step_control(world, &step.value, timeout_seconds).await +} + +#[when(expr = "I perform manual control of transactions for all wallets no time-out")] +async fn step_manual_control_transactions_no_time_out( + world: &mut CucumberWorld, + step: &Step, +) -> StepResult { + perform_manual_step_control(world, &step.value, u64::MAX).await +} + +#[when( + expr = "I perform continuous transactions on user wallets with {int} coin split outputs of {int} LGO, {int} transactions of {int} LGO each for {int} cycles with {int} epochs headroom" +)] +#[expect( + clippy::too_many_arguments, + reason = "Cucumber step captures map directly to step arguments" +)] +async fn step_continuous_user_wallets( + world: &mut CucumberWorld, + step: &Step, + coin_split_outputs: usize, + coin_split_value: u64, + transactions: usize, + value: u64, + cycles: usize, + epochs_headroom: u32, +) -> StepResult { + info!( + target: TARGET, + "Starting continuous user wallet transactions: coin_split_outputs={coin_split_outputs}, coin_split_value={coin_split_value}, transactions={transactions}, value={value}, cycles={cycles}" + ); + + execute_continuous_round_robin_user_wallets( + world, + &step.value, + coin_split_outputs, + coin_split_value, + transactions, + value, + cycles, + epochs_headroom, + ) + .await + .inspect_err(|e| { + warn!(target: TARGET, "Step `{}` error: {e}", step.value); + })?; + + info!(target: TARGET, "Completed continuous user wallet transactions step"); + + Ok(()) +} + +#[when( + expr = "I perform continuous transactions on user wallets with {int} coin split outputs of {int} LGO, {int} transactions of {int} LGO each for {int} cycles and timeout of {int} seconds with {int} epochs headroom" +)] +#[expect( + clippy::too_many_arguments, + reason = "Cucumber step captures map directly to step function arguments." +)] +async fn step_continuous_user_wallets_with_timeout( + world: &mut CucumberWorld, + step: &Step, + coin_split_outputs: usize, + coin_split_value: u64, + transactions: usize, + value: u64, + cycles: usize, + timeout_seconds: u64, + epochs_headroom: u32, +) -> StepResult { + timeout( + Duration::from_secs(timeout_seconds), + step_continuous_user_wallets( + world, + step, + coin_split_outputs, + coin_split_value, + transactions, + value, + cycles, + epochs_headroom, + ), + ) + .await + .map_err(|_| StepError::Timeout { + message: format!( + "continuous user wallet transactions did not finish within {timeout_seconds} seconds" + ), + })? +} + +#[when( + expr = "I perform {int} coin split transactions for each user wallet with {int} outputs of {int} LGO each" +)] +async fn step_coin_split_transactions_for_each_user_wallet( + world: &mut CucumberWorld, + step: &Step, + splits_per_wallet: usize, + outputs: usize, + value: u64, +) -> StepResult { + execute_coin_splits_all_user_wallets(world, &step.value, splits_per_wallet, outputs, value) + .await + .inspect_err(|e| { + warn!(target: TARGET, "Step `{}` error: {e}", step.value); + })?; + + Ok(()) +} + +#[when(expr = "I verify each wallet has minimum {int} outputs {string} in {int} seconds")] +async fn step_verify_each_wallet_minimum_outputs( + world: &mut CucumberWorld, + step: &Step, + min_outputs: usize, + wallet_state_type: String, + timeout_seconds: u64, +) -> StepResult { + verify_min_outputs_all_user_wallets( + world, + &step.value, + min_outputs, + timeout_seconds, + parse_wallet_output_state(&wallet_state_type) + .inspect_err(|e| { + warn!(target: TARGET, "Step `{}` error: {e}", step.value); + }) + .map_err(|e| StepError::InvalidArgument { + message: e.to_string(), + })?, + ) + .await + .inspect_err(|e| { + warn!(target: TARGET, "Step `{}` error: {e}", step.value); + })?; + + Ok(()) +} + +#[when( + expr = "I perform {int} stress continuous cycles with {int} transactions of {int} LGO to the next user wallet with {int} epochs headroom" +)] +async fn step_perform_stress_continuous_cycles_next_user_wallet( + world: &mut CucumberWorld, + step: &Step, + cycles: usize, + num_transactions: usize, + value: u64, + epochs_headroom: u32, +) -> StepResult { + execute_continuous_next_wallet_user_wallet( + world, + &step.value, + &ManualCommand::ContinuousNextWalletUserWallets { + cycles, + num_transactions, + value, + epochs_headroom, + }, + ) + .await + .inspect_err(|e| { + warn!(target: TARGET, "Step `{}` error: {e}", step.value); + })?; + + Ok(()) +} diff --git a/tests/src/cucumber/steps/manual_transactions/tracked_transactions.rs b/tests/src/cucumber/steps/transactions/tracked_transactions.rs similarity index 98% rename from tests/src/cucumber/steps/manual_transactions/tracked_transactions.rs rename to tests/src/cucumber/steps/transactions/tracked_transactions.rs index 2701f7fea..5e42b9c1f 100644 --- a/tests/src/cucumber/steps/manual_transactions/tracked_transactions.rs +++ b/tests/src/cucumber/steps/transactions/tracked_transactions.rs @@ -18,8 +18,7 @@ use crate::{ cucumber::{ error::StepError, steps::{ - TARGET, - manual_transactions::utils::create_and_submit_transaction_hashes_with_utxo_cache, + TARGET, transactions::utils::create_and_submit_transaction_hashes_with_utxo_cache, }, wallet::sync::{WalletSendReadiness, wait_wallet_send_ready}, world::{CucumberWorld, WalletType}, diff --git a/tests/src/cucumber/steps/manual_transactions/utils.rs b/tests/src/cucumber/steps/transactions/utils.rs similarity index 97% rename from tests/src/cucumber/steps/manual_transactions/utils.rs rename to tests/src/cucumber/steps/transactions/utils.rs index 42a73fb40..e8c662e3a 100644 --- a/tests/src/cucumber/steps/manual_transactions/utils.rs +++ b/tests/src/cucumber/steps/transactions/utils.rs @@ -30,7 +30,7 @@ pub(crate) use crate::cucumber::wallet::{ }; use crate::cucumber::{ error::{StepError, StepResult}, - steps::{TARGET, manual_transactions::faucet::FaucetTask}, + steps::{TARGET, transactions::faucet::FaucetTask}, world::CucumberWorld, }; diff --git a/tests/src/cucumber/steps/zone/actions/channel.rs b/tests/src/cucumber/steps/zone/actions/channel.rs new file mode 100644 index 000000000..d4704c1d8 --- /dev/null +++ b/tests/src/cucumber/steps/zone/actions/channel.rs @@ -0,0 +1,477 @@ +use super::{ + AtomicZoneDepositRequest, CucumberWorld, Duration, Ed25519Key, Inscription, Keys, Metadata, + PublishDeadline, PublishResult, SequencerCheckpoint, Step, StepError, StepResult, TxHash, Utxo, + WalletInfo, WalletReservedInputs, ZONE_CHANNEL_DEPOSIT_THRESHOLD, + ZONE_CHANNEL_WITHDRAW_THRESHOLD, ZoneDeposit, ZoneTestError, build_zone_deposit, + build_zone_deposit_from_values, current_available_utxos_for_wallet, log_step_error, + make_inscription, publish_atomic_zone_withdraw, submit_atomic_zone_deposit, + submit_zone_channel_split, submit_zone_deposit, submit_zone_withdraw, timeout, zone_step_error, +}; + +pub(in super::super) async fn submit_zone_channel_config( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: &str, + transaction_alias: String, + authorized_aliases: Vec, + posting_timeframe: u32, + posting_timeout: u32, +) -> StepResult { + let handle = log_step_error(step, world.zone.sequencer_client(sequencer_alias))?; + let mut ordered_aliases = vec![sequencer_alias.to_owned()]; + + for alias in authorized_aliases { + if ordered_aliases.iter().any(|existing| existing == &alias) { + continue; + } + + ordered_aliases.push(alias); + } + + let authorized_keys = ordered_aliases + .into_iter() + .map(|alias| { + world + .zone + .sequencer_signing_key(&alias) + .map(Ed25519Key::public_key) + }) + .collect::, _>>()?; + + let mut checkpoint_rx = world + .zone + .checkpoint_receiver(sequencer_alias) + .ok_or_else(|| StepError::LogicalError { + message: format!("Zone sequencer '{sequencer_alias}' has no checkpoint watch"), + })?; + checkpoint_rx.mark_unchanged(); + + let ((result, post_call_checkpoint), _signed_tx) = handle + .channel_config( + Keys::new_unchecked(authorized_keys), + posting_timeframe.into(), + posting_timeout.into(), + ZONE_CHANNEL_WITHDRAW_THRESHOLD, + ZONE_CHANNEL_DEPOSIT_THRESHOLD, + ) + .await + .map_err(|error| StepError::LogicalError { + message: format!("Zone channel_config failed: {error}"), + })?; + + // Sanity-check the inline checkpoint already mentions our tx; the + // event-stream watcher below also catches it once the drive task + // re-publishes its checkpoint after the next block. + let tx_hash = result.inscription_id(); + let checkpoint = if post_call_checkpoint + .pending_txs + .iter() + .any(|(hash, _)| *hash == tx_hash) + { + post_call_checkpoint + } else { + timeout( + Duration::from_secs(30), + wait_for_checkpoint_with_tx(&mut checkpoint_rx, tx_hash), + ) + .await + .map_err(|_| StepError::LogicalError { + message: format!( + "timed out waiting for sequencer '{sequencer_alias}' checkpoint to include {tx_hash:?}", + ), + })? + .map_err(|message| StepError::LogicalError { message })? + }; + + world + .zone + .set_latest_checkpoint_for(sequencer_alias, checkpoint.clone()); + world + .zone + .remember_checkpoint(format!("{transaction_alias}_CHECKPOINT"), checkpoint); + world.remember_submitted_transaction(transaction_alias, tx_hash); + + Ok(()) +} + +pub(in super::super) fn stop_zone_sequencer( + world: &mut CucumberWorld, + sequencer_alias: impl AsRef, +) -> StepResult { + world.zone.stop_sequencer(sequencer_alias.as_ref())?; + + Ok(()) +} + +pub(in super::super) fn save_zone_checkpoint( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: impl AsRef, + checkpoint_alias: String, +) -> StepResult { + let sequencer_alias = sequencer_alias.as_ref(); + let checkpoint = log_step_error(step, world.zone.current_checkpoint_for(sequencer_alias))?; + + world.zone.remember_checkpoint(checkpoint_alias, checkpoint); + + Ok(()) +} + +pub(in super::super) fn remember_published_zone_message( + world: &mut CucumberWorld, + sequencer_alias: &str, + message_alias: String, + payload: Inscription, + result: &PublishResult, +) { + let checkpoint = world.zone.current_checkpoint_for(sequencer_alias).ok(); + world.zone.remember_zone_message( + message_alias, + payload, + Some(result.inscription_id()), + Some(sequencer_alias), + checkpoint, + ); +} + +async fn wait_for_checkpoint_with_tx( + rx: &mut tokio::sync::watch::Receiver>, + tx_hash: TxHash, +) -> Result { + loop { + let snapshot = rx.borrow_and_update().clone(); + if let Some(checkpoint) = snapshot + && checkpoint + .pending_txs + .iter() + .any(|(hash, _)| *hash == tx_hash) + { + return Ok(checkpoint); + } + + rx.changed() + .await + .map_err(|error| format!("checkpoint watch closed: {error}"))?; + } +} + +fn resolve_zone_wallet( + world: &CucumberWorld, + sequencer_alias: &str, +) -> Result { + let wallet_name = world.zone.sequencer_default_wallet_name(sequencer_alias)?; + + world.resolve_wallet(wallet_name) +} + +fn record_zone_wallet_submission( + world: &CucumberWorld, + wallet_name: &str, + tx_hash: TxHash, + reserved_inputs: Vec, +) -> StepResult { + world.with_wallets_mut(|wallets| { + wallets.record_wallet_reservation( + wallet_name.to_owned(), + tx_hash, + WalletReservedInputs::new(reserved_inputs, Vec::new()), + 0, + ); + }) +} + +pub(in super::super) async fn submit_zone_deposit_transaction( + world: &mut CucumberWorld, + step: &Step, + transaction_alias: String, + channel_alias: String, + amount: u64, + metadata: Metadata, +) -> StepResult { + let node_url = log_step_error(step, world.zone_node_url_for_sequencer(&channel_alias))?; + let wallet = log_step_error(step, resolve_zone_wallet(world, &channel_alias))?; + let public_key = log_step_error(step, wallet.public_key())?; + let available_utxos = log_step_error( + step, + current_available_utxos_for_wallet(world, &step.value, &wallet.wallet_name).await, + )?; + let ZoneDeposit { + deposit, + reserved_inputs, + channel_notes, + } = build_zone_deposit( + available_utxos, + world.zone.sequencer_channel_id(&channel_alias)?, + amount, + metadata, + ) + .map_err(|error| zone_step_error(step, &error))?; + + let response = submit_zone_deposit(&node_url, &deposit, public_key) + .await + .map_err(|error| zone_step_error(step, &error))?; + + world + .zone + .remember_deposit_channel_notes(transaction_alias.clone(), channel_notes); + world + .zone + .remember_submitted_deposit(transaction_alias.clone(), deposit, amount); + record_zone_wallet_submission(world, &wallet.wallet_name, response, reserved_inputs)?; + world.remember_submitted_transaction(transaction_alias, response); + + Ok(()) +} + +/// Submits a multi-input channel deposit that consumes one wallet note per +/// listed value, exercising the channel wallet's per-note value tracking. +pub(in super::super) async fn submit_zone_multi_deposit_transaction( + world: &mut CucumberWorld, + step: &Step, + transaction_alias: String, + channel_alias: String, + input_values: Vec, + metadata: Metadata, +) -> StepResult { + let node_url = log_step_error(step, world.zone_node_url_for_sequencer(&channel_alias))?; + let wallet = log_step_error(step, resolve_zone_wallet(world, &channel_alias))?; + let public_key = log_step_error(step, wallet.public_key())?; + let available_utxos = log_step_error( + step, + current_available_utxos_for_wallet(world, &step.value, &wallet.wallet_name).await, + )?; + let amount: u64 = input_values.iter().sum(); + let ZoneDeposit { + deposit, + reserved_inputs, + channel_notes, + } = build_zone_deposit_from_values( + available_utxos, + world.zone.sequencer_channel_id(&channel_alias)?, + &input_values, + metadata, + ) + .map_err(|error| zone_step_error(step, &error))?; + + let response = submit_zone_deposit(&node_url, &deposit, public_key) + .await + .map_err(|error| zone_step_error(step, &error))?; + + world + .zone + .remember_deposit_channel_notes(transaction_alias.clone(), channel_notes); + world + .zone + .remember_submitted_deposit(transaction_alias.clone(), deposit, amount); + record_zone_wallet_submission(world, &wallet.wallet_name, response, reserved_inputs)?; + world.remember_submitted_transaction(transaction_alias, response); + + Ok(()) +} + +pub(in super::super) async fn submit_zone_channel_split_transaction( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: &str, + deposit_alias: &str, + dust_count: usize, + transaction_alias: String, +) -> StepResult { + let node_url = log_step_error(step, world.zone_node_url_for_sequencer(sequencer_alias))?; + let wallet = log_step_error(step, resolve_zone_wallet(world, sequencer_alias))?; + let public_key = log_step_error(step, wallet.public_key())?; + let channel_id = world.zone.sequencer_channel_id(sequencer_alias)?; + let signing_key = + log_step_error(step, world.zone.sequencer_signing_key(sequencer_alias))?.clone(); + let input_note = *log_step_error( + step, + world.zone.resolve_deposit_channel_notes(deposit_alias), + )? + .first() + .ok_or_else(|| StepError::LogicalError { + message: format!("Zone deposit '{deposit_alias}' created no channel notes to split"), + })?; + + let tx_hash = submit_zone_channel_split( + &node_url, + channel_id, + &signing_key, + public_key, + input_note, + dust_count, + ) + .await + .map_err(|error| zone_step_error(step, &error))?; + + world.remember_submitted_transaction(transaction_alias, tx_hash); + + Ok(()) +} + +pub(in super::super) async fn submit_atomic_zone_deposit_transaction( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: &str, + transaction_alias: String, + message_alias: String, + amount: u64, + metadata: Metadata, +) -> StepResult { + let node_url = log_step_error(step, world.zone_node_url_for_sequencer(sequencer_alias))?; + let wallet = log_step_error(step, resolve_zone_wallet(world, sequencer_alias))?; + let public_key = log_step_error(step, wallet.public_key())?; + let available_utxos = log_step_error( + step, + current_available_utxos_for_wallet(world, &step.value, &wallet.wallet_name).await, + )?; + let sequencer = log_step_error(step, world.zone.sequencer_client(sequencer_alias))?; + let inscription_data = make_inscription(&format!("Mint {amount} to Alice")); + + let submission = submit_atomic_zone_deposit( + &node_url, + sequencer, + AtomicZoneDepositRequest { + channel_id: world.zone.sequencer_channel_id(sequencer_alias)?, + funding_public_key: public_key, + available_utxos, + amount, + metadata, + inscription_data: inscription_data.clone(), + }, + ) + .await + .map_err(|error| zone_step_error(step, &error))?; + + world + .zone + .remember_submitted_deposit(transaction_alias.clone(), submission.deposit, amount); + remember_published_zone_message( + world, + sequencer_alias, + message_alias, + inscription_data, + &submission.publish, + ); + record_zone_wallet_submission( + world, + &wallet.wallet_name, + submission.publish.inscription_id(), + submission.reserved_inputs, + )?; + world.remember_submitted_transaction(transaction_alias, submission.publish.inscription_id()); + + Ok(()) +} + +pub(in super::super) async fn submit_zone_withdraw_transaction( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: &str, + transaction_alias: String, + message_alias: String, + amount: u64, +) -> StepResult { + let wallet = log_step_error(step, resolve_zone_wallet(world, sequencer_alias))?; + let public_key = log_step_error(step, wallet.public_key())?; + let sequencer = log_step_error(step, world.zone.sequencer_client(sequencer_alias))?; + let inscription_data = make_inscription(&format!("Burn {amount}")); + + let submission = submit_zone_withdraw( + sequencer, + world.zone.sequencer_channel_id(sequencer_alias)?, + public_key, + amount, + inscription_data.clone(), + ) + .await + .map_err(|error| zone_step_error(step, &error))?; + + world + .zone + .remember_submitted_withdraw(transaction_alias.clone(), submission.withdraw); + remember_published_zone_message( + world, + sequencer_alias, + message_alias, + inscription_data, + &submission.publish, + ); + world.remember_submitted_transaction(transaction_alias, submission.publish.inscription_id()); + + Ok(()) +} + +/// Action wrapper for the new `publish_atomic_withdraw` SDK API. Mirrors +/// [`submit_zone_withdraw_transaction`] but uses the high-level fire-and-forget +/// flow: SDK fills the withdraw nonce, locates its own accredited-key index, +/// builds the bundled `MantleTx`, signs locally, and submits. +/// +/// `withdraw_rows` carries one `(alias, outputs)` per `WithdrawArg`; each +/// withdraw is remembered under its own alias so multi-withdraw bundles can +/// be asserted per-withdraw via the indexer step. `bundle_alias` is remembered +/// as the bundle's tx hash for `zone transaction "..." is finalized`. +pub(in super::super) async fn publish_atomic_zone_withdraw_transaction( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: &str, + bundle_alias: String, + message_alias: String, + withdraw_rows: Vec<(String, Vec)>, +) -> StepResult { + let wallet = log_step_error(step, resolve_zone_wallet(world, sequencer_alias))?; + let public_key = log_step_error(step, wallet.public_key())?; + let total: u64 = withdraw_rows + .iter() + .flat_map(|(_, outputs)| outputs.iter()) + .sum(); + let inscription_data = make_inscription(&format!("Burn {total}")); + let outputs_per_arg: Vec> = withdraw_rows + .iter() + .map(|(_, outputs)| outputs.clone()) + .collect(); + + let submission = { + let sequencer = log_step_error(step, world.zone.sequencer_client(sequencer_alias))?.clone(); + + publish_atomic_zone_withdraw( + &sequencer, + public_key, + outputs_per_arg, + inscription_data.clone(), + PublishDeadline::from_now(Duration::from_mins(3)), + ) + .await + .map_err(|error| zone_step_error(step, &error))? + }; + + // A bundle carries a single `ChannelWithdrawOp` that releases every + // recipient note the transfer created, regardless of how many withdraw args + // were passed. Remember that one op under each row alias so per-withdraw + // indexer assertions all resolve to the same finalized op. + let [withdraw_op] = submission.withdraws.as_slice() else { + return Err(zone_step_error( + step, + &ZoneTestError::SubmitWithdraw { + message: format!( + "atomic withdraw bundle produced {} withdraw ops, expected exactly 1", + submission.withdraws.len(), + ), + }, + )); + }; + for (alias, _) in &withdraw_rows { + world + .zone + .remember_submitted_withdraw(alias.clone(), withdraw_op.clone()); + } + remember_published_zone_message( + world, + sequencer_alias, + message_alias, + inscription_data, + &submission.publish, + ); + world.remember_submitted_transaction(bundle_alias, submission.publish.inscription_id()); + + Ok(()) +} diff --git a/tests/src/cucumber/steps/zone/actions/cluster.rs b/tests/src/cucumber/steps/zone/actions/cluster.rs new file mode 100644 index 000000000..92f7f41e5 --- /dev/null +++ b/tests/src/cucumber/steps/zone/actions/cluster.rs @@ -0,0 +1,117 @@ +use super::{ + CucumberWorld, Duration, HashMap, NodesToStartUnordered, NonZero, Step, StepResult, TARGET, + WalletStartInfo, ZONE_SECURITY_PARAM, ZoneNodeResourcesRow, keygen, + set_deployment_config_override, start_node, start_nodes_order_respecting_dependencies, warn, +}; + +pub(in super::super) fn register_zone_sequencers_with_shared_key( + world: &mut CucumberWorld, + source_alias: &str, + aliases: Vec, +) -> StepResult { + let signing_key = world.zone.sequencer_signing_key(source_alias)?.clone(); + + for alias in aliases { + world.zone.register_sequencer(alias, signing_key.clone()); + } + + Ok(()) +} + +pub(in super::super) async fn start_nodes_with_zone_resources( + world: &mut CucumberWorld, + step: &Step, + rows: Vec, +) -> StepResult { + apply_zone_timing_defaults(world, &step.value)?; + + let nodes = collect_zone_nodes_to_start(&rows); + let nodes = start_nodes_order_respecting_dependencies( + nodes, + world.nodes_info.keys().cloned().collect(), + ) + .inspect_err(|error| { + warn!(target: TARGET, "Step `{}` error: {error}", step.value); + })?; + + for (node_name, wallet_start_info, mut initial_peers) in nodes { + initial_peers.sort(); + initial_peers.dedup(); + + start_node( + world, + &step.value, + &node_name, + &wallet_start_info, + &initial_peers, + false, + &[], + ) + .await?; + } + + register_zone_resources(world, rows) +} + +fn apply_zone_timing_defaults(world: &mut CucumberWorld, step: &str) -> StepResult { + world.set_cryptarchia_security_param( + NonZero::new(ZONE_SECURITY_PARAM).expect("zone security parameter is non-zero"), + ); + world.set_prolonged_bootstrap_period(Duration::ZERO); + + set_deployment_config_override(world, step, "time.slot_duration", "seconds(1)")?; + set_deployment_config_override( + world, + step, + "cryptarchia.slot_activation_coeff.numerator", + "1", + )?; + set_deployment_config_override( + world, + step, + "cryptarchia.slot_activation_coeff.denominator", + "2", + ) +} + +fn collect_zone_nodes_to_start(rows: &[ZoneNodeResourcesRow]) -> NodesToStartUnordered { + let mut nodes = HashMap::new(); + + for row in rows { + let entry = nodes + .entry(row.node_name.clone()) + .or_insert_with(|| (Vec::new(), Vec::new())); + + entry.0.push(WalletStartInfo { + wallet_name: row.wallet_name.clone(), + account_index: row.account_index, + }); + + if let Some(peer) = &row.connected_to { + entry.1.push(peer.clone()); + } + } + + nodes +} + +fn register_zone_resources( + world: &mut CucumberWorld, + rows: Vec, +) -> StepResult { + for row in rows { + for alias in row.sequencers { + if !world.zone.has_sequencer(&alias) { + world.zone.register_sequencer(alias.clone(), keygen()); + } + + world.zone.attach_sequencer_resources( + &alias, + row.node_name.clone(), + row.wallet_name.clone(), + )?; + } + } + + Ok(()) +} diff --git a/tests/src/cucumber/steps/zone/actions/mod.rs b/tests/src/cucumber/steps/zone/actions/mod.rs new file mode 100644 index 000000000..9f50cd2c3 --- /dev/null +++ b/tests/src/cucumber/steps/zone/actions/mod.rs @@ -0,0 +1,141 @@ +use std::{collections::HashMap, num::NonZero, time::Duration}; + +use cucumber::gherkin::Step; +use futures::future::join_all; +use lb_common_http_client::CommonHttpClient; +use lb_core::mantle::{ + TxHash, Utxo, + gas::GasCost, + ops::channel::{config::Keys, deposit::Metadata, inscribe::Inscription}, +}; +use lb_key_management_system_service::keys::Ed25519Key; +use lb_testing_framework::NodeHttpClient; +use lb_zone_sdk::{ + adapter::NodeHttpClient as ZoneNodeHttpClient, + sequencer::{FundingConfig, ZoneSequencer}, +}; +use tokio::{ + sync::broadcast, + task::JoinHandle, + time::{error::Elapsed, timeout}, +}; +use tracing::warn; + +use super::{ + AtomicZoneDepositRequest, CustomRepublishDeps, DiscardedPayloads, PolicyRuntime, + PublishDeadline, ZoneAccountBalances, ZoneDeposit, ZoneTestError, build_zone_deposit, + build_zone_deposit_from_values, ensure_zone_transactions_included, + errors::{log_step_error, zone_step_error}, + keygen, publish_atomic_zone_withdraw, publish_message_with_retry, + runner::{Event, PublishResult, SequencerCheckpoint, SequencerClient}, + sequencer_config, sequencer_config_with_pending_submit_depth, start_balance_aware_policy, + start_custom_republish_policy, start_republish_lineage_policy, start_sequencer_event_loop, + start_sorted_conflict_policy, + steps::DEFAULT_ZONE_SEQUENCER, + submit_atomic_zone_deposit, submit_zone_channel_split, submit_zone_deposit, + submit_zone_withdraw, + tables::{ConcurrentZoneMessageRow, ZoneNodeResourcesRow, group_zone_messages_by_sequencer}, +}; +use crate::{ + common::{ + mantle_inscription::make_inscription, manual_cluster::wait_for_height, + wallet::WalletReservedInputs, + }, + cucumber::{ + error::{StepError, StepResult}, + steps::{ + TARGET, + nodes::{ + NodesToStartUnordered, WalletStartInfo, + config_override::set_deployment_config_override, start_node, + start_nodes_order_respecting_dependencies, + }, + }, + wallet::sync::current_available_utxos_for_wallet, + world::{CucumberWorld, WalletInfo, ZoneReaderConfig}, + }, +}; + +const ZONE_CHANNEL_WITHDRAW_THRESHOLD: u16 = 1; +const ZONE_CHANNEL_DEPOSIT_THRESHOLD: u16 = 1; +const SEQUENCER_READY_TIMEOUT: Duration = Duration::from_mins(2); +const SEQUENCER_READY_POLL_TIMEOUT: Duration = Duration::from_secs(10); +const SEQUENCER_READY_HEIGHT_ADVANCE_TIMEOUT: Duration = Duration::from_secs(30); +const ZONE_SECURITY_PARAM: u32 = 5; +// These high-volume scenarios can move storage prices before all queued +// publishes are included. Keep their test funding comfortably above the +// public 12% default so they exercise sequencing rather than fee starvation. +const ZONE_TEST_PRIORITY_FEE_PERCENT: u64 = 50; + +pub(super) enum DriveMode { + Passive { + republish_orphans: bool, + }, + RepublishLineage { + planned: Vec, + }, + Sorted { + discarded: DiscardedPayloads, + }, + BalanceAware { + initial_balances: ZoneAccountBalances, + planned_payloads: Vec, + }, + CustomRepublish { + deps: Box, + }, +} + +impl DriveMode { + pub(super) const fn passive() -> Self { + Self::Passive { + republish_orphans: false, + } + } + + pub(super) const fn passive_republish_orphans() -> Self { + Self::Passive { + republish_orphans: true, + } + } +} + +struct PublishedZoneMessage { + alias: String, + payload: Inscription, + result: PublishResult, +} + +struct StartedSequencerRuntime { + task: JoinHandle<()>, + client: SequencerClient, + events: broadcast::Receiver, + checkpoint_rx: tokio::sync::watch::Receiver>, + ready_rx: tokio::sync::watch::Receiver, + channel_view_rx: tokio::sync::watch::Receiver, + turn_to_write_rx: tokio::sync::watch::Receiver, + tx_status_rx: broadcast::Receiver, + discarded_payloads: Option, +} + +mod channel; +mod cluster; +mod publishing; +mod sequencer; + +pub(super) use channel::{ + publish_atomic_zone_withdraw_transaction, remember_published_zone_message, + save_zone_checkpoint, stop_zone_sequencer, submit_atomic_zone_deposit_transaction, + submit_zone_channel_config, submit_zone_channel_split_transaction, + submit_zone_deposit_transaction, submit_zone_multi_deposit_transaction, + submit_zone_withdraw_transaction, +}; +pub(super) use cluster::{ + register_zone_sequencers_with_shared_key, start_nodes_with_zone_resources, +}; +pub(super) use publishing::{ + initialize_zone_indexer, publish_zone_messages, publish_zone_messages_concurrently, +}; +pub(super) use sequencer::{ + start_named_sequencer, start_named_sequencer_with_pending_submit_depth, +}; diff --git a/tests/src/cucumber/steps/zone/actions/publishing.rs b/tests/src/cucumber/steps/zone/actions/publishing.rs new file mode 100644 index 000000000..ac41d6aaf --- /dev/null +++ b/tests/src/cucumber/steps/zone/actions/publishing.rs @@ -0,0 +1,129 @@ +use super::{ + ConcurrentZoneMessageRow, CucumberWorld, DEFAULT_ZONE_SEQUENCER, Duration, Inscription, + PublishDeadline, PublishedZoneMessage, Step, StepError, StepResult, ZoneReaderConfig, + ensure_zone_transactions_included, group_zone_messages_by_sequencer, join_all, log_step_error, + publish_message_with_retry, remember_published_zone_message, zone_step_error, +}; + +pub(in super::super) fn initialize_zone_indexer( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: impl AsRef, +) -> StepResult { + let sequencer_alias = sequencer_alias.as_ref(); + let node_url = log_step_error(step, world.zone_node_url_for_sequencer(sequencer_alias))?; + let indexer = ZoneReaderConfig { + channel_id: world.zone.sequencer_channel_id(sequencer_alias)?, + node_url, + }; + + world.zone.set_indexer(indexer); + + Ok(()) +} + +pub(in super::super) async fn publish_zone_messages( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: impl AsRef, + rows: Vec<(String, Inscription)>, +) -> StepResult { + let sequencer_alias = sequencer_alias.as_ref().to_owned(); + let node = log_step_error( + step, + world.zone_node_http_client_for_sequencer(&sequencer_alias), + )?; + + let published = { + let sequencer = + log_step_error(step, world.zone.sequencer_client(&sequencer_alias))?.clone(); + + let publish_deadline = PublishDeadline::from_now(Duration::from_mins(3)); + let mut published = Vec::with_capacity(rows.len()); + + for (alias, payload) in &rows { + let result = publish_message_with_retry(&sequencer, payload, publish_deadline) + .await + .map_err(|error| zone_step_error(step, &error))?; + + ensure_zone_transactions_included( + &node, + &[result.inscription_id()], + Duration::from_mins(3), + ) + .await + .map_err(|error| zone_step_error(step, &error))?; + + published.push(PublishedZoneMessage { + alias: alias.clone(), + payload: payload.clone(), + result, + }); + } + + published + }; + + for message in published { + remember_published_zone_message( + world, + &sequencer_alias, + message.alias, + message.payload, + &message.result, + ); + } + + Ok(()) +} + +pub(in super::super) async fn publish_zone_messages_concurrently( + world: &mut CucumberWorld, + step: &Step, + rows: Vec, +) -> StepResult { + let grouped = group_zone_messages_by_sequencer(&rows); + let handles = grouped + .keys() + .map(|sequencer_alias| { + log_step_error(step, world.zone.sequencer_client(sequencer_alias)) + .map(|handle| (sequencer_alias.clone(), handle.clone())) + }) + .collect::, _>>()?; + + join_all(handles.into_iter().map(|(sequencer_alias, handle)| { + let payloads = grouped[&sequencer_alias] + .iter() + .map(|message| message.payload.clone()) + .collect::>(); + + async move { + for payload in payloads { + handle.publish(payload).await.map_err(|error| { + StepError::LogicalError { + message: format!( + "Zone concurrent publish failed for sequencer '{sequencer_alias}': {error}" + ), + } + })?; + } + + Ok::<(), StepError>(()) + } + })) + .await + .into_iter() + .collect::, _>>()?; + + for row in rows { + world + .zone + .remember_zone_message(row.message_alias, row.payload, None, None, None); + } + + if world.zone.indexer().is_err() { + initialize_zone_indexer(world, step, DEFAULT_ZONE_SEQUENCER)?; + } + + Ok(()) +} diff --git a/tests/src/cucumber/steps/zone/actions/sequencer.rs b/tests/src/cucumber/steps/zone/actions/sequencer.rs new file mode 100644 index 000000000..7c273a9d8 --- /dev/null +++ b/tests/src/cucumber/steps/zone/actions/sequencer.rs @@ -0,0 +1,201 @@ +use super::{ + CommonHttpClient, CucumberWorld, DiscardedPayloads, DriveMode, Elapsed, FundingConfig, GasCost, + NodeHttpClient, PolicyRuntime, SEQUENCER_READY_HEIGHT_ADVANCE_TIMEOUT, + SEQUENCER_READY_POLL_TIMEOUT, SEQUENCER_READY_TIMEOUT, SequencerCheckpoint, + StartedSequencerRuntime, Step, StepError, StepResult, ZONE_TEST_PRIORITY_FEE_PERCENT, + ZoneNodeHttpClient, ZoneSequencer, log_step_error, sequencer_config, + sequencer_config_with_pending_submit_depth, start_balance_aware_policy, + start_custom_republish_policy, start_republish_lineage_policy, start_sequencer_event_loop, + start_sorted_conflict_policy, timeout, wait_for_height, +}; + +pub(in super::super) async fn start_named_sequencer( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: impl AsRef, + checkpoint: Option, + mode: DriveMode, +) -> StepResult { + let funding = log_step_error(step, sequencer_funding(world, sequencer_alias.as_ref()))?; + start_named_sequencer_with_config( + world, + step, + sequencer_alias, + checkpoint, + mode, + sequencer_config(funding), + ) + .await +} + +/// Fund sequencer transactions from the node's own funding wallet. +fn sequencer_funding( + world: &CucumberWorld, + sequencer_alias: &str, +) -> Result { + let node_name = world.zone.sequencer_node_name(sequencer_alias)?; + let funding_pk = world.funding_wallet(node_name)?.public_key()?; + Ok(FundingConfig { + funding_pk, + change_pk: None, + max_tx_fee: GasCost::new(u64::MAX), + priority_fee_percent: ZONE_TEST_PRIORITY_FEE_PERCENT, + }) +} + +pub(in super::super) async fn start_named_sequencer_with_pending_submit_depth( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: impl AsRef, + checkpoint: Option, + mode: DriveMode, + max_pending_publish_depth: usize, +) -> StepResult { + let funding = log_step_error(step, sequencer_funding(world, sequencer_alias.as_ref()))?; + let config = sequencer_config_with_pending_submit_depth(max_pending_publish_depth, funding); + + start_named_sequencer_with_config(world, step, sequencer_alias, checkpoint, mode, config).await +} + +async fn start_named_sequencer_with_config( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: impl AsRef, + checkpoint: Option, + mode: DriveMode, + config: lb_zone_sdk::sequencer::SequencerConfig, +) -> StepResult { + let sequencer_alias = sequencer_alias.as_ref().to_owned(); + let signing_key = + log_step_error(step, world.zone.sequencer_signing_key(&sequencer_alias))?.clone(); + let node_client = log_step_error( + step, + world.zone_node_http_client_for_sequencer(&sequencer_alias), + )?; + let node_url = log_step_error(step, world.zone_node_url_for_sequencer(&sequencer_alias))?; + let sequencer = ZoneSequencer::init_with_config( + world.zone.sequencer_channel_id(&sequencer_alias)?, + signing_key, + ZoneNodeHttpClient::new(CommonHttpClient::new(None), node_url), + config, + checkpoint, + ); + + let runtime = start_sequencer_runtime(sequencer, mode); + let mut ready_rx = runtime.ready_rx.clone(); + + if let Err(error) = + wait_for_sequencer_ready(&sequencer_alias, &node_client, &mut ready_rx).await + { + runtime.task.abort(); + return Err(error); + } + + world.zone.set_sequencer_runtime( + sequencer_alias, + runtime.client, + runtime.task, + runtime.events, + runtime.checkpoint_rx, + runtime.channel_view_rx, + runtime.turn_to_write_rx, + runtime.tx_status_rx, + runtime.discarded_payloads, + ); + + Ok(()) +} + +async fn wait_for_sequencer_ready( + sequencer_alias: &str, + node_client: &NodeHttpClient, + ready_rx: &mut tokio::sync::watch::Receiver, +) -> StepResult { + timeout(SEQUENCER_READY_TIMEOUT, async { + let mut last_height = node_client.consensus_info().await?.cryptarchia_info.height; + + loop { + let poll = timeout(SEQUENCER_READY_POLL_TIMEOUT, async { + loop { + if ready_rx.changed().await.is_err() { + return Err(()); + } + if *ready_rx.borrow() { + return Ok(()); + } + } + }) + .await; + if matches!(poll, Ok(Ok(()))) { + return Ok(()); + } + + let _ = wait_for_height( + node_client, + last_height.saturating_add(1), + SEQUENCER_READY_HEIGHT_ADVANCE_TIMEOUT, + ) + .await; + + last_height = node_client + .consensus_info() + .await? + .cryptarchia_info + .height + .max(last_height); + } + }) + .await + .map_err(|_: Elapsed| StepError::Timeout { + message: format!( + "Sequencer `{sequencer_alias}` did not become ready within {} seconds", + SEQUENCER_READY_TIMEOUT.as_secs() + ), + })? +} + +fn from_policy_runtime( + rt: PolicyRuntime, + discarded_payloads: Option, +) -> StartedSequencerRuntime { + StartedSequencerRuntime { + task: rt.task, + client: rt.client, + events: rt.events, + checkpoint_rx: rt.checkpoint_rx, + ready_rx: rt.ready_rx, + channel_view_rx: rt.channel_view_rx, + turn_to_write_rx: rt.turn_to_write_rx, + tx_status_rx: rt.tx_status_rx, + discarded_payloads, + } +} + +fn start_sequencer_runtime( + sequencer: ZoneSequencer, + mode: DriveMode, +) -> StartedSequencerRuntime { + match mode { + DriveMode::Passive { republish_orphans } => from_policy_runtime( + start_sequencer_event_loop(sequencer, republish_orphans), + None, + ), + DriveMode::RepublishLineage { planned } => { + from_policy_runtime(start_republish_lineage_policy(sequencer, planned), None) + } + DriveMode::Sorted { discarded } => from_policy_runtime( + start_sorted_conflict_policy(sequencer, &discarded), + Some(discarded), + ), + DriveMode::BalanceAware { + initial_balances, + planned_payloads, + } => from_policy_runtime( + start_balance_aware_policy(sequencer, initial_balances, planned_payloads), + None, + ), + DriveMode::CustomRepublish { deps } => { + from_policy_runtime(start_custom_republish_policy(sequencer, *deps), None) + } + } +} diff --git a/tests/src/cucumber/steps/manual_zone/assertions.rs b/tests/src/cucumber/steps/zone/assertions.rs similarity index 99% rename from tests/src/cucumber/steps/manual_zone/assertions.rs rename to tests/src/cucumber/steps/zone/assertions.rs index fc5f37942..f246b2eb3 100644 --- a/tests/src/cucumber/steps/manual_zone/assertions.rs +++ b/tests/src/cucumber/steps/zone/assertions.rs @@ -5,7 +5,7 @@ use std::{ use lb_core::mantle::ops::channel::inscribe::Inscription; -use super::support::{ +use super::{ DiscardedPayloads, ZoneTestError, replay_finalized_history, replayed_inscription_payloads, }; use crate::cucumber::{ diff --git a/tests/src/cucumber/steps/manual_zone/errors.rs b/tests/src/cucumber/steps/zone/errors.rs similarity index 94% rename from tests/src/cucumber/steps/manual_zone/errors.rs rename to tests/src/cucumber/steps/zone/errors.rs index 7d89c4314..c0d9c98cf 100644 --- a/tests/src/cucumber/steps/manual_zone/errors.rs +++ b/tests/src/cucumber/steps/zone/errors.rs @@ -1,7 +1,7 @@ use cucumber::gherkin::Step; use tracing::warn; -use super::support::ZoneTestError; +use super::ZoneTestError; use crate::cucumber::{error::StepError, steps::TARGET}; pub(super) fn zone_step_error(step: &Step, error: &ZoneTestError) -> StepError { diff --git a/tests/src/cucumber/steps/zone/mod.rs b/tests/src/cucumber/steps/zone/mod.rs new file mode 100644 index 000000000..60c4e786f --- /dev/null +++ b/tests/src/cucumber/steps/zone/mod.rs @@ -0,0 +1,26 @@ +mod actions; +mod assertions; +mod errors; +mod operations; +pub mod runner; +pub mod steps; +mod tables; + +use operations::{ + AtomicZoneDepositRequest, CustomRepublishDeps, DiscardedPayloads, PolicyRuntime, + PublishDeadline, ZoneAccountBalances, ZoneDeposit, ZoneTestError, balance_update_payload, + build_zone_deposit, build_zone_deposit_from_values, collect_indexed_messages, + collect_indexed_messages_exactly_once, ensure_zone_transactions_included, keygen, + parse_balance_payload, publish_atomic_zone_withdraw, publish_message_with_retry, + replay_finalized_history, replayed_inscription_payloads, sequencer_config, + sequencer_config_with_pending_submit_depth, start_balance_aware_policy, + start_custom_republish_policy, start_republish_lineage_policy, start_sequencer_event_loop, + start_sorted_conflict_policy, submit_atomic_zone_deposit, submit_zone_channel_split, + submit_zone_deposit, submit_zone_withdraw, wait_for_channel_transfer_input_count, + wait_for_channel_view, wait_for_channel_wallet_counts, wait_for_channel_wallet_note, + wait_for_deposit, wait_for_exact_indexed_payload_count, + wait_for_finalized_deposit_via_sequencer_and_collect_mempool_pending, + wait_for_finalized_withdraw_via_sequencer_and_collect_mempool_pending, wait_for_lib_advance, + wait_for_on_chain_statuses_and_collect_mempool_pending, wait_for_transactions_finalized, + wait_for_turn_to_write, wait_for_tx_status_lifecycle, wait_for_withdraw, +}; diff --git a/tests/src/cucumber/steps/zone/operations/atomic.rs b/tests/src/cucumber/steps/zone/operations/atomic.rs new file mode 100644 index 000000000..f422f513a --- /dev/null +++ b/tests/src/cucumber/steps/zone/operations/atomic.rs @@ -0,0 +1,199 @@ +use super::*; + +/// Builds the funding transfer that creates the note consumed by an atomic +/// zone deposit. +/// Generous fee margin for the atomic `[Transfer, Deposit, Inscribe]` +/// transaction. The mandatory fee (execution + size-based storage gas) is +/// roughly 2k and varies with input count and change-note presence, so a +/// tight margin intermittently underfunds the tx — which is permanently +/// invalid and silently evicted at block assembly. Matches +/// `MAX_ZONE_DEPOSIT_TX_FEE`; the excess above the mandatory fee is a tip. +const ATOMIC_DEPOSIT_FEE_MARGIN: u64 = 10_000; + +pub(super) fn build_atomic_deposit_transfer( + available_utxos: Vec, + funding_public_key: ZkPublicKey, + amount: Value, +) -> Result<(TransferOp, Vec), ZoneTestError> { + let deposit_note = Note::new(amount, funding_public_key); + let funded_transfer = build_wallet_funded_transfer( + available_utxos, + vec![deposit_note], + funding_public_key, + ATOMIC_DEPOSIT_FEE_MARGIN, + ) + .map_err(|error| ZoneTestError::BuildAtomicDeposit { + message: error.to_string(), + })?; + + Ok(funded_transfer.into_parts()) +} + +/// Points the channel deposit at the note created by the atomic funding +/// transfer, keeping both operations in the same transaction. +pub(super) fn build_atomic_deposit_op( + channel_id: ChannelId, + metadata: Metadata, + transfer: &TransferOp, +) -> Result { + let deposit_note_id = transfer + .outputs + .utxo_by_index(0, transfer) + .ok_or_else(|| ZoneTestError::BuildAtomicDeposit { + message: "transfer did not produce the deposit note".to_owned(), + })? + .id(); + + Ok(DepositOp { + channel_id, + inputs: Inputs::new([deposit_note_id]), + metadata, + }) +} + +/// Submits a channel withdraw signed by the active zone sequencer and publishes +/// the withdraw inscription as part of the same SDK flow. +/// +/// The withdraw pays a single note of `amount` back to `funding_public_key` +/// (self-withdraw). Inputs are selected automatically by the SDK +/// (`WithdrawInputs::Auto`, best-fit largest-first, capped at 255 inputs). +pub async fn submit_zone_withdraw( + client: &SequencerClient, + _channel_id: ChannelId, + funding_public_key: ZkPublicKey, + amount: Value, + inscription_data: Inscription, +) -> Result { + let (result, _cp) = client + .publish_atomic_withdraw( + inscription_data, + vec![WithdrawArg { + outputs: Outputs::new([Note::new(amount, funding_public_key)]), + }], + WithdrawInputs::Auto, + ) + .await + .map_err(|error| ZoneTestError::SubmitWithdraw { + message: error.to_string(), + })?; + + let PendingTx::AtomicWithdraw(info) = result.tx else { + return Err(ZoneTestError::SubmitWithdraw { + message: "publish_atomic_withdraw returned a non-AtomicWithdraw publish result" + .to_owned(), + }); + }; + let withdraw = info + .withdraws + .first() + .ok_or_else(|| ZoneTestError::SubmitWithdraw { + message: "atomic withdraw bundle had no withdraw ops".to_owned(), + })? + .op + .clone(); + + Ok(ZoneWithdrawSubmission { + withdraw, + publish: PublishResult { + tx: PendingTx::AtomicWithdraw(info), + }, + }) +} + +/// Result of publishing an atomic inscription+withdraw bundle. Carries every +/// withdraw op produced by the SDK (one per `WithdrawArg`, in submission +/// order) so a multi-withdraw scenario can match each by its outputs. +pub struct ZoneAtomicWithdrawSubmission { + pub withdraws: Vec, + pub publish: PublishResult, +} + +/// Publishes an atomic inscription+withdraw bundle through the runner. +/// Returns every withdraw op (with the nonce filled by the SDK) from the +/// publish call's return value, so downstream cucumber assertions can +/// match each withdraw by its outputs. +/// +/// `outputs_per_arg` carries one entry per `WithdrawArg`; each inner `Vec` +/// becomes that arg's `Outputs` (one `Note::new(amount, funding_pk)` per +/// listed amount). Exercises the SDK API at full width: multiple args, with +/// any arg able to carry multiple output notes. +pub async fn publish_atomic_zone_withdraw( + client: &SequencerClient, + funding_public_key: ZkPublicKey, + outputs_per_arg: Vec>, + inscription_data: Inscription, + _deadline: PublishDeadline, +) -> Result { + if outputs_per_arg.is_empty() { + return Err(ZoneTestError::SubmitWithdraw { + message: "publish_atomic_zone_withdraw requires at least one withdraw arg".to_owned(), + }); + } + let withdraw_args: Vec = outputs_per_arg + .iter() + .map(|amounts| { + Ok::(WithdrawArg { + outputs: Outputs::try_new( + amounts + .iter() + .map(|amount| Note::new(*amount, funding_public_key)) + .collect::>(), + )?, + }) + }) + .collect::, ZoneTestError>>()?; + + let (result, _cp) = client + .publish_atomic_withdraw(inscription_data, withdraw_args, WithdrawInputs::Auto) + .await + .map_err(|error| ZoneTestError::SubmitWithdraw { + message: error.to_string(), + })?; + + let PendingTx::AtomicWithdraw(info) = result.tx else { + return Err(ZoneTestError::SubmitWithdraw { + message: "publish_atomic_withdraw returned a non-AtomicWithdraw publish result" + .to_owned(), + }); + }; + if info.withdraws.is_empty() { + return Err(ZoneTestError::SubmitWithdraw { + message: "atomic withdraw bundle had no withdraw ops".to_owned(), + }); + } + Ok(ZoneAtomicWithdrawSubmission { + withdraws: info.withdraws.iter().map(|w| w.op.clone()).collect(), + publish: PublishResult { + tx: PendingTx::AtomicWithdraw(info), + }, + }) +} + +/// Asks the node wallet service to sign a Mantle transaction for the requested +/// ZK keys. +pub(super) async fn sign_tx_zk( + node_url: &Url, + tx: &RawMantleTx, + public_keys: Vec, +) -> Result { + let request_url = + node_url + .join("wallet/sign/zk") + .map_err(|error| ZoneTestError::SignTransaction { + message: error.to_string(), + })?; + let response: WalletSignTxZkResponseBody = CommonHttpClient::new(None) + .post( + request_url, + &WalletSignTxZkRequestBody { + tx_hash: tx.hash(), + pks: ZkPublicKeys::try_from(public_keys)?, + }, + ) + .await + .map_err(|error| ZoneTestError::SignTransaction { + message: error.to_string(), + })?; + + Ok(response.sig) +} diff --git a/tests/src/cucumber/steps/zone/operations/custom_policy.rs b/tests/src/cucumber/steps/zone/operations/custom_policy.rs new file mode 100644 index 000000000..ba0a8779e --- /dev/null +++ b/tests/src/cucumber/steps/zone/operations/custom_policy.rs @@ -0,0 +1,168 @@ +use super::{ + ChannelId, ChannelUpdateTx, Ed25519Key, Event, HashSet, Inscription, MsgId, NodeHttpClient, + PolicyRuntime, SequencerChannelView, VecDeque, ZkPublicKey, ZoneNodeHttpClient, ZoneSequencer, + build_funded_custom_tx, channel_inscriptions, finalized_inscriptions, runner, + to_policy_runtime, warn, +}; + +pub struct CustomRepublishDeps { + pub node_client: NodeHttpClient, + pub channel_id: ChannelId, + pub signing_key: Ed25519Key, + pub funding_pk: ZkPublicKey, + pub batches: VecDeque>, +} + +pub fn start_custom_republish_policy( + sequencer: ZoneSequencer, + deps: CustomRepublishDeps, +) -> PolicyRuntime { + let view_rx = sequencer.subscribe_channel_view(); + let policy = CustomRepublishPolicy { + deps, + view_rx, + pending: HashSet::new(), + finalized: HashSet::new(), + chain_tip: None, + ready: false, + }; + to_policy_runtime(runner::spawn(sequencer, policy)) +} + +/// [`OrphanRepublishPolicy`] for the custom-tx flow: orphans that are +/// neither in `pending` nor finalized are rebuilt and re-submitted. +struct CustomRepublishPolicy { + deps: CustomRepublishDeps, + view_rx: tokio::sync::watch::Receiver, + pending: HashSet, + finalized: HashSet, + /// Where our own submitted chain ends; reset on orphans so rebuilds + /// chain from the channel tip instead. + chain_tip: Option, + /// No submissions until ready — a fail-fast submit would leak its + /// funding reservation. + ready: bool, +} + +impl CustomRepublishPolicy { + async fn submit( + &mut self, + sequencer: &mut ZoneSequencer, + payloads: Vec, + ) -> bool + where + Node: lb_zone_sdk::adapter::Node + Clone + Send + Sync + 'static, + { + let parent = self + .chain_tip + .unwrap_or_else(|| self.view_rx.borrow().tip_message); + let built = build_funded_custom_tx( + &self.deps.node_client, + self.deps.channel_id, + &self.deps.signing_key, + self.deps.funding_pk, + &payloads, + parent, + ) + .await; + let (signed_tx, msg_id) = match built { + Ok(built) => built, + Err(error) => { + warn!(%error, "Failed to build custom zone tx"); + return false; + } + }; + match sequencer.handle().submit_signed_tx(signed_tx, msg_id) { + Ok((_result, _checkpoint)) => { + self.pending.extend(payloads); + self.chain_tip = Some(msg_id); + true + } + Err(error) => { + warn!(%error, "Failed to submit custom zone tx"); + false + } + } + } + + fn entry_payloads(&self, entry: &ChannelUpdateTx) -> Vec { + match entry { + ChannelUpdateTx::Custom(tx) => channel_inscriptions(tx, self.deps.channel_id) + .into_iter() + .map(|info| info.payload) + .collect(), + typed => typed + .inscription() + .map(|info| info.payload.clone()) + .into_iter() + .collect(), + } + } +} + +impl runner::Policy for CustomRepublishPolicy +where + Node: lb_zone_sdk::adapter::Node + Clone + Send + Sync + 'static, +{ + async fn on_event(&mut self, sequencer: &mut ZoneSequencer, event: &Event) { + let (channel_update, finalized) = match event { + Event::Ready => { + self.ready = true; + (None, None) + } + Event::BlocksProcessed { + channel_update, + finalized, + .. + } => (Some(channel_update), Some(finalized)), + _ => return, + }; + + if let Some(finalized) = finalized { + self.finalized + .extend(finalized_inscriptions(finalized).map(|info| info.payload.clone())); + } + + if let Some(channel_update) = channel_update { + let orphaned: HashSet = channel_update + .orphaned + .iter() + .flat_map(|entry| self.entry_payloads(entry)) + .collect(); + let adopted: Vec = channel_update + .adopted + .iter() + .flat_map(|entry| self.entry_payloads(entry)) + .collect(); + for payload in &orphaned { + self.pending.remove(payload); + } + self.pending.extend(adopted); + + let republish: Vec = orphaned + .into_iter() + .filter(|payload| { + !self.pending.contains(payload) && !self.finalized.contains(payload) + }) + .collect(); + if self.ready && !republish.is_empty() { + self.chain_tip = None; + if !self.submit(sequencer, republish.clone()).await { + self.deps.batches.push_back(republish); + } + } + } + + // One attempt per batch per event: a failed submission stops the + // drain and is retried on the next event. + while self.ready { + let Some(batch) = self.deps.batches.pop_front() else { + break; + }; + if !self.submit(sequencer, batch.clone()).await { + self.deps.batches.push_front(batch); + break; + } + } + } +} diff --git a/tests/src/cucumber/steps/zone/operations/mod.rs b/tests/src/cucumber/steps/zone/operations/mod.rs new file mode 100644 index 000000000..21864554c --- /dev/null +++ b/tests/src/cucumber/steps/zone/operations/mod.rs @@ -0,0 +1,275 @@ +//! Zone SDK test helpers shared by Cucumber steps. +//! +//! The helpers in this module keep the feature steps focused on scenario +//! intent: start a zone-backed node, run sequencers, publish messages, observe +//! the indexer, and submit the channel operations that the zone layer relies +//! on. + +use std::{ + collections::{BTreeSet, HashMap, HashSet, VecDeque}, + sync::{Arc, LazyLock}, + time::{Duration, Instant}, +}; + +use lb_common_http_client::{CommonHttpClient, Slot}; +use lb_core::{ + mantle::{ + Note, Op, OpProof, RawMantleTx, Utxo, Value, + gas::GasCost, + ledger::{Inputs, Outputs, OutputsError}, + ops::{ + OpId as _, + channel::{ + ChannelId, MsgId, + channel_transfer::ChannelTransferOp, + deposit::{DepositOp, Metadata}, + inscribe::{Inscription, InscriptionOp}, + withdraw::ChannelWithdrawOp, + }, + transfer::TransferOp, + }, + traits::Hashable as _, + transactions::{OpsProofs, builder::MantleTxBuilder, states::Unverified}, + }, + proofs::channel_multi_sig_proof::{ChannelMultiSigProof, IndexedSignature}, +}; +use lb_http_api_common::bodies::{ + channel::{ChannelDepositRequestBody, ChannelDepositResponseBody}, + wallet::{ + fund::WalletFundRequestBody, + sign::{WalletSignTxZkRequestBody, WalletSignTxZkResponseBody}, + }, +}; +use lb_key_management_system_service::keys::{Ed25519Key, ZkPublicKey, ZkPublicKeys, ZkSignature}; +use lb_node::SignedMantleTx; +use lb_testing_framework::NodeHttpClient; +use lb_zone_sdk::{ + adapter::NodeHttpClient as ZoneNodeHttpClient, + sequencer::{ZoneSequencer, channel_inscriptions}, +}; +use rand::{Rng as _, thread_rng}; +use reqwest::Url; +use tokio::{ + task::JoinHandle, + time::{sleep, timeout}, +}; +use tracing::warn; + +use super::runner::{ + self, ChannelUpdate, ChannelUpdateTx, Event, FinalizedOp, FinalizedTx, FundingConfig, + InscriptionId, InscriptionInfo, PendingTx, PublishResult, SequencerChannelView, + SequencerCheckpoint, SequencerClient, SequencerConfig, TurnNotification, TxStatus, + TxStatusUpdate, WithdrawArg, WithdrawInputs, +}; + +/// Inscriptions in the just-finalized txs — the permanent, settled part of the +/// channel. Once a payload finalizes it's on chain for good, so a policy pins +/// these and never re-homes a finalized payload when it later drops off a +/// non-canonical branch. +fn finalized_inscriptions(finalized: &[FinalizedTx]) -> impl Iterator { + finalized + .iter() + .flat_map(|tx| tx.ops.iter()) + .filter_map(|op| match op { + FinalizedOp::Inscription(info) => Some(info), + FinalizedOp::Deposit(_) + | FinalizedOp::Withdraw(_) + | FinalizedOp::Config(_) + | FinalizedOp::ChannelTransfer(_) => None, + }) +} +use crate::{ + common::{ + chain::wait_for_transactions_inclusion, mantle_inscription::make_inscription, + wallet::build_wallet_funded_transfer, + }, + cucumber::world::ZoneReaderConfig, +}; + +#[derive(Debug, thiserror::Error)] +pub enum ZoneTestError { + #[error("timed out waiting for zone sequencer to accept a publish request")] + PublishTimeout, + #[error("zone indexer request failed: {message}")] + Indexer { message: String }, + #[error("timed out waiting for zone indexer to return all messages")] + IndexerTimeout, + #[error("zone indexer returned {actual} copies of '{payload}', expected {expected}")] + IndexedPayloadCountMismatch { + payload: String, + expected: usize, + actual: usize, + }, + #[error("timed out waiting for zone transactions to appear on the canonical chain")] + InclusionTimeout, + #[error("failed to fetch consensus info while checking finalized transactions: {message}")] + Consensus { message: String }, + #[error("failed to fetch block while checking finalized transactions: {message}")] + Block { message: String }, + #[error("timed out waiting for zone transactions to finalize")] + FinalizationTimeout, + #[error("channel wallet request failed: {message}")] + ChannelWallet { message: String }, + #[error("timed out waiting for a channel wallet note")] + ChannelWalletTimeout, + #[error("timed out waiting for zone LIB to advance")] + LibAdvanceTimeout, + #[error("timed out waiting for zone sequencer channel view condition: {message}")] + ChannelViewTimeout { message: String }, + #[error("failed to find a funding note with exact value {value}")] + MissingExactFundingNote { value: Value }, + #[error("failed to submit zone deposit: {message}")] + SubmitDeposit { message: String }, + #[error("failed to submit zone channel split transfer: {message}")] + SplitTransfer { message: String }, + #[error("failed to sign zone transaction: {message}")] + SignTransaction { message: String }, + #[error("failed to build atomic zone deposit transaction: {message}")] + BuildAtomicDeposit { message: String }, + #[error("failed to submit atomic zone deposit transaction: {message}")] + SubmitAtomicDeposit { message: String }, + #[error("failed to submit zone withdraw transaction: {message}")] + SubmitWithdraw { message: String }, + #[error("timed out waiting for zone withdraw to appear in the indexer")] + WithdrawTimeout, + #[error("failed to build custom zone transaction: {message}")] + BuildCustomTx { message: String }, + #[error("failed to submit custom zone transaction: {message}")] + SubmitCustomTx { message: String }, + #[error("zone sequencer event stream stopped before observing the expected event")] + SequencerStopped, + #[error(transparent)] + BoundedError(#[from] lb_utils::bounded::BoundedError), + #[error(transparent)] + OutputsError(#[from] OutputsError), +} + +/// Result of an atomic deposit scenario where a deposit and zone inscription +/// are submitted as one Mantle transaction. +pub struct AtomicZoneDepositSubmission { + pub deposit: DepositOp, + pub publish: PublishResult, + pub reserved_inputs: Vec, +} + +pub struct AtomicZoneDepositRequest { + pub channel_id: ChannelId, + pub funding_public_key: ZkPublicKey, + pub available_utxos: Vec, + pub amount: Value, + pub inscription_data: Inscription, + pub metadata: Metadata, +} + +/// Result of a withdraw scenario where the zone sequencer signs the channel +/// withdraw and publishes the accompanying inscription. +pub struct ZoneWithdrawSubmission { + pub withdraw: ChannelWithdrawOp, + pub publish: PublishResult, +} + +pub struct ZoneDeposit { + pub deposit: DepositOp, + pub reserved_inputs: Vec, + /// The channel notes the deposit re-creates (1:1 with its inputs). Computed + /// deterministically from the deposit's `op_id` and its input notes so a + /// later channel transfer can spend them without waiting on the indexer. + pub channel_notes: Vec, +} + +/// The channel notes a deposit re-creates: one per input, same note (value + +/// pk), re-homed under the deposit's `op_id`. Matches the ledger's deposit +/// execution (`DepositOp::outputs` re-creates inputs 1:1) and the zone-sdk's +/// channel-note derivation. +fn recreated_channel_notes(deposit: &DepositOp, reserved_inputs: &[Utxo]) -> Vec { + let op_id = deposit.op_id(); + reserved_inputs + .iter() + .enumerate() + .map(|(index, utxo)| Utxo::new(op_id, index, utxo.note)) + .collect() +} + +pub type DiscardedPayloads = Arc>>; +pub type ZoneAccountBalances = HashMap; + +/// Shared deadline for a publish attempt and the matching event wait so the +/// whole operation has one timeout budget. +#[derive(Clone, Copy)] +pub struct PublishDeadline { + started_at: Instant, + timeout: Duration, +} + +impl PublishDeadline { + #[must_use] + pub fn from_now(timeout: Duration) -> Self { + Self { + started_at: Instant::now(), + timeout, + } + } + + fn is_expired(self) -> bool { + self.started_at.elapsed() > self.timeout + } +} + +/// Bundle returned from policy starters so callers can wire the cucumber +/// world. Wraps [`runner::Runtime`] — events and checkpoints are exposed +/// uniformly across all policies because the policy runs inline on the +/// drive task; the event mpsc is purely for test observation. +pub struct PolicyRuntime { + pub task: JoinHandle<()>, + pub client: SequencerClient, + pub events: tokio::sync::broadcast::Receiver, + pub checkpoint_rx: tokio::sync::watch::Receiver>, + pub ready_rx: tokio::sync::watch::Receiver, + pub channel_view_rx: tokio::sync::watch::Receiver, + pub turn_to_write_rx: tokio::sync::watch::Receiver, + pub tx_status_rx: tokio::sync::broadcast::Receiver, +} + +fn to_policy_runtime(rt: runner::Runtime) -> PolicyRuntime { + PolicyRuntime { + task: rt.task, + client: rt.client, + events: rt.event_rx, + checkpoint_rx: rt.checkpoint_rx, + ready_rx: rt.ready_rx, + channel_view_rx: rt.channel_view_rx, + turn_to_write_rx: rt.turn_to_write_rx, + tx_status_rx: rt.tx_status_rx, + } +} + +mod atomic; +mod custom_policy; +mod observation; +mod policies; +mod transactions; + +use atomic::{build_atomic_deposit_op, build_atomic_deposit_transfer, sign_tx_zk}; +pub(super) use atomic::{publish_atomic_zone_withdraw, submit_zone_withdraw}; +pub(super) use custom_policy::{CustomRepublishDeps, start_custom_republish_policy}; +pub(super) use observation::{ + balance_update_payload, collect_indexed_messages, collect_indexed_messages_exactly_once, + ensure_zone_transactions_included, keygen, parse_balance_payload, publish_message_with_retry, + replay_finalized_history, replayed_inscription_payloads, sequencer_config, + sequencer_config_with_pending_submit_depth, wait_for_channel_transfer_input_count, + wait_for_channel_view, wait_for_channel_wallet_counts, wait_for_channel_wallet_note, + wait_for_deposit, wait_for_exact_indexed_payload_count, + wait_for_finalized_deposit_via_sequencer_and_collect_mempool_pending, + wait_for_finalized_withdraw_via_sequencer_and_collect_mempool_pending, wait_for_lib_advance, + wait_for_on_chain_statuses_and_collect_mempool_pending, wait_for_transactions_finalized, + wait_for_turn_to_write, wait_for_tx_status_lifecycle, wait_for_withdraw, +}; +pub(super) use policies::{ + start_balance_aware_policy, start_republish_lineage_policy, start_sequencer_event_loop, + start_sorted_conflict_policy, +}; +use transactions::build_funded_custom_tx; +pub(super) use transactions::{ + build_zone_deposit, build_zone_deposit_from_values, submit_atomic_zone_deposit, + submit_zone_channel_split, submit_zone_deposit, +}; diff --git a/tests/src/cucumber/steps/zone/operations/observation.rs b/tests/src/cucumber/steps/zone/operations/observation.rs new file mode 100644 index 000000000..fa1252f55 --- /dev/null +++ b/tests/src/cucumber/steps/zone/operations/observation.rs @@ -0,0 +1,685 @@ +use super::*; + +/// Creates a scenario-local sequencer key. +#[must_use] +pub fn keygen() -> Ed25519Key { + let mut key_bytes = [0u8; 32]; + thread_rng().fill(&mut key_bytes); + Ed25519Key::from_bytes(&key_bytes) +} + +/// Encodes a balance-affecting zone payload used by balance-aware sequencer +/// scenarios. +#[must_use] +pub fn balance_update_payload(uuid: &str, account: &str, delta: i64) -> Inscription { + make_inscription(&format!("{uuid}:{account}:{delta}")) +} + +/// Parses a balance-affecting payload in the same format produced by +/// [`balance_update_payload`]. +pub fn parse_balance_payload(payload: &Inscription) -> Option<(String, String, i64)> { + let payload = std::str::from_utf8(payload.as_slice()).ok()?; + let parts = payload.splitn(3, ':').collect::>(); + let [uuid, account, delta] = parts.as_slice() else { + return None; + }; + + Some(( + (*uuid).to_owned(), + (*account).to_owned(), + delta.parse().ok()?, + )) +} + +/// Uses a short resubmit interval so retry-sensitive zone scenarios settle +/// quickly enough for CI. +#[must_use] +pub const fn sequencer_config(funding: FundingConfig) -> SequencerConfig { + SequencerConfig { + resubmit_interval: Duration::from_secs(3), + min_slots_remaining_in_turn: 2, + ..SequencerConfig::new(funding) + } +} + +/// Uses the same retry profile while overriding pending publish submit depth. +#[must_use] +pub const fn sequencer_config_with_pending_submit_depth( + max_pending_publish_depth: usize, + funding: FundingConfig, +) -> SequencerConfig { + SequencerConfig { + max_pending_publish_depth, + ..sequencer_config(funding) + } +} + +/// Publishes a zone payload through the runner and returns the SDK's +/// [`PublishResult`] inline. Retries transient publish errors until the +/// deadline elapses. No "wait for event" — the SDK accepts the publish +/// inline (funding it via the node when configured) and the runner forwards +/// the call through the drive task. +pub async fn publish_message_with_retry( + client: &SequencerClient, + data: &Inscription, + deadline: PublishDeadline, +) -> Result { + loop { + if deadline.is_expired() { + return Err(ZoneTestError::PublishTimeout); + } + match client.publish(data.clone()).await { + Ok((result, _cp)) => return Ok(result), + Err(error) => { + warn!(error = %error, "Zone sequencer publish failed, retrying"); + sleep(Duration::from_millis(500)).await; + } + } + } +} + +/// Waits until every tx in `tx_hashes` reports [`TxStatus::OnChain`] on the +/// sequencer's status stream, collecting the tx hashes seen as +/// [`TxStatus::PendingMempool`] along the way. Own publishes don't echo in +/// [`ChannelUpdate::adopted`] on chain extension (the sequencer already +/// tracks them), so the per-tx status stream is where "landed on chain, not +/// yet finalized" is observable. +pub async fn wait_for_on_chain_statuses_and_collect_mempool_pending( + statuses: &mut tokio::sync::broadcast::Receiver, + tx_hashes: &[InscriptionId], + duration: Duration, +) -> Result, ZoneTestError> { + timeout(duration, async { + let mut on_chain: HashSet = HashSet::new(); + let mut mempool_pending = HashSet::new(); + + while on_chain.len() < tx_hashes.len() { + let update = match statuses.recv().await { + Ok(update) => update, + Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => { + warn!("status subscriber lagged by {n}, recovering"); + continue; + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => { + return Err(ZoneTestError::SequencerStopped); + } + }; + match update.status { + TxStatus::PendingMempool => { + mempool_pending.insert(update.tx_hash); + } + TxStatus::OnChain(_) if tx_hashes.contains(&update.tx_hash) => { + on_chain.insert(update.tx_hash); + } + _ => {} + } + } + + Ok(mempool_pending) + }) + .await + .map_err(|_| ZoneTestError::PublishTimeout)? +} + +pub async fn wait_for_tx_status_lifecycle( + tx_status_rx: &mut tokio::sync::broadcast::Receiver, + tx_hashes: &[InscriptionId], + statuses: &[TxStatus], + duration: Duration, +) -> Result<(), ZoneTestError> { + let mut remaining: HashSet<(InscriptionId, TxStatus)> = tx_hashes + .iter() + .flat_map(|tx_hash| statuses.iter().map(move |status| (*tx_hash, *status))) + .collect(); + + timeout(duration, async { + while !remaining.is_empty() { + let update = match tx_status_rx.recv().await { + Ok(update) => update, + Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => { + warn!("tx-status subscriber lagged by {n}, recovering"); + continue; + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => { + return Err(ZoneTestError::SequencerStopped); + } + }; + remaining.remove(&(update.tx_hash, update.status)); + if remaining.is_empty() { + return Ok(()); + } + } + Ok(()) + }) + .await + .map_err(|_| ZoneTestError::IndexerTimeout)? +} + +/// Waits until the subscribed channel view satisfies the supplied predicate. +pub async fn wait_for_channel_view( + view_rx: &mut tokio::sync::watch::Receiver, + duration: Duration, + predicate: impl Fn(&SequencerChannelView) -> bool + Send + Sync, +) -> Result { + timeout(duration, async { + loop { + let current = view_rx.borrow().clone(); + if predicate(¤t) { + return Ok(current); + } + + view_rx + .changed() + .await + .map_err(|error| ZoneTestError::Indexer { + message: format!("channel view sender closed: {error}"), + })?; + } + }) + .await + .map_err(|_| ZoneTestError::ChannelViewTimeout { + message: format!( + "condition not reached within {} seconds", + duration.as_secs() + ), + })? +} + +/// Waits until the sequencer emits a turn-to-write notification. +pub async fn wait_for_turn_to_write( + turn_rx: &mut tokio::sync::watch::Receiver, + duration: Duration, +) -> Result { + timeout(duration, async { + loop { + let current = turn_rx.borrow().clone(); + if current.our_turn_to_write { + return Ok(current); + } + + turn_rx + .changed() + .await + .map_err(|error| ZoneTestError::Indexer { + message: format!("turn-to-write sender closed: {error}"), + })?; + } + }) + .await + .map_err(|_| ZoneTestError::ChannelViewTimeout { + message: format!( + "turn to write not reached within {} seconds", + duration.as_secs() + ), + })? +} + +/// Replays the channel's finalized history by cold-starting a fresh +/// read-only sequencer: a random signing key that is not part of the channel +/// rotation, so the instance can never publish or repost anything — +/// inscription posting is turn-gated. Finalized txs are collected from the +/// backfill events until the sequencer reports `Ready`, then the instance is +/// dropped; each call observes a fresh snapshot up to the LIB at connect +/// time. +pub async fn replay_finalized_history( + reader: &ZoneReaderConfig, +) -> Result, ZoneTestError> { + let node = ZoneNodeHttpClient::new(CommonHttpClient::new(None), reader.node_url.clone()); + // Placeholder funding: the reader never publishes (random key, posting is + // turn-gated), so the funding wallet is never exercised. + let funding = FundingConfig { + funding_pk: lb_groth16::Fr::from(1u64).into(), + change_pk: None, + max_tx_fee: GasCost::new(u64::MAX), + priority_fee_percent: FundingConfig::DEFAULT_PRIORITY_FEE_PERCENT, + }; + let mut sequencer = ZoneSequencer::init(reader.channel_id, keygen(), node, funding, None); + + timeout(Duration::from_mins(3), async { + let mut finalized = Vec::new(); + loop { + match sequencer.next_event().await { + Event::BlocksProcessed { + finalized: batch, .. + } => finalized.extend(batch), + Event::Ready => return finalized, + Event::MempoolPending(_) | Event::TurnNotification { .. } => {} + } + } + }) + .await + .map_err(|_| ZoneTestError::IndexerTimeout) +} + +/// Ordered inscription payloads within a finalized-history replay. +pub fn replayed_inscription_payloads(history: &[FinalizedTx]) -> Vec { + finalized_inscriptions(history) + .map(|info| info.payload.clone()) + .collect() +} + +/// Collects indexed block payloads until all expected messages have appeared. +/// +/// The returned order is the finalized on-chain order, which lets assertions +/// decide whether ordering matters for the scenario. +pub async fn collect_indexed_messages( + reader: &ZoneReaderConfig, + expected_messages: &[Inscription], + duration: Duration, +) -> Result, ZoneTestError> { + let expected: HashSet = expected_messages.iter().cloned().collect(); + + timeout(duration, async { + loop { + let payloads = replayed_inscription_payloads(&replay_finalized_history(reader).await?); + let mut seen: HashSet = HashSet::new(); + let mut ordered: Vec = Vec::new(); + for payload in payloads { + if expected.contains(&payload) && seen.insert(payload.clone()) { + ordered.push(payload); + } + } + + if seen == expected { + return Ok(ordered); + } + + sleep(Duration::from_millis(500)).await; + } + }) + .await + .map_err(|_| ZoneTestError::IndexerTimeout)? +} + +/// Replays the finalized history until it exactly matches the expected +/// message sequence without duplicates. +pub async fn collect_indexed_messages_exactly_once( + reader: &ZoneReaderConfig, + expected_messages: &[Inscription], + duration: Duration, +) -> Result, ZoneTestError> { + let expected: HashSet = expected_messages.iter().cloned().collect(); + + timeout(duration, async { + loop { + let ordered: Vec = + replayed_inscription_payloads(&replay_finalized_history(reader).await?) + .into_iter() + .filter(|payload| expected.contains(payload)) + .collect(); + + if ordered == expected_messages { + return Ok(ordered); + } + + sleep(Duration::from_millis(500)).await; + } + }) + .await + .map_err(|_| ZoneTestError::IndexerTimeout)? +} + +/// Waits until the finalized history contains exactly `expected_count` copies +/// of one payload after a short settle period. +/// +/// This intentionally counts duplicate payload bytes, which is required for +/// shared-payload zone tests where each inscription has the same data but a +/// distinct transaction lineage. +pub async fn wait_for_exact_indexed_payload_count( + reader: &ZoneReaderConfig, + expected_payload: Inscription, + expected_count: usize, + duration: Duration, +) -> Result<(), ZoneTestError> { + timeout(duration, async { + loop { + let count = count_indexed_payload(reader, &expected_payload).await?; + + if count >= expected_count { + sleep(Duration::from_secs(30)).await; + + let final_count = count_indexed_payload(reader, &expected_payload).await?; + if final_count == expected_count { + return Ok(()); + } + + return Err(ZoneTestError::IndexedPayloadCountMismatch { + payload: String::from_utf8_lossy(expected_payload.as_slice()).to_string(), + expected: expected_count, + actual: final_count, + }); + } + + sleep(Duration::from_millis(500)).await; + } + }) + .await + .map_err(|_| ZoneTestError::IndexerTimeout)? +} + +async fn count_indexed_payload( + reader: &ZoneReaderConfig, + expected_payload: &Inscription, +) -> Result { + Ok( + replayed_inscription_payloads(&replay_finalized_history(reader).await?) + .iter() + .filter(|payload| *payload == expected_payload) + .count(), + ) +} + +/// Polls until the wallet holds exactly the given number of finalized and +/// unfinalized notes — an exact-count check that catches double-counting. +pub async fn wait_for_channel_wallet_counts( + client: &SequencerClient, + finalized: usize, + unfinalized: usize, + duration: Duration, +) -> Result<(), ZoneTestError> { + timeout(duration, async { + loop { + let view = + client + .channel_wallet() + .await + .map_err(|error| ZoneTestError::ChannelWallet { + message: error.to_string(), + })?; + if view.finalized.len() == finalized && view.unfinalized.len() == unfinalized { + return Ok(()); + } + sleep(Duration::from_millis(500)).await; + } + }) + .await + .map_err(|_| ZoneTestError::ChannelWalletTimeout)? +} + +/// Polls the sequencer's channel wallet until a note of `value` is present. +/// With `finalized_only`, only the finalized layer counts. +pub async fn wait_for_channel_wallet_note( + client: &SequencerClient, + value: Value, + finalized_only: bool, + duration: Duration, +) -> Result<(), ZoneTestError> { + timeout(duration, async { + loop { + let view = + client + .channel_wallet() + .await + .map_err(|error| ZoneTestError::ChannelWallet { + message: error.to_string(), + })?; + let unfinalized = (!finalized_only) + .then_some(view.unfinalized.iter()) + .into_iter() + .flatten(); + if view + .finalized + .iter() + .chain(unfinalized) + .any(|note| note.value == value) + { + return Ok(()); + } + sleep(Duration::from_millis(500)).await; + } + }) + .await + .map_err(|_| ZoneTestError::ChannelWalletTimeout)? +} + +/// Waits until the finalized channel history contains the expected channel +/// deposit, including its amount. +pub async fn wait_for_deposit( + reader: &ZoneReaderConfig, + expected: &DepositOp, + expected_amount: Value, + duration: Duration, +) -> Result<(), ZoneTestError> { + poll_replayed_history_until(reader, duration, ZoneTestError::IndexerTimeout, |op| { + matches!(op, FinalizedOp::Deposit(deposit) + if deposit.inputs == expected.inputs + && deposit.amount == expected_amount + && deposit.metadata == expected.metadata) + }) + .await +} + +/// Waits until the finalized channel history contains the expected withdraw. +pub async fn wait_for_withdraw( + reader: &ZoneReaderConfig, + expected: &ChannelWithdrawOp, + timeout_duration: Duration, +) -> Result<(), ZoneTestError> { + poll_replayed_history_until( + reader, + timeout_duration, + ZoneTestError::WithdrawTimeout, + |op| matches!(op, FinalizedOp::Withdraw(withdraw) if withdraw.op.inputs == expected.inputs), + ) + .await +} + +/// Waits until the finalized channel history contains a channel transfer whose +/// input set has exactly `expected_inputs` notes. +/// +/// This is the on-chain record of the withdrawal's note selection: in the +/// dust-flood scenario it proves best-fit largest-first consumed a single +/// covering note rather than sweeping the >255-note dust flood into a transfer +/// the ledger would reject. +pub async fn wait_for_channel_transfer_input_count( + reader: &ZoneReaderConfig, + expected_inputs: usize, + timeout_duration: Duration, +) -> Result<(), ZoneTestError> { + poll_replayed_history_until( + reader, + timeout_duration, + ZoneTestError::IndexerTimeout, + move |op| { + matches!(op, FinalizedOp::ChannelTransfer(transfer) + if transfer.op.inputs.len() == expected_inputs) + }, + ) + .await +} + +async fn poll_replayed_history_until( + reader: &ZoneReaderConfig, + duration: Duration, + timeout_error: ZoneTestError, + mut predicate: impl FnMut(&FinalizedOp) -> bool, +) -> Result<(), ZoneTestError> { + timeout(duration, async { + loop { + let history = replay_finalized_history(reader).await?; + if history + .iter() + .flat_map(|tx| tx.ops.iter()) + .any(&mut predicate) + { + return Ok(()); + } + + sleep(Duration::from_millis(500)).await; + } + }) + .await + .map_err(|_| timeout_error)? +} + +/// Waits until the sequencer's event stream surfaces the expected deposit +/// in [`Event::BlocksProcessed::finalized`] (matched by `inputs`, `amount`, +/// and `metadata`) while collecting any mempool-pending events. Drains the +/// events channel as it goes — call this after any earlier event consumers in +/// the scenario have moved past the relevant publish events. +pub async fn wait_for_finalized_deposit_via_sequencer_and_collect_mempool_pending( + events: &mut tokio::sync::broadcast::Receiver, + expected: &DepositOp, + expected_amount: Value, + duration: Duration, +) -> Result, ZoneTestError> { + poll_sequencer_finalized_until_and_collect_mempool_pending( + events, + duration, + ZoneTestError::IndexerTimeout, + |op| { + matches!(op, FinalizedOp::Deposit(d) + if d.inputs == expected.inputs + && d.amount == expected_amount + && d.metadata == expected.metadata) + }, + ) + .await +} + +/// Waits until the sequencer's event stream surfaces the expected withdraw +/// (matched by `outputs`) while collecting any mempool-pending events. Drains +/// the events channel as it goes. +pub async fn wait_for_finalized_withdraw_via_sequencer_and_collect_mempool_pending( + events: &mut tokio::sync::broadcast::Receiver, + expected: &ChannelWithdrawOp, + duration: Duration, +) -> Result, ZoneTestError> { + poll_sequencer_finalized_until_and_collect_mempool_pending( + events, + duration, + ZoneTestError::WithdrawTimeout, + |op| matches!(op, FinalizedOp::Withdraw(w) if w.op.inputs == expected.inputs), + ) + .await +} + +async fn poll_sequencer_finalized_until_and_collect_mempool_pending( + events: &mut tokio::sync::broadcast::Receiver, + duration: Duration, + timeout_error: ZoneTestError, + mut predicate: impl FnMut(&FinalizedOp) -> bool, +) -> Result, ZoneTestError> { + timeout(duration, async { + let mut mempool_pending = HashSet::new(); + loop { + let event = match events.recv().await { + Ok(event) => event, + Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => { + warn!("event subscriber lagged by {n}, recovering"); + continue; + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => { + return Err(ZoneTestError::SequencerStopped); + } + }; + if let Event::MempoolPending(tx_hash) = event { + mempool_pending.insert(tx_hash); + continue; + } + let Event::BlocksProcessed { finalized, .. } = event else { + continue; + }; + for tx in finalized { + if tx.ops.iter().any(&mut predicate) { + return Ok(mempool_pending); + } + } + } + }) + .await + .map_err(|_| timeout_error)? +} + +/// Waits until node mempool/chain observation confirms the submitted zone +/// transactions reached the canonical chain. +pub async fn ensure_zone_transactions_included( + client: &NodeHttpClient, + tx_hashes: &[InscriptionId], + duration: Duration, +) -> Result<(), ZoneTestError> { + let included = wait_for_transactions_inclusion(client, tx_hashes, duration).await; + + if included { + return Ok(()); + } + + Err(ZoneTestError::InclusionTimeout) +} + +/// Walks back from LIB until every expected zone transaction is found in the +/// finalized chain. +pub async fn wait_for_transactions_finalized( + node_url: Url, + tx_hashes: &[InscriptionId], + duration: Duration, +) -> Result<(), ZoneTestError> { + let client = CommonHttpClient::new(None); + let expected: HashSet<_> = tx_hashes.iter().copied().collect(); + + timeout(duration, async { + loop { + let info = client + .consensus_info(node_url.clone()) + .await + .map_err(|error| ZoneTestError::Consensus { + message: error.to_string(), + })?; + + let mut found = HashSet::new(); + let mut current = info.cryptarchia_info.lib; + + while let Some(block) = client + .get_block_by_id(node_url.clone(), current) + .await + .map_err(|error| ZoneTestError::Block { + message: error.to_string(), + })? + { + for tx in &block.transactions { + let hash = tx.mantle_tx().hash(); + if expected.contains(&hash) { + found.insert(hash); + } + } + + current = block.header.parent_block; + } + + if found == expected { + return Ok(()); + } + + sleep(Duration::from_millis(500)).await; + } + }) + .await + .map_err(|_| ZoneTestError::FinalizationTimeout)? +} + +/// Waits for LIB movement after a restart so stale-checkpoint scenarios can +/// distinguish old local state from new canonical chain progress. +pub async fn wait_for_lib_advance( + client: &NodeHttpClient, + initial_lib_slot: Slot, + duration: Duration, +) -> Result<(), ZoneTestError> { + timeout(duration, async { + loop { + let info = client + .consensus_info() + .await + .map_err(|error| ZoneTestError::Consensus { + message: error.to_string(), + })?; + + if info.cryptarchia_info.lib_slot > initial_lib_slot { + return Ok(()); + } + + sleep(Duration::from_millis(500)).await; + } + }) + .await + .map_err(|_| ZoneTestError::LibAdvanceTimeout)? +} diff --git a/tests/src/cucumber/steps/zone/operations/policies.rs b/tests/src/cucumber/steps/zone/operations/policies.rs new file mode 100644 index 000000000..00c293cfa --- /dev/null +++ b/tests/src/cucumber/steps/zone/operations/policies.rs @@ -0,0 +1,569 @@ +use super::{ + Arc, BTreeSet, ChannelUpdate, ChannelUpdateTx, DiscardedPayloads, Event, FinalizedTx, HashMap, + HashSet, Inscription, InscriptionInfo, LazyLock, MsgId, PolicyRuntime, SequencerChannelView, + VecDeque, ZoneAccountBalances, ZoneNodeHttpClient, ZoneSequencer, finalized_inscriptions, + parse_balance_payload, runner, to_policy_runtime, warn, +}; + +/// Spawn a sequencer drive task with a no-op policy. Step bodies drive +/// publishes via [`SequencerClient`]; events flow to `PolicyRuntime.events`. +/// If `republish_orphans` is set, the [`OrphanRepublishPolicy`] runs inline +/// inside the drive loop. +pub fn start_sequencer_event_loop( + sequencer: ZoneSequencer, + republish_orphans: bool, +) -> PolicyRuntime { + if republish_orphans { + to_policy_runtime(runner::spawn(sequencer, OrphanRepublishPolicy::default())) + } else { + to_policy_runtime(runner::spawn(sequencer, runner::PassivePolicy)) + } +} + +/// Drives a competing-sequencer policy that publishes `planned` once ready and +/// re-publishes its own orphans (tracked by intent lineage) until they land — +/// correct even when payloads repeat. +pub fn start_republish_lineage_policy( + sequencer: ZoneSequencer, + planned: Vec, +) -> PolicyRuntime { + let policy = RepublishLineagePolicy { + planned, + published_initial: false, + lineage: LineageTracker::default(), + }; + to_policy_runtime(runner::spawn(sequencer, policy)) +} + +/// Drives a policy that republishes orphaned balance updates only when the +/// local canonical view can still apply the update without going negative, +/// and lays planned balance updates whenever it's our turn to write. +pub fn start_balance_aware_policy( + sequencer: ZoneSequencer, + initial_balances: ZoneAccountBalances, + planned_payloads: Vec, +) -> PolicyRuntime { + let view_rx = sequencer.subscribe_channel_view(); + let policy = BalanceAwarePolicy { + balances: BalanceAwareState::new(initial_balances), + planned: VecDeque::from(planned_payloads), + view_rx, + }; + to_policy_runtime(runner::spawn(sequencer, policy)) +} + +/// Drives a deterministic conflict policy used by tests that expect the final +/// zone chain to converge to sorted payload order. +pub fn start_sorted_conflict_policy( + sequencer: ZoneSequencer, + discarded: &DiscardedPayloads, +) -> PolicyRuntime { + let policy = SortedConflictPolicy { + state: SortedConflictState::new(Arc::clone(discarded)), + }; + to_policy_runtime(runner::spawn(sequencer, policy)) +} + +/// Inline policy: republish orphaned inscriptions not already back on the +/// canonical chain. Plain inscriptions only — bundles re-prepare themselves. +/// Assumes unique payloads; for repeating payloads see +/// [`RepublishLineagePolicy`]. +/// +/// Tracks canonical on-chain state keyed by id, decided by payload (see +/// `on_event`): a dead twin's id leaves while a live twin keeps the payload +/// covered, so a payload still on chain is never re-homed. +#[derive(Default)] +struct OrphanRepublishPolicy { + /// Canonical on-chain inscriptions by id (adopted-unfinalized + finalized). + /// Finalized entries are added and never removed — they can't be orphaned — + /// so this one set is the whole on-chain view. + canonical: HashMap, +} + +impl runner::Policy for OrphanRepublishPolicy +where + Node: lb_zone_sdk::adapter::Node + Clone + Send + Sync + 'static, +{ + async fn on_event(&mut self, sequencer: &mut ZoneSequencer, event: &Event) { + let Event::BlocksProcessed { + channel_update, + finalized, + .. + } = event + else { + return; + }; + // 1. Remove orphaned by id — a dead twin leaves, a live twin stays. + for entry in &channel_update.orphaned { + if let Some(info) = entry.inscription() { + self.canonical.remove(&info.this_msg); + } + } + // 2. Add finalized by id (permanent — finalized can't be orphaned). + for info in finalized_inscriptions(finalized) { + self.canonical.insert(info.this_msg, info.payload.clone()); + } + // 3. Add adopted by id (this block's new canonical). + for info in channel_update + .adopted + .iter() + .filter_map(ChannelUpdateTx::inscription) + { + self.canonical.insert(info.this_msg, info.payload.clone()); + } + // 4. Republish orphaned whose payload no canonical id still carries. + for entry in &channel_update.orphaned { + let ChannelUpdateTx::Inscription(info) = entry else { + continue; + }; + if self + .canonical + .values() + .any(|payload| *payload == info.payload) + { + continue; + } + if let Err(error) = sequencer.handle().publish(info.payload.clone()).await { + warn!(%error, "Failed to re-publish orphaned zone payload"); + } + } + } +} + +/// Tracks our published inscriptions by intent lineage, so republishing works +/// even when payloads repeat (identical bytes published as distinct messages). +/// +/// Each original publish is its own intent, rooted at its `this_msg`; every +/// republish we issue for an orphaned member is recorded under the same root. +/// An intent is "live" while any of its `this_msg`s is on the channel +/// (`adopted`) or in flight as a publish/republish we issued. Identical +/// payloads form distinct intents (distinct `this_msg`s), so each lands once, +/// and other sequencers' inscriptions are never in our map, so we never +/// republish theirs. +#[derive(Default)] +struct LineageTracker { + /// Every `this_msg` we've published (originals + republishes) → intent + /// root. + intent_root: HashMap, + /// Per intent root, the `this_msg`s currently pending (in the + /// non-finalized channel view). + pending: HashMap>, + /// Intent roots that have finalized — permanently landed, so the intent is + /// considered live forever and never re-homed again. + finalized_roots: HashSet, +} + +impl LineageTracker { + /// Record an original publish as its own intent, in flight. + fn record_publish(&mut self, this_msg: MsgId) { + self.intent_root.insert(this_msg, this_msg); + self.pending.entry(this_msg).or_default().insert(this_msg); + } + + /// Record a republish of `orphan` as a new live member of its intent. + fn record_republish(&mut self, orphan: MsgId, republished: MsgId) { + let root = self.intent_root.get(&orphan).copied().unwrap_or(orphan); + self.intent_root.insert(republished, root); + self.pending.entry(root).or_default().insert(republished); + } + + /// Fold a delta into per-intent liveness — only our `msg_id`s are relevant. + /// Adopted members become live; orphaned members stop being live. + fn observe(&mut self, channel_update: &ChannelUpdate) { + for info in channel_update + .adopted + .iter() + .filter_map(ChannelUpdateTx::inscription) + { + if let Some(&root) = self.intent_root.get(&info.this_msg) { + self.pending.entry(root).or_default().insert(info.this_msg); + } + } + for entry in &channel_update.orphaned { + if let ChannelUpdateTx::Inscription(info) = entry + && let Some(&root) = self.intent_root.get(&info.this_msg) + && let Some(members) = self.pending.get_mut(&root) + { + members.remove(&info.this_msg); + } + } + } + + /// Pin the intents of any finalized `this_msg`s of ours as permanently + /// live — once a member finalizes the payload is on chain for good. + fn observe_finalized(&mut self, finalized: impl Iterator) { + for this_msg in finalized { + if let Some(&root) = self.intent_root.get(&this_msg) { + self.finalized_roots.insert(root); + } + } + } + + /// True if `this_msg` is one of ours. + fn is_ours(&self, this_msg: &MsgId) -> bool { + self.intent_root.contains_key(this_msg) + } + + /// True if the intent of `this_msg` has finalized, or still has a live + /// member. + fn intent_live(&self, this_msg: &MsgId) -> bool { + let root = self.intent_root.get(this_msg).copied().unwrap_or(*this_msg); + self.finalized_roots.contains(&root) + || self + .pending + .get(&root) + .is_some_and(|members| !members.is_empty()) + } +} + +/// Inline republish policy for channels whose payloads can repeat. Publishes +/// its own `planned` payloads once the sequencer is ready, then republishes any +/// of *our* orphans whose intent has no live member, tracking msg-id lineage +/// (the payload can't identify the message when it repeats). Owning the +/// publishes is what gives the policy its outbox: every `this_msg` it sends is +/// recorded. +struct RepublishLineagePolicy { + planned: Vec, + published_initial: bool, + lineage: LineageTracker, +} + +impl runner::Policy for RepublishLineagePolicy +where + Node: lb_zone_sdk::adapter::Node + Clone + Send + Sync + 'static, +{ + async fn on_event(&mut self, sequencer: &mut ZoneSequencer, event: &Event) { + match event { + Event::Ready if !self.published_initial => { + self.published_initial = true; + for payload in self.planned.clone() { + match sequencer.handle().publish(payload).await { + Ok((result, _checkpoint)) => { + self.lineage + .record_publish(result.tx.inscription().this_msg); + } + Err(error) => warn!(%error, "Failed to publish planned zone payload"), + } + } + } + Event::BlocksProcessed { + channel_update, + finalized, + .. + } => { + self.lineage + .observe_finalized(finalized_inscriptions(finalized).map(|i| i.this_msg)); + self.lineage.observe(channel_update); + for entry in &channel_update.orphaned { + let ChannelUpdateTx::Inscription(info) = entry else { + continue; + }; + if !self.lineage.is_ours(&info.this_msg) + || self.lineage.intent_live(&info.this_msg) + { + continue; + } + match sequencer.handle().publish(info.payload.clone()).await { + Ok((result, _checkpoint)) => { + self.lineage + .record_republish(info.this_msg, result.tx.inscription().this_msg); + } + Err(error) => warn!(%error, "Failed to re-publish orphaned zone payload"), + } + } + } + _ => {} + } + } +} + +/// Inline policy: republish orphans only when the local balance view still +/// allows it; publish planned payloads as soon as it's our turn to write. +/// +/// The balance view is rebuilt from the full delta — every orphaned op is +/// removed and every adopted op applied — so affordability reflects all +/// inscriptions on the channel. Removing an orphan we never applied (never- +/// landed pending) is a no-op, and an already-adopted op is skipped because its +/// id is already in the applied set after `record_adopted_payloads`. +struct BalanceAwarePolicy { + balances: BalanceAwareState, + planned: VecDeque, + view_rx: tokio::sync::watch::Receiver, +} + +impl runner::Policy for BalanceAwarePolicy +where + Node: lb_zone_sdk::adapter::Node + Clone + Send + Sync + 'static, +{ + async fn on_event(&mut self, sequencer: &mut ZoneSequencer, event: &Event) { + if let Event::BlocksProcessed { + channel_update, + finalized, + .. + } = event + { + self.balances.record_finalized_payloads(finalized); + let ChannelUpdate { orphaned, adopted } = channel_update; + let orphaned_inscriptions: Vec = orphaned + .iter() + .filter_map(|o| match o { + ChannelUpdateTx::Inscription(i) => Some(i.clone()), + ChannelUpdateTx::AtomicWithdraw(_) + | ChannelUpdateTx::Custom(_) + | ChannelUpdateTx::Config(_) => None, + }) + .collect(); + self.balances + .remove_orphaned_payloads(&orphaned_inscriptions); + self.balances.record_adopted_payloads(adopted); + for info in orphaned_inscriptions { + if !self.balances.should_republish(&info.payload) { + continue; + } + if let Err(error) = sequencer.handle().publish(info.payload.clone()).await { + warn!(%error, "Failed to re-publish balance-aware zone payload"); + continue; + } + self.balances.record_republished_payload(&info.payload); + } + } + + if !self.view_rx.borrow().our_turn_to_write { + return; + } + while let Some(payload) = self.planned.pop_front() { + if !self.balances.should_republish(&payload) { + continue; + } + if let Err(error) = sequencer.handle().publish(payload.clone()).await { + warn!(%error, "Failed to publish planned balance-aware zone payload"); + self.planned.push_front(payload); + break; + } + self.balances.record_republished_payload(&payload); + } + } +} + +/// Inline policy: republish orphans only when they preserve sorted-payload +/// order; otherwise mark them as discarded. +/// +/// The full delta lets us rebuild the on-chain payload set each update (drop +/// orphaned, add adopted), so the order floor we gate republishing on falls +/// back correctly when the highest payload is orphaned. +struct SortedConflictPolicy { + state: SortedConflictState, +} + +impl runner::Policy for SortedConflictPolicy +where + Node: lb_zone_sdk::adapter::Node + Clone + Send + Sync + 'static, +{ + async fn on_event(&mut self, sequencer: &mut ZoneSequencer, event: &Event) { + let Event::BlocksProcessed { + channel_update, + finalized, + .. + } = event + else { + return; + }; + // Pin finalized payloads first. + self.state.record_finalized(finalized); + let ChannelUpdate { orphaned, adopted } = channel_update; + let orphaned_inscriptions: Vec<&InscriptionInfo> = orphaned + .iter() + .filter_map(|o| match o { + ChannelUpdateTx::Inscription(i) => Some(i), + ChannelUpdateTx::AtomicWithdraw(_) + | ChannelUpdateTx::Custom(_) + | ChannelUpdateTx::Config(_) => None, + }) + .collect(); + + // Rebuild on-chain state from this delta before deciding anything. + self.state.revert_orphaned(&orphaned_inscriptions); + self.state.record_adoptions(adopted).await; + + let readopted: HashSet<&Inscription> = adopted + .iter() + .filter_map(|tx| tx.inscription().map(|i| &i.payload)) + .collect(); + + // Consider this round's fresh orphans together with everything parked, + // in sorted order (a `BTreeSet` iterates ascending). A payload parked + // under a higher floor on another branch then slots in ahead of a higher + // fresh orphan instead of being locked out, and the chain stays sorted. + // Finalized payloads are excluded — they're already permanently landed. + let mut candidates: BTreeSet = orphaned_inscriptions + .iter() + .map(|i| i.payload.clone()) + .filter(|payload| !readopted.contains(payload) && !self.state.is_finalized(payload)) + .collect(); + candidates.extend(self.state.discarded_snapshot().await); + + for payload in candidates { + if self.state.is_finalized(&payload) { + continue; + } + if self.state.preserves_order(&payload) { + if let Err(error) = sequencer.handle().publish(payload.clone()).await { + warn!(%error, "Failed to re-publish sorted zone payload"); + continue; + } + self.state.record_published_payload(payload).await; + } else { + self.state.discard(payload).await; + } + } + } +} + +struct BalanceAwareState { + initial_balances: ZoneAccountBalances, + applied: HashMap>, + finalized: HashSet, +} + +impl BalanceAwareState { + fn new(initial_balances: ZoneAccountBalances) -> Self { + Self { + initial_balances, + applied: HashMap::new(), + finalized: HashSet::new(), + } + } + + /// Pin finalized payloads. + fn record_finalized_payloads(&mut self, finalized: &[FinalizedTx]) { + for inscription in finalized_inscriptions(finalized) { + if let Some((uuid, _, _)) = parse_balance_payload(&inscription.payload) { + self.finalized.insert(uuid); + } + self.record_applied_payload(&inscription.payload); + } + } + + fn record_applied_payload(&mut self, payload: &Inscription) { + let Some((uuid, account, delta)) = parse_balance_payload(payload) else { + return; + }; + + self.applied.entry(account).or_default().insert(uuid, delta); + } + + fn remove_orphaned_payloads(&mut self, orphaned: &[InscriptionInfo]) { + for inscription in orphaned { + let Some((uuid, account, _)) = parse_balance_payload(&inscription.payload) else { + continue; + }; + + // A finalized delta is permanent — never drop it on an orphan. + if self.finalized.contains(&uuid) { + continue; + } + + if let Some(account_updates) = self.applied.get_mut(&account) { + account_updates.remove(&uuid); + } + } + } + + fn record_adopted_payloads(&mut self, adopted: &[ChannelUpdateTx]) { + for info in adopted.iter().filter_map(ChannelUpdateTx::inscription) { + self.record_applied_payload(&info.payload); + } + } + + fn should_republish(&self, payload: &Inscription) -> bool { + let Some((uuid, account, delta)) = parse_balance_payload(payload) else { + return false; + }; + + if self.finalized.contains(&uuid) || self.account_updates(&account).contains_key(&uuid) { + return false; + } + + self.available_balance(&account) + delta >= 0 + } + + fn record_republished_payload(&mut self, payload: &Inscription) { + self.record_applied_payload(payload); + } + + fn available_balance(&self, account: &str) -> i64 { + self.initial_balances.get(account).copied().unwrap_or(0) + + self.account_updates(account).values().sum::() + } + + fn account_updates(&self, account: &str) -> &HashMap { + self.applied.get(account).unwrap_or(&EMPTY_BALANCE_UPDATES) + } +} + +static EMPTY_BALANCE_UPDATES: LazyLock> = LazyLock::new(HashMap::new); + +struct SortedConflictState { + /// The local channel view: pending (non-finalized) payloads plus the + /// pinned finalized base, kept as the ordering floor. + channel_view: BTreeSet, + discarded: DiscardedPayloads, + finalized: HashSet, +} + +impl SortedConflictState { + fn new(discarded: DiscardedPayloads) -> Self { + Self { + channel_view: BTreeSet::new(), + discarded, + finalized: HashSet::new(), + } + } + + /// Pin finalized payloads into the channel view permanently. + fn record_finalized(&mut self, finalized: &[FinalizedTx]) { + for inscription in finalized_inscriptions(finalized) { + self.finalized.insert(inscription.payload.clone()); + self.channel_view.insert(inscription.payload.clone()); + } + } + + fn is_finalized(&self, payload: &Inscription) -> bool { + self.finalized.contains(payload) + } + + /// Drop orphaned payloads from the channel view — the order floor falls + /// back to the max of whatever remains. Finalized payloads stay put. + fn revert_orphaned(&mut self, orphaned: &[&InscriptionInfo]) { + for inscription in orphaned { + if self.finalized.contains(&inscription.payload) { + continue; + } + self.channel_view.remove(&inscription.payload); + } + } + + async fn record_adoptions(&mut self, adopted: &[ChannelUpdateTx]) { + for info in adopted.iter().filter_map(ChannelUpdateTx::inscription) { + self.discarded.lock().await.remove(&info.payload); + self.channel_view.insert(info.payload.clone()); + } + } + + async fn record_published_payload(&mut self, payload: Inscription) { + self.discarded.lock().await.remove(&payload); + self.channel_view.insert(payload); + } + + fn preserves_order(&self, payload: &Inscription) -> bool { + self.channel_view.last().is_none_or(|max| payload >= max) + } + + async fn discard(&self, payload: Inscription) { + self.discarded.lock().await.insert(payload); + } + + async fn discarded_snapshot(&self) -> Vec { + self.discarded.lock().await.iter().cloned().collect() + } +} diff --git a/tests/src/cucumber/steps/zone/operations/transactions.rs b/tests/src/cucumber/steps/zone/operations/transactions.rs new file mode 100644 index 000000000..296e49a35 --- /dev/null +++ b/tests/src/cucumber/steps/zone/operations/transactions.rs @@ -0,0 +1,297 @@ +use super::*; + +/// Builds a regular channel deposit for an existing funding note with the +/// exact deposit value. +pub fn build_zone_deposit( + available_utxos: Vec, + channel_id: ChannelId, + amount: Value, + metadata: Metadata, +) -> Result { + let note = available_utxos + .into_iter() + .find(|utxo| utxo.note.value == amount) + .ok_or(ZoneTestError::MissingExactFundingNote { value: amount })?; + + let deposit = DepositOp { + channel_id, + inputs: Inputs::new([note.id()]), + metadata, + }; + let reserved_inputs = vec![note]; + let channel_notes = recreated_channel_notes(&deposit, &reserved_inputs); + Ok(ZoneDeposit { + deposit, + reserved_inputs, + channel_notes, + }) +} + +/// Build a deposit that consumes one wallet note per listed value, in order. +/// A deposit re-creates its inputs 1:1 as channel notes, so this yields a +/// multi-input deposit whose recreated notes carry those exact per-note values +/// — used to cover the channel wallet's per-note tracking. +pub fn build_zone_deposit_from_values( + available_utxos: Vec, + channel_id: ChannelId, + input_values: &[Value], + metadata: Metadata, +) -> Result { + let mut remaining = available_utxos; + let mut reserved_inputs = Vec::new(); + for &value in input_values { + let index = remaining + .iter() + .position(|utxo| utxo.note.value == value) + .ok_or(ZoneTestError::MissingExactFundingNote { value })?; + reserved_inputs.push(remaining.remove(index)); + } + + let input_ids: Vec<_> = reserved_inputs.iter().map(Utxo::id).collect(); + let deposit = DepositOp { + channel_id, + inputs: Inputs::try_new(input_ids).map_err(|error| ZoneTestError::SubmitDeposit { + message: format!("deposit input set exceeds bound: {error:?}"), + })?, + metadata, + }; + let channel_notes = recreated_channel_notes(&deposit, &reserved_inputs); + Ok(ZoneDeposit { + deposit, + reserved_inputs, + channel_notes, + }) +} + +/// Generous cap on channel transaction fees at genesis gas prices; actual +/// fees are a few hundred gas units for these small transactions. +const MAX_ZONE_DEPOSIT_TX_FEE: u64 = 10_000; + +/// Submits a regular channel deposit through the node wallet API. +pub async fn submit_zone_deposit( + node_url: &Url, + deposit: &DepositOp, + funding_public_key: ZkPublicKey, +) -> Result { + let body = ChannelDepositRequestBody { + tip: None, + deposit: deposit.clone(), + change_public_key: funding_public_key, + funding_public_keys: vec![funding_public_key], + max_tx_fee: MAX_ZONE_DEPOSIT_TX_FEE.into(), + }; + + let request_url = + node_url + .join("/channel/deposit") + .map_err(|error| ZoneTestError::SubmitDeposit { + message: error.to_string(), + })?; + + let response: ChannelDepositResponseBody = CommonHttpClient::new(None) + .post(request_url, &body) + .await + .map_err(|error| ZoneTestError::SubmitDeposit { + message: error.to_string(), + })?; + + Ok(response.hash) +} + +/// Splits one channel note into `dust_count` value-1 dust notes via a raw, +/// node-funded `ChannelTransfer`, submitted straight to the node mempool (no +/// zone-sdk involvement — mirrors how deposits are submitted). +/// +/// The transfer is value-preserving, so the input note's value must equal +/// `dust_count` (every output is value 1). The channel authorizes the transfer +/// with the sequencer's accredited key at index 0 (single-signer channel, +/// `transfer_threshold == 1`); the node appends and proves the fee transfer. +pub async fn submit_zone_channel_split( + node_url: &Url, + channel_id: ChannelId, + signing_key: &Ed25519Key, + funding_pk: ZkPublicKey, + input_note: Utxo, + dust_count: usize, +) -> Result { + let input_value = input_note.note.value; + if input_value != dust_count as u64 { + return Err(ZoneTestError::SplitTransfer { + message: format!( + "channel note value {input_value} must equal dust count {dust_count} \ + (each dust note is value 1)" + ), + }); + } + + let outputs = + Outputs::try_new(vec![Note::new(1, funding_pk); dust_count]).map_err(|error| { + ZoneTestError::SplitTransfer { + message: format!("dust outputs exceed bound: {error:?}"), + } + })?; + let transfer = ChannelTransferOp { + channel_id, + inputs: Inputs::new([input_note.id()]), + outputs, + }; + + let tx_builder = MantleTxBuilder::new() + .push_op(Op::ChannelTransfer(transfer)) + .map_err(|error| ZoneTestError::SplitTransfer { + message: format!("too many ops: {error}"), + })?; + + let node = NodeHttpClient::from_url(node_url.clone()); + let response = node + .fund_tx(WalletFundRequestBody { + tip: None, + priority_fee_percent: 0, + tx_builder, + change_public_key: funding_pk, + funding_public_keys: vec![funding_pk], + max_tx_fee: GasCost::new(u64::MAX), + }) + .await + .map_err(|error| ZoneTestError::SplitTransfer { + message: format!("funding failed: {error}"), + })?; + + // The channel multi-sig proves the transfer over the funded tx hash; the + // funding appends its own fee transfer proof as the last op. + let funded_tx = response.funded_tx; + let tx_hash = funded_tx.hash(); + let signature = signing_key.sign_payload(tx_hash.as_signing_bytes().as_ref()); + let proof = ChannelMultiSigProof::try_new([IndexedSignature::new(0, signature)].into()) + .map_err(|error| ZoneTestError::SplitTransfer { + message: format!("multi-sig proof assembly failed: {error:?}"), + })?; + let mut ops_proofs = OpsProofs::new_unchecked(vec![OpProof::ChannelMultiSigProof(proof)]); + if let Some(transfer_proof) = response.transfer_proof { + ops_proofs + .try_push(transfer_proof) + .map_err(|error| ZoneTestError::SplitTransfer { + message: format!("too many operation proofs: {error:?}"), + })?; + } + + let signed_tx = SignedMantleTx::new(funded_tx, ops_proofs); + node.submit_transaction(&signed_tx) + .await + .map_err(|error| ZoneTestError::SplitTransfer { + message: format!("submit failed: {error}"), + })?; + + Ok(tx_hash) +} + +/// Builds and submits a single transaction that both creates the deposit note +/// and publishes the zone inscription that consumes it. +pub async fn submit_atomic_zone_deposit( + node_url: &Url, + client: &SequencerClient, + request: AtomicZoneDepositRequest, +) -> Result { + let AtomicZoneDepositRequest { + channel_id, + funding_public_key, + available_utxos, + amount, + metadata, + inscription_data, + } = request; + let (transfer, reserved_inputs) = + build_atomic_deposit_transfer(available_utxos, funding_public_key, amount)?; + let deposit = build_atomic_deposit_op(channel_id, metadata, &transfer)?; + + let (tx, msg_id, sequencer_sig) = client + .prepare_tx( + [Op::Transfer(transfer), Op::ChannelDeposit(deposit.clone())].into(), + inscription_data, + ) + .await + .map_err(|error| ZoneTestError::BuildAtomicDeposit { + message: error.to_string(), + })?; + + let user_sig = sign_tx_zk(node_url, &tx, vec![funding_public_key]).await?; + let signed_tx = SignedMantleTx::new( + tx, + [ + OpProof::ZkSig(user_sig.clone()), + OpProof::ZkSig(user_sig), + OpProof::Ed25519Sig(sequencer_sig), + ] + .into(), + ); + + let (result, _cp) = client + .submit_signed_tx(signed_tx, msg_id) + .await + .map_err(|error| ZoneTestError::SubmitAtomicDeposit { + message: error.to_string(), + })?; + + Ok(AtomicZoneDepositSubmission { + deposit, + publish: result, + reserved_inputs, + }) +} + +pub(super) async fn build_funded_custom_tx( + node_client: &NodeHttpClient, + channel_id: ChannelId, + signing_key: &Ed25519Key, + funding_pk: ZkPublicKey, + payloads: &[Inscription], + mut parent: MsgId, +) -> Result<(SignedMantleTx, MsgId), ZoneTestError> { + let signer = signing_key.public_key(); + let mut tx_builder = MantleTxBuilder::new(); + for payload in payloads { + let op = InscriptionOp { + channel_id, + inscription: payload.clone(), + parent, + signer, + }; + parent = op.id(); + tx_builder = tx_builder + .push_op(Op::ChannelInscribe(op)) + .map_err(|error| ZoneTestError::BuildCustomTx { + message: format!("too many ops: {error}"), + })?; + } + + let response = node_client + .fund_tx(WalletFundRequestBody { + tip: None, + priority_fee_percent: 0, + tx_builder, + change_public_key: funding_pk, + funding_public_keys: vec![funding_pk], + max_tx_fee: GasCost::new(u64::MAX), + }) + .await + .map_err(|error| ZoneTestError::SubmitCustomTx { + message: format!("funding failed: {error}"), + })?; + + // Funding appends the fee transfer as the last op; every inscription is + // proven by the sequencer key over the funded tx hash. + let funded_tx = response.funded_tx; + let signature = signing_key.sign_payload(funded_tx.hash().as_signing_bytes().as_ref()); + let mut ops_proofs = + OpsProofs::new_unchecked(vec![OpProof::Ed25519Sig(signature); payloads.len()]); + if let Some(proof) = response.transfer_proof { + ops_proofs + .try_push(proof) + .map_err(|error| ZoneTestError::BuildCustomTx { + message: format!("too many operation proofs: {error:?}"), + })?; + } + let signed_tx = SignedMantleTx::new(funded_tx, ops_proofs); + + Ok((signed_tx, parent)) +} diff --git a/tests/src/cucumber/steps/manual_zone/runner.rs b/tests/src/cucumber/steps/zone/runner.rs similarity index 100% rename from tests/src/cucumber/steps/manual_zone/runner.rs rename to tests/src/cucumber/steps/zone/runner.rs diff --git a/tests/src/cucumber/steps/zone/steps/assertions.rs b/tests/src/cucumber/steps/zone/steps/assertions.rs new file mode 100644 index 000000000..4231242de --- /dev/null +++ b/tests/src/cucumber/steps/zone/steps/assertions.rs @@ -0,0 +1,602 @@ +use super::{ + CONCURRENT_DUPLICATE_SETTLE_SECS, CucumberWorld, DEFAULT_ZONE_SEQUENCER, Duration, HashMap, + HashSet, Inscription, Step, StepError, StepResult, TxSource, TxStatus, assert_sorted_outcome, + collect_indexed_messages, collect_indexed_messages_exactly_once, + ensure_zone_transactions_included, log_step_error, make_inscription, parse_balance_payload, + scan_indexer_for_payloads, single_column_table, wait_for_channel_transfer_input_count, + wait_for_channel_wallet_counts, wait_for_channel_wallet_note, wait_for_deposit, + wait_for_exact_indexed_payload_count, + wait_for_finalized_deposit_via_sequencer_and_collect_mempool_pending, + wait_for_finalized_withdraw_via_sequencer_and_collect_mempool_pending, + wait_for_indexer_unordered, wait_for_transactions_finalized, wait_for_tx_status_lifecycle, + wait_for_withdraw, wait_until_sorted_conflict_settles, zone_step_error, +}; + +#[cucumber::then(expr = "all zone messages are safe in {int} seconds")] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require `&mut World` as the first parameter" +)] +async fn step_all_zone_messages_are_safe( + world: &mut CucumberWorld, + step: &Step, + timeout_seconds: u64, +) -> StepResult { + let inscription_ids = log_step_error(step, world.zone.ordered_inscription_ids())?; + + if !world.zone.has_published_messages() { + return Err(StepError::LogicalError { + message: "No zone messages have been published".to_owned(), + }); + } + + let node = log_step_error(step, world.zone_node_http_client())?; + + ensure_zone_transactions_included( + &node, + &inscription_ids, + Duration::from_secs(timeout_seconds), + ) + .await + .map_err(|error| zone_step_error(step, &error)) +} + +#[cucumber::then(expr = "all zone messages are finalized in {int} seconds")] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require `&mut World` as the first parameter" +)] +async fn step_all_zone_messages_are_finalized( + world: &mut CucumberWorld, + step: &Step, + timeout_seconds: u64, +) -> StepResult { + let inscription_ids = log_step_error(step, world.zone.ordered_inscription_ids())?; + + if !world.zone.has_published_messages() { + return Err(StepError::LogicalError { + message: "No zone messages have been published".to_owned(), + }); + } + + let node_url = log_step_error(step, world.zone_node_url())?; + + wait_for_transactions_finalized( + node_url, + &inscription_ids, + Duration::from_secs(timeout_seconds), + ) + .await + .map_err(|error| zone_step_error(step, &error)) +} + +#[cucumber::then( + expr = "sequencer {string} emits the full transaction lifecycle for zone messages in {int} seconds:" +)] +#[cucumber::when( + expr = "sequencer {string} emits the full transaction lifecycle for zone messages in {int} seconds:" +)] +async fn step_sequencer_emits_full_transaction_lifecycle( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, + timeout_seconds: u64, +) -> StepResult { + let aliases = single_column_table(step, "alias", "zone message aliases")?; + let tx_hashes = log_step_error(step, world.zone.message_tx_hashes_for_aliases(&aliases))?; + let mut tx_status_rx = log_step_error( + step, + world.zone.take_sequencer_tx_status_rx(&sequencer_alias), + )?; + + wait_for_tx_status_lifecycle( + &mut tx_status_rx, + &tx_hashes, + &[ + TxStatus::AcceptedLocally, + TxStatus::PendingMempool, + TxStatus::OnChain(TxSource::Local), + TxStatus::Finalized(TxSource::Local), + ], + Duration::from_secs(timeout_seconds), + ) + .await + .map_err(|error| zone_step_error(step, &error)) +} + +#[cucumber::then("the zone indexer returns messages in this order:")] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require `&mut World` as the first parameter" +)] +async fn step_zone_indexer_returns_messages_in_order( + world: &mut CucumberWorld, + step: &Step, +) -> StepResult { + let aliases = single_column_table(step, "alias", "zone message aliases")?; + let expected = log_step_error(step, world.zone.message_payloads_for_aliases(&aliases))?; + let indexer = log_step_error(step, world.zone.indexer())?; + + let actual = collect_indexed_messages(indexer, &expected, Duration::from_mins(3)) + .await + .map_err(|error| zone_step_error(step, &error))?; + + if actual == expected { + return Ok(()); + } + + Err(StepError::LogicalError { + message: format!( + "Zone indexer returned messages in unexpected order: expected {} messages, got {}", + expected.len(), + actual.len() + ), + }) +} + +#[cucumber::then(expr = "the zone indexer returns messages in any order in {int} seconds:")] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require `&mut World` as the first parameter" +)] +async fn step_zone_indexer_returns_messages_in_any_order( + world: &mut CucumberWorld, + step: &Step, + timeout_seconds: u64, +) -> StepResult { + let aliases = single_column_table(step, "alias", "zone message aliases")?; + let expected = log_step_error(step, world.zone.message_payloads_for_aliases(&aliases))?; + let expected = expected.into_iter().collect::>(); + let indexer = log_step_error(step, world.zone.indexer())?; + + wait_for_indexer_unordered(indexer, &expected, Duration::from_secs(timeout_seconds)) + .await + .map_err(|error| zone_step_error(step, &error))?; + + Ok(()) +} + +#[cucumber::then("the zone indexer returns each of these messages exactly once in this order:")] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require `&mut World` as the first parameter" +)] +async fn step_zone_indexer_returns_messages_exactly_once_in_order( + world: &mut CucumberWorld, + step: &Step, +) -> StepResult { + let aliases = single_column_table(step, "alias", "zone message aliases")?; + let expected = log_step_error(step, world.zone.message_payloads_for_aliases(&aliases))?; + let indexer = log_step_error(step, world.zone.indexer())?; + + let actual = collect_indexed_messages_exactly_once(indexer, &expected, Duration::from_mins(3)) + .await + .map_err(|error| zone_step_error(step, &error))?; + + if actual == expected { + return Ok(()); + } + + Err(StepError::LogicalError { + message: format!( + "Zone indexer returned duplicate or out-of-order messages: expected {} messages, got {}", + expected.len(), + actual.len() + ), + }) +} + +#[cucumber::then(expr = "zone transaction {string} is included in {int} seconds")] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require `&mut World` as the first parameter" +)] +async fn step_zone_transaction_is_included( + world: &mut CucumberWorld, + step: &Step, + transaction_alias: String, + timeout_seconds: u64, +) -> StepResult { + let tx_hash = world.resolve_submitted_transaction(&transaction_alias)?; + let node = log_step_error(step, world.zone_node_http_client())?; + + ensure_zone_transactions_included(&node, &[tx_hash], Duration::from_secs(timeout_seconds)) + .await + .map_err(|error| zone_step_error(step, &error)) +} + +#[cucumber::then(expr = "zone transaction {string} is finalized in {int} seconds")] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require `&mut World` as the first parameter" +)] +async fn step_zone_transaction_is_finalized( + world: &mut CucumberWorld, + step: &Step, + transaction_alias: String, + timeout_seconds: u64, +) -> StepResult { + let tx_hash = world.resolve_submitted_transaction(&transaction_alias)?; + let node_url = log_step_error(step, world.zone_node_url())?; + + wait_for_transactions_finalized(node_url, &[tx_hash], Duration::from_secs(timeout_seconds)) + .await + .map_err(|error| zone_step_error(step, &error)) +} + +#[cucumber::then(expr = "the zone indexer returns finalized deposit {string} in {int} seconds")] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require `&mut World` as the first parameter" +)] +async fn step_zone_indexer_returns_finalized_deposit( + world: &mut CucumberWorld, + step: &Step, + deposit_alias: String, + timeout_seconds: u64, +) -> StepResult { + let (deposit, amount) = world + .zone + .resolve_submitted_deposit(&deposit_alias)? + .clone(); + let indexer = log_step_error(step, world.zone.indexer())?; + + wait_for_deposit( + indexer, + &deposit, + amount, + Duration::from_secs(timeout_seconds), + ) + .await + .map_err(|error| zone_step_error(step, &error)) +} + +#[cucumber::then(expr = "the zone indexer returns finalized withdraw {string} in {int} seconds")] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require `&mut World` as the first parameter" +)] +async fn step_zone_indexer_returns_finalized_withdraw( + world: &mut CucumberWorld, + step: &Step, + withdraw_alias: String, + timeout_seconds: u64, +) -> StepResult { + let withdraw = world + .zone + .resolve_submitted_withdraw(&withdraw_alias)? + .clone(); + let indexer = log_step_error(step, world.zone.indexer())?; + + wait_for_withdraw(indexer, &withdraw, Duration::from_secs(timeout_seconds)) + .await + .map_err(|error| zone_step_error(step, &error)) +} + +#[cucumber::then( + expr = "the zone indexer returns a finalized channel transfer consuming {int} inputs in {int} seconds" +)] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require `&mut World` as the first parameter" +)] +async fn step_zone_indexer_returns_finalized_channel_transfer_input_count( + world: &mut CucumberWorld, + step: &Step, + expected_inputs: usize, + timeout_seconds: u64, +) -> StepResult { + let indexer = log_step_error(step, world.zone.indexer())?; + + wait_for_channel_transfer_input_count( + indexer, + expected_inputs, + Duration::from_secs(timeout_seconds), + ) + .await + .map_err(|error| zone_step_error(step, &error)) +} + +#[cucumber::then( + expr = "the channel wallet of {string} contains a note of value {int} in {int} seconds" +)] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require `&mut World` as the first parameter" +)] +async fn step_channel_wallet_contains_note( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, + value: u64, + timeout_seconds: u64, +) -> StepResult { + let client = log_step_error(step, world.zone.sequencer_client(&sequencer_alias))?; + wait_for_channel_wallet_note(client, value, false, Duration::from_secs(timeout_seconds)) + .await + .map_err(|error| zone_step_error(step, &error)) +} + +#[cucumber::then( + expr = "the channel wallet of {string} contains a finalized note of value {int} in {int} seconds" +)] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require `&mut World` as the first parameter" +)] +async fn step_channel_wallet_contains_finalized_note( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, + value: u64, + timeout_seconds: u64, +) -> StepResult { + let client = log_step_error(step, world.zone.sequencer_client(&sequencer_alias))?; + wait_for_channel_wallet_note(client, value, true, Duration::from_secs(timeout_seconds)) + .await + .map_err(|error| zone_step_error(step, &error)) +} + +#[cucumber::then( + expr = "the channel wallet of {string} has exactly {int} finalized and {int} unfinalized notes in {int} seconds" +)] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require `&mut World` as the first parameter" +)] +async fn step_channel_wallet_has_exact_counts( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, + finalized: usize, + unfinalized: usize, + timeout_seconds: u64, +) -> StepResult { + let client = log_step_error(step, world.zone.sequencer_client(&sequencer_alias))?; + wait_for_channel_wallet_counts( + client, + finalized, + unfinalized, + Duration::from_secs(timeout_seconds), + ) + .await + .map_err(|error| zone_step_error(step, &error)) +} + +#[cucumber::then(expr = "sequencer {string} finalizes deposit {string} in {int} seconds")] +async fn step_zone_sequencer_finalizes_deposit( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, + deposit_alias: String, + timeout_seconds: u64, +) -> StepResult { + let (deposit, amount) = world + .zone + .resolve_submitted_deposit(&deposit_alias)? + .clone(); + let events = log_step_error(step, world.zone.sequencer_events_mut(&sequencer_alias))?; + + let mempool_pending = wait_for_finalized_deposit_via_sequencer_and_collect_mempool_pending( + events, + &deposit, + amount, + Duration::from_secs(timeout_seconds), + ) + .await + .map_err(|error| zone_step_error(step, &error))?; + world + .zone + .record_mempool_pending(sequencer_alias.clone(), mempool_pending); + Ok(()) +} + +#[cucumber::then(expr = "sequencer {string} finalizes withdraw {string} in {int} seconds")] +async fn step_zone_sequencer_finalizes_withdraw( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, + withdraw_alias: String, + timeout_seconds: u64, +) -> StepResult { + let withdraw = world + .zone + .resolve_submitted_withdraw(&withdraw_alias)? + .clone(); + let events = log_step_error(step, world.zone.sequencer_events_mut(&sequencer_alias))?; + + let mempool_pending = wait_for_finalized_withdraw_via_sequencer_and_collect_mempool_pending( + events, + &withdraw, + Duration::from_secs(timeout_seconds), + ) + .await + .map_err(|error| zone_step_error(step, &error))?; + world + .zone + .record_mempool_pending(sequencer_alias.clone(), mempool_pending); + Ok(()) +} + +#[cucumber::then( + expr = "the zone indexer returns all zone messages exactly once in any order in {int} seconds" +)] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require `&mut World` as the first parameter" +)] +async fn step_zone_indexer_returns_all_messages_exactly_once_any_order( + world: &mut CucumberWorld, + step: &Step, + timeout_seconds: u64, +) -> StepResult { + let expected_set = published_payload_set(world, step)?; + let indexer = log_step_error(step, world.zone.indexer())?; + + let seen = + wait_for_indexer_unordered(indexer, &expected_set, Duration::from_secs(timeout_seconds)) + .await + .map_err(|error| zone_step_error(step, &error))?; + + tokio::time::sleep(Duration::from_secs(CONCURRENT_DUPLICATE_SETTLE_SECS)).await; + + let all_payloads = scan_indexer_for_payloads(indexer, &expected_set) + .await + .map_err(|error| zone_step_error(step, &error))?; + + ensure_indexed_payloads_match_once(&expected_set, &seen, &all_payloads) +} + +fn published_payload_set( + world: &CucumberWorld, + step: &Step, +) -> Result, StepError> { + let expected_payloads = log_step_error(step, world.zone.published_message_payloads())?; + + Ok(expected_payloads.into_iter().collect()) +} + +fn ensure_indexed_payloads_match_once( + expected: &HashSet, + seen: &HashSet, + all_payloads: &[Inscription], +) -> StepResult { + let unique: HashSet<&Inscription> = all_payloads.iter().collect(); + + if unique.len() != all_payloads.len() { + return Err(StepError::LogicalError { + message: format!( + "Duplicate inscriptions detected on chain: expected {} unique, got {} total", + unique.len(), + all_payloads.len() + ), + }); + } + + if unique.len() != expected.len() || seen != expected { + return Err(StepError::LogicalError { + message: format!( + "Zone indexer did not return the expected message set: expected {}, got {}", + expected.len(), + unique.len() + ), + }); + } + + Ok(()) +} + +#[cucumber::then( + expr = "the zone indexer returns {int} copies of zone message {string} in {int} seconds" +)] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require `&mut World` as the first parameter" +)] +async fn step_zone_indexer_returns_payload_count( + world: &mut CucumberWorld, + step: &Step, + expected_count: usize, + payload: String, + timeout_seconds: u64, +) -> StepResult { + let indexer = log_step_error(step, world.zone.indexer())?; + wait_for_exact_indexed_payload_count( + indexer, + make_inscription(&payload), + expected_count, + Duration::from_secs(timeout_seconds), + ) + .await + .map_err(|error| zone_step_error(step, &error)) +} + +#[cucumber::then(expr = "zone balance updates keep all accounts non-negative after {int} seconds")] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require `&mut World` as the first parameter" +)] +async fn step_zone_balance_updates_keep_accounts_non_negative( + world: &mut CucumberWorld, + step: &Step, + settle_seconds: u64, +) -> StepResult { + tokio::time::sleep(Duration::from_secs(settle_seconds)).await; + + let mut balances = world.zone.zone_account_balances()?; + let expected_set = published_payload_set(world, step)?; + let indexer = log_step_error(step, world.zone.indexer())?; + let on_chain = scan_indexer_for_payloads(indexer, &expected_set) + .await + .map_err(|error| zone_step_error(step, &error))?; + + apply_indexed_balance_updates(&mut balances, &on_chain); + + ensure_balances_non_negative(&balances) +} + +#[cucumber::then( + expr = "the zone indexer preserves per-sequencer order and converges without duplicates in {int} seconds" +)] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require `&mut World` as the first parameter" +)] +async fn step_zone_indexer_preserves_per_sequencer_order_without_duplicates( + world: &mut CucumberWorld, + step: &Step, + timeout_seconds: u64, +) -> StepResult { + let expected_set = published_payload_set(world, step)?; + let total = world.zone.sorted_total_payloads()?; + let expected_by_sequencer = world.zone.sorted_expected_by_sequencer()?; + let discarded = log_step_error(step, world.zone.discarded_payloads(DEFAULT_ZONE_SEQUENCER))?; + let indexer = log_step_error(step, world.zone.indexer())?; + + let on_chain = wait_until_sorted_conflict_settles( + indexer, + &expected_set, + &discarded, + total, + Duration::from_secs(timeout_seconds), + ) + .await + .map_err(|error| zone_step_error(step, &error))?; + + let discarded_snapshot = discarded.lock().await.clone(); + assert_sorted_outcome( + &on_chain, + &discarded_snapshot, + total, + &expected_by_sequencer, + ) +} + +fn apply_indexed_balance_updates(balances: &mut HashMap, payloads: &[Inscription]) { + for payload in payloads { + let Some((_, account, delta)) = parse_balance_payload(payload) else { + continue; + }; + + *balances.entry(account).or_default() += delta; + } +} + +fn ensure_balances_non_negative(balances: &HashMap) -> StepResult { + let negative = balances + .iter() + .filter(|(_, balance)| **balance < 0) + .map(|(account, balance)| format!("{account}={balance}")) + .collect::>(); + + if negative.is_empty() { + return Ok(()); + } + + Err(StepError::LogicalError { + message: format!( + "Zone account balances went negative: {}", + negative.join(", ") + ), + }) +} diff --git a/tests/src/cucumber/steps/zone/steps/channel.rs b/tests/src/cucumber/steps/zone/steps/channel.rs new file mode 100644 index 000000000..e2043a8d1 --- /dev/null +++ b/tests/src/cucumber/steps/zone/steps/channel.rs @@ -0,0 +1,281 @@ +use super::{ + CucumberWorld, Step, StepResult, publish_atomic_zone_withdraw_transaction, + save_zone_checkpoint, single_column_table, start_named_sequencer_with_startup, + submit_atomic_zone_deposit_transaction, submit_zone_channel_config, + submit_zone_channel_split_transaction, submit_zone_deposit_transaction, + submit_zone_multi_deposit_transaction, submit_zone_withdraw_transaction, when, + zone_atomic_withdraw_rows, zone_config_row, +}; + +#[when(expr = "I save current checkpoint of sequencer {string} as {string}")] +fn step_save_zone_checkpoint( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, + checkpoint_alias: String, +) -> StepResult { + save_zone_checkpoint(world, step, sequencer_alias, checkpoint_alias) +} + +#[when(expr = "I restart zone sequencer {string} from checkpoint {string}")] +async fn step_restart_zone_sequencer_from_checkpoint( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, + checkpoint_alias: String, +) -> StepResult { + let checkpoint = world.zone.resolve_checkpoint(checkpoint_alias)?; + let startup = world.zone.sequencer_startup_for(&sequencer_alias); + + start_named_sequencer_with_startup(world, step, &sequencer_alias, Some(checkpoint), startup) + .await +} + +#[when(expr = "I restart zone sequencer {string} fresh")] +async fn step_restart_zone_sequencer_fresh( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, +) -> StepResult { + let startup = world.zone.sequencer_startup_for(&sequencer_alias); + start_named_sequencer_with_startup(world, step, &sequencer_alias, None, startup).await +} + +#[when(expr = "sequencer {string} submits zone config transaction {string} authorizing:")] +async fn step_submit_zone_channel_config_transaction( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, + transaction_alias: String, +) -> StepResult { + let authorized_aliases = + single_column_table(step, "alias", "authorized zone sequencer aliases")?; + + submit_zone_channel_config( + world, + step, + &sequencer_alias, + transaction_alias, + authorized_aliases, + 0, + 0, + ) + .await +} + +#[when( + expr = "sequencer {string} submits zone config transaction {string} with posting timeframe {int} and timeout {int} authorizing:" +)] +async fn step_submit_zone_channel_config_transaction_with_posting_window( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, + transaction_alias: String, + posting_timeframe: u32, + posting_timeout: u32, +) -> StepResult { + let authorized_aliases = + single_column_table(step, "alias", "authorized zone sequencer aliases")?; + + submit_zone_channel_config( + world, + step, + &sequencer_alias, + transaction_alias, + authorized_aliases, + posting_timeframe, + posting_timeout, + ) + .await +} + +#[when(expr = "sequencer {string} submits zone config transaction:")] +async fn step_submit_zone_channel_config_transaction_from_table( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, +) -> StepResult { + let row = zone_config_row(step)?; + + submit_zone_channel_config( + world, + step, + &sequencer_alias, + row.config_name, + row.authorized_sequencers, + row.posting_timeframe, + row.posting_timeout, + ) + .await +} + +#[when( + expr = "I submit zone deposit transaction {string} into channel of {string} of {int} with metadata {string}" +)] +async fn step_submit_zone_deposit_transaction( + world: &mut CucumberWorld, + step: &Step, + transaction_alias: String, + channel_alias: String, + amount: u64, + metadata: String, +) -> StepResult { + submit_zone_deposit_transaction( + world, + step, + transaction_alias, + channel_alias, + amount, + metadata + .into_bytes() + .try_into() + .expect("Metadata too large for deposit op."), + ) + .await +} + +#[when( + expr = "I submit zone deposit transaction {string} into channel of {string} consuming notes valued {string} with metadata {string}" +)] +async fn step_submit_zone_multi_deposit_transaction( + world: &mut CucumberWorld, + step: &Step, + transaction_alias: String, + channel_alias: String, + values: String, + metadata: String, +) -> StepResult { + let input_values = values + .split(',') + .map(|part| part.trim().parse::()) + .collect::, _>>() + .expect("deposit note values must be a comma-separated list of integers"); + submit_zone_multi_deposit_transaction( + world, + step, + transaction_alias, + channel_alias, + input_values, + metadata + .into_bytes() + .try_into() + .expect("Metadata too large for deposit op."), + ) + .await +} + +#[when( + expr = "I submit zone deposit transaction {string} into channel of {string} consuming {int} notes of value {int} with metadata {string}" +)] +async fn step_submit_zone_bulk_deposit_transaction( + world: &mut CucumberWorld, + step: &Step, + transaction_alias: String, + channel_alias: String, + count: usize, + value: u64, + metadata: String, +) -> StepResult { + let input_values = vec![value; count]; + submit_zone_multi_deposit_transaction( + world, + step, + transaction_alias, + channel_alias, + input_values, + metadata + .into_bytes() + .try_into() + .expect("Metadata too large for deposit op."), + ) + .await +} + +#[when(expr = "sequencer {string} splits deposit {string} into {int} dust notes as {string}")] +async fn step_submit_zone_channel_split_transaction( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, + deposit_alias: String, + dust_count: usize, + transaction_alias: String, +) -> StepResult { + submit_zone_channel_split_transaction( + world, + step, + &sequencer_alias, + &deposit_alias, + dust_count, + transaction_alias, + ) + .await +} + +#[when( + expr = "sequencer {string} submits atomic zone deposit transaction {string} with inscription {string} of {int} with metadata {string}" +)] +async fn step_submit_atomic_zone_deposit_transaction( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, + transaction_alias: String, + message_alias: String, + amount: u64, + metadata: String, +) -> StepResult { + submit_atomic_zone_deposit_transaction( + world, + step, + &sequencer_alias, + transaction_alias, + message_alias, + amount, + metadata + .into_bytes() + .try_into() + .expect("Metadata too large for deposit op."), + ) + .await +} + +#[when( + expr = "sequencer {string} submits zone withdraw transaction {string} with inscription {string} of {int}" +)] +async fn step_submit_zone_withdraw_transaction( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, + transaction_alias: String, + message_alias: String, + amount: u64, +) -> StepResult { + submit_zone_withdraw_transaction( + world, + step, + &sequencer_alias, + transaction_alias, + message_alias, + amount, + ) + .await +} + +#[when(expr = "sequencer {string} publishes atomic withdraw {string} with inscription {string}:")] +async fn step_publish_atomic_zone_withdraw_transaction( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, + bundle_alias: String, + message_alias: String, +) -> StepResult { + let withdraw_rows = zone_atomic_withdraw_rows(step)?; + publish_atomic_zone_withdraw_transaction( + world, + step, + &sequencer_alias, + bundle_alias, + message_alias, + withdraw_rows, + ) + .await +} diff --git a/tests/src/cucumber/steps/zone/steps/lifecycle.rs b/tests/src/cucumber/steps/zone/steps/lifecycle.rs new file mode 100644 index 000000000..1e6362a4d --- /dev/null +++ b/tests/src/cucumber/steps/zone/steps/lifecycle.rs @@ -0,0 +1,131 @@ +use super::{ + CucumberWorld, DriveMode, Duration, Step, StepError, StepResult, ZoneSequencerStartup, given, + initialize_zone_indexer, log_step_error, parse_optional_submit_depth, + register_zone_sequencers_with_shared_key, single_column_table, start_named_sequencer, + start_named_sequencer_with_startup, start_nodes_with_zone_resources, stop_zone_sequencer, + wait_for_lib_advance, when, zone_account_balances, zone_node_resource_rows, + zone_sequencer_start_rows, zone_step_error, +}; + +#[given("I start nodes with wallet and sequencer resources:")] +#[when("I start nodes with wallet and sequencer resources:")] +async fn step_start_nodes_with_wallet_and_sequencer_resources( + world: &mut CucumberWorld, + step: &Step, +) -> StepResult { + let rows = zone_node_resource_rows(step)?; + + start_nodes_with_zone_resources(world, step, rows).await +} + +#[given(expr = "the following zone sequencers share the signing key of {string}:")] +#[expect( + clippy::needless_pass_by_value, + reason = "Cucumber string captures are provided as owned `String`s" +)] +fn step_zone_sequencers_share_signing_key( + world: &mut CucumberWorld, + step: &Step, + source_alias: String, +) -> StepResult { + let aliases = single_column_table(step, "alias", "zone sequencer aliases")?; + register_zone_sequencers_with_shared_key(world, &source_alias, aliases) +} + +#[given("the following zone account balances exist:")] +fn step_zone_account_balances(world: &mut CucumberWorld, step: &Step) -> StepResult { + let balances = zone_account_balances(step)? + .into_iter() + .map(|row| (row.account, row.balance)) + .collect(); + + world.zone.set_zone_account_balances(balances); + + Ok(()) +} + +#[when(expr = "I start zone sequencer {string}")] +async fn step_start_zone_sequencer( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, +) -> StepResult { + start_named_sequencer(world, step, sequencer_alias, None, DriveMode::passive()).await +} + +#[when(expr = "I start zone sequencer {string} with indexer")] +async fn step_start_zone_sequencer_with_indexer( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, +) -> StepResult { + start_sequencer_with_indexer(world, step, &sequencer_alias).await +} + +async fn start_sequencer_with_indexer( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: &str, +) -> StepResult { + start_named_sequencer(world, step, sequencer_alias, None, DriveMode::passive()).await?; + initialize_zone_indexer(world, step, sequencer_alias) +} + +#[when("I start zone sequencers:")] +async fn step_start_zone_sequencers(world: &mut CucumberWorld, step: &Step) -> StepResult { + for row in zone_sequencer_start_rows(step)? { + let alias = row.alias; + let startup = ZoneSequencerStartup { + pending_submit_depth: parse_optional_submit_depth(step, &row.pending_submit_depth)?, + passive_republish_orphans: row.passive_republish_orphans, + }; + world.zone.set_sequencer_startup(&alias, startup); + + start_named_sequencer_with_startup(world, step, &alias, None, startup).await?; + + if row.indexer { + initialize_zone_indexer(world, step, &alias)?; + } + } + + Ok(()) +} + +#[when(expr = "I stop zone sequencer {string}")] +fn step_stop_zone_sequencer( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, +) -> StepResult { + let _ = step; + stop_zone_sequencer(world, sequencer_alias) +} + +#[cucumber::when(expr = "the zone LIB advances in {int} seconds")] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require `&mut World` as the first parameter" +)] +async fn step_zone_lib_advances( + world: &mut CucumberWorld, + step: &Step, + timeout_seconds: u64, +) -> StepResult { + let client = log_step_error(step, world.zone_node_http_client())?; + let initial_lib_slot = client + .consensus_info() + .await + .map_err(|error| StepError::LogicalError { + message: format!("Failed to fetch zone consensus info: {error}"), + })? + .cryptarchia_info + .lib_slot; + + wait_for_lib_advance( + &client, + initial_lib_slot, + Duration::from_secs(timeout_seconds), + ) + .await + .map_err(|error| zone_step_error(step, &error)) +} diff --git a/tests/src/cucumber/steps/zone/steps/mod.rs b/tests/src/cucumber/steps/zone/steps/mod.rs new file mode 100644 index 000000000..50b7a6f2d --- /dev/null +++ b/tests/src/cucumber/steps/zone/steps/mod.rs @@ -0,0 +1,126 @@ +use std::{ + collections::{HashMap, HashSet, VecDeque}, + sync::Arc, + time::Duration, +}; + +use cucumber::{gherkin::Step, given, when}; +use lb_core::mantle::ops::channel::inscribe::Inscription; + +use super::{ + CustomRepublishDeps, PublishDeadline, + actions::{ + DriveMode, initialize_zone_indexer, publish_atomic_zone_withdraw_transaction, + publish_zone_messages, publish_zone_messages_concurrently, + register_zone_sequencers_with_shared_key, remember_published_zone_message, + save_zone_checkpoint, start_named_sequencer, + start_named_sequencer_with_pending_submit_depth, start_nodes_with_zone_resources, + stop_zone_sequencer, submit_atomic_zone_deposit_transaction, submit_zone_channel_config, + submit_zone_channel_split_transaction, submit_zone_deposit_transaction, + submit_zone_multi_deposit_transaction, submit_zone_withdraw_transaction, + }, + assertions::{ + assert_sorted_outcome, scan_indexer_for_payloads, wait_for_indexer_unordered, + wait_until_sorted_conflict_settles, + }, + balance_update_payload, collect_indexed_messages, collect_indexed_messages_exactly_once, + ensure_zone_transactions_included, + errors::{log_step_error, zone_step_error}, + parse_balance_payload, publish_message_with_retry, + runner::{TxSource, TxStatus}, + tables::{ + ConcurrentZoneMessageRow, GeneratedZoneMessageBatch, concurrent_zone_message_rows, + custom_tx_rows, generated_zone_message_batches, generated_zone_message_sequencers, + group_zone_messages_by_sequencer, zone_account_balances, zone_atomic_withdraw_rows, + zone_balance_rows, zone_config_row, zone_message_rows, zone_node_resource_rows, + zone_sequencer_start_rows, zone_sequencing_state_row, + }, + wait_for_channel_transfer_input_count, wait_for_channel_view, wait_for_channel_wallet_counts, + wait_for_channel_wallet_note, wait_for_deposit, wait_for_exact_indexed_payload_count, + wait_for_finalized_deposit_via_sequencer_and_collect_mempool_pending, + wait_for_finalized_withdraw_via_sequencer_and_collect_mempool_pending, wait_for_lib_advance, + wait_for_on_chain_statuses_and_collect_mempool_pending, wait_for_transactions_finalized, + wait_for_turn_to_write, wait_for_tx_status_lifecycle, wait_for_withdraw, +}; +use crate::{ + common::mantle_inscription::make_inscription, + cucumber::{ + error::{StepError, StepResult}, + steps::parse_steps::single_column_table, + world::{CucumberWorld, ZoneSequencerStartup}, + }, +}; + +pub(super) const DEFAULT_ZONE_SEQUENCER: &str = "SEQ_A"; + +fn parse_submit_depth(step: &Step, value: &str) -> Result { + let value = value.trim(); + if matches!(value.to_lowercase().as_str(), "unlimited" | "none") { + return Ok(usize::MAX); + } + + let inner = value + .strip_prefix("Some(") + .and_then(|rest| rest.strip_suffix(')')) + .unwrap_or(value); + + let limit = inner + .parse::() + .map_err(|error| StepError::LogicalError { + message: format!( + "Invalid pending submit depth '{value}' in step '{}': {error}", + step.value + ), + })?; + + Ok(limit) +} + +fn parse_optional_submit_depth(step: &Step, value: &str) -> Result, StepError> { + let value = value.trim(); + if value.eq_ignore_ascii_case("default") { + return Ok(None); + } + + parse_submit_depth(step, value).map(Some) +} + +const fn passive_mode_for_startup(startup: ZoneSequencerStartup) -> DriveMode { + if startup.passive_republish_orphans { + DriveMode::passive_republish_orphans() + } else { + DriveMode::passive() + } +} + +async fn start_named_sequencer_with_startup( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: &str, + checkpoint: Option, + startup: ZoneSequencerStartup, +) -> StepResult { + let mode = passive_mode_for_startup(startup); + if let Some(submit_depth) = startup.pending_submit_depth { + start_named_sequencer_with_pending_submit_depth( + world, + step, + sequencer_alias, + checkpoint, + mode, + submit_depth, + ) + .await + } else { + start_named_sequencer(world, step, sequencer_alias, checkpoint, mode).await + } +} + +const CONCURRENT_DUPLICATE_SETTLE_SECS: u64 = 30; + +mod assertions; +mod channel; +mod lifecycle; +mod policies; +mod publishing; +mod sequencing; diff --git a/tests/src/cucumber/steps/zone/steps/policies.rs b/tests/src/cucumber/steps/zone/steps/policies.rs new file mode 100644 index 000000000..c638235dc --- /dev/null +++ b/tests/src/cucumber/steps/zone/steps/policies.rs @@ -0,0 +1,259 @@ +use super::{ + Arc, ConcurrentZoneMessageRow, CucumberWorld, DEFAULT_ZONE_SEQUENCER, DriveMode, + GeneratedZoneMessageBatch, HashMap, HashSet, Inscription, Step, StepResult, + balance_update_payload, concurrent_zone_message_rows, generated_zone_message_batches, + generated_zone_message_sequencers, group_zone_messages_by_sequencer, initialize_zone_indexer, + make_inscription, publish_zone_messages_concurrently, start_named_sequencer, + start_named_sequencer_with_pending_submit_depth, when, zone_balance_rows, +}; + +#[when("the following zone messages are published concurrently with republish policy:")] +async fn step_publish_zone_messages_concurrently_with_republish_policy( + world: &mut CucumberWorld, + step: &Step, +) -> StepResult { + let rows = concurrent_zone_message_rows(step)?; + publish_zone_messages_with_republish_policy(world, step, rows).await +} + +#[when( + expr = "each listed zone sequencer publishes {int} generated zone messages concurrently with republish policy:" +)] +async fn step_publish_generated_zone_messages_concurrently_with_republish_policy( + world: &mut CucumberWorld, + step: &Step, + messages_per_sequencer: usize, +) -> StepResult { + let rows = build_generated_zone_message_rows( + generated_zone_message_batches(step)?, + messages_per_sequencer, + ); + + publish_zone_messages_with_republish_policy(world, step, rows).await +} + +#[when( + expr = "each listed zone sequencer publishes {int} copies of zone message {string} concurrently with republish policy:" +)] +async fn step_publish_repeated_zone_messages_concurrently_with_republish_policy( + world: &mut CucumberWorld, + step: &Step, + copies_per_sequencer: usize, + payload: String, +) -> StepResult { + let rows = build_repeated_zone_message_rows( + generated_zone_message_sequencers(step)?, + copies_per_sequencer, + &payload, + ); + + publish_zone_messages_with_republish_policy(world, step, rows).await +} + +fn build_generated_zone_message_rows( + batches: Vec, + messages_per_sequencer: usize, +) -> Vec { + let mut builder = GeneratedZoneMessages::default(); + + for batch in batches { + builder.append_numbered_payloads(batch, messages_per_sequencer); + } + + builder.finish() +} + +fn build_repeated_zone_message_rows( + sequencer_aliases: Vec, + copies_per_sequencer: usize, + payload: &str, +) -> Vec { + let mut builder = GeneratedZoneMessages::default(); + let payload = make_inscription(payload); + + for sequencer_alias in sequencer_aliases { + builder.append_repeated_payloads(sequencer_alias, copies_per_sequencer, &payload); + } + + builder.finish() +} + +#[derive(Default)] +struct GeneratedZoneMessages { + next_message_number: usize, + rows: Vec, +} + +impl GeneratedZoneMessages { + fn append_numbered_payloads(&mut self, batch: GeneratedZoneMessageBatch, count: usize) { + let GeneratedZoneMessageBatch { + sequencer_alias, + data_prefix, + } = batch; + + for payload_number in 1..=count { + self.push( + sequencer_alias.clone(), + make_inscription(&format!("{data_prefix}{payload_number}")), + ); + } + } + + fn append_repeated_payloads( + &mut self, + sequencer_alias: String, + count: usize, + payload: &Inscription, + ) { + for _ in 1..count { + self.push(sequencer_alias.clone(), payload.clone()); + } + + if count > 0 { + self.push(sequencer_alias, payload.clone()); + } + } + + fn push(&mut self, sequencer_alias: String, payload: Inscription) { + self.next_message_number += 1; + + self.rows.push(ConcurrentZoneMessageRow { + sequencer_alias, + message_alias: format!("MSG_{}", self.next_message_number), + payload, + }); + } + + fn finish(self) -> Vec { + self.rows + } +} + +async fn publish_zone_messages_with_republish_policy( + world: &mut CucumberWorld, + step: &Step, + rows: Vec, +) -> StepResult { + let grouped = group_zone_messages_by_sequencer(&rows); + + for (sequencer_alias, messages) in &grouped { + let planned = messages + .iter() + .map(|message| message.payload.clone()) + .collect(); + start_named_sequencer_with_pending_submit_depth( + world, + step, + sequencer_alias, + None, + DriveMode::RepublishLineage { planned }, + usize::MAX, + ) + .await?; + } + + // Each sequencer's lineage policy owns publishing its own copies; the step + // only records the messages so the indexer assertions can find them. + for row in rows { + world + .zone + .remember_zone_message(row.message_alias, row.payload, None, None, None); + } + if world.zone.indexer().is_err() { + initialize_zone_indexer(world, step, DEFAULT_ZONE_SEQUENCER)?; + } + Ok(()) +} + +#[when("the following zone messages are published concurrently with sorted conflict policy:")] +async fn step_publish_zone_messages_concurrently_with_sorted_policy( + world: &mut CucumberWorld, + step: &Step, +) -> StepResult { + let rows = concurrent_zone_message_rows(step)?; + let grouped = group_zone_messages_by_sequencer(&rows); + let discarded = Arc::new(tokio::sync::Mutex::new(HashSet::new())); + + for sequencer_alias in grouped.keys() { + start_named_sequencer( + world, + step, + sequencer_alias, + None, + DriveMode::Sorted { + discarded: Arc::clone(&discarded), + }, + ) + .await?; + } + + world.zone.set_sorted_total_payloads(rows.len()); + world.zone.set_sorted_expected_by_sequencer( + grouped + .iter() + .map(|(sequencer_alias, messages)| { + ( + sequencer_alias.clone(), + messages + .iter() + .map(|message| message.payload.clone()) + .collect(), + ) + }) + .collect(), + ); + + publish_zone_messages_concurrently(world, step, rows).await +} + +#[when("the following zone balance updates are published concurrently with balance-aware policy:")] +async fn step_publish_zone_balance_updates_with_balance_policy( + world: &mut CucumberWorld, + step: &Step, +) -> StepResult { + let rows = zone_balance_rows(step)?; + let initial_balances = world.zone.zone_account_balances()?; + let grouped = rows.iter().fold( + HashMap::>::new(), + |mut grouped, row| { + let payload = balance_update_payload(&row.message_alias, &row.account, row.delta); + grouped + .entry(row.sequencer_alias.clone()) + .or_default() + .push((row.message_alias.clone(), payload)); + grouped + }, + ); + + for (sequencer_alias, planned) in &grouped { + start_named_sequencer( + world, + step, + sequencer_alias, + None, + DriveMode::BalanceAware { + initial_balances: initial_balances.clone(), + planned_payloads: planned.iter().map(|(_, payload)| payload.clone()).collect(), + }, + ) + .await?; + } + + for messages in grouped.values() { + for (message_alias, payload) in messages { + world.zone.remember_zone_message( + message_alias.clone(), + payload.clone(), + None, + None, + None, + ); + } + } + + if world.zone.indexer().is_err() { + initialize_zone_indexer(world, step, DEFAULT_ZONE_SEQUENCER)?; + } + + Ok(()) +} diff --git a/tests/src/cucumber/steps/zone/steps/publishing.rs b/tests/src/cucumber/steps/zone/steps/publishing.rs new file mode 100644 index 000000000..1dc248b22 --- /dev/null +++ b/tests/src/cucumber/steps/zone/steps/publishing.rs @@ -0,0 +1,306 @@ +use super::{ + CucumberWorld, CustomRepublishDeps, DEFAULT_ZONE_SEQUENCER, DriveMode, Duration, HashSet, + Inscription, PublishDeadline, Step, StepError, StepResult, VecDeque, custom_tx_rows, + log_step_error, make_inscription, publish_message_with_retry, publish_zone_messages, + remember_published_zone_message, start_named_sequencer, wait_for_channel_view, + wait_for_indexer_unordered, when, zone_message_rows, zone_step_error, +}; + +#[when("I publish the following zone messages:")] +async fn step_publish_zone_messages(world: &mut CucumberWorld, step: &Step) -> StepResult { + publish_zone_messages( + world, + step, + DEFAULT_ZONE_SEQUENCER, + zone_message_rows(step)?, + ) + .await +} + +#[when(expr = "sequencer {string} publishes the following zone messages:")] +async fn step_publish_zone_messages_for_sequencer( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, +) -> StepResult { + publish_zone_messages(world, step, sequencer_alias, zone_message_rows(step)?).await +} + +/// Publishing while the sequencer's node is down must be rejected: with +/// funding configured, building a transaction needs the node's wallet, so +/// the sequencer fails fast with `Unavailable` once the stream drop is +/// noticed (or surfaces the funding error in the brief window before). A +/// fresh `Ready` event fires once the node is back and a live block +/// confirms the reconnect. +#[cucumber::then( + expr = "publishing zone message with data {string} via sequencer {string} fails while the node is down" +)] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require the world as the first `&mut` argument" +)] +async fn step_publish_fails_while_node_down( + world: &mut CucumberWorld, + step: &Step, + data: String, + sequencer_alias: String, +) -> StepResult { + let _ = step; + let payload = make_inscription(&data); + let client = world.zone.sequencer_client(&sequencer_alias)?.clone(); + match client.publish(payload).await { + Ok(_) => Err(StepError::LogicalError { + message: format!( + "Zone publish unexpectedly succeeded for sequencer '{sequencer_alias}' while its node is down" + ), + }), + Err(_expected) => Ok(()), + } +} + +#[when( + expr = "I submit zone message {string} to sequencer {string} with data {string} immediately" +)] +async fn step_publish_single_zone_message_for_sequencer( + world: &mut CucumberWorld, + step: &Step, + message_alias: String, + sequencer_alias: String, + data: String, +) -> StepResult { + let _ = step; + let payload = make_inscription(&data); + let handle = world.zone.sequencer_client(&sequencer_alias)?.clone(); + + let (published, _checkpoint) = handle + .publish(payload.clone()) + .await + .map_err(|error| StepError::LogicalError { + message: format!( + "Zone publish failed for sequencer '{sequencer_alias}' and message '{message_alias}': {error}" + ), + })?; + + remember_published_zone_message(world, &sequencer_alias, message_alias, payload, &published); + + Ok(()) +} + +#[when( + expr = "sequencer {string} submits zone message {string} with data {string} to queue immediately" +)] +async fn step_publish_single_zone_message_to_queue_for_sequencer( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, + message_alias: String, + data: String, +) -> StepResult { + step_publish_single_zone_message_for_sequencer( + world, + step, + message_alias, + sequencer_alias, + data, + ) + .await +} + +#[when( + "the following custom transactions are published concurrently with custom republish policy:" +)] +async fn step_publish_custom_txs_concurrently_with_policy( + world: &mut CucumberWorld, + step: &Step, +) -> StepResult { + let rows = custom_tx_rows(step)?; + let mut expected_payloads = Vec::new(); + + for row in &rows { + let batches: VecDeque> = (0..row.transactions) + .map(|tx_index| { + (0..row.inscriptions) + .map(|entry_index| { + make_inscription(&format!( + "custom-{}-{tx_index}-{entry_index}", + row.sequencer_alias + )) + }) + .collect() + }) + .collect(); + expected_payloads.extend(batches.iter().flatten().cloned()); + + let node_client = log_step_error( + step, + world.zone_node_http_client_for_sequencer(&row.sequencer_alias), + )?; + let node_name = world + .zone + .sequencer_node_name(&row.sequencer_alias)? + .to_owned(); + let funding_pk = world.funding_wallet(&node_name)?.public_key()?; + let deps = CustomRepublishDeps { + node_client, + channel_id: world.zone.sequencer_channel_id(&row.sequencer_alias)?, + signing_key: world + .zone + .sequencer_signing_key(&row.sequencer_alias)? + .clone(), + funding_pk, + batches, + }; + + start_named_sequencer( + world, + step, + &row.sequencer_alias, + None, + DriveMode::CustomRepublish { + deps: Box::new(deps), + }, + ) + .await?; + } + + world + .zone + .remember_expected_custom_payloads(expected_payloads); + Ok(()) +} + +#[cucumber::then(expr = "the zone indexer returns all custom payloads in {int} seconds")] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require `&mut World` as the first parameter" +)] +async fn step_zone_indexer_returns_all_custom_payloads( + world: &mut CucumberWorld, + step: &Step, + timeout_seconds: u64, +) -> StepResult { + let expected: HashSet = world + .zone + .expected_custom_payloads() + .iter() + .cloned() + .collect(); + if expected.is_empty() { + return Err(StepError::LogicalError { + message: "no custom transactions were planned".to_owned(), + }); + } + let indexer = log_step_error(step, world.zone.indexer())?; + + wait_for_indexer_unordered(indexer, &expected, Duration::from_secs(timeout_seconds)) + .await + .map_err(|error| zone_step_error(step, &error))?; + Ok(()) +} + +#[when( + expr = "I submit zone message {string} to sequencer {string} with data {string} on its turn" +)] +async fn step_publish_single_zone_message_for_sequencer_on_turn( + world: &mut CucumberWorld, + step: &Step, + message_alias: String, + sequencer_alias: String, + data: String, +) -> StepResult { + let payload = make_inscription(&data); + let handle = world.zone.sequencer_client(&sequencer_alias)?.clone(); + let mut view_rx = world.zone.sequencer_channel_view_rx(&sequencer_alias)?; + + wait_for_channel_view(&mut view_rx, Duration::from_mins(3), |view| { + view.our_turn_to_write + && view.authorized_key_index.is_some() + && view.authorized_key_index == view.own_key_index + }) + .await + .map_err(|error| zone_step_error(step, &error))?; + + let published = publish_message_with_retry( + &handle, + &payload, + PublishDeadline::from_now(Duration::from_mins(3)), + ) + .await + .map_err(|error| zone_step_error(step, &error))?; + + remember_published_zone_message(world, &sequencer_alias, message_alias, payload, &published); + + Ok(()) +} + +#[when(expr = "sequencer {string} submits the following zone messages to queue immediately:")] +async fn step_publish_zone_messages_to_queue_for_sequencer( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, +) -> StepResult { + let rows = zone_message_rows(step)?; + let handle = world.zone.sequencer_client(&sequencer_alias)?.clone(); + + for (message_alias, payload) in rows { + let (published, _checkpoint) = handle + .publish(payload.clone()) + .await + .map_err(|error| StepError::LogicalError { + message: format!( + "Zone publish failed for sequencer '{sequencer_alias}' and message '{message_alias}': {error}" + ), + })?; + + remember_published_zone_message( + world, + &sequencer_alias, + message_alias, + payload, + &published, + ); + } + + Ok(()) +} + +/// Publish via [`SequencerClient`] and record each inscription id, without +/// waiting for on-chain inclusion. +/// +/// Unlike `publishes the following zone messages` (which polls the node to +/// confirm inclusion), this performs no node HTTP calls, so it can be issued +/// while the node is down: each publish is accepted locally and posted on +/// reconnect. Recording the ids (the publish returns them locally) lets later +/// `... are finalized` / indexer assertions track the messages once the node is +/// back. +#[when( + expr = "sequencer {string} submits the following zone messages without waiting for inclusion:" +)] +async fn step_publish_zone_messages_without_inclusion_for_sequencer( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, +) -> StepResult { + let rows = zone_message_rows(step)?; + let handle = world.zone.sequencer_client(&sequencer_alias)?.clone(); + + for (message_alias, payload) in rows { + let (published, _checkpoint) = handle + .publish(payload.clone()) + .await + .map_err(|error| StepError::LogicalError { + message: format!( + "Zone publish failed for sequencer '{sequencer_alias}' and message '{message_alias}': {error}" + ), + })?; + remember_published_zone_message( + world, + &sequencer_alias, + message_alias, + payload, + &published, + ); + } + + Ok(()) +} diff --git a/tests/src/cucumber/steps/zone/steps/sequencing.rs b/tests/src/cucumber/steps/zone/steps/sequencing.rs new file mode 100644 index 000000000..9d9a36fff --- /dev/null +++ b/tests/src/cucumber/steps/zone/steps/sequencing.rs @@ -0,0 +1,271 @@ +use super::{ + CucumberWorld, Duration, Step, StepError, StepResult, log_step_error, single_column_table, + wait_for_channel_view, wait_for_on_chain_statuses_and_collect_mempool_pending, + wait_for_turn_to_write, zone_sequencing_state_row, zone_step_error, +}; + +#[cucumber::then( + expr = "sequencer {string} reaches sequencing state OWN_KEY_INDEX {int} NOT_OUR_TURN with {int} pending publish txs in {int} seconds" +)] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require `&mut World` as the first parameter" +)] +async fn step_sequencer_reaches_sequencing_state_not_our_turn( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, + own_key_index: usize, + pending_publish_txs: usize, + timeout_seconds: u64, +) -> StepResult { + wait_for_sequencing_state( + world, + step, + &sequencer_alias, + own_key_index, + false, + pending_publish_txs, + timeout_seconds, + ) + .await +} + +#[cucumber::then( + expr = "sequencer {string} reaches sequencing state OWN_KEY_INDEX {int} OUR_TURN with {int} pending publish txs in {int} seconds" +)] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require `&mut World` as the first parameter" +)] +async fn step_sequencer_reaches_sequencing_state_our_turn( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, + own_key_index: usize, + pending_publish_txs: usize, + timeout_seconds: u64, +) -> StepResult { + wait_for_sequencing_state( + world, + step, + &sequencer_alias, + own_key_index, + true, + pending_publish_txs, + timeout_seconds, + ) + .await +} + +#[cucumber::then(expr = "sequencer {string} reaches sequencing state:")] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require `&mut World` as the first parameter" +)] +async fn step_sequencer_reaches_sequencing_state_from_table( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, +) -> StepResult { + let row = zone_sequencing_state_row(step)?; + + wait_for_sequencing_state( + world, + step, + &sequencer_alias, + row.own_key_index, + row.is_our_turn, + row.pending_transactions, + row.timeout_seconds, + ) + .await +} + +async fn wait_for_sequencing_state( + world: &CucumberWorld, + step: &Step, + sequencer_alias: &str, + own_key_index: usize, + is_our_turn: bool, + pending_publish_txs: usize, + timeout_seconds: u64, +) -> StepResult { + let _handle = log_step_error(step, world.zone.sequencer_client(sequencer_alias))?.clone(); + let mut view_rx = log_step_error(step, world.zone.sequencer_channel_view_rx(sequencer_alias))?; + + wait_for_channel_view( + &mut view_rx, + Duration::from_secs(timeout_seconds), + move |view| { + view.own_key_index == Some(own_key_index as u16) + && view.authorized_key_index.is_some() + && view.our_turn_to_write == is_our_turn + && (is_our_turn || view.authorized_key_index != view.own_key_index) + && (!is_our_turn || view.authorized_key_index == Some(own_key_index as u16)) + && view.pending_publish_txs == pending_publish_txs + }, + ) + .await + .map_err(|error| zone_step_error(step, &error))?; + + Ok(()) +} + +#[cucumber::then( + expr = "sequencer {string} is notified it is their turn to write in {int} seconds" +)] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require `&mut World` as the first parameter" +)] +async fn step_sequencer_notified_turn_to_write( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, + timeout_seconds: u64, +) -> StepResult { + let mut turn_rx = log_step_error( + step, + world.zone.sequencer_turn_to_write_rx(&sequencer_alias), + )?; + wait_for_turn_to_write(&mut turn_rx, Duration::from_secs(timeout_seconds)) + .await + .map_err(|error| zone_step_error(step, &error))?; + + Ok(()) +} + +#[cucumber::then( + expr = "sequencer {string} emits published events for queued zone messages on its turn in {int} seconds:" +)] +async fn step_sequencer_emits_published_events_for_queued_zone_messages_on_turn( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, + timeout_seconds: u64, +) -> StepResult { + let aliases = single_column_table(step, "alias", "zone message aliases")?; + let tx_hashes = log_step_error(step, world.zone.message_tx_hashes_for_aliases(&aliases))?; + let mut view_rx = log_step_error(step, world.zone.sequencer_channel_view_rx(&sequencer_alias))?; + wait_for_channel_view(&mut view_rx, Duration::from_secs(timeout_seconds), |view| { + view.our_turn_to_write + }) + .await + .map_err(|error| zone_step_error(step, &error))?; + + // The messages were remembered when queued; wait until they're mined + // (`OnChain`, not yet finalized) via the per-tx status stream. + let mut statuses = log_step_error( + step, + world.zone.take_sequencer_tx_status_rx(&sequencer_alias), + )?; + let mempool_pending = wait_for_on_chain_statuses_and_collect_mempool_pending( + &mut statuses, + &tx_hashes, + Duration::from_secs(timeout_seconds), + ) + .await + .map_err(|error| zone_step_error(step, &error))?; + + world + .zone + .record_mempool_pending(sequencer_alias.clone(), mempool_pending); + + Ok(()) +} + +#[cucumber::then(expr = "sequencer {string} observed mempool pending events for zone messages:")] +#[expect( + clippy::unused_async, + reason = "Cucumber step functions are async even when assertion is synchronous" +)] +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require `&mut World` as the first parameter" +)] +async fn step_sequencer_emitted_mempool_pending_events_for_zone_messages( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, +) -> StepResult { + let aliases = single_column_table(step, "alias", "zone message aliases")?; + let tx_hashes = log_step_error(step, world.zone.message_tx_hashes_for_aliases(&aliases))?; + + for (alias, tx_hash) in aliases.iter().zip(tx_hashes.iter()) { + if !world + .zone + .has_observed_mempool_pending(&sequencer_alias, tx_hash) + { + return Err(StepError::LogicalError { + message: format!( + "Sequencer '{sequencer_alias}' did not emit mempool pending event for zone message '{alias}'" + ), + }); + } + } + + Ok(()) +} + +#[expect( + clippy::needless_pass_by_ref_mut, + reason = "Cucumber step functions require `&mut World` as the first parameter" +)] +#[cucumber::then(expr = "sequencer {string} has {int} pending publish txs in {int} seconds")] +async fn step_sequencer_has_pending_publish_txs( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, + pending_publish_txs: usize, + timeout_seconds: u64, +) -> StepResult { + let mut view_rx = log_step_error(step, world.zone.sequencer_channel_view_rx(&sequencer_alias))?; + + wait_for_channel_view( + &mut view_rx, + Duration::from_secs(timeout_seconds), + move |view| view.pending_publish_txs == pending_publish_txs, + ) + .await + .map_err(|error| zone_step_error(step, &error))?; + + Ok(()) +} + +#[cucumber::then( + expr = "sequencer {string} publishes {string} immediately while in turn in {int} seconds" +)] +async fn step_sequencer_publishes_immediately_while_in_turn( + world: &mut CucumberWorld, + step: &Step, + sequencer_alias: String, + message_alias: String, + timeout_seconds: u64, +) -> StepResult { + // The message was remembered when submitted; wait until it's mined + // (`OnChain`, not yet finalized) via the per-tx status stream. + let tx_hashes = log_step_error( + step, + world + .zone + .message_tx_hashes_for_aliases(std::slice::from_ref(&message_alias)), + )?; + let mut statuses = log_step_error( + step, + world.zone.take_sequencer_tx_status_rx(&sequencer_alias), + )?; + let mempool_pending = wait_for_on_chain_statuses_and_collect_mempool_pending( + &mut statuses, + &tx_hashes, + Duration::from_secs(timeout_seconds), + ) + .await + .map_err(|error| zone_step_error(step, &error))?; + + world + .zone + .record_mempool_pending(sequencer_alias.clone(), mempool_pending); + + Ok(()) +} diff --git a/tests/src/cucumber/steps/manual_zone/tables.rs b/tests/src/cucumber/steps/zone/tables.rs similarity index 100% rename from tests/src/cucumber/steps/manual_zone/tables.rs rename to tests/src/cucumber/steps/zone/tables.rs diff --git a/tests/src/cucumber/world.rs b/tests/src/cucumber/world.rs index fdf39b05d..81f151e50 100644 --- a/tests/src/cucumber/world.rs +++ b/tests/src/cucumber/world.rs @@ -61,10 +61,10 @@ use crate::{ error::{StepError, StepResult}, fee_reserve::{SCENARIO_FEE_ACCOUNT_NAME, ScenarioFeeState}, steps::{ - manual_zone::runner::{ + tokio_console::profile::TokioConsoleProfile, + zone::runner::{ Event, InscriptionId, SequencerCheckpoint, SequencerClient, TxStatusUpdate, }, - tokio_console::profile::TokioConsoleProfile, }, utils::{make_builder, shared_host_bin_path}, wallet::snapshot::WalletSnapshot,