From 2d8c9f10990704de19b3fdceb117f75f58902688 Mon Sep 17 00:00:00 2001 From: gusto Date: Thu, 23 Mar 2023 10:32:54 +0200 Subject: [PATCH] Tree overlay for simulation app (#101) * Add simulations crate * Firs working runner * Cleanupt * Extract overlay from runner * Fix off id calls * Fix testing values * WIP: Tree overlay for simulation app * Build tree overlay from depth and size * Tree overlay generation tests * WIP: Committee role in layout * Use commtittee role when constructing node * Pairprogramming role calculation * WIP: Extract step runner to node * Aggregate times from different overlay layers * Runner for overlay with layers * Use StepRng in runner tests * Decouple role from node * Define leader steps and behaviour * Get possible node ids from the overlay * Handle unknown state in flat overlay * Use solver type for deserialization * Remove refcell from nodes hashmap * Create layout from the provided node ids * Break runner loops into seperate functions * Use for loop to collect node step times * Revert leader times collection * Add interegion latency tests --------- Co-authored-by: danielsanchezq --- simulations/Cargo.toml | 1 - simulations/src/bin/app.rs | 21 +- simulations/src/node/carnot.rs | 113 ++++++-- simulations/src/node/mod.rs | 7 +- simulations/src/overlay/flat.rs | 24 +- simulations/src/overlay/mod.rs | 35 ++- simulations/src/overlay/tree.rs | 253 ++++++++++++++++++ simulations/src/runner.rs | 458 +++++++++++++++++++++++--------- 8 files changed, 734 insertions(+), 178 deletions(-) create mode 100644 simulations/src/overlay/tree.rs diff --git a/simulations/Cargo.toml b/simulations/Cargo.toml index ba058de7..ec78c7ca 100644 --- a/simulations/Cargo.toml +++ b/simulations/Cargo.toml @@ -14,4 +14,3 @@ clap = { version = "4", features = ["derive"] } [target.'cfg(target_arch = "wasm32")'.dependencies] getrandom = { version = "0.2", features = ["js"] } - diff --git a/simulations/src/bin/app.rs b/simulations/src/bin/app.rs index 61569104..e28ae386 100644 --- a/simulations/src/bin/app.rs +++ b/simulations/src/bin/app.rs @@ -10,7 +10,7 @@ use simulations::{ Node, StepTime, }, overlay::{flat::FlatOverlay, Overlay}, - runner::{ConsensusRunner, LayoutNodes}, + runner::ConsensusRunner, }; /// Simple program to greet a person @@ -96,7 +96,7 @@ pub fn main() -> Result<(), Box> { _, Config< ::Settings, - ::Settings, + >::Settings, CarnotStep, >, >(std::fs::File::open(config)?)?; @@ -107,23 +107,10 @@ pub fn main() -> Result<(), Box> { let layout = overlay.layout(&node_ids, &mut rng); let leaders = overlay.leaders(&node_ids, 1, &mut rng).collect(); - let carnot_steps: Vec<_> = cfg - .steps - .iter() - .copied() - .map(|step| { - ( - LayoutNodes::Leader, - step, - Box::new(|times: &[StepTime]| *times.iter().max().unwrap()) - as Box StepTime>, - ) - }) - .collect(); - let mut runner: simulations::runner::ConsensusRunner = ConsensusRunner::new(&mut rng, layout, leaders, cfg.node_settings); - runner.run(&carnot_steps) + runner.run(Box::new(|times: &[StepTime]| *times.iter().max().unwrap()) + as Box StepTime>) } }; diff --git a/simulations/src/node/carnot.rs b/simulations/src/node/carnot.rs index 996b9d3a..54bd3b6c 100644 --- a/simulations/src/node/carnot.rs +++ b/simulations/src/node/carnot.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use std::hash::Hash; use std::rc::Rc; +use std::time::Duration; // crates use rand::prelude::SmallRng; use rand::{Rng, SeedableRng}; @@ -10,12 +11,28 @@ use crate::network::Network; use crate::node::{Node, NodeId, StepTime}; use crate::overlay::{Committee, Layout}; +pub type RootCommitteeReceiverSolver = fn(&mut SmallRng, NodeId, &[NodeId], &Network) -> StepTime; + pub type ParentCommitteeReceiverSolver = fn(&mut SmallRng, NodeId, &Committee, &Network) -> StepTime; pub type ChildCommitteeReceiverSolver = fn(&mut SmallRng, NodeId, &[&Committee], &Network) -> StepTime; +fn leader_receive_proposal( + rng: &mut SmallRng, + node: NodeId, + leader_nodes: &[NodeId], + network: &Network, +) -> StepTime { + assert!(!leader_nodes.is_empty()); + leader_nodes + .iter() + .filter_map(|&sender| network.send_message_cost(rng, sender, node)) + .max() + .unwrap() +} + fn receive_proposal( rng: &mut SmallRng, node: NodeId, @@ -24,6 +41,7 @@ fn receive_proposal( ) -> StepTime { assert!(!committee.is_empty()); committee + .nodes .iter() .filter_map(|&sender| network.send_message_cost(rng, sender, node)) .max() @@ -41,6 +59,7 @@ fn receive_commit( .iter() .filter_map(|committee| { committee + .nodes .iter() .filter_map(|&sender| network.send_message_cost(rng, sender, node)) .max() @@ -51,10 +70,13 @@ fn receive_commit( #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, serde::Serialize, serde::Deserialize)] pub enum CarnotStep { + RootReceiveProposal, ReceiveProposal, ValidateProposal, + LeaderReceiveVote, ReceiveVote, ValidateVote, + Ignore, } impl core::str::FromStr for CarnotStep { @@ -76,6 +98,8 @@ pub enum CarnotStepSolver { Plain(StepTime), ParentCommitteeReceiverSolver(ParentCommitteeReceiverSolver), ChildCommitteeReceiverSolver(ChildCommitteeReceiverSolver), + LeaderToCommitteeReceiverSolver(ChildCommitteeReceiverSolver), + RootCommitteeReceiverSolver(RootCommitteeReceiverSolver), } #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, serde::Serialize, serde::Deserialize)] @@ -83,6 +107,8 @@ pub enum CarnotStepSolverType { Plain(StepTime), ParentCommitteeReceiverSolver, ChildCommitteeReceiverSolver, + LeaderToCommitteeReceiverSolver, + RootCommitteeReceiverSolver, } impl CarnotStepSolverType { @@ -95,6 +121,12 @@ impl CarnotStepSolverType { Self::ChildCommitteeReceiverSolver => { CarnotStepSolver::ChildCommitteeReceiverSolver(receive_commit) } + Self::LeaderToCommitteeReceiverSolver => { + CarnotStepSolver::LeaderToCommitteeReceiverSolver(receive_commit) + } + Self::RootCommitteeReceiverSolver => { + CarnotStepSolver::RootCommitteeReceiverSolver(leader_receive_proposal) + } } } } @@ -104,15 +136,29 @@ pub struct CarnotNodeSettings { pub steps_costs: HashMap, pub network: Network, pub layout: Layout, + pub leaders: Vec, } +#[derive(Clone)] pub struct CarnotNode { id: NodeId, rng: SmallRng, settings: Rc, } +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, serde::Serialize, serde::Deserialize)] +pub enum CarnotRole { + Leader, + Root, + Intermediate, + Leaf, +} + pub const CARNOT_STEPS_COSTS: &[(CarnotStep, CarnotStepSolverType)] = &[ + ( + CarnotStep::RootReceiveProposal, + CarnotStepSolverType::RootCommitteeReceiverSolver, + ), ( CarnotStep::ReceiveProposal, CarnotStepSolverType::ParentCommitteeReceiverSolver, @@ -121,6 +167,10 @@ pub const CARNOT_STEPS_COSTS: &[(CarnotStep, CarnotStepSolverType)] = &[ CarnotStep::ValidateProposal, CarnotStepSolverType::Plain(StepTime::from_secs(1)), ), + ( + CarnotStep::LeaderReceiveVote, + CarnotStepSolverType::LeaderToCommitteeReceiverSolver, + ), ( CarnotStep::ReceiveVote, CarnotStepSolverType::ChildCommitteeReceiverSolver, @@ -129,12 +179,17 @@ pub const CARNOT_STEPS_COSTS: &[(CarnotStep, CarnotStepSolverType)] = &[ CarnotStep::ValidateVote, CarnotStepSolverType::Plain(StepTime::from_secs(1)), ), + ( + CarnotStep::Ignore, + CarnotStepSolverType::Plain(StepTime::from_secs(0)), + ), ]; -pub const CARNOT_LEADER_STEPS: &[CarnotStep] = &[CarnotStep::ReceiveVote, CarnotStep::ValidateVote]; +pub const CARNOT_LEADER_STEPS: &[CarnotStep] = + &[CarnotStep::LeaderReceiveVote, CarnotStep::ValidateVote]; pub const CARNOT_ROOT_STEPS: &[CarnotStep] = &[ - CarnotStep::ReceiveProposal, + CarnotStep::RootReceiveProposal, CarnotStep::ValidateProposal, CarnotStep::ReceiveVote, CarnotStep::ValidateVote, @@ -150,32 +205,39 @@ pub const CARNOT_INTERMEDIATE_STEPS: &[CarnotStep] = &[ pub const CARNOT_LEAF_STEPS: &[CarnotStep] = &[CarnotStep::ReceiveProposal, CarnotStep::ValidateProposal]; +pub const CARNOT_UNKNOWN_MESSAGE_RECEIVED_STEPS: &[CarnotStep] = &[CarnotStep::Ignore]; + impl Node for CarnotNode { type Settings = Rc; - type Step = CarnotStep; fn new(rng: &mut R, id: NodeId, settings: Self::Settings) -> Self { let rng = SmallRng::from_rng(rng).unwrap(); - Self { id, settings, rng } + Self { id, rng, settings } } fn id(&self) -> usize { self.id } - fn run_step(&mut self, step: Self::Step) -> StepTime { + fn run_steps(&mut self, steps: &[CarnotStep]) -> StepTime { use CarnotStepSolver::*; - match self.settings.steps_costs.get(&step) { - Some(step) => match step.to_solver() { + + let mut overall_steps_time = Duration::ZERO; + for step in steps.iter() { + let step_time = match self.settings.steps_costs.get(step).unwrap().to_solver() { Plain(t) => t, - ParentCommitteeReceiverSolver(solver) => solver( - &mut self.rng, - self.id, - self.settings + ParentCommitteeReceiverSolver(solver) => { + match self + .settings .layout - .parent_nodes(self.settings.layout.committee(self.id)), - &self.settings.network, - ), + .parent_nodes(self.settings.layout.committee(self.id)) + { + Some(parent) => { + solver(&mut self.rng, self.id, &parent, &self.settings.network) + } + None => Duration::ZERO, + } + } ChildCommitteeReceiverSolver(solver) => solver( &mut self.rng, self.id, @@ -185,8 +247,27 @@ impl Node for CarnotNode { .children_nodes(self.settings.layout.committee(self.id)), &self.settings.network, ), - }, - None => panic!("Unknown step: {step:?}"), + RootCommitteeReceiverSolver(solver) => solver( + &mut self.rng, + self.id, + &self.settings.leaders, + &self.settings.network, + ), + LeaderToCommitteeReceiverSolver(solver) => { + let committees: Vec<&Committee> = + self.settings.layout.committees.values().collect(); + solver( + &mut self.rng, + self.id, + &committees[..], + &self.settings.network, + ) + } + }; + + overall_steps_time += step_time } + + overall_steps_time } } diff --git a/simulations/src/node/mod.rs b/simulations/src/node/mod.rs index 51d675b3..67a058ed 100644 --- a/simulations/src/node/mod.rs +++ b/simulations/src/node/mod.rs @@ -4,16 +4,17 @@ pub mod carnot; use std::time::Duration; // crates use rand::Rng; + +use self::carnot::CarnotStep; // internal pub type NodeId = usize; pub type CommitteeId = usize; pub type StepTime = Duration; -pub trait Node { +pub trait Node: Clone { type Settings; - type Step; fn new(rng: &mut R, id: NodeId, settings: Self::Settings) -> Self; fn id(&self) -> NodeId; - fn run_step(&mut self, steps: Self::Step) -> StepTime; + fn run_steps(&mut self, steps: &[CarnotStep]) -> StepTime; } diff --git a/simulations/src/overlay/flat.rs b/simulations/src/overlay/flat.rs index 148d0f33..36478cdd 100644 --- a/simulations/src/overlay/flat.rs +++ b/simulations/src/overlay/flat.rs @@ -1,21 +1,26 @@ -use rand::prelude::IteratorRandom; -use rand::Rng; // std // crates +use rand::prelude::IteratorRandom; +use rand::Rng; // internal use super::Overlay; +use crate::node::carnot::{CarnotNode, CarnotRole}; use crate::node::NodeId; use crate::overlay::{Committee, Layout}; pub struct FlatOverlay; -impl Overlay for FlatOverlay { +impl Overlay for FlatOverlay { type Settings = (); fn new(_settings: Self::Settings) -> Self { Self } + fn nodes(&self) -> Vec { + (0..10).collect() + } + fn leaders( &self, nodes: &[NodeId], @@ -27,10 +32,17 @@ impl Overlay for FlatOverlay { } fn layout(&self, nodes: &[NodeId], _rng: &mut R) -> Layout { - let committees = - std::iter::once((0, nodes.iter().copied().collect::())).collect(); + let committees = std::iter::once(( + 0, + Committee { + nodes: nodes.iter().copied().collect(), + role: CarnotRole::Leader, + }, + )) + .collect(); let parent = std::iter::once((0, 0)).collect(); let children = std::iter::once((0, vec![0])).collect(); - Layout::new(committees, parent, children) + let layers = std::iter::once((0, vec![0])).collect(); + Layout::new(committees, parent, children, layers) } } diff --git a/simulations/src/overlay/mod.rs b/simulations/src/overlay/mod.rs index fc594655..3b0c09d9 100644 --- a/simulations/src/overlay/mod.rs +++ b/simulations/src/overlay/mod.rs @@ -1,21 +1,35 @@ pub mod flat; +pub mod tree; // std use std::collections::{BTreeSet, HashMap}; // crates use rand::Rng; // internal -use crate::node::{CommitteeId, NodeId}; +use crate::node::{carnot::CarnotRole, CommitteeId, Node, NodeId}; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct Committee { + pub nodes: BTreeSet, + pub role: CarnotRole, +} + +impl Committee { + #[must_use] + pub fn is_empty(&self) -> bool { + self.nodes.len() == 0 + } +} -pub type Committee = BTreeSet; pub type Leaders = BTreeSet; -#[derive(Debug, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct Layout { pub committees: HashMap, pub from_committee: HashMap, pub parent: HashMap, pub children: HashMap>, + pub layers: HashMap>, } impl Layout { @@ -23,11 +37,13 @@ impl Layout { committees: HashMap, parent: HashMap, children: HashMap>, + layers: HashMap>, ) -> Self { let from_committee = committees .iter() .flat_map(|(&committee_id, committee)| { committee + .nodes .iter() .map(move |&node_id| (node_id, committee_id)) }) @@ -37,6 +53,7 @@ impl Layout { from_committee, parent, children, + layers, } } @@ -48,12 +65,13 @@ impl Layout { &self.committees[&committee_id] } - pub fn parent(&self, committee_id: CommitteeId) -> CommitteeId { - self.parent[&committee_id] + pub fn parent(&self, committee_id: CommitteeId) -> Option { + self.parent.get(&committee_id).copied() } - pub fn parent_nodes(&self, committee_id: CommitteeId) -> &Committee { - &self.committees[&self.parent(committee_id)] + pub fn parent_nodes(&self, committee_id: CommitteeId) -> Option { + self.parent(committee_id) + .map(|c| self.committees[&c].clone()) } pub fn children(&self, committee_id: CommitteeId) -> &[CommitteeId] { @@ -72,10 +90,11 @@ impl Layout { } } -pub trait Overlay { +pub trait Overlay { type Settings; fn new(settings: Self::Settings) -> Self; + fn nodes(&self) -> Vec; fn leaders( &self, nodes: &[NodeId], diff --git a/simulations/src/overlay/tree.rs b/simulations/src/overlay/tree.rs new file mode 100644 index 00000000..88c79da3 --- /dev/null +++ b/simulations/src/overlay/tree.rs @@ -0,0 +1,253 @@ +// std +use std::collections::HashMap; +// crates +use rand::seq::IteratorRandom; +// internal +use super::{Committee, Layout, Overlay}; +use crate::node::{ + carnot::{CarnotNode, CarnotRole}, + NodeId, +}; + +pub enum TreeType { + FullBinaryTree, +} + +pub struct TreeSettings { + pub tree_type: TreeType, + pub committee_size: usize, + pub depth: usize, +} + +pub struct TreeOverlay { + settings: TreeSettings, +} + +struct TreeProperties { + committee_count: usize, + node_count: usize, +} + +impl TreeOverlay { + fn build_full_binary_tree( + node_ids: &[NodeId], + rng: &mut R, + settings: &TreeSettings, + ) -> Layout { + let properties = get_tree_properties(settings); + + // For full binary tree to be formed from existing nodes + // a certain unique node count needs to be provided. + assert!(properties.node_count <= node_ids.len()); + + let mut committees = HashMap::new(); + let mut parents = HashMap::new(); + let mut children = HashMap::new(); + let mut layers = HashMap::new(); + + for (committee_id, nodes) in node_ids + .iter() + .choose_multiple(rng, properties.node_count) + .chunks(settings.committee_size) + .enumerate() + { + let mut has_children = false; + let left_child_id = 2 * committee_id + 1; + let right_child_id = left_child_id + 1; + + // Check for leaf nodes. + if right_child_id <= properties.committee_count { + children.insert(committee_id, vec![left_child_id, right_child_id]); + has_children = true; + } + + // Root node has no parent. + if committee_id > 0 { + let parent_id = get_parent_id(committee_id); + parents.insert(committee_id, parent_id); + } + + let role = match (committee_id, has_children) { + (0, _) => CarnotRole::Root, + (_, true) => CarnotRole::Intermediate, + (_, false) => CarnotRole::Leaf, + }; + + let committee = Committee { + nodes: nodes.iter().copied().copied().collect(), + role, + }; + + committees.insert(committee_id, committee); + + layers + .entry(get_layer(committee_id)) + .or_insert_with(Vec::new) + .push(committee_id); + } + + Layout::new(committees, parents, children, layers) + } +} + +impl Overlay for TreeOverlay { + type Settings = TreeSettings; + + fn new(settings: Self::Settings) -> Self { + Self { settings } + } + + fn nodes(&self) -> Vec { + let properties = get_tree_properties(&self.settings); + (0..properties.node_count).collect() + } + + fn leaders( + &self, + nodes: &[NodeId], + size: usize, + rng: &mut R, + ) -> Box> { + let leaders = nodes.iter().copied().choose_multiple(rng, size).into_iter(); + Box::new(leaders) + } + + fn layout(&self, nodes: &[NodeId], rng: &mut R) -> Layout { + match self.settings.tree_type { + TreeType::FullBinaryTree => Self::build_full_binary_tree(nodes, rng, &self.settings), + } + } +} + +fn get_tree_properties(settings: &TreeSettings) -> TreeProperties { + let committee_count = committee_count(settings.depth); + let node_count = committee_count * settings.committee_size; + TreeProperties { + committee_count, + node_count, + } +} + +/// Returns the number of nodes in the whole tree. +/// `depth` parameter assumes that roots is included. +fn committee_count(depth: usize) -> usize { + (1 << depth) - 1 +} + +fn get_parent_id(id: usize) -> usize { + (id - 1 + id % 2) / 2 +} + +/// Get a layer in a tree of a given committee id. +fn get_layer(id: usize) -> usize { + (id as f64 + 1.).log2().floor() as usize +} + +#[cfg(test)] +mod tests { + use super::*; + use rand::rngs::mock::StepRng; + + #[test] + fn build_full_depth_1() { + let mut rng = StepRng::new(1, 0); + let overlay = TreeOverlay::new(TreeSettings { + tree_type: TreeType::FullBinaryTree, + depth: 1, + committee_size: 1, + }); + let nodes = overlay.nodes(); + let layout = overlay.layout(&nodes, &mut rng); + assert_eq!(layout.committees.len(), 1); + assert!(layout.children.is_empty()); + assert!(layout.parent.is_empty()); + } + + #[test] + fn build_full_depth_3() { + let mut rng = StepRng::new(1, 0); + let overlay = TreeOverlay::new(TreeSettings { + tree_type: TreeType::FullBinaryTree, + depth: 3, + committee_size: 1, + }); + let nodes = overlay.nodes(); + let layout = overlay.layout(&nodes, &mut rng); + assert_eq!(layout.children[&0], vec![1, 2]); + assert_eq!(layout.parent[&1], 0); + assert_eq!(layout.parent[&2], 0); + + assert_eq!(layout.children[&1], vec![3, 4]); + assert_eq!(layout.children[&2], vec![5, 6]); + + assert_eq!(layout.parent[&3], 1); + assert_eq!(layout.children.get(&3), None); + + assert_eq!(layout.parent[&4], 1); + assert_eq!(layout.children.get(&4), None); + + assert_eq!(layout.parent[&5], 2); + assert_eq!(layout.children.get(&5), None); + + assert_eq!(layout.parent[&6], 2); + assert_eq!(layout.children.get(&6), None); + } + + #[test] + fn build_full_committee_size() { + let mut rng = StepRng::new(1, 0); + let overlay = TreeOverlay::new(TreeSettings { + tree_type: TreeType::FullBinaryTree, + depth: 10, + committee_size: 10, + }); + let nodes = overlay.nodes(); + let layout = overlay.layout(&nodes, &mut rng); + + // 2^h - 1 + assert_eq!(layout.committees.len(), 1023); + + let root_nodes = &layout.committees[&0].nodes; + assert_eq!(root_nodes.len(), 10); + assert_eq!(root_nodes.first(), Some(&0)); + assert_eq!(root_nodes.last(), Some(&9)); + + let last_nodes = &layout.committees[&1022].nodes; + assert_eq!(last_nodes.len(), 10); + assert_eq!(last_nodes.first(), Some(&10220)); + assert_eq!(last_nodes.last(), Some(&10229)); + } + + #[test] + fn check_committee_role() { + let mut rng = StepRng::new(1, 0); + let overlay = TreeOverlay::new(TreeSettings { + tree_type: TreeType::FullBinaryTree, + depth: 3, + committee_size: 1, + }); + let nodes = overlay.nodes(); + let layout = overlay.layout(&nodes, &mut rng); + + assert_eq!(layout.committees[&0].role, CarnotRole::Root); + assert_eq!(layout.committees[&1].role, CarnotRole::Intermediate); + assert_eq!(layout.committees[&2].role, CarnotRole::Intermediate); + assert_eq!(layout.committees[&3].role, CarnotRole::Leaf); + assert_eq!(layout.committees[&6].role, CarnotRole::Leaf); + } + + #[test] + fn check_layers() { + let mut rng = StepRng::new(1, 0); + let overlay = TreeOverlay::new(TreeSettings { + tree_type: TreeType::FullBinaryTree, + depth: 4, + committee_size: 1, + }); + let nodes = overlay.nodes(); + let layout = overlay.layout(&nodes, &mut rng); + assert_eq!(layout.layers[&0], vec![0]); + assert_eq!(layout.layers[&1], vec![1, 2]); + assert_eq!(layout.layers[&2], vec![3, 4, 5, 6]); + } +} diff --git a/simulations/src/runner.rs b/simulations/src/runner.rs index 64653e5d..8f50b0c2 100644 --- a/simulations/src/runner.rs +++ b/simulations/src/runner.rs @@ -1,16 +1,17 @@ -// 1 - Leader forwards a proposal - Leader builds proposal -// 2 - Every committee member receives the proposal and validates it -// - +use crate::node::carnot::{ + CarnotRole, CARNOT_INTERMEDIATE_STEPS, CARNOT_LEADER_STEPS, CARNOT_LEAF_STEPS, + CARNOT_ROOT_STEPS, CARNOT_UNKNOWN_MESSAGE_RECEIVED_STEPS, +}; use crate::node::{Node, NodeId, StepTime}; use crate::overlay::Layout; use rand::Rng; +use std::collections::HashMap; use std::time::Duration; -pub struct ConsensusRunner { - nodes: Vec, - layout: Layout, +pub struct ConsensusRunner { + nodes: HashMap, leaders: Vec, + layout: Layout, } #[allow(dead_code)] @@ -19,14 +20,7 @@ pub struct Report { round_time: Duration, } -#[derive(Copy, Clone)] -pub enum LayoutNodes { - Leader, - Committee, - LeafCommittee, -} - -pub type ExecutionSteps = [(LayoutNodes, S, Box StepTime>)]; +type Reducer = Box StepTime>; impl ConsensusRunner where @@ -40,7 +34,10 @@ where ) -> Self { let nodes = layout .node_ids() - .map(|id| N::new(&mut rng, id, node_settings.clone())) + .map(|id| { + let node = N::new(&mut rng, id, node_settings.clone()); + (id, node) + }) .collect(); Self { nodes, @@ -49,166 +46,373 @@ where } } - pub fn node_ids(&self) -> impl Iterator + '_ { - self.nodes.iter().map(Node::id) - } - - pub fn run(&mut self, execution: &ExecutionSteps) -> Report - where - N::Step: Clone, - { + pub fn run(&mut self, reducer: Reducer) -> Report { let leaders = &self.leaders; let layout = &self.layout; - let round_time = execution + let mut leader_times = leaders .iter() - .map(|(layout_node, step, reducer)| { - let times: Vec = match layout_node { - LayoutNodes::Leader => leaders - .iter() - .map(|&leader| self.nodes[leader].run_step(step.clone())) - .collect(), - LayoutNodes::Committee => { - let non_leaf_committees = layout - .children - .iter() - .filter_map(|(id, children)| (!children.is_empty()).then_some(id)); - - non_leaf_committees - .flat_map(|committee_id| { - layout - .committees - .get(committee_id) - .unwrap() - .iter() - .map(|&node| self.nodes[node].run_step(step.clone())) - .max() - }) - .collect() - } - LayoutNodes::LeafCommittee => { - let leaf_committees = - layout.children.iter().filter_map(|(id, children)| { - (children.is_empty() || (layout.parent(*id) == *id)).then_some(id) - }); - - leaf_committees - .flat_map(|committee_id| { - layout - .committees - .get(committee_id) - .unwrap() - .iter() - .map(|&node| self.nodes[node].run_step(step.clone())) - .max() - }) - .collect() - } - }; - - reducer(×) + .map(|leader_node| { + vec![self + .nodes + .get_mut(leader_node) + .unwrap() + .run_steps(CARNOT_LEADER_STEPS)] }) - .sum(); + .collect(); + + let mut layer_times = Vec::new(); + for layer_nodes in layout + .layers + .values() + .map(|committees| get_layer_nodes(committees, layout)) + { + let times: Vec = layer_nodes + .iter() + .map(|(committee_id, node_id)| { + let steps = match layout.committees[committee_id].role { + CarnotRole::Root => CARNOT_ROOT_STEPS, + CarnotRole::Intermediate => CARNOT_INTERMEDIATE_STEPS, + CarnotRole::Leaf => CARNOT_LEAF_STEPS, + _ => { + // TODO: Should leader act as a leaf in a flat overlay? + CARNOT_UNKNOWN_MESSAGE_RECEIVED_STEPS + } + }; + self.nodes.get_mut(node_id).unwrap().run_steps(steps) + }) + .collect(); + + layer_times.push(times) + } + + layer_times.append(&mut leader_times); + let round_time = layer_times.iter().map(|d| reducer(d)).sum(); Report { round_time } } } +fn get_layer_nodes(layer_committees: &[NodeId], layout: &Layout) -> Vec<(NodeId, NodeId)> { + layer_committees + .iter() + .flat_map(|committee_id| get_committee_nodes(committee_id, layout)) + .collect() +} + +fn get_committee_nodes(committee: &NodeId, layout: &Layout) -> Vec<(NodeId, NodeId)> { + layout.committees[committee] + .nodes + .clone() + .into_iter() + .map(|node_id| (*committee, node_id)) + .collect() +} + #[cfg(test)] mod test { - use crate::network::behaviour::NetworkBehaviour; - use crate::network::regions::{Region, RegionsData}; - use crate::network::Network; - use crate::node::carnot::{ - CarnotNode, CarnotNodeSettings, CARNOT_LEADER_STEPS, CARNOT_LEAF_STEPS, CARNOT_STEPS_COSTS, + use crate::{ + network::{ + behaviour::NetworkBehaviour, + regions::{Region, RegionsData}, + Network, + }, + node::{ + carnot::{CarnotNode, CarnotNodeSettings, CARNOT_STEPS_COSTS}, + NodeId, StepTime, + }, + overlay::{ + flat::FlatOverlay, + tree::{TreeOverlay, TreeSettings, TreeType}, + Overlay, + }, + runner::{ConsensusRunner, Reducer}, }; - use crate::node::{NodeId, StepTime}; - use crate::overlay::flat::FlatOverlay; - use crate::overlay::Overlay; - use crate::runner::{ConsensusRunner, LayoutNodes}; - use rand::rngs::SmallRng; - use rand::{Rng, SeedableRng}; - use std::rc::Rc; - use std::time::Duration; + use rand::{rngs::mock::StepRng, Rng}; + use std::{collections::HashMap, rc::Rc, time::Duration}; - fn setup_runner( + fn setup_runner>( mut rng: &mut R, overlay: &O, ) -> ConsensusRunner { - let regions = std::iter::once((Region::Europe, (0..10).collect())).collect(); + let node_ids = overlay.nodes(); + let layout = overlay.layout(&node_ids, &mut rng); + let leaders: Vec = overlay.leaders(&node_ids, 1, &mut rng).collect(); + + let regions = std::iter::once((Region::Europe, node_ids.clone())).collect(); let network_behaviour = std::iter::once(( (Region::Europe, Region::Europe), NetworkBehaviour::new(Duration::from_millis(100), 0.0), )) .collect(); - let node_ids: Vec = (0..10).collect(); - let layout = overlay.layout(&node_ids, &mut rng); - let leaders = overlay.leaders(&node_ids, 1, &mut rng).collect(); + let node_settings: CarnotNodeSettings = CarnotNodeSettings { steps_costs: CARNOT_STEPS_COSTS.iter().cloned().collect(), network: Network::new(RegionsData::new(regions, network_behaviour)), layout: overlay.layout(&node_ids, &mut rng), + leaders: leaders.clone(), }; ConsensusRunner::new(&mut rng, layout, leaders, Rc::new(node_settings)) } #[test] - fn test_run_flat_single_leader_steps() { - let mut rng = SmallRng::seed_from_u64(0); + fn run_flat_single_leader_steps() { + let mut rng = StepRng::new(1, 0); let overlay = FlatOverlay::new(()); let mut runner = setup_runner(&mut rng, &overlay); - let carnot_steps: Vec<_> = CARNOT_LEADER_STEPS - .iter() - .copied() - .map(|step| { - ( - LayoutNodes::Leader, - step, - Box::new(|times: &[StepTime]| *times.iter().max().unwrap()) - as Box StepTime>, - ) - }) - .collect(); - assert_eq!( Duration::from_millis(1100), - runner.run(&carnot_steps).round_time + runner + .run(Box::new(|times: &[StepTime]| *times.iter().max().unwrap()) as Reducer) + .round_time ); } #[test] - fn test_run_flat_single_leader_single_committee() { - let mut rng = SmallRng::seed_from_u64(0); - let overlay = FlatOverlay::new(()); + fn run_tree_committee_1() { + let mut rng = StepRng::new(1, 0); + + let overlay = TreeOverlay::new(TreeSettings { + tree_type: TreeType::FullBinaryTree, + depth: 3, + committee_size: 1, + }); let mut runner: ConsensusRunner = setup_runner(&mut rng, &overlay); - let leader_steps = CARNOT_LEADER_STEPS.iter().copied().map(|step| { - ( - LayoutNodes::Leader, - step, - Box::new(|times: &[StepTime]| *times.iter().max().unwrap()) - as Box StepTime>, - ) - }); + // # Leader (1 node): + // + // - 100ms - LeaderReceiveVote, + // - 1s - ValidateVote, + // + // Expected times [1.1s] - let committee_steps = CARNOT_LEAF_STEPS.iter().copied().map(|step| { - ( - LayoutNodes::LeafCommittee, - step, - Box::new(|times: &[StepTime]| *times.iter().max().unwrap()) - as Box StepTime>, - ) - }); + // # Root (1 node): + // + // - 100ms - RootReceiveProposal, + // - 1s - ValidateProposal, + // - 100ms - ReceiveVote, + // - 1s - ValidateVote, + // + // Expected times [2.2s] - let carnot_steps: Vec<_> = leader_steps.chain(committee_steps).collect(); + // # Intermediary (2 nodes): + // + // - 100ms - ReceiveProposal, + // - 1s - ValidateProposal, + // - 100ms - ReceiveVote, + // - 1s - ValidateVote, + // + // Expected times [2.2s, 2.2s] + + // # Leaf (4 nodes): + // + // - 100ms - ReceiveProposal + // - 1s - ValidateProposal + // + // Expected times [1.1s, 1.1s, 1.1s, 1.1s] assert_eq!( - Duration::from_millis(2200), - runner.run(&carnot_steps).round_time + Duration::from_millis(6600), + runner + .run(Box::new(|times: &[StepTime]| *times.iter().max().unwrap()) as Reducer) + .round_time + ); + } + + #[test] + fn run_tree_committee_100() { + let mut rng = StepRng::new(1, 0); + + let overlay = TreeOverlay::new(TreeSettings { + tree_type: TreeType::FullBinaryTree, + depth: 3, + committee_size: 100, + }); + + let mut runner: ConsensusRunner = setup_runner(&mut rng, &overlay); + + assert_eq!( + Duration::from_millis(6600), + runner + .run(Box::new(|times: &[StepTime]| *times.iter().max().unwrap()) as Reducer) + .round_time + ); + } + + #[test] + fn run_tree_network_config_1() { + let mut rng = StepRng::new(1, 0); + + let overlay = TreeOverlay::new(TreeSettings { + tree_type: TreeType::FullBinaryTree, + depth: 3, + committee_size: 1, + }); + + let node_ids = overlay.nodes(); + let layout = overlay.layout(&node_ids, &mut rng); + // Normaly a leaders would be selected randomly, here, we're assuming it's NodeID 1. + // let leaders: Vec = overlay.leaders(&node_ids, 1, &mut rng).collect(); + let leaders = vec![1]; + + let first_five = &node_ids[..5]; + let rest = &node_ids[5..]; + + // 0 + // 1 2 + // 3 4 5 6 + // + // # Leader - NodeID 1 + // + // Sends vote to all committees. + // + // LeaderReceiveVote: + // - 100ms - Asia - Asia (1 to 0, 1, 2, 3, 4) + // - 500ms - Asia - Europe (1 to 5, 6) + // + // ValidateVote: 1s + // + // Expected times: [1.5s] + + // # Root - NodeID 0 + // + // Sends vote to child committees. + // + // RootReceiveProposal: + // - 100ms - Asia - Asia (0 to 1) + // + // ReceiveVote: + // - 100ms - Asia - Asia (0 to 1, 2) + // + // No network: + // - 1s - ValidateVote + // - 1s - ValidateProposal + // + // Expected times: [2.2s] + + // # Intermediary - NodeID 1, 2: + // + // ReceiveVote: + // - 100ms - Asia - Asia (1 to 3, 4) + // - 500ms - Asia - Europe (2 to 5, 6) + // + // ReceiveProposal: + // - 100ms - Asia - Asia (1 to 0, 2 to 0) + // + // No network: + // - 1s - ValidateVote + // - 1s - ValidateProposal + // + // Expected times [2.2s, 2.6s] + + // # Leaf - NodeID 3, 4, 5, 6 + // + // ReceiveProposal: + // - 100ms - Asia - Asia ( 3, 4 to 1) + // - 500ms - Asia - Europe ( 5, 6 to 2) + // + // No network: + // - 1s - ValidateProposal + // + // Expected times [1.1s, 1.1s, 1.5s, 1.5s] + + let regions = HashMap::from([ + (Region::Asia, first_five.to_vec()), + (Region::Europe, rest.to_vec()), + ]); + + let network_behaviour = HashMap::from([ + ( + (Region::Asia, Region::Asia), + NetworkBehaviour::new(Duration::from_millis(100), 0.0), + ), + ( + (Region::Asia, Region::Europe), + NetworkBehaviour::new(Duration::from_millis(500), 0.0), + ), + ( + (Region::Europe, Region::Europe), + NetworkBehaviour::new(Duration::from_millis(100), 0.0), + ), + ]); + + let node_settings: CarnotNodeSettings = CarnotNodeSettings { + steps_costs: CARNOT_STEPS_COSTS.iter().cloned().collect(), + network: Network::new(RegionsData::new(regions, network_behaviour)), + layout: overlay.layout(&node_ids, &mut rng), + leaders: leaders.clone(), + }; + + let mut runner = + ConsensusRunner::::new(&mut rng, layout, leaders, Rc::new(node_settings)); + + assert_eq!( + Duration::from_millis(7800), + runner + .run(Box::new(|times: &[StepTime]| *times.iter().max().unwrap()) as Reducer) + .round_time + ); + } + + #[test] + fn run_tree_network_config_100() { + let mut rng = StepRng::new(1, 0); + + let overlay = TreeOverlay::new(TreeSettings { + tree_type: TreeType::FullBinaryTree, + depth: 3, + committee_size: 100, + }); + + let node_ids = overlay.nodes(); + let layout = overlay.layout(&node_ids, &mut rng); + // Normaly a leaders would be selected randomly, here, we're assuming it's NodeID 1. + // let leaders: Vec = overlay.leaders(&node_ids, 1, &mut rng).collect(); + let leaders = vec![1]; + + let two_thirds = node_ids.len() as f32 * 0.66; + let rest = &node_ids[two_thirds as usize..]; + let two_thirds = &node_ids[..two_thirds as usize]; + + let regions = HashMap::from([ + (Region::Asia, two_thirds.to_vec()), + (Region::Europe, rest.to_vec()), + ]); + + let network_behaviour = HashMap::from([ + ( + (Region::Asia, Region::Asia), + NetworkBehaviour::new(Duration::from_millis(100), 0.0), + ), + ( + (Region::Asia, Region::Europe), + NetworkBehaviour::new(Duration::from_millis(500), 0.0), + ), + ( + (Region::Europe, Region::Europe), + NetworkBehaviour::new(Duration::from_millis(100), 0.0), + ), + ]); + + let node_settings: CarnotNodeSettings = CarnotNodeSettings { + steps_costs: CARNOT_STEPS_COSTS.iter().cloned().collect(), + network: Network::new(RegionsData::new(regions, network_behaviour)), + layout: overlay.layout(&node_ids, &mut rng), + leaders: leaders.clone(), + }; + + let mut runner = + ConsensusRunner::::new(&mut rng, layout, leaders, Rc::new(node_settings)); + + assert_eq!( + Duration::from_millis(7800), + runner + .run(Box::new(|times: &[StepTime]| *times.iter().max().unwrap()) as Reducer) + .round_time ); } }