From ff2c1b4271e07647f739690137448fdb780c642f Mon Sep 17 00:00:00 2001 From: Daniel Sanchez <3danimanimal@gmail.com> Date: Thu, 7 Nov 2024 06:05:56 +0100 Subject: [PATCH] Restructure crates (#24) * Rename to netrunner * Refactor sims structure * Add missing workspace cargo * Rebase changes * Rebase changes * Rebase changes * Fix tests --- .gitignore | 2 +- simlib/Cargo.toml | 7 ++++ simlib/mixnet-sims/Cargo.toml | 25 ++++++++++++++ .../bin/app => simlib/mixnet-sims/src}/log.rs | 0 .../app => simlib/mixnet-sims/src}/main.rs | 33 ++++++++----------- .../src/node/mix/consensus_streams.rs | 0 .../mixnet-sims}/src/node/mix/lottery.rs | 0 .../mixnet-sims}/src/node/mix/mod.rs | 18 +++++----- .../mixnet-sims}/src/node/mix/scheduler.rs | 0 .../mixnet-sims}/src/node/mix/state.rs | 2 +- .../src/node/mix/stream_wrapper.rs | 0 simlib/mixnet-sims/src/node/mod.rs | 1 + .../netrunner}/Cargo.lock | 0 .../netrunner}/Cargo.toml | 18 +--------- .../netrunner}/README.md | 0 .../netrunner}/config/carnot_dev.json | 0 .../netrunner}/config/mixnode.json | 0 .../netrunner}/src/lib.rs | 0 .../netrunner}/src/network/behaviour.rs | 0 .../netrunner}/src/network/mod.rs | 0 .../netrunner}/src/network/regions.rs | 0 .../netrunner}/src/node/dummy_streaming.rs | 0 .../netrunner}/src/node/mod.rs | 1 - .../netrunner}/src/output_processors/mod.rs | 0 .../netrunner}/src/runner/async_runner.rs | 0 .../netrunner}/src/runner/mod.rs | 0 .../netrunner}/src/runner/sync_runner.rs | 0 .../netrunner}/src/settings.rs | 0 .../netrunner}/src/streaming/io.rs | 0 .../netrunner}/src/streaming/mod.rs | 16 --------- .../netrunner}/src/streaming/naive.rs | 0 .../netrunner}/src/streaming/polars.rs | 0 .../src/streaming/runtime_subscriber.rs | 0 .../src/streaming/settings_subscriber.rs | 12 +++---- .../netrunner}/src/warding/mod.rs | 0 .../netrunner}/src/warding/ttf.rs | 0 36 files changed, 62 insertions(+), 73 deletions(-) create mode 100644 simlib/Cargo.toml create mode 100644 simlib/mixnet-sims/Cargo.toml rename {network-runner/src/bin/app => simlib/mixnet-sims/src}/log.rs (100%) rename {network-runner/src/bin/app => simlib/mixnet-sims/src}/main.rs (86%) rename {network-runner => simlib/mixnet-sims}/src/node/mix/consensus_streams.rs (100%) rename {network-runner => simlib/mixnet-sims}/src/node/mix/lottery.rs (100%) rename {network-runner => simlib/mixnet-sims}/src/node/mix/mod.rs (98%) rename {network-runner => simlib/mixnet-sims}/src/node/mix/scheduler.rs (100%) rename {network-runner => simlib/mixnet-sims}/src/node/mix/state.rs (99%) rename {network-runner => simlib/mixnet-sims}/src/node/mix/stream_wrapper.rs (100%) create mode 100644 simlib/mixnet-sims/src/node/mod.rs rename {network-runner => simlib/netrunner}/Cargo.lock (100%) rename {network-runner => simlib/netrunner}/Cargo.toml (55%) rename {network-runner => simlib/netrunner}/README.md (100%) rename {network-runner => simlib/netrunner}/config/carnot_dev.json (100%) rename {network-runner => simlib/netrunner}/config/mixnode.json (100%) rename {network-runner => simlib/netrunner}/src/lib.rs (100%) rename {network-runner => simlib/netrunner}/src/network/behaviour.rs (100%) rename {network-runner => simlib/netrunner}/src/network/mod.rs (100%) rename {network-runner => simlib/netrunner}/src/network/regions.rs (100%) rename {network-runner => simlib/netrunner}/src/node/dummy_streaming.rs (100%) rename {network-runner => simlib/netrunner}/src/node/mod.rs (99%) rename {network-runner => simlib/netrunner}/src/output_processors/mod.rs (100%) rename {network-runner => simlib/netrunner}/src/runner/async_runner.rs (100%) rename {network-runner => simlib/netrunner}/src/runner/mod.rs (100%) rename {network-runner => simlib/netrunner}/src/runner/sync_runner.rs (100%) rename {network-runner => simlib/netrunner}/src/settings.rs (100%) rename {network-runner => simlib/netrunner}/src/streaming/io.rs (100%) rename {network-runner => simlib/netrunner}/src/streaming/mod.rs (95%) rename {network-runner => simlib/netrunner}/src/streaming/naive.rs (100%) rename {network-runner => simlib/netrunner}/src/streaming/polars.rs (100%) rename {network-runner => simlib/netrunner}/src/streaming/runtime_subscriber.rs (100%) rename {network-runner => simlib/netrunner}/src/streaming/settings_subscriber.rs (95%) rename {network-runner => simlib/netrunner}/src/warding/mod.rs (100%) rename {network-runner => simlib/netrunner}/src/warding/ttf.rs (100%) diff --git a/.gitignore b/.gitignore index d8a5322..34df463 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,5 @@ __pycache__/ *$py.class *.so simulation -network-runner/target +simlib/netrunner/target .idea/ \ No newline at end of file diff --git a/simlib/Cargo.toml b/simlib/Cargo.toml new file mode 100644 index 0000000..cee8ef1 --- /dev/null +++ b/simlib/Cargo.toml @@ -0,0 +1,7 @@ +[workspace] +members = [ + "netrunner", + "mixnet-sims" + +] +resolver = "2" \ No newline at end of file diff --git a/simlib/mixnet-sims/Cargo.toml b/simlib/mixnet-sims/Cargo.toml new file mode 100644 index 0000000..a7ca1f0 --- /dev/null +++ b/simlib/mixnet-sims/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "mixnet-sims" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1.0.93" +clap = { version = "4.5.20", features = ["derive"] } +crossbeam = "0.8.4" +ctrlc = "3.4" +parking_lot = "0.12.3" +rand = "0.8" +serde = { version = "1.0.214", features = ["derive"] } +log = "0.4.22" +serde_json = "1.0.132" +tracing = "0.1.40" +tracing-subscriber = { version = "0.3", features = ["json", "env-filter", "tracing-log"] } +netrunner = { path = "../netrunner" } +nomos-tracing = { git = "https://github.com/logos-co/nomos-node.git", rev = "cc5fef6" } +nomos-mix = { git = "https://github.com/logos-co/nomos-node", rev = "e095964", package = "nomos-mix" } +nomos-mix-message = { git = "https://github.com/logos-co/nomos-node", rev = "e095964", package = "nomos-mix-message" } +futures = "0.3.31" +rand_chacha = "0.3" +multiaddr = "0.18" + diff --git a/network-runner/src/bin/app/log.rs b/simlib/mixnet-sims/src/log.rs similarity index 100% rename from network-runner/src/bin/app/log.rs rename to simlib/mixnet-sims/src/log.rs diff --git a/network-runner/src/bin/app/main.rs b/simlib/mixnet-sims/src/main.rs similarity index 86% rename from network-runner/src/bin/app/main.rs rename to simlib/mixnet-sims/src/main.rs index 99abf0b..f2a77e1 100644 --- a/network-runner/src/bin/app/main.rs +++ b/simlib/mixnet-sims/src/main.rs @@ -4,26 +4,22 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; // crates +use crate::node::mix::state::{MixnodeRecord, MixnodeState}; +use crate::node::mix::{MixMessage, MixnodeSettings}; use anyhow::Ok; use clap::Parser; use crossbeam::channel; +use netrunner::network::behaviour::create_behaviours; +use netrunner::network::regions::{create_regions, RegionsData}; +use netrunner::network::{InMemoryNetworkInterface, Network}; +use netrunner::node::{NodeId, NodeIdExt}; +use netrunner::output_processors::Record; +use netrunner::runner::{BoxedNode, SimulationRunnerHandle}; +use netrunner::streaming::{io::IOSubscriber, naive::NaiveSubscriber, StreamType}; use nomos_mix::message_blend::{ CryptographicProcessorSettings, MessageBlendSettings, TemporalSchedulerSettings, }; use nomos_mix::persistent_transmission::PersistentTransmissionSettings; -use nomos_simulations_network_runner::network::behaviour::create_behaviours; -use nomos_simulations_network_runner::network::regions::{create_regions, RegionsData}; -use nomos_simulations_network_runner::network::{InMemoryNetworkInterface, Network}; -use nomos_simulations_network_runner::node::mix::state::{MixnodeRecord, MixnodeState}; -use nomos_simulations_network_runner::node::mix::{MixMessage, MixNode, MixnodeSettings}; -use nomos_simulations_network_runner::node::{NodeId, NodeIdExt}; -use nomos_simulations_network_runner::output_processors::Record; -use nomos_simulations_network_runner::runner::{BoxedNode, SimulationRunnerHandle}; -#[cfg(feature = "polars")] -use nomos_simulations_network_runner::streaming::polars::PolarsSubscriber; -use nomos_simulations_network_runner::streaming::{ - io::IOSubscriber, naive::NaiveSubscriber, StreamType, -}; use parking_lot::Mutex; use rand::prelude::IteratorRandom; use rand::rngs::SmallRng; @@ -32,9 +28,11 @@ use rand::SeedableRng; use serde::de::DeserializeOwned; use serde::Serialize; // internal -use nomos_simulations_network_runner::{runner::SimulationRunner, settings::SimulationSettings}; -mod log; +use crate::node::mix::MixNode; +use netrunner::{runner::SimulationRunner, settings::SimulationSettings}; +mod log; +mod node; /// Main simulation wrapper /// Pipes together the cli arguments with the execution #[derive(Parser)] @@ -195,11 +193,6 @@ where let settings = stream_settings.unwrap_io(); runner.simulate_and_subscribe::>(settings)? } - #[cfg(feature = "polars")] - Some(StreamType::Polars) => { - let settings = stream_settings.unwrap_polars(); - runner.simulate_and_subscribe::>(settings)? - } None => runner.simulate()?, }; diff --git a/network-runner/src/node/mix/consensus_streams.rs b/simlib/mixnet-sims/src/node/mix/consensus_streams.rs similarity index 100% rename from network-runner/src/node/mix/consensus_streams.rs rename to simlib/mixnet-sims/src/node/mix/consensus_streams.rs diff --git a/network-runner/src/node/mix/lottery.rs b/simlib/mixnet-sims/src/node/mix/lottery.rs similarity index 100% rename from network-runner/src/node/mix/lottery.rs rename to simlib/mixnet-sims/src/node/mix/lottery.rs diff --git a/network-runner/src/node/mix/mod.rs b/simlib/mixnet-sims/src/node/mix/mod.rs similarity index 98% rename from network-runner/src/node/mix/mod.rs rename to simlib/mixnet-sims/src/node/mix/mod.rs index 694378e..241d3e5 100644 --- a/network-runner/src/node/mix/mod.rs +++ b/simlib/mixnet-sims/src/node/mix/mod.rs @@ -1,18 +1,18 @@ -mod lottery; -mod consensus_streams; -mod scheduler; +pub mod consensus_streams; +pub mod lottery; +pub mod scheduler; pub mod state; -mod stream_wrapper; +pub mod stream_wrapper; -use super::{Node, NodeId}; -use crate::{ - network::{InMemoryNetworkInterface, NetworkInterface, PayloadSize}, - warding::WardCondition, -}; use crossbeam::channel; use futures::Stream; use lottery::StakeLottery; use multiaddr::Multiaddr; +use netrunner::node::{Node, NodeId}; +use netrunner::{ + network::{InMemoryNetworkInterface, NetworkInterface, PayloadSize}, + warding::WardCondition, +}; use nomos_mix::{ membership::Membership, message_blend::{ diff --git a/network-runner/src/node/mix/scheduler.rs b/simlib/mixnet-sims/src/node/mix/scheduler.rs similarity index 100% rename from network-runner/src/node/mix/scheduler.rs rename to simlib/mixnet-sims/src/node/mix/scheduler.rs diff --git a/network-runner/src/node/mix/state.rs b/simlib/mixnet-sims/src/node/mix/state.rs similarity index 99% rename from network-runner/src/node/mix/state.rs rename to simlib/mixnet-sims/src/node/mix/state.rs index 688d39a..c1d57f2 100644 --- a/network-runner/src/node/mix/state.rs +++ b/simlib/mixnet-sims/src/node/mix/state.rs @@ -2,7 +2,7 @@ use std::any::Any; use serde::Serialize; -use crate::{ +use netrunner::{ node::NodeId, output_processors::{Record, RecordType, Runtime}, settings::SimulationSettings, diff --git a/network-runner/src/node/mix/stream_wrapper.rs b/simlib/mixnet-sims/src/node/mix/stream_wrapper.rs similarity index 100% rename from network-runner/src/node/mix/stream_wrapper.rs rename to simlib/mixnet-sims/src/node/mix/stream_wrapper.rs diff --git a/simlib/mixnet-sims/src/node/mod.rs b/simlib/mixnet-sims/src/node/mod.rs new file mode 100644 index 0000000..bd1b04a --- /dev/null +++ b/simlib/mixnet-sims/src/node/mod.rs @@ -0,0 +1 @@ +pub mod mix; diff --git a/network-runner/Cargo.lock b/simlib/netrunner/Cargo.lock similarity index 100% rename from network-runner/Cargo.lock rename to simlib/netrunner/Cargo.lock diff --git a/network-runner/Cargo.toml b/simlib/netrunner/Cargo.toml similarity index 55% rename from network-runner/Cargo.toml rename to simlib/netrunner/Cargo.toml index d7abed9..6d8b2fd 100644 --- a/network-runner/Cargo.toml +++ b/simlib/netrunner/Cargo.toml @@ -1,22 +1,16 @@ [package] -name = "nomos-simulations-network-runner" +name = "netrunner" version = "0.1.0" edition = "2021" -[[bin]] -name = "simulation" -path = "src/bin/app/main.rs" - # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] anyhow = "1" blake2 = "0.10" -bls-signatures = "0.14" digest = "0.10" csv = "1" clap = { version = "4", features = ["derive"] } -ctrlc = "3.4" chrono = { version = "0.4", features = ["serde"] } crc32fast = "1.3" crossbeam = { version = "0.8.2", features = ["crossbeam-channel"] } @@ -24,10 +18,8 @@ fixed-slice-deque = "0.1.0-beta2" futures = "0.3" humantime = "2.1" humantime-serde = "1" -nomos-tracing = { git = "https://github.com/logos-co/nomos-node.git", rev = "cc5fef6" } once_cell = "1.17" parking_lot = "0.12" -polars = { version = "0.27", features = ["serde", "object", "json", "csv-file", "parquet", "dtype-struct"], optional = true } rand = { version = "0.8", features = ["small_rng"] } rayon = "1.7" scopeguard = "1" @@ -36,14 +28,6 @@ serde_with = "2.3" serde_json = "1.0" thiserror = "1" tracing = { version = "0.1", default-features = false, features = ["log", "attributes"] } -tracing-subscriber = { version = "0.3", features = ["json", "env-filter", "tracing-log"]} -nomos-mix = { git = "https://github.com/logos-co/nomos-node", rev = "e095964", package = "nomos-mix" } -nomos-mix-message = { git = "https://github.com/logos-co/nomos-node", rev = "e095964", package = "nomos-mix-message" } -rand_chacha = "0.3" -multiaddr = "0.18" [target.'cfg(target_arch = "wasm32")'.dependencies] getrandom = { version = "0.2", features = ["js"] } - -[features] -polars = ["dep:polars"] diff --git a/network-runner/README.md b/simlib/netrunner/README.md similarity index 100% rename from network-runner/README.md rename to simlib/netrunner/README.md diff --git a/network-runner/config/carnot_dev.json b/simlib/netrunner/config/carnot_dev.json similarity index 100% rename from network-runner/config/carnot_dev.json rename to simlib/netrunner/config/carnot_dev.json diff --git a/network-runner/config/mixnode.json b/simlib/netrunner/config/mixnode.json similarity index 100% rename from network-runner/config/mixnode.json rename to simlib/netrunner/config/mixnode.json diff --git a/network-runner/src/lib.rs b/simlib/netrunner/src/lib.rs similarity index 100% rename from network-runner/src/lib.rs rename to simlib/netrunner/src/lib.rs diff --git a/network-runner/src/network/behaviour.rs b/simlib/netrunner/src/network/behaviour.rs similarity index 100% rename from network-runner/src/network/behaviour.rs rename to simlib/netrunner/src/network/behaviour.rs diff --git a/network-runner/src/network/mod.rs b/simlib/netrunner/src/network/mod.rs similarity index 100% rename from network-runner/src/network/mod.rs rename to simlib/netrunner/src/network/mod.rs diff --git a/network-runner/src/network/regions.rs b/simlib/netrunner/src/network/regions.rs similarity index 100% rename from network-runner/src/network/regions.rs rename to simlib/netrunner/src/network/regions.rs diff --git a/network-runner/src/node/dummy_streaming.rs b/simlib/netrunner/src/node/dummy_streaming.rs similarity index 100% rename from network-runner/src/node/dummy_streaming.rs rename to simlib/netrunner/src/node/dummy_streaming.rs diff --git a/network-runner/src/node/mod.rs b/simlib/netrunner/src/node/mod.rs similarity index 99% rename from network-runner/src/node/mod.rs rename to simlib/netrunner/src/node/mod.rs index 83f652f..2de43cd 100644 --- a/network-runner/src/node/mod.rs +++ b/simlib/netrunner/src/node/mod.rs @@ -1,6 +1,5 @@ #[cfg(test)] pub mod dummy_streaming; -pub mod mix; // std use std::{ diff --git a/network-runner/src/output_processors/mod.rs b/simlib/netrunner/src/output_processors/mod.rs similarity index 100% rename from network-runner/src/output_processors/mod.rs rename to simlib/netrunner/src/output_processors/mod.rs diff --git a/network-runner/src/runner/async_runner.rs b/simlib/netrunner/src/runner/async_runner.rs similarity index 100% rename from network-runner/src/runner/async_runner.rs rename to simlib/netrunner/src/runner/async_runner.rs diff --git a/network-runner/src/runner/mod.rs b/simlib/netrunner/src/runner/mod.rs similarity index 100% rename from network-runner/src/runner/mod.rs rename to simlib/netrunner/src/runner/mod.rs diff --git a/network-runner/src/runner/sync_runner.rs b/simlib/netrunner/src/runner/sync_runner.rs similarity index 100% rename from network-runner/src/runner/sync_runner.rs rename to simlib/netrunner/src/runner/sync_runner.rs diff --git a/network-runner/src/settings.rs b/simlib/netrunner/src/settings.rs similarity index 100% rename from network-runner/src/settings.rs rename to simlib/netrunner/src/settings.rs diff --git a/network-runner/src/streaming/io.rs b/simlib/netrunner/src/streaming/io.rs similarity index 100% rename from network-runner/src/streaming/io.rs rename to simlib/netrunner/src/streaming/io.rs diff --git a/network-runner/src/streaming/mod.rs b/simlib/netrunner/src/streaming/mod.rs similarity index 95% rename from network-runner/src/streaming/mod.rs rename to simlib/netrunner/src/streaming/mod.rs index 946cf94..fad48dc 100644 --- a/network-runner/src/streaming/mod.rs +++ b/simlib/netrunner/src/streaming/mod.rs @@ -11,8 +11,6 @@ use crate::output_processors::{Record, RecordType, Runtime}; pub mod io; pub mod naive; -#[cfg(feature = "polars")] -pub mod polars; pub mod runtime_subscriber; pub mod settings_subscriber; @@ -84,8 +82,6 @@ pub enum StreamType { #[default] IO, Naive, - #[cfg(feature = "polars")] - Polars, } impl FromStr for StreamType { @@ -95,8 +91,6 @@ impl FromStr for StreamType { match s.trim().to_ascii_lowercase().as_str() { "io" => Ok(Self::IO), "naive" => Ok(Self::Naive), - #[cfg(feature = "polars")] - "polars" => Ok(Self::Polars), tag => Err(format!( "Invalid {tag} streaming type, only [naive, polars] are supported", )), @@ -119,8 +113,6 @@ impl<'de> serde::Deserialize<'de> for StreamType { pub enum StreamSettings { Naive(naive::NaiveSettings), IO(io::IOStreamSettings), - #[cfg(feature = "polars")] - Polars(polars::PolarsSettings), } impl Default for StreamSettings { @@ -143,14 +135,6 @@ impl StreamSettings { _ => panic!("unwrap io failed"), } } - - #[cfg(feature = "polars")] - pub fn unwrap_polars(self) -> polars::PolarsSettings { - match self { - StreamSettings::Polars(settings) => settings, - _ => panic!("unwrap polars failed"), - } - } } pub struct SubscriberHandle { diff --git a/network-runner/src/streaming/naive.rs b/simlib/netrunner/src/streaming/naive.rs similarity index 100% rename from network-runner/src/streaming/naive.rs rename to simlib/netrunner/src/streaming/naive.rs diff --git a/network-runner/src/streaming/polars.rs b/simlib/netrunner/src/streaming/polars.rs similarity index 100% rename from network-runner/src/streaming/polars.rs rename to simlib/netrunner/src/streaming/polars.rs diff --git a/network-runner/src/streaming/runtime_subscriber.rs b/simlib/netrunner/src/streaming/runtime_subscriber.rs similarity index 100% rename from network-runner/src/streaming/runtime_subscriber.rs rename to simlib/netrunner/src/streaming/runtime_subscriber.rs diff --git a/network-runner/src/streaming/settings_subscriber.rs b/simlib/netrunner/src/streaming/settings_subscriber.rs similarity index 95% rename from network-runner/src/streaming/settings_subscriber.rs rename to simlib/netrunner/src/streaming/settings_subscriber.rs index 2d3aa07..59105ce 100644 --- a/network-runner/src/streaming/settings_subscriber.rs +++ b/simlib/netrunner/src/streaming/settings_subscriber.rs @@ -101,9 +101,10 @@ where #[cfg(test)] mod tests { + use std::collections::HashSet; use std::{collections::HashMap, time::Duration}; - use consensus_engine::View; + type View = usize; use crate::{ network::{ @@ -123,7 +124,7 @@ mod tests { use super::*; #[derive(Debug, Clone, Serialize)] struct SettingsRecord { - states: HashMap, + states: HashSet, } impl TryFrom<&SimulationState> for SettingsRecord { @@ -131,12 +132,7 @@ mod tests { fn try_from(value: &SimulationState) -> Result { Ok(Self { - states: value - .nodes - .read() - .iter() - .map(|node| (node.id(), node.current_view())) - .collect(), + states: value.nodes.read().iter().map(|node| node.id()).collect(), }) } } diff --git a/network-runner/src/warding/mod.rs b/simlib/netrunner/src/warding/mod.rs similarity index 100% rename from network-runner/src/warding/mod.rs rename to simlib/netrunner/src/warding/mod.rs diff --git a/network-runner/src/warding/ttf.rs b/simlib/netrunner/src/warding/ttf.rs similarity index 100% rename from network-runner/src/warding/ttf.rs rename to simlib/netrunner/src/warding/ttf.rs