diff --git a/.env.example b/.env.example index 4f0955f2..9fd1edf4 100644 --- a/.env.example +++ b/.env.example @@ -6,4 +6,4 @@ DOCKER_COMPOSE_ETCDCTL_ENDPOINTS=etcd:2379 DOCKER_COMPOSE_ETCDCTL_API=3 DOCKER_COMPOSE_BOOSTRAP_NET_NODE_KEY=1000000000000000000000000000000000000000000000000000000000000000 DOCKER_COMPOSE_OVERLAY_NODES=$DOCKER_COMPOSE_BOOSTRAP_NET_NODE_KEY -DOCKER_COMPOSE_NET_INITIAL_PEERS=/dns/bootstrap/tcp/3000 +DOCKER_COMPOSE_NET_INITIAL_PEERS=/dns/bootstrap/udp/3000/quic-v1 diff --git a/.env.testnet b/.env.testnet index bc05fb6b..4f80beb7 100644 --- a/.env.testnet +++ b/.env.testnet @@ -6,4 +6,4 @@ DOCKER_COMPOSE_ETCDCTL_ENDPOINTS=etcd:2379 DOCKER_COMPOSE_ETCDCTL_API=3 DOCKER_COMPOSE_BOOSTRAP_NET_NODE_KEY=1000000000000000000000000000000000000000000000000000000000000000 DOCKER_COMPOSE_OVERLAY_NODES=1000000000000000000000000000000000000000000000000000000000000000 -DOCKER_COMPOSE_NET_INITIAL_PEERS=/dns/bootstrap/tcp/3000 +DOCKER_COMPOSE_NET_INITIAL_PEERS=/dns/bootstrap/udp/3000/quic-v1 diff --git a/Cargo.toml b/Cargo.toml index ce6f3384..f1744e0e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,15 +19,11 @@ members = [ "nomos-cli", "nomos-utils", "nodes/nomos-node", - "nodes/mixnode", + "mixnet", "simulations", "consensus/carnot-engine", "consensus/cryptarchia-engine", "ledger/cryptarchia-ledger", "tests", - "mixnet/node", - "mixnet/client", - "mixnet/protocol", - "mixnet/topology", ] resolver = "2" diff --git a/ci/Jenkinsfile.nightly.integration b/ci/Jenkinsfile.nightly.integration index 9dff6b1a..e3757df4 100644 --- a/ci/Jenkinsfile.nightly.integration +++ b/ci/Jenkinsfile.nightly.integration @@ -53,10 +53,6 @@ pipeline { /* To prevent rebuilding node for each test, tests are defined here */ def tests = ['ten_nodes_happy', 'two_nodes_happy', 'ten_nodes_one_down'] - if (FEATURE == 'libp2p') { - tests.add('mixnet') - } - def report = runBuildAndTestsForFeature(FEATURE, tests) writeFile(file: "${WORKSPACE}/report.txt", text: report) } diff --git a/compose.static.yml b/compose.static.yml index fb17ec89..4b604d0e 100644 --- a/compose.static.yml +++ b/compose.static.yml @@ -7,14 +7,10 @@ services: context: . dockerfile: testnet/Dockerfile ports: - - "3000:3000/tcp" + - "3000:3000/udp" - "18080:18080/tcp" volumes: - ./testnet:/etc/nomos - depends_on: - - mix-node-0 - - mix-node-1 - - mix-node-2 environment: - BOOTSTRAP_NODE_KEY=${DOCKER_COMPOSE_BOOSTRAP_NET_NODE_KEY:-1000000000000000000000000000000000000000000000000000000000000000} - LIBP2P_NODE_MASK=${DOCKER_COMPOSE_LIBP2P_NODE_KEY_MASK:-2000000000000000000000000000000000000000000000000000000000000000} @@ -32,11 +28,8 @@ services: depends_on: - bootstrap - etcd - - mix-node-0 - - mix-node-1 - - mix-node-2 ports: - - "3001:3000/tcp" + - "3001:3000/udp" - "18081:18080/tcp" environment: - LIBP2P_REPLICAS=3 @@ -45,7 +38,7 @@ services: - LIBP2P_NODE_MASK=${DOCKER_COMPOSE_LIBP2P_NODE_KEY_MASK:-2000000000000000000000000000000000000000000000000000000000000000} - OVERLAY_NODES=${DOCKER_COMPOSE_OVERLAY_NODES:-1000000000000000000000000000000000000000000000000000000000000000} - OVERLAY_SUPER_MAJORITY_THRESHOLD=${DOCKER_COMPOSE_SUPER_MAJORITY_THRESHOLD:-1} - - NET_INITIAL_PEERS=${DOCKER_COMPOSE_NET_INITIAL_PEERS:-/dns/bootstrap/tcp/3000} + - NET_INITIAL_PEERS=${DOCKER_COMPOSE_NET_INITIAL_PEERS:-/dns/bootstrap/udp/3000/quic-v1} entrypoint: /etc/nomos/scripts/run_nomos_node.sh libp2p-node-2: @@ -58,11 +51,8 @@ services: depends_on: - bootstrap - etcd - - mix-node-0 - - mix-node-1 - - mix-node-2 ports: - - "3002:3000/tcp" + - "3002:3000/udp" - "18082:18080/tcp" environment: - LIBP2P_REPLICAS=3 @@ -71,7 +61,7 @@ services: - LIBP2P_NODE_MASK=${DOCKER_COMPOSE_LIBP2P_NODE_KEY_MASK:-2000000000000000000000000000000000000000000000000000000000000000} - OVERLAY_NODES=${DOCKER_COMPOSE_OVERLAY_NODES:-1000000000000000000000000000000000000000000000000000000000000000} - OVERLAY_SUPER_MAJORITY_THRESHOLD=${DOCKER_COMPOSE_SUPER_MAJORITY_THRESHOLD:-1} - - NET_INITIAL_PEERS=${DOCKER_COMPOSE_NET_INITIAL_PEERS:-/dns/bootstrap/tcp/3000} + - NET_INITIAL_PEERS=${DOCKER_COMPOSE_NET_INITIAL_PEERS:-/dns/bootstrap/udp/3000/quic-v1} entrypoint: /etc/nomos/scripts/run_nomos_node.sh libp2p-node-3: @@ -84,11 +74,8 @@ services: depends_on: - bootstrap - etcd - - mix-node-0 - - mix-node-1 - - mix-node-2 ports: - - "3003:3000/tcp" + - "3003:3000/udp" - "18083:18080/tcp" environment: - LIBP2P_REPLICAS=3 @@ -97,48 +84,9 @@ services: - LIBP2P_NODE_MASK=${DOCKER_COMPOSE_LIBP2P_NODE_KEY_MASK:-2000000000000000000000000000000000000000000000000000000000000000} - OVERLAY_NODES=${DOCKER_COMPOSE_OVERLAY_NODES:-1000000000000000000000000000000000000000000000000000000000000000} - OVERLAY_SUPER_MAJORITY_THRESHOLD=${DOCKER_COMPOSE_SUPER_MAJORITY_THRESHOLD:-1} - - NET_INITIAL_PEERS=${DOCKER_COMPOSE_NET_INITIAL_PEERS:-/dns/bootstrap/tcp/3000} + - NET_INITIAL_PEERS=${DOCKER_COMPOSE_NET_INITIAL_PEERS:-/dns/bootstrap/udp/3000/quic-v1} entrypoint: /etc/nomos/scripts/run_nomos_node.sh - mix-node-0: - container_name: mix_node_0 - build: - context: . - dockerfile: testnet/Dockerfile - volumes: - - ./testnet:/etc/nomos - ports: - - "7707:7777/tcp" - - "7708:7778/tcp" - entrypoint: /usr/bin/mixnode - command: /etc/nomos/mixnode_config.yaml - - mix-node-1: - container_name: mix_node_1 - build: - context: . - dockerfile: testnet/Dockerfile - volumes: - - ./testnet:/etc/nomos - ports: - - "7717:7777/tcp" - - "7718:7778/tcp" - entrypoint: /usr/bin/mixnode - command: /etc/nomos/mixnode_config.yaml - - mix-node-2: - container_name: mix_node_2 - build: - context: . - dockerfile: testnet/Dockerfile - volumes: - - ./testnet:/etc/nomos - ports: - - "7727:7777/tcp" - - "7728:7778/tcp" - entrypoint: /usr/bin/mixnode - command: /etc/nomos/mixnode_config.yaml - chatbot: container_name: chatbot build: diff --git a/compose.yml b/compose.yml index d44668b1..2511356b 100644 --- a/compose.yml +++ b/compose.yml @@ -5,7 +5,7 @@ services: context: . dockerfile: testnet/Dockerfile ports: - - "3000:3000/tcp" + - "3000:3000/udp" - "18080:18080/tcp" volumes: - ./testnet:/etc/nomos @@ -27,9 +27,6 @@ services: depends_on: - bootstrap - etcd - - mix-node-0 - - mix-node-1 - - mix-node-2 environment: - LIBP2P_REPLICAS=${DOCKER_COMPOSE_LIBP2P_REPLICAS:-1} - ETCDCTL_ENDPOINTS=${DOCKER_COMPOSE_ETCDCTL_ENDPOINTS:-etcd:2379} @@ -37,36 +34,9 @@ services: - LIBP2P_NODE_MASK=${DOCKER_COMPOSE_LIBP2P_NODE_KEY_MASK:-2000000000000000000000000000000000000000000000000000000000000000} - OVERLAY_NODES=${DOCKER_COMPOSE_OVERLAY_NODES:-1000000000000000000000000000000000000000000000000000000000000000} - OVERLAY_SUPER_MAJORITY_THRESHOLD=${DOCKER_COMPOSE_SUPER_MAJORITY_THRESHOLD:-1} - - NET_INITIAL_PEERS=${DOCKER_COMPOSE_NET_INITIAL_PEERS:-/dns/bootstrap/tcp/3000} + - NET_INITIAL_PEERS=${DOCKER_COMPOSE_NET_INITIAL_PEERS:-/dns/bootstrap/udp/3000/quic-v1} entrypoint: /etc/nomos/scripts/run_nomos_node.sh - mix-node-0: - build: - context: . - dockerfile: testnet/Dockerfile - volumes: - - ./testnet:/etc/nomos - entrypoint: /usr/bin/mixnode - command: /etc/nomos/mixnode_config.yaml - - mix-node-1: - build: - context: . - dockerfile: testnet/Dockerfile - volumes: - - ./testnet:/etc/nomos - entrypoint: /usr/bin/mixnode - command: /etc/nomos/mixnode_config.yaml - - mix-node-2: - build: - context: . - dockerfile: testnet/Dockerfile - volumes: - - ./testnet:/etc/nomos - entrypoint: /usr/bin/mixnode - command: /etc/nomos/mixnode_config.yaml - etcd: image: quay.io/coreos/etcd:v3.4.15 ports: diff --git a/consensus/carnot-engine/Cargo.toml b/consensus/carnot-engine/Cargo.toml index 324d9e27..6b6af110 100644 --- a/consensus/carnot-engine/Cargo.toml +++ b/consensus/carnot-engine/Cargo.toml @@ -15,7 +15,7 @@ rand = "0.8" rand_chacha = "0.3" thiserror = "1" fraction = { version = "0.13" } -nomos-utils = { path = "../../nomos-utils", optional = true } +nomos-utils = { path = "../../nomos-utils" } utoipa = { version = "4.0", optional = true } serde_json = { version = "1.0", optional = true } diff --git a/consensus/carnot-engine/src/overlay/branch_overlay.rs b/consensus/carnot-engine/src/overlay/branch_overlay.rs index 18eac14e..f074d4df 100644 --- a/consensus/carnot-engine/src/overlay/branch_overlay.rs +++ b/consensus/carnot-engine/src/overlay/branch_overlay.rs @@ -188,7 +188,9 @@ fn build_committee_from_nodes_with_size( #[cfg(test)] mod tests { - use crate::overlay::{FisherYatesShuffle, RoundRobin}; + use nomos_utils::fisheryates::FisherYatesShuffle; + + use crate::overlay::RoundRobin; use super::*; const ENTROPY: [u8; 32] = [0; 32]; diff --git a/consensus/carnot-engine/src/overlay/membership.rs b/consensus/carnot-engine/src/overlay/membership.rs index e174ebd3..ab4c33e7 100644 --- a/consensus/carnot-engine/src/overlay/membership.rs +++ b/consensus/carnot-engine/src/overlay/membership.rs @@ -1,33 +1,7 @@ -// std - -// crates -use rand::{Rng, SeedableRng}; -use rand_chacha::ChaCha20Rng; - // internal use crate::overlay::CommitteeMembership; use crate::NodeId; - -#[derive(Copy, Clone, Debug, Eq, PartialEq)] -pub struct FisherYatesShuffle { - entropy: [u8; 32], -} - -impl FisherYatesShuffle { - pub fn new(entropy: [u8; 32]) -> Self { - Self { entropy } - } - - pub fn shuffle(elements: &mut [T], entropy: [u8; 32]) { - let mut rng = ChaCha20Rng::from_seed(entropy); - // Implementation of fisher yates shuffling - // https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle - for i in (1..elements.len()).rev() { - let j = rng.gen_range(0..=i); - elements.swap(i, j); - } - } -} +use nomos_utils::fisheryates::FisherYatesShuffle; impl CommitteeMembership for FisherYatesShuffle { fn reshape_committees(&self, nodes: &mut [NodeId]) { diff --git a/consensus/carnot-engine/src/overlay/mod.rs b/consensus/carnot-engine/src/overlay/mod.rs index 91d3daa5..9a8eb17e 100644 --- a/consensus/carnot-engine/src/overlay/mod.rs +++ b/consensus/carnot-engine/src/overlay/mod.rs @@ -53,8 +53,10 @@ pub trait CommitteeMembership: Clone { #[cfg(test)] mod tests { + use nomos_utils::fisheryates::FisherYatesShuffle; + use super::*; - use crate::overlay::{FisherYatesShuffle, RoundRobin}; + use crate::overlay::RoundRobin; const ENTROPY: [u8; 32] = [0; 32]; diff --git a/consensus/carnot-engine/src/overlay/random_beacon.rs b/consensus/carnot-engine/src/overlay/random_beacon.rs index 16c26e89..fd13b56f 100644 --- a/consensus/carnot-engine/src/overlay/random_beacon.rs +++ b/consensus/carnot-engine/src/overlay/random_beacon.rs @@ -1,6 +1,7 @@ -use crate::overlay::{CommitteeMembership, FisherYatesShuffle}; +use crate::overlay::CommitteeMembership; use crate::types::*; use bls_signatures::{PrivateKey, PublicKey, Serialize, Signature}; +use nomos_utils::fisheryates::FisherYatesShuffle; use rand::{seq::SliceRandom, SeedableRng}; use serde::{Deserialize, Serialize as SerdeSerialize}; use sha2::{Digest, Sha256}; diff --git a/consensus/carnot-engine/src/overlay/tree_overlay/overlay.rs b/consensus/carnot-engine/src/overlay/tree_overlay/overlay.rs index 8155238a..b969fbb2 100644 --- a/consensus/carnot-engine/src/overlay/tree_overlay/overlay.rs +++ b/consensus/carnot-engine/src/overlay/tree_overlay/overlay.rs @@ -231,8 +231,9 @@ where #[cfg(test)] mod tests { + use nomos_utils::fisheryates::FisherYatesShuffle; + use crate::overlay::leadership::RoundRobin; - use crate::overlay::membership::FisherYatesShuffle; use crate::Overlay; use super::*; diff --git a/mixnet/Cargo.toml b/mixnet/Cargo.toml new file mode 100644 index 00000000..7f00e182 --- /dev/null +++ b/mixnet/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "mixnet" +version = "0.0.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rand = "0.8" +rand_distr = "0.4" +nomos-utils = { path = "../nomos-utils" } +thiserror = "1.0.57" +tokio = { version = "1.36.0", features = ["sync"] } +serde = { version = "1.0.197", features = ["derive"] } +sphinx-packet = "0.1.0" +nym-sphinx-addressing = { package = "nym-sphinx-addressing", git = "https://github.com/nymtech/nym", tag = "v1.1.22" } +tracing = "0.1.40" +uuid = { version = "1.7.0", features = ["v4"] } +futures = "0.3" + +[dev-dependencies] +tokio = { version = "1.36.0", features = ["test-util"] } diff --git a/mixnet/README.md b/mixnet/README.md deleted file mode 100644 index 78efa28f..00000000 --- a/mixnet/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# Mixnet - -## Components - -- `node`: A mixnode implementation that will be assigned to one of the mixnet layers and will be responsible for receiving packets and forwarding them to the next mixnet layer. -- `client`: A mixclient implementation - - which splits a message into multiple Sphinx packets, constructs mix routes for them, and sends the packets to the first mixnode in each route. - - which receives Sphinx packets from a mixnode and reconstructs a message. - -## Recommended Architecture - -```mermaid -flowchart LR - subgraph layer-1 - mixnode-1-1 - mixnode-1-2 - end - subgraph layer-2 - mixnode-2-1 - mixnode-2-2 - end - subgraph layer-3 - mixnode-3-1 - mixnode-3-2 - end - mixnode-1-1 --> mixnode-2-1 - mixnode-1-1 --> mixnode-2-2 - mixnode-1-2 --> mixnode-2-1 - mixnode-1-2 --> mixnode-2-2 - mixnode-2-1 --> mixnode-3-1 - mixnode-2-1 --> mixnode-3-2 - mixnode-2-2 --> mixnode-3-1 - mixnode-2-2 --> mixnode-3-2 - mixclient-sender-1 --> mixnode-1-1 - mixclient-sender-1 --> mixnode-1-2 - mixnode-3-1 --> mixclient-senderreceiver-1 - mixnode-3-2 --> mixclient-senderreceiver-2 -``` - -The mix `node` component can be integrated into a separate application, for example, so that it can be run independently with mixclients for better reliability or privacy. - -The mix `client` component is also designed to be integrated into any application that wants to send/receive packets to/from the mixnet. -The `client` can be configured with one of modes described [below](#mixclient-modes). - -## Mixclient Modes - -- `Sender`: A mixclient only sends Sphinx packets to mixnodes, but doesn't receive any packets from mixnodes. -- `SenderReceiver`: A mixclient not only sends Sphinx packets to mixnodes, but also receive packets from mixnodes. - - Due to the design of mixnet, mixclients always receive packets from mixnodes in the last mixnet layer. - - Currently, only 1:1 mapping is supported. In other words, multiple mixclients cannot listen from the same mixnode. It also means that it is recommended that a single node operator runs both a mixnode and a mixclient. diff --git a/mixnet/client/Cargo.toml b/mixnet/client/Cargo.toml deleted file mode 100644 index 85d839d8..00000000 --- a/mixnet/client/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -[package] -name = "mixnet-client" -version = "0.1.0" -edition = "2021" - -[dependencies] -serde = { version = "1.0", features = ["derive"] } -tracing = "0.1.37" -tokio = { version = "1.32", features = ["net"] } -sphinx-packet = "0.1.0" -nym-sphinx = { package = "nym-sphinx", git = "https://github.com/nymtech/nym", tag = "v1.1.22" } -# Using an older version, since `nym-sphinx` depends on `rand` v0.7.3. -rand = "0.7.3" -mixnet-protocol = { path = "../protocol" } -mixnet-topology = { path = "../topology" } -mixnet-util = { path = "../util" } -futures = "0.3.28" -thiserror = "1" - -[dev-dependencies] -serde_yaml = "0.9.25" diff --git a/mixnet/client/src/config.rs b/mixnet/client/src/config.rs deleted file mode 100644 index 4d42273a..00000000 --- a/mixnet/client/src/config.rs +++ /dev/null @@ -1,93 +0,0 @@ -use std::{net::ToSocketAddrs, time::Duration}; - -use futures::{stream, StreamExt}; -use mixnet_topology::MixnetTopology; -use serde::{Deserialize, Serialize}; - -use crate::{receiver::Receiver, MessageStream, MixnetClientError}; - -#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] -pub struct MixnetClientConfig { - pub mode: MixnetClientMode, - pub topology: MixnetTopology, - #[serde(default = "MixnetClientConfig::default_connection_pool_size")] - pub connection_pool_size: usize, - #[serde(default = "MixnetClientConfig::default_max_retries")] - pub max_retries: usize, - #[serde(default = "MixnetClientConfig::default_retry_delay")] - pub retry_delay: std::time::Duration, -} - -impl MixnetClientConfig { - /// Creates a new `MixnetClientConfig` with default values. - pub fn new(mode: MixnetClientMode, topology: MixnetTopology) -> Self { - Self { - mode, - topology, - connection_pool_size: Self::default_connection_pool_size(), - max_retries: Self::default_max_retries(), - retry_delay: Self::default_retry_delay(), - } - } - - const fn default_connection_pool_size() -> usize { - 256 - } - - const fn default_max_retries() -> usize { - 3 - } - - const fn default_retry_delay() -> Duration { - Duration::from_secs(5) - } -} - -#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] -pub enum MixnetClientMode { - Sender, - SenderReceiver(String), -} - -impl MixnetClientMode { - pub(crate) async fn run(&self) -> Result { - match self { - Self::Sender => Ok(stream::empty().boxed()), - Self::SenderReceiver(node_address) => { - let mut addrs = node_address - .to_socket_addrs() - .map_err(|e| MixnetClientError::MixnetNodeAddressError(e.to_string()))?; - let socket_addr = addrs - .next() - .ok_or(MixnetClientError::MixnetNodeAddressError( - "No address provided".into(), - ))?; - Ok(Receiver::new(socket_addr).run().await?.boxed()) - } - } - } -} - -#[cfg(test)] -mod tests { - use mixnet_topology::MixnetTopology; - - use crate::{MixnetClientConfig, MixnetClientMode}; - - #[test] - fn default_config_serde() { - let yaml = " - mode: Sender - topology: - layers: [] - "; - let conf: MixnetClientConfig = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - conf, - MixnetClientConfig::new( - MixnetClientMode::Sender, - MixnetTopology { layers: Vec::new() } - ) - ); - } -} diff --git a/mixnet/client/src/error.rs b/mixnet/client/src/error.rs deleted file mode 100644 index 302363c6..00000000 --- a/mixnet/client/src/error.rs +++ /dev/null @@ -1,26 +0,0 @@ -use mixnet_protocol::ProtocolError; -use nym_sphinx::addressing::nodes::NymNodeRoutingAddressError; - -#[derive(thiserror::Error, Debug)] -pub enum MixnetClientError { - #[error("invalid node address: {0}")] - MixnetNodeAddressError(String), - #[error("mixnet node connect error")] - MixnetNodeConnectError, - #[error("mixnode stream has been closed")] - MixnetNodeStreamClosed, - #[error("unexpected stream body received")] - UnexpectedStreamBody, - #[error("invalid payload")] - InvalidPayload, - #[error("invalid fragment")] - InvalidFragment, - #[error("invalid routing address: {0}")] - InvalidRoutingAddress(#[from] NymNodeRoutingAddressError), - #[error("{0}")] - Protocol(#[from] ProtocolError), - #[error("{0}")] - Message(#[from] nym_sphinx::message::NymMessageError), -} - -pub type Result = core::result::Result; diff --git a/mixnet/client/src/lib.rs b/mixnet/client/src/lib.rs deleted file mode 100644 index ec871eb1..00000000 --- a/mixnet/client/src/lib.rs +++ /dev/null @@ -1,46 +0,0 @@ -pub mod config; -pub mod error; -pub use error::*; -mod receiver; -mod sender; - -use std::time::Duration; - -pub use config::MixnetClientConfig; -pub use config::MixnetClientMode; -use futures::stream::BoxStream; -use mixnet_util::ConnectionPool; -use rand::Rng; -use sender::Sender; - -// A client for sending packets to Mixnet and receiving packets from Mixnet. -pub struct MixnetClient { - mode: MixnetClientMode, - sender: Sender, -} - -pub type MessageStream = BoxStream<'static, Result>>; - -impl MixnetClient { - pub fn new(config: MixnetClientConfig, rng: R) -> Self { - let cache = ConnectionPool::new(config.connection_pool_size); - Self { - mode: config.mode, - sender: Sender::new( - config.topology, - cache, - rng, - config.max_retries, - config.retry_delay, - ), - } - } - - pub async fn run(&self) -> Result { - self.mode.run().await - } - - pub fn send(&mut self, msg: Vec, total_delay: Duration) -> Result<()> { - self.sender.send(msg, total_delay) - } -} diff --git a/mixnet/client/src/receiver.rs b/mixnet/client/src/receiver.rs deleted file mode 100644 index 885d64ea..00000000 --- a/mixnet/client/src/receiver.rs +++ /dev/null @@ -1,132 +0,0 @@ -use std::net::SocketAddr; - -use futures::{stream, Stream, StreamExt}; -use mixnet_protocol::Body; -use nym_sphinx::{ - chunking::{fragment::Fragment, reconstruction::MessageReconstructor}, - message::{NymMessage, PaddedMessage}, - Payload, -}; -use tokio::net::TcpStream; - -use super::error::*; -use crate::MixnetClientError; - -// Receiver accepts TCP connections to receive incoming payloads from the Mixnet. -pub struct Receiver { - node_address: SocketAddr, -} - -impl Receiver { - pub fn new(node_address: SocketAddr) -> Self { - Self { node_address } - } - - pub async fn run(&self) -> Result>> + Send + 'static> { - let Ok(socket) = TcpStream::connect(self.node_address).await else { - return Err(MixnetClientError::MixnetNodeConnectError); - }; - - Ok(Self::message_stream(Box::pin(Self::fragment_stream( - socket, - )))) - } - - fn fragment_stream(socket: TcpStream) -> impl Stream> + Send + 'static { - stream::unfold(socket, move |mut socket| { - async move { - let Ok(body) = Body::read(&mut socket).await else { - // TODO: Maybe this is a hard error and the stream is corrupted? In that case stop the stream - return Some((Err(MixnetClientError::MixnetNodeStreamClosed), socket)); - }; - - match body { - Body::SphinxPacket(_) => { - Some((Err(MixnetClientError::UnexpectedStreamBody), socket)) - } - Body::FinalPayload(payload) => { - Some((Self::fragment_from_payload(payload), socket)) - } - _ => unreachable!(), - } - } - }) - } - - fn message_stream( - fragment_stream: impl Stream> + Send + Unpin + 'static, - ) -> impl Stream>> + Send + 'static { - // MessageReconstructor buffers all received fragments - // and eventually returns reconstructed messages. - let message_reconstructor: MessageReconstructor = Default::default(); - - stream::unfold( - (fragment_stream, message_reconstructor), - |(mut fragment_stream, mut message_reconstructor)| async move { - let result = - Self::reconstruct_message(&mut fragment_stream, &mut message_reconstructor) - .await; - Some((result, (fragment_stream, message_reconstructor))) - }, - ) - } - - fn fragment_from_payload(payload: Payload) -> Result { - let Ok(payload_plaintext) = payload.recover_plaintext() else { - return Err(MixnetClientError::InvalidPayload); - }; - let Ok(fragment) = Fragment::try_from_bytes(&payload_plaintext) else { - return Err(MixnetClientError::InvalidPayload); - }; - Ok(fragment) - } - - async fn reconstruct_message( - fragment_stream: &mut (impl Stream> + Send + Unpin + 'static), - message_reconstructor: &mut MessageReconstructor, - ) -> Result> { - // Read fragments until at least one message is fully reconstructed. - while let Some(next) = fragment_stream.next().await { - match next { - Ok(fragment) => { - if let Some(message) = - Self::try_reconstruct_message(fragment, message_reconstructor)? - { - return Ok(message); - } - } - Err(e) => { - return Err(e); - } - } - } - - // fragment_stream closed before messages are fully reconstructed - Err(MixnetClientError::MixnetNodeStreamClosed) - } - - fn try_reconstruct_message( - fragment: Fragment, - message_reconstructor: &mut MessageReconstructor, - ) -> Result>> { - let reconstruction_result = message_reconstructor.insert_new_fragment(fragment); - match reconstruction_result { - Some((padded_message, _)) => { - let message = Self::remove_padding(padded_message)?; - Ok(Some(message)) - } - None => Ok(None), - } - } - - fn remove_padding(msg: Vec) -> Result> { - let padded_message = PaddedMessage::new_reconstructed(msg); - // we need this because PaddedMessage.remove_padding requires it for other NymMessage types. - let dummy_num_mix_hops = 0; - - match padded_message.remove_padding(dummy_num_mix_hops)? { - NymMessage::Plain(msg) => Ok(msg), - _ => Err(MixnetClientError::InvalidFragment), - } - } -} diff --git a/mixnet/client/src/sender.rs b/mixnet/client/src/sender.rs deleted file mode 100644 index a3450fd3..00000000 --- a/mixnet/client/src/sender.rs +++ /dev/null @@ -1,231 +0,0 @@ -use std::{net::SocketAddr, time::Duration}; - -use mixnet_protocol::{Body, ProtocolError}; -use mixnet_topology::MixnetTopology; -use mixnet_util::ConnectionPool; -use nym_sphinx::{ - addressing::nodes::NymNodeRoutingAddress, chunking::fragment::Fragment, message::NymMessage, - params::PacketSize, Delay, Destination, DestinationAddressBytes, NodeAddressBytes, - IDENTIFIER_LENGTH, PAYLOAD_OVERHEAD_SIZE, -}; -use rand::{distributions::Uniform, prelude::Distribution, Rng}; -use sphinx_packet::{route, SphinxPacket, SphinxPacketBuilder}; - -use super::error::*; - -// Sender splits messages into Sphinx packets and sends them to the Mixnet. -pub struct Sender { - //TODO: handle topology update - topology: MixnetTopology, - pool: ConnectionPool, - max_retries: usize, - retry_delay: Duration, - rng: R, -} - -impl Sender { - pub fn new( - topology: MixnetTopology, - pool: ConnectionPool, - rng: R, - max_retries: usize, - retry_delay: Duration, - ) -> Self { - Self { - topology, - rng, - pool, - max_retries, - retry_delay, - } - } - - pub fn send(&mut self, msg: Vec, total_delay: Duration) -> Result<()> { - let destination = self.topology.random_destination(&mut self.rng)?; - let destination = Destination::new( - DestinationAddressBytes::from_bytes(destination.address.as_bytes()), - [0; IDENTIFIER_LENGTH], // TODO: use a proper SURBIdentifier if we need SURB - ); - - self.pad_and_split_message(msg) - .into_iter() - .map(|fragment| self.build_sphinx_packet(fragment, &destination, total_delay)) - .collect::>>()? - .into_iter() - .for_each(|(packet, first_node)| { - let pool = self.pool.clone(); - let max_retries = self.max_retries; - let retry_delay = self.retry_delay; - tokio::spawn(async move { - if let Err(e) = Self::send_packet( - &pool, - max_retries, - retry_delay, - Box::new(packet), - first_node.address, - ) - .await - { - tracing::error!("failed to send packet to the first node: {e}"); - } - }); - }); - - Ok(()) - } - - fn pad_and_split_message(&mut self, msg: Vec) -> Vec { - let nym_message = NymMessage::new_plain(msg); - - // TODO: add PUBLIC_KEY_SIZE for encryption for the destination, - // if we're going to encrypt final payloads for the destination. - // TODO: add ACK_OVERHEAD if we need SURB-ACKs. - // https://github.com/nymtech/nym/blob/3748ab77a132143d5fd1cd75dd06334d33294815/common/nymsphinx/src/message.rs#L181-L181 - let plaintext_size_per_packet = PacketSize::RegularPacket.plaintext_size(); - - nym_message - .pad_to_full_packet_lengths(plaintext_size_per_packet) - .split_into_fragments(&mut self.rng, plaintext_size_per_packet) - } - - fn build_sphinx_packet( - &mut self, - fragment: Fragment, - destination: &Destination, - total_delay: Duration, - ) -> Result<(sphinx_packet::SphinxPacket, route::Node)> { - let route = self.topology.random_route(&mut self.rng)?; - - let delays: Vec = - RandomDelayIterator::new(&mut self.rng, route.len() as u64, total_delay) - .map(|d| Delay::new_from_millis(d.as_millis() as u64)) - .collect(); - - // TODO: encrypt the payload for the destination, if we want - // https://github.com/nymtech/nym/blob/3748ab77a132143d5fd1cd75dd06334d33294815/common/nymsphinx/src/preparer/payload.rs#L70 - let payload = fragment.into_bytes(); - - let packet = SphinxPacketBuilder::new() - .with_payload_size(payload.len() + PAYLOAD_OVERHEAD_SIZE) - .build_packet(payload, &route, destination, &delays) - .map_err(ProtocolError::InvalidSphinxPacket)?; - - let first_mixnode = route.first().cloned().expect("route is not empty"); - - Ok((packet, first_mixnode)) - } - - async fn send_packet( - pool: &ConnectionPool, - max_retries: usize, - retry_delay: Duration, - packet: Box, - addr: NodeAddressBytes, - ) -> Result<()> { - let addr = SocketAddr::from(NymNodeRoutingAddress::try_from(addr)?); - tracing::debug!("Sending a Sphinx packet to the node: {addr:?}"); - - let mu: std::sync::Arc> = - pool.get_or_init(&addr).await?; - let arc_socket = mu.clone(); - - let body = Body::SphinxPacket(packet); - - if let Err(e) = { - let mut socket = mu.lock().await; - body.write(&mut *socket).await - } { - tracing::error!("Failed to send packet to {addr} with error: {e}. Retrying..."); - return mixnet_protocol::retry_backoff( - addr, - max_retries, - retry_delay, - body, - arc_socket, - ) - .await - .map_err(Into::into); - } - Ok(()) - } -} - -struct RandomDelayIterator { - rng: R, - remaining_delays: u64, - remaining_time: u64, - avg_delay: u64, -} - -impl RandomDelayIterator { - fn new(rng: R, total_delays: u64, total_time: Duration) -> Self { - let total_time = total_time.as_millis() as u64; - RandomDelayIterator { - rng, - remaining_delays: total_delays, - remaining_time: total_time, - avg_delay: total_time / total_delays, - } - } -} - -impl Iterator for RandomDelayIterator -where - R: Rng, -{ - type Item = Duration; - - fn next(&mut self) -> Option { - if self.remaining_delays == 0 { - return None; - } - - self.remaining_delays -= 1; - - if self.remaining_delays == 1 { - return Some(Duration::from_millis(self.remaining_time)); - } - - // Calculate bounds to avoid extreme values - let upper_bound = (self.avg_delay as f64 * 1.5) - // guarantee that we don't exceed the remaining time and promise the delay we return is - // at least 1ms. - .min(self.remaining_time.saturating_sub(self.remaining_delays) as f64); - let lower_bound = (self.avg_delay as f64 * 0.5).min(upper_bound); - - let delay = Uniform::new_inclusive(lower_bound, upper_bound).sample(&mut self.rng) as u64; - self.remaining_time = self.remaining_time.saturating_sub(delay); - - Some(Duration::from_millis(delay)) - } -} - -#[cfg(test)] -mod tests { - use std::time::Duration; - - use super::RandomDelayIterator; - - const TOTAL_DELAYS: u64 = 3; - - #[test] - fn test_random_delay_iter_zero_total_time() { - let mut delays = RandomDelayIterator::new(rand::thread_rng(), TOTAL_DELAYS, Duration::ZERO); - for _ in 0..TOTAL_DELAYS { - assert!(delays.next().is_some()); - } - assert!(delays.next().is_none()); - } - - #[test] - fn test_random_delay_iter_small_total_time() { - let mut delays = - RandomDelayIterator::new(rand::thread_rng(), TOTAL_DELAYS, Duration::from_millis(1)); - let mut d = Duration::ZERO; - for _ in 0..TOTAL_DELAYS { - d += delays.next().unwrap(); - } - assert!(delays.next().is_none()); - assert_eq!(d, Duration::from_millis(1)); - } -} diff --git a/mixnet/node/Cargo.toml b/mixnet/node/Cargo.toml deleted file mode 100644 index f1fed044..00000000 --- a/mixnet/node/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "mixnet-node" -version = "0.1.0" -edition = "2021" - -[dependencies] -serde = { version = "1.0", features = ["derive"] } -tracing = "0.1.37" -tokio = { version = "1.32", features = ["net", "time", "signal"] } -thiserror = "1" -sphinx-packet = "0.1.0" -nym-sphinx = { package = "nym-sphinx", git = "https://github.com/nymtech/nym", tag = "v1.1.22" } -mixnet-protocol = { path = "../protocol" } -mixnet-topology = { path = "../topology" } -mixnet-util = { path = "../util" } - -[dev-dependencies] -tokio = {version = "1.32", features =["full"]} \ No newline at end of file diff --git a/mixnet/node/src/client_notifier.rs b/mixnet/node/src/client_notifier.rs deleted file mode 100644 index 2a277689..00000000 --- a/mixnet/node/src/client_notifier.rs +++ /dev/null @@ -1,49 +0,0 @@ -use std::net::SocketAddr; - -use mixnet_protocol::Body; -use tokio::{ - net::{TcpListener, TcpStream}, - sync::mpsc, -}; - -pub struct ClientNotifier {} - -impl ClientNotifier { - pub async fn run( - listen_address: SocketAddr, - mut rx: mpsc::Receiver, - ) -> super::Result<()> { - let listener = TcpListener::bind(listen_address) - .await - .map_err(super::ProtocolError::IO)?; - tracing::info!("Listening mixnet client connections: {listen_address}"); - - // Currently, handling only a single incoming connection - // TODO: consider handling multiple clients - loop { - match listener.accept().await { - Ok((socket, remote_addr)) => { - tracing::debug!("Accepted incoming client connection from {remote_addr}"); - - if let Err(e) = Self::handle_connection(socket, &mut rx).await { - tracing::error!("failed to handle conn: {e}"); - } - } - Err(e) => tracing::warn!("Failed to accept incoming client connection: {e}"), - } - } - } - - async fn handle_connection( - mut socket: TcpStream, - rx: &mut mpsc::Receiver, - ) -> super::Result<()> { - while let Some(body) = rx.recv().await { - if let Err(e) = body.write(&mut socket).await { - return Err(super::MixnetNodeError::Client(e)); - } - } - tracing::debug!("body receiver closed"); - Ok(()) - } -} diff --git a/mixnet/node/src/config.rs b/mixnet/node/src/config.rs deleted file mode 100644 index ae33f603..00000000 --- a/mixnet/node/src/config.rs +++ /dev/null @@ -1,61 +0,0 @@ -use std::{ - net::{Ipv4Addr, SocketAddr, SocketAddrV4}, - time::Duration, -}; - -use nym_sphinx::{PrivateKey, PublicKey}; -use serde::{Deserialize, Serialize}; -use sphinx_packet::crypto::{PRIVATE_KEY_SIZE, PUBLIC_KEY_SIZE}; - -#[derive(Serialize, Deserialize, Copy, Clone, Debug)] -pub struct MixnetNodeConfig { - /// A listen address for receiving Sphinx packets - pub listen_address: SocketAddr, - /// An listen address for communicating with mixnet clients - pub client_listen_address: SocketAddr, - /// A key for decrypting Sphinx packets - pub private_key: [u8; PRIVATE_KEY_SIZE], - /// The size of the connection pool. - #[serde(default = "MixnetNodeConfig::default_connection_pool_size")] - pub connection_pool_size: usize, - /// The maximum number of retries. - #[serde(default = "MixnetNodeConfig::default_max_retries")] - pub max_retries: usize, - /// The retry delay between retries. - #[serde(default = "MixnetNodeConfig::default_retry_delay")] - pub retry_delay: Duration, -} - -impl Default for MixnetNodeConfig { - fn default() -> Self { - Self { - listen_address: SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(127, 0, 0, 1), 7777)), - client_listen_address: SocketAddr::V4(SocketAddrV4::new( - Ipv4Addr::new(127, 0, 0, 1), - 7778, - )), - private_key: PrivateKey::new().to_bytes(), - connection_pool_size: 255, - max_retries: 3, - retry_delay: Duration::from_secs(5), - } - } -} - -impl MixnetNodeConfig { - const fn default_connection_pool_size() -> usize { - 255 - } - - const fn default_max_retries() -> usize { - 3 - } - - const fn default_retry_delay() -> Duration { - Duration::from_secs(5) - } - - pub fn public_key(&self) -> [u8; PUBLIC_KEY_SIZE] { - *PublicKey::from(&PrivateKey::from(self.private_key)).as_bytes() - } -} diff --git a/mixnet/node/src/lib.rs b/mixnet/node/src/lib.rs deleted file mode 100644 index f79525ca..00000000 --- a/mixnet/node/src/lib.rs +++ /dev/null @@ -1,313 +0,0 @@ -mod client_notifier; -pub mod config; - -use std::{collections::HashMap, net::SocketAddr, time::Duration}; - -use client_notifier::ClientNotifier; -pub use config::MixnetNodeConfig; -use mixnet_protocol::{Body, ProtocolError}; -use mixnet_topology::MixnetNodeId; -use nym_sphinx::{ - addressing::nodes::{NymNodeRoutingAddress, NymNodeRoutingAddressError}, - Delay, DestinationAddressBytes, NodeAddressBytes, PrivateKey, -}; -pub use sphinx_packet::crypto::PRIVATE_KEY_SIZE; -use sphinx_packet::{crypto::PUBLIC_KEY_SIZE, ProcessedPacket, SphinxPacket}; -use tokio::{ - net::{TcpListener, TcpStream}, - sync::mpsc, -}; - -pub type Result = core::result::Result; - -#[derive(Debug, thiserror::Error)] -pub enum MixnetNodeError { - #[error("{0}")] - Protocol(#[from] ProtocolError), - #[error("invalid routing address: {0}")] - InvalidRoutingAddress(#[from] NymNodeRoutingAddressError), - #[error("send error: {0}")] - PacketSendError(#[from] tokio::sync::mpsc::error::SendError), - #[error("send error: fail to send {0} to client")] - ClientSendError(#[from] tokio::sync::mpsc::error::TrySendError), - #[error("client: {0}")] - Client(ProtocolError), -} - -// A mix node that routes packets in the Mixnet. -pub struct MixnetNode { - config: MixnetNodeConfig, -} - -impl MixnetNode { - pub fn new(config: MixnetNodeConfig) -> Self { - Self { config } - } - - pub fn id(&self) -> MixnetNodeId { - self.public_key() - } - - pub fn public_key(&self) -> [u8; PUBLIC_KEY_SIZE] { - self.config.public_key() - } - - const CLIENT_NOTI_CHANNEL_SIZE: usize = 100; - - pub async fn run(self) -> Result<()> { - tracing::info!("Public key: {:?}", self.public_key()); - - // Spawn a ClientNotifier - let (client_tx, client_rx) = mpsc::channel(Self::CLIENT_NOTI_CHANNEL_SIZE); - tokio::spawn(async move { - if let Err(e) = ClientNotifier::run(self.config.client_listen_address, client_rx).await - { - tracing::error!("failed to run client notifier: {e}"); - } - }); - - //TODO: Accepting ad-hoc TCP conns for now. Improve conn handling. - let listener = TcpListener::bind(self.config.listen_address) - .await - .map_err(ProtocolError::IO)?; - tracing::info!( - "Listening mixnet node connections: {}", - self.config.listen_address - ); - - let (tx, rx) = mpsc::unbounded_channel(); - - let packet_forwarder = PacketForwarder::new(tx.clone(), rx, self.config); - - tokio::spawn(async move { - packet_forwarder.run().await; - }); - - let runner = MixnetNodeRunner { - config: self.config, - client_tx, - packet_tx: tx, - }; - - loop { - tokio::select! { - res = listener.accept() => { - match res { - Ok((socket, remote_addr)) => { - tracing::debug!("Accepted incoming connection from {remote_addr:?}"); - - let runner = runner.clone(); - tokio::spawn(async move { - if let Err(e) = runner.handle_connection(socket).await { - tracing::error!("failed to handle conn: {e}"); - } - }); - } - Err(e) => tracing::warn!("Failed to accept incoming connection: {e}"), - } - } - _ = tokio::signal::ctrl_c() => { - tracing::info!("Shutting down..."); - return Ok(()); - } - } - } - } -} - -#[derive(Clone)] -struct MixnetNodeRunner { - config: MixnetNodeConfig, - client_tx: mpsc::Sender, - packet_tx: mpsc::UnboundedSender, -} - -impl MixnetNodeRunner { - async fn handle_connection(&self, mut socket: TcpStream) -> Result<()> { - loop { - let body = Body::read(&mut socket).await?; - let this = self.clone(); - tokio::spawn(async move { - if let Err(e) = this.handle_body(body).await { - tracing::error!("failed to handle body: {e}"); - } - }); - } - } - - async fn handle_body(&self, pkt: Body) -> Result<()> { - match pkt { - Body::SphinxPacket(packet) => self.handle_sphinx_packet(packet).await, - Body::FinalPayload(payload) => { - self.forward_body_to_client_notifier(Body::FinalPayload(payload)) - .await - } - _ => unreachable!(), - } - } - - async fn handle_sphinx_packet(&self, packet: Box) -> Result<()> { - match packet - .process(&PrivateKey::from(self.config.private_key)) - .map_err(ProtocolError::InvalidSphinxPacket)? - { - ProcessedPacket::ForwardHop(packet, next_node_addr, delay) => { - self.forward_packet_to_next_hop(Body::SphinxPacket(packet), next_node_addr, delay) - .await - } - ProcessedPacket::FinalHop(destination_addr, _, payload) => { - self.forward_payload_to_destination(Body::FinalPayload(payload), destination_addr) - .await - } - } - } - - async fn forward_body_to_client_notifier(&self, body: Body) -> Result<()> { - // TODO: Decrypt the final payload using the private key, if it's encrypted - - // Do not wait when the channel is full or no receiver exists - self.client_tx.try_send(body)?; - Ok(()) - } - - async fn forward_packet_to_next_hop( - &self, - packet: Body, - next_node_addr: NodeAddressBytes, - delay: Delay, - ) -> Result<()> { - tracing::debug!("Delaying the packet for {delay:?}"); - tokio::time::sleep(delay.to_duration()).await; - - self.forward(packet, NymNodeRoutingAddress::try_from(next_node_addr)?) - .await - } - - async fn forward_payload_to_destination( - &self, - payload: Body, - destination_addr: DestinationAddressBytes, - ) -> Result<()> { - tracing::debug!("Forwarding final payload to destination mixnode"); - - self.forward( - payload, - NymNodeRoutingAddress::try_from_bytes(&destination_addr.as_bytes())?, - ) - .await - } - - async fn forward(&self, pkt: Body, to: NymNodeRoutingAddress) -> Result<()> { - let addr = SocketAddr::from(to); - - self.packet_tx.send(Packet::new(addr, pkt))?; - Ok(()) - } -} - -struct PacketForwarder { - config: MixnetNodeConfig, - packet_rx: mpsc::UnboundedReceiver, - packet_tx: mpsc::UnboundedSender, - connections: HashMap, -} - -impl PacketForwarder { - pub fn new( - packet_tx: mpsc::UnboundedSender, - packet_rx: mpsc::UnboundedReceiver, - config: MixnetNodeConfig, - ) -> Self { - Self { - packet_tx, - packet_rx, - connections: HashMap::with_capacity(config.connection_pool_size), - config, - } - } - - pub async fn run(mut self) { - loop { - tokio::select! { - pkt = self.packet_rx.recv() => { - if let Some(pkt) = pkt { - self.send(pkt).await; - } else { - unreachable!("Packet channel should not be closed, because PacketForwarder is also holding the send half"); - } - }, - _ = tokio::signal::ctrl_c() => { - tracing::info!("Shutting down packet forwarder task..."); - return; - } - } - } - } - - async fn try_send(&mut self, target: SocketAddr, body: &Body) -> Result<()> { - if let std::collections::hash_map::Entry::Vacant(e) = self.connections.entry(target) { - match TcpStream::connect(target).await { - Ok(tcp) => { - e.insert(tcp); - } - Err(e) => { - tracing::error!("failed to connect to {}: {e}", target); - return Err(MixnetNodeError::Protocol(e.into())); - } - } - } - Ok(body - .write(self.connections.get_mut(&target).unwrap()) - .await?) - } - - async fn send(&mut self, pkt: Packet) { - if let Err(err) = self.try_send(pkt.target, &pkt.body).await { - match err { - MixnetNodeError::Protocol(ProtocolError::IO(e)) - if e.kind() == std::io::ErrorKind::Unsupported => - { - tracing::error!("fail to send message to {}: {e}", pkt.target); - } - _ => self.handle_retry(pkt), - } - } - } - - fn handle_retry(&self, mut pkt: Packet) { - if pkt.retry_count < self.config.max_retries { - let delay = Duration::from_millis( - (self.config.retry_delay.as_millis() as u64).pow(pkt.retry_count as u32), - ); - let tx = self.packet_tx.clone(); - tokio::spawn(async move { - tokio::time::sleep(delay).await; - pkt.retry_count += 1; - if let Err(e) = tx.send(pkt) { - tracing::error!("fail to enqueue retry message: {e}"); - } - }); - } else { - tracing::error!( - "fail to send message to {}: reach maximum retries", - pkt.target - ); - } - } -} - -pub struct Packet { - target: SocketAddr, - body: Body, - retry_count: usize, -} - -impl Packet { - fn new(target: SocketAddr, body: Body) -> Self { - Self { - target, - body, - retry_count: 0, - } - } -} diff --git a/mixnet/protocol/Cargo.toml b/mixnet/protocol/Cargo.toml deleted file mode 100644 index d0673195..00000000 --- a/mixnet/protocol/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "mixnet-protocol" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -tokio = { version = "1.32", features = ["sync", "net", "time"] } -sphinx-packet = "0.1.0" -futures = "0.3" -tokio-util = { version = "0.7", features = ["io", "io-util"] } -thiserror = "1" diff --git a/mixnet/protocol/src/lib.rs b/mixnet/protocol/src/lib.rs deleted file mode 100644 index af6fdf7c..00000000 --- a/mixnet/protocol/src/lib.rs +++ /dev/null @@ -1,149 +0,0 @@ -use sphinx_packet::{payload::Payload, SphinxPacket}; - -use std::{io::ErrorKind, net::SocketAddr, sync::Arc, time::Duration}; - -use tokio::{ - io::{self, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}, - net::TcpStream, - sync::Mutex, -}; - -pub type Result = core::result::Result; - -#[derive(Debug, thiserror::Error)] -pub enum ProtocolError { - #[error("Unknown body type {0}")] - UnknownBodyType(u8), - #[error("{0}")] - InvalidSphinxPacket(sphinx_packet::Error), - #[error("{0}")] - InvalidPayload(sphinx_packet::Error), - #[error("{0}")] - IO(#[from] io::Error), - #[error("fail to send packet, reach maximum retries {0}")] - ReachMaxRetries(usize), -} - -#[non_exhaustive] -pub enum Body { - SphinxPacket(Box), - FinalPayload(Payload), -} - -impl Body { - pub fn new_sphinx(packet: Box) -> Self { - Self::SphinxPacket(packet) - } - - pub fn new_final_payload(payload: Payload) -> Self { - Self::FinalPayload(payload) - } - - fn variant_as_u8(&self) -> u8 { - match self { - Self::SphinxPacket(_) => 0, - Self::FinalPayload(_) => 1, - } - } - - pub async fn read(reader: &mut R) -> Result - where - R: AsyncRead + Unpin, - { - let id = reader.read_u8().await?; - match id { - 0 => Self::read_sphinx_packet(reader).await, - 1 => Self::read_final_payload(reader).await, - id => Err(ProtocolError::UnknownBodyType(id)), - } - } - - fn sphinx_packet_from_bytes(data: &[u8]) -> Result { - SphinxPacket::from_bytes(data) - .map(|packet| Self::new_sphinx(Box::new(packet))) - .map_err(ProtocolError::InvalidPayload) - } - - async fn read_sphinx_packet(reader: &mut R) -> Result - where - R: AsyncRead + Unpin, - { - let size = reader.read_u64().await?; - let mut buf = vec![0; size as usize]; - reader.read_exact(&mut buf).await?; - Self::sphinx_packet_from_bytes(&buf) - } - - pub fn final_payload_from_bytes(data: &[u8]) -> Result { - Payload::from_bytes(data) - .map(Self::new_final_payload) - .map_err(ProtocolError::InvalidPayload) - } - - async fn read_final_payload(reader: &mut R) -> Result - where - R: AsyncRead + Unpin, - { - let size = reader.read_u64().await?; - let mut buf = vec![0; size as usize]; - reader.read_exact(&mut buf).await?; - - Self::final_payload_from_bytes(&buf) - } - - pub async fn write(&self, writer: &mut W) -> Result<()> - where - W: AsyncWrite + Unpin + ?Sized, - { - let variant = self.variant_as_u8(); - writer.write_u8(variant).await?; - match self { - Self::SphinxPacket(packet) => { - let data = packet.to_bytes(); - writer.write_u64(data.len() as u64).await?; - writer.write_all(&data).await?; - } - Self::FinalPayload(payload) => { - let data = payload.as_bytes(); - writer.write_u64(data.len() as u64).await?; - writer.write_all(data).await?; - } - } - Ok(()) - } -} - -pub async fn retry_backoff( - peer_addr: SocketAddr, - max_retries: usize, - retry_delay: Duration, - body: Body, - socket: Arc>, -) -> Result<()> { - for idx in 0..max_retries { - // backoff - let wait = Duration::from_millis((retry_delay.as_millis() as u64).pow(idx as u32)); - tokio::time::sleep(wait).await; - let mut socket = socket.lock().await; - match body.write(&mut *socket).await { - Ok(_) => return Ok(()), - Err(e) => { - match &e { - ProtocolError::IO(err) => { - match err.kind() { - ErrorKind::Unsupported => return Err(e), - _ => { - // update the connection - if let Ok(tcp) = TcpStream::connect(peer_addr).await { - *socket = tcp; - } - } - } - } - _ => return Err(e), - } - } - } - } - Err(ProtocolError::ReachMaxRetries(max_retries)) -} diff --git a/mixnet/src/address.rs b/mixnet/src/address.rs new file mode 100644 index 00000000..faae16d8 --- /dev/null +++ b/mixnet/src/address.rs @@ -0,0 +1,53 @@ +use std::net::SocketAddr; + +use nym_sphinx_addressing::nodes::NymNodeRoutingAddress; +use serde::{Deserialize, Serialize}; + +use crate::error::MixnetError; + +/// Represents an address of mix node. +/// +/// This just contains a single [`SocketAddr`], but has conversion functions +/// for various address types defined in the `sphinx-packet` crate. +#[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq)] +pub struct NodeAddress(SocketAddr); + +impl From for NodeAddress { + fn from(address: SocketAddr) -> Self { + Self(address) + } +} + +impl From for SocketAddr { + fn from(address: NodeAddress) -> Self { + address.0 + } +} + +impl TryInto for NodeAddress { + type Error = MixnetError; + + fn try_into(self) -> Result { + Ok(NymNodeRoutingAddress::from(SocketAddr::from(self)).try_into()?) + } +} + +impl TryFrom for NodeAddress { + type Error = MixnetError; + + fn try_from(value: sphinx_packet::route::NodeAddressBytes) -> Result { + Ok(Self::from(SocketAddr::from( + NymNodeRoutingAddress::try_from(value)?, + ))) + } +} + +impl TryFrom for NodeAddress { + type Error = MixnetError; + + fn try_from(value: sphinx_packet::route::DestinationAddressBytes) -> Result { + Self::try_from(sphinx_packet::route::NodeAddressBytes::from_bytes( + value.as_bytes(), + )) + } +} diff --git a/mixnet/src/client.rs b/mixnet/src/client.rs new file mode 100644 index 00000000..dac6327e --- /dev/null +++ b/mixnet/src/client.rs @@ -0,0 +1,183 @@ +use std::{collections::VecDeque, num::NonZeroU8}; + +use rand::rngs::OsRng; +use serde::{Deserialize, Serialize}; +use tokio::sync::mpsc; + +use crate::{error::MixnetError, packet::Packet, poisson::Poisson, topology::MixnetTopology}; + +/// Mix client implementation that is used to schedule messages to be sent to the mixnet. +/// Messages inserted to the [`MessageQueue`] are scheduled according to the Poisson interals +/// and returns from [`MixClient.next()`] when it is ready to be sent to the mixnet. +/// If there is no messages inserted to the [`MessageQueue`], cover packets are generated and +/// returned from [`MixClient.next()`]. +pub struct MixClient { + packet_rx: mpsc::UnboundedReceiver, +} + +struct MixClientRunner { + config: MixClientConfig, + poisson: Poisson, + message_queue: mpsc::Receiver>, + real_packet_queue: VecDeque, + packet_tx: mpsc::UnboundedSender, +} + +/// Mix client configuration +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct MixClientConfig { + /// Mixnet topology + pub topology: MixnetTopology, + /// Poisson rate for packet emissions (per minute) + pub emission_rate_per_min: f64, + /// Packet redundancy for passive retransmission + pub redundancy: NonZeroU8, +} + +const MESSAGE_QUEUE_SIZE: usize = 256; + +/// Queue for sending messages to [`MixClient`] +pub type MessageQueue = mpsc::Sender>; + +impl MixClient { + /// Creates a [`MixClient`] and a [`MessageQueue`]. + /// + /// This returns [`MixnetError`] if the given `config` is invalid. + pub fn new(config: MixClientConfig) -> Result<(Self, MessageQueue), MixnetError> { + let poisson = Poisson::new(config.emission_rate_per_min)?; + let (tx, rx) = mpsc::channel(MESSAGE_QUEUE_SIZE); + let (packet_tx, packet_rx) = mpsc::unbounded_channel(); + + MixClientRunner { + config, + poisson, + message_queue: rx, + real_packet_queue: VecDeque::new(), + packet_tx, + } + .run(); + + Ok((Self { packet_rx }, tx)) + } + + /// Returns a next [`Packet`] to be emitted, if it exists and the Poisson timer is done. + pub async fn next(&mut self) -> Option { + self.packet_rx.recv().await + } +} + +impl MixClientRunner { + fn run(mut self) { + tokio::spawn(async move { + let mut delay = tokio::time::sleep(self.poisson.interval(&mut OsRng)); + loop { + let next_deadline = delay.deadline() + self.poisson.interval(&mut OsRng); + delay.await; + delay = tokio::time::sleep_until(next_deadline); + + match self.next_packet().await { + Ok(packet) => { + // packet_tx is always expected to be not closed/dropped. + self.packet_tx.send(packet).unwrap(); + } + Err(e) => { + tracing::error!( + "failed to find a next packet to emit. skipping to the next turn: {e}" + ); + } + } + } + }); + } + + const DROP_COVER_MSG: &'static [u8] = b"drop cover"; + + async fn next_packet(&mut self) -> Result { + if let Some(packet) = self.real_packet_queue.pop_front() { + return Ok(packet); + } + + match self.message_queue.try_recv() { + Ok(msg) => { + for packet in Packet::build_real(msg, &self.config.topology)? { + for _ in 0..self.config.redundancy.get() { + self.real_packet_queue.push_back(packet.clone()); + } + } + Ok(self + .real_packet_queue + .pop_front() + .expect("real packet queue should not be empty")) + } + Err(_) => { + let mut packets = Packet::build_drop_cover( + Vec::from(Self::DROP_COVER_MSG), + &self.config.topology, + )?; + Ok(packets.pop().expect("drop cover should not be empty")) + } + } + } +} + +#[cfg(test)] +mod tests { + use std::{num::NonZeroU8, time::Instant}; + + use crate::{ + client::MixClientConfig, + topology::{ + tests::{gen_entropy, gen_mixnodes}, + MixnetTopology, + }, + }; + + use super::MixClient; + + #[tokio::test] + async fn poisson_emission() { + let emission_rate_per_min = 60.0; + let (mut client, _) = MixClient::new(MixClientConfig { + topology: MixnetTopology::new(gen_mixnodes(10), 3, 2, gen_entropy()).unwrap(), + emission_rate_per_min, + redundancy: NonZeroU8::new(3).unwrap(), + }) + .unwrap(); + + let mut ts = Instant::now(); + let mut intervals = Vec::new(); + for _ in 0..30 { + assert!(client.next().await.is_some()); + let now = Instant::now(); + intervals.push(now - ts); + ts = now; + } + + let avg_sec = intervals.iter().map(|d| d.as_secs()).sum::() / intervals.len() as u64; + let expected_avg_sec = (60.0 / emission_rate_per_min) as u64; + assert!( + avg_sec.abs_diff(expected_avg_sec) <= 1, + "{avg_sec} -{expected_avg_sec}" + ); + } + + #[tokio::test] + async fn real_packet_emission() { + let (mut client, msg_queue) = MixClient::new(MixClientConfig { + topology: MixnetTopology::new(gen_mixnodes(10), 3, 2, gen_entropy()).unwrap(), + emission_rate_per_min: 360.0, + redundancy: NonZeroU8::new(3).unwrap(), + }) + .unwrap(); + + msg_queue.send("hello".as_bytes().into()).await.unwrap(); + + // Check if the next 3 packets are the same, according to the redundancy + let packet = client.next().await.unwrap(); + assert_eq!(packet, client.next().await.unwrap()); + assert_eq!(packet, client.next().await.unwrap()); + + // Check if the next packet is different (drop cover) + assert_ne!(packet, client.next().await.unwrap()); + } +} diff --git a/mixnet/src/error.rs b/mixnet/src/error.rs new file mode 100644 index 00000000..627dde28 --- /dev/null +++ b/mixnet/src/error.rs @@ -0,0 +1,34 @@ +/// Mixnet Errors +#[derive(thiserror::Error, Debug)] +pub enum MixnetError { + /// Invalid topology size + #[error("invalid mixnet topology size")] + InvalidTopologySize, + /// Invalid packet flag + #[error("invalid packet flag")] + InvalidPacketFlag, + /// Invalid fragment header + #[error("invalid fragment header")] + InvalidFragmentHeader, + /// Invalid fragment set ID + #[error("invalid fragment set ID: {0}")] + InvalidFragmentSetId(#[from] uuid::Error), + /// Invalid fragment ID + #[error("invalid fragment ID")] + InvalidFragmentId, + /// Message too long + #[error("message too long: {0} bytes")] + MessageTooLong(usize), + /// Invalid message + #[error("invalid message")] + InvalidMessage, + /// Node address error + #[error("node address error: {0}")] + NodeAddressError(#[from] nym_sphinx_addressing::nodes::NymNodeRoutingAddressError), + /// Sphinx packet error + #[error("sphinx packet error: {0}")] + SphinxPacketError(#[from] sphinx_packet::Error), + /// Exponential distribution error + #[error("exponential distribution error: {0}")] + ExponentialError(#[from] rand_distr::ExpError), +} diff --git a/mixnet/src/fragment.rs b/mixnet/src/fragment.rs new file mode 100644 index 00000000..5799eefc --- /dev/null +++ b/mixnet/src/fragment.rs @@ -0,0 +1,280 @@ +use std::collections::HashMap; + +use sphinx_packet::{constants::PAYLOAD_SIZE, payload::PAYLOAD_OVERHEAD_SIZE}; +use uuid::Uuid; + +use crate::error::MixnetError; + +pub(crate) struct FragmentSet(Vec); + +impl FragmentSet { + const MAX_PLAIN_PAYLOAD_SIZE: usize = PAYLOAD_SIZE - PAYLOAD_OVERHEAD_SIZE; + const CHUNK_SIZE: usize = Self::MAX_PLAIN_PAYLOAD_SIZE - FragmentHeader::SIZE; + + pub(crate) fn new(msg: &[u8]) -> Result { + // For now, we don't support more than `u8::MAX + 1` fragments. + // If needed, we can devise the FragmentSet chaining to support larger messages, like Nym. + let last_fragment_id = FragmentId::try_from(Self::num_chunks(msg) - 1) + .map_err(|_| MixnetError::MessageTooLong(msg.len()))?; + let set_id = FragmentSetId::new(); + + Ok(FragmentSet( + msg.chunks(Self::CHUNK_SIZE) + .enumerate() + .map(|(i, chunk)| Fragment { + header: FragmentHeader { + set_id, + last_fragment_id, + fragment_id: FragmentId::try_from(i) + .expect("i is always in the right range"), + }, + body: Vec::from(chunk), + }) + .collect(), + )) + } + + fn num_chunks(msg: &[u8]) -> usize { + msg.len().div_ceil(Self::CHUNK_SIZE) + } +} + +impl AsRef> for FragmentSet { + fn as_ref(&self) -> &Vec { + &self.0 + } +} + +#[derive(PartialEq, Eq, Debug, Clone)] +pub(crate) struct Fragment { + header: FragmentHeader, + body: Vec, +} + +impl Fragment { + pub(crate) fn bytes(&self) -> Vec { + let mut out = Vec::with_capacity(FragmentHeader::SIZE + self.body.len()); + out.extend(self.header.bytes()); + out.extend(&self.body); + out + } + + pub(crate) fn from_bytes(value: &[u8]) -> Result { + Ok(Self { + header: FragmentHeader::from_bytes(&value[0..FragmentHeader::SIZE])?, + body: value[FragmentHeader::SIZE..].to_vec(), + }) + } +} + +#[derive(PartialEq, Eq, Hash, Debug, Clone, Copy)] +struct FragmentSetId(Uuid); + +impl FragmentSetId { + const SIZE: usize = 16; + + fn new() -> Self { + Self(Uuid::new_v4()) + } +} + +#[derive(PartialEq, Eq, Hash, Debug, Clone, Copy)] +struct FragmentId(u8); + +impl FragmentId { + const SIZE: usize = std::mem::size_of::(); +} + +impl TryFrom for FragmentId { + type Error = MixnetError; + + fn try_from(id: usize) -> Result { + if id > u8::MAX as usize { + return Err(MixnetError::InvalidFragmentId); + } + Ok(Self(id as u8)) + } +} + +impl From for usize { + fn from(id: FragmentId) -> Self { + id.0 as usize + } +} + +#[derive(PartialEq, Eq, Debug, Clone)] +struct FragmentHeader { + set_id: FragmentSetId, + last_fragment_id: FragmentId, + fragment_id: FragmentId, +} + +impl FragmentHeader { + const SIZE: usize = FragmentSetId::SIZE + 2 * FragmentId::SIZE; + + fn bytes(&self) -> [u8; Self::SIZE] { + let mut out = [0u8; Self::SIZE]; + out[0..FragmentSetId::SIZE].copy_from_slice(self.set_id.0.as_bytes()); + out[FragmentSetId::SIZE] = self.last_fragment_id.0; + out[FragmentSetId::SIZE + FragmentId::SIZE] = self.fragment_id.0; + out + } + + fn from_bytes(value: &[u8]) -> Result { + if value.len() != Self::SIZE { + return Err(MixnetError::InvalidFragmentHeader); + } + + Ok(Self { + set_id: FragmentSetId(Uuid::from_slice(&value[0..FragmentSetId::SIZE])?), + last_fragment_id: FragmentId(value[FragmentSetId::SIZE]), + fragment_id: FragmentId(value[FragmentSetId::SIZE + FragmentId::SIZE]), + }) + } +} + +pub struct MessageReconstructor { + fragment_sets: HashMap, +} + +impl MessageReconstructor { + pub fn new() -> Self { + Self { + fragment_sets: HashMap::new(), + } + } + + /// Adds a fragment to the reconstructor and tries to reconstruct a message from the fragment set. + /// This returns `None` if the message has not been reconstructed yet. + pub fn add_and_reconstruct(&mut self, fragment: Fragment) -> Option> { + let set_id = fragment.header.set_id; + let reconstructed_msg = self + .fragment_sets + .entry(set_id) + .or_insert(FragmentSetReconstructor::new( + fragment.header.last_fragment_id, + )) + .add(fragment) + .try_reconstruct_message()?; + // A message has been reconstructed completely from the fragment set. + // Delete the fragment set from the reconstructor. + self.fragment_sets.remove(&set_id); + Some(reconstructed_msg) + } +} + +struct FragmentSetReconstructor { + last_fragment_id: FragmentId, + fragments: HashMap, + // For mem optimization, accumulates the expected message size + // whenever a new fragment is added to the `fragments`. + message_size: usize, +} + +impl FragmentSetReconstructor { + fn new(last_fragment_id: FragmentId) -> Self { + Self { + last_fragment_id, + fragments: HashMap::new(), + message_size: 0, + } + } + + fn add(&mut self, fragment: Fragment) -> &mut Self { + self.message_size += fragment.body.len(); + if let Some(old_fragment) = self.fragments.insert(fragment.header.fragment_id, fragment) { + // In the case when a new fragment replaces the old one, adjust the `meesage_size`. + // e.g. The same fragment has been received multiple times. + self.message_size -= old_fragment.body.len(); + } + self + } + + /// Merges all fragments gathered if possible + fn try_reconstruct_message(&self) -> Option> { + (self.fragments.len() - 1 == self.last_fragment_id.into()).then(|| { + let mut msg = Vec::with_capacity(self.message_size); + for id in 0..=self.last_fragment_id.0 { + msg.extend(&self.fragments.get(&FragmentId(id)).unwrap().body); + } + msg + }) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + + use rand::RngCore; + + use super::*; + + #[test] + fn fragment_header() { + let header = FragmentHeader { + set_id: FragmentSetId::new(), + last_fragment_id: FragmentId(19), + fragment_id: FragmentId(0), + }; + let bz = header.bytes(); + assert_eq!(FragmentHeader::SIZE, bz.len()); + assert_eq!(header, FragmentHeader::from_bytes(bz.as_slice()).unwrap()); + } + + #[test] + fn fragment() { + let fragment = Fragment { + header: FragmentHeader { + set_id: FragmentSetId::new(), + last_fragment_id: FragmentId(19), + fragment_id: FragmentId(0), + }, + body: vec![1, 2, 3, 4], + }; + let bz = fragment.bytes(); + assert_eq!(FragmentHeader::SIZE + fragment.body.len(), bz.len()); + assert_eq!(fragment, Fragment::from_bytes(bz.as_slice()).unwrap()); + } + + #[test] + fn fragment_set() { + let mut msg = vec![0u8; FragmentSet::CHUNK_SIZE * 3 + FragmentSet::CHUNK_SIZE / 2]; + rand::thread_rng().fill_bytes(&mut msg); + + assert_eq!(4, FragmentSet::num_chunks(&msg)); + + let set = FragmentSet::new(&msg).unwrap(); + assert_eq!(4, set.as_ref().iter().len()); + assert_eq!( + 1, + HashSet::::from_iter( + set.as_ref().iter().map(|fragment| fragment.header.set_id) + ) + .len() + ); + set.as_ref() + .iter() + .enumerate() + .for_each(|(i, fragment)| assert_eq!(i, fragment.header.fragment_id.0 as usize)); + } + + #[test] + fn message_reconstructor() { + let mut msg = vec![0u8; FragmentSet::CHUNK_SIZE * 2]; + rand::thread_rng().fill_bytes(&mut msg); + + let set = FragmentSet::new(&msg).unwrap(); + + let mut reconstructor = MessageReconstructor::new(); + let mut fragments = set.as_ref().iter(); + assert_eq!( + None, + reconstructor.add_and_reconstruct(fragments.next().unwrap().clone()) + ); + assert_eq!( + Some(msg), + reconstructor.add_and_reconstruct(fragments.next().unwrap().clone()) + ); + } +} diff --git a/mixnet/src/lib.rs b/mixnet/src/lib.rs new file mode 100644 index 00000000..c835d847 --- /dev/null +++ b/mixnet/src/lib.rs @@ -0,0 +1,17 @@ +//! Mixnet +// #![deny(missing_docs, warnings)] +// #![forbid(unsafe_code)] + +/// Mix node address +pub mod address; +/// Mix client +pub mod client; +/// Mixnet errors +pub mod error; +mod fragment; +/// Mix node +pub mod node; +pub mod packet; +mod poisson; +/// Mixnet topology +pub mod topology; diff --git a/mixnet/src/node.rs b/mixnet/src/node.rs new file mode 100644 index 00000000..5bd6903a --- /dev/null +++ b/mixnet/src/node.rs @@ -0,0 +1,191 @@ +use rand::rngs::OsRng; +use serde::{Deserialize, Serialize}; +use sphinx_packet::crypto::{PrivateKey, PRIVATE_KEY_SIZE}; +use tokio::sync::mpsc; + +use crate::{ + error::MixnetError, + fragment::{Fragment, MessageReconstructor}, + packet::{Message, Packet, PacketBody}, + poisson::Poisson, +}; + +/// Mix node implementation that returns Sphinx packets which needs to be forwarded to next mix nodes, +/// or messages reconstructed from Sphinx packets delivered through all mix layers. +pub struct MixNode { + output_rx: mpsc::UnboundedReceiver, +} + +struct MixNodeRunner { + _config: MixNodeConfig, + encryption_private_key: PrivateKey, + poisson: Poisson, + packet_queue: mpsc::Receiver, + message_reconstructor: MessageReconstructor, + output_tx: mpsc::UnboundedSender, +} + +/// Mix node configuration +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct MixNodeConfig { + /// Private key for decrypting Sphinx packets + pub encryption_private_key: [u8; PRIVATE_KEY_SIZE], + /// Poisson delay rate per minutes + pub delay_rate_per_min: f64, +} + +const PACKET_QUEUE_SIZE: usize = 256; + +/// Queue for sending packets to [`MixNode`] +pub type PacketQueue = mpsc::Sender; + +impl MixNode { + /// Creates a [`MixNode`] and a [`PacketQueue`]. + /// + /// This returns [`MixnetError`] if the given `config` is invalid. + pub fn new(config: MixNodeConfig) -> Result<(Self, PacketQueue), MixnetError> { + let encryption_private_key = PrivateKey::from(config.encryption_private_key); + let poisson = Poisson::new(config.delay_rate_per_min)?; + let (packet_tx, packet_rx) = mpsc::channel(PACKET_QUEUE_SIZE); + let (output_tx, output_rx) = mpsc::unbounded_channel(); + + let mixnode_runner = MixNodeRunner { + _config: config, + encryption_private_key, + poisson, + packet_queue: packet_rx, + message_reconstructor: MessageReconstructor::new(), + output_tx, + }; + tokio::spawn(mixnode_runner.run()); + + Ok((Self { output_rx }, packet_tx)) + } + + /// Returns a next `[Output]` to be emitted, if it exists and the Poisson delay is done (if necessary). + pub async fn next(&mut self) -> Option { + self.output_rx.recv().await + } +} + +impl MixNodeRunner { + async fn run(mut self) { + loop { + if let Some(packet) = self.packet_queue.recv().await { + if let Err(e) = self.process_packet(packet) { + tracing::error!("failed to process packet. skipping it: {e}"); + } + } + } + } + + fn process_packet(&mut self, packet: PacketBody) -> Result<(), MixnetError> { + match packet { + PacketBody::SphinxPacket(packet) => self.process_sphinx_packet(packet.as_ref()), + PacketBody::Fragment(fragment) => self.process_fragment(fragment.as_ref()), + } + } + + fn process_sphinx_packet(&self, packet: &[u8]) -> Result<(), MixnetError> { + let output = Output::Forward(PacketBody::process_sphinx_packet( + packet, + &self.encryption_private_key, + )?); + let delay = self.poisson.interval(&mut OsRng); + let output_tx = self.output_tx.clone(); + tokio::spawn(async move { + tokio::time::sleep(delay).await; + // output_tx is always expected to be not closed/dropped. + output_tx.send(output).unwrap(); + }); + Ok(()) + } + + fn process_fragment(&mut self, fragment: &[u8]) -> Result<(), MixnetError> { + if let Some(msg) = self + .message_reconstructor + .add_and_reconstruct(Fragment::from_bytes(fragment)?) + { + match Message::from_bytes(&msg)? { + Message::Real(msg) => { + let output = Output::ReconstructedMessage(msg.into_boxed_slice()); + self.output_tx + .send(output) + .expect("output channel shouldn't be closed"); + } + Message::DropCover(_) => { + tracing::debug!("Drop cover message has been reconstructed. Dropping it..."); + } + } + } + Ok(()) + } +} + +/// Output that [`MixNode::next`] returns. +#[derive(Debug, PartialEq, Eq)] +pub enum Output { + /// Packet to be forwarded to the next mix node + Forward(Packet), + /// Message reconstructed from [`Packet`]s + ReconstructedMessage(Box<[u8]>), +} + +#[cfg(test)] +mod tests { + use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + + use sphinx_packet::crypto::PublicKey; + + use crate::{ + packet::Packet, + topology::{tests::gen_entropy, MixNodeInfo, MixnetTopology}, + }; + + use super::*; + + #[tokio::test] + async fn mixnode() { + let encryption_private_key = PrivateKey::new(); + let node_info = MixNodeInfo::new( + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 1000u16).into(), + *PublicKey::from(&encryption_private_key).as_bytes(), + ) + .unwrap(); + + let topology = MixnetTopology::new( + (0..2).map(|_| node_info.clone()).collect(), + 2, + 1, + gen_entropy(), + ) + .unwrap(); + let (mut mixnode, packet_queue) = MixNode::new(MixNodeConfig { + encryption_private_key: encryption_private_key.to_bytes(), + delay_rate_per_min: 60.0, + }) + .unwrap(); + + let msg = "hello".as_bytes().to_vec(); + let packets = Packet::build_real(msg.clone(), &topology).unwrap(); + let num_packets = packets.len(); + + for packet in packets.into_iter() { + packet_queue.send(packet.body()).await.unwrap(); + } + + for _ in 0..num_packets { + match mixnode.next().await.unwrap() { + Output::Forward(packet_to) => { + packet_queue.send(packet_to.body()).await.unwrap(); + } + Output::ReconstructedMessage(_) => unreachable!(), + } + } + + assert_eq!( + Output::ReconstructedMessage(msg.into_boxed_slice()), + mixnode.next().await.unwrap() + ); + } +} diff --git a/mixnet/src/packet.rs b/mixnet/src/packet.rs new file mode 100644 index 00000000..f9787ce7 --- /dev/null +++ b/mixnet/src/packet.rs @@ -0,0 +1,226 @@ +use std::{io, u8}; + +use futures::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; +use sphinx_packet::{crypto::PrivateKey, header::delays::Delay}; + +use crate::{ + address::NodeAddress, + error::MixnetError, + fragment::{Fragment, FragmentSet}, + topology::MixnetTopology, +}; + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct Packet { + address: NodeAddress, + body: PacketBody, +} + +impl Packet { + fn new(processed_packet: sphinx_packet::ProcessedPacket) -> Result { + match processed_packet { + sphinx_packet::ProcessedPacket::ForwardHop(packet, addr, _) => Ok(Packet { + address: addr.try_into()?, + body: PacketBody::from(packet.as_ref()), + }), + sphinx_packet::ProcessedPacket::FinalHop(addr, _, payload) => Ok(Packet { + address: addr.try_into()?, + body: PacketBody::try_from(payload)?, + }), + } + } + + pub(crate) fn build_real( + msg: Vec, + topology: &MixnetTopology, + ) -> Result, MixnetError> { + Self::build(Message::Real(msg), topology) + } + + pub(crate) fn build_drop_cover( + msg: Vec, + topology: &MixnetTopology, + ) -> Result, MixnetError> { + Self::build(Message::DropCover(msg), topology) + } + + fn build(msg: Message, topology: &MixnetTopology) -> Result, MixnetError> { + let destination = topology.choose_destination(); + + let fragment_set = FragmentSet::new(&msg.bytes())?; + let mut packets = Vec::with_capacity(fragment_set.as_ref().len()); + for fragment in fragment_set.as_ref().iter() { + let route = topology.gen_route(); + if route.is_empty() { + // Create a packet that will be directly sent to the mix destination + packets.push(Packet { + address: NodeAddress::try_from(destination.address)?, + body: PacketBody::from(fragment), + }); + } else { + // Use dummy delays because mixnodes will ignore this value and generate delay randomly by themselves. + let delays = vec![Delay::new_from_nanos(0); route.len()]; + packets.push(Packet { + address: NodeAddress::try_from(route[0].address)?, + body: PacketBody::from(&sphinx_packet::SphinxPacket::new( + fragment.bytes(), + &route, + &destination, + &delays, + )?), + }); + } + } + Ok(packets) + } + + pub fn address(&self) -> NodeAddress { + self.address + } + + pub fn body(self) -> PacketBody { + self.body + } +} + +#[derive(Clone, PartialEq, Eq, Debug)] +pub enum PacketBody { + SphinxPacket(Vec), + Fragment(Vec), +} + +impl From<&sphinx_packet::SphinxPacket> for PacketBody { + fn from(packet: &sphinx_packet::SphinxPacket) -> Self { + Self::SphinxPacket(packet.to_bytes()) + } +} + +impl From<&Fragment> for PacketBody { + fn from(fragment: &Fragment) -> Self { + Self::Fragment(fragment.bytes()) + } +} + +impl TryFrom for PacketBody { + type Error = MixnetError; + + fn try_from(payload: sphinx_packet::payload::Payload) -> Result { + Ok(Self::Fragment(payload.recover_plaintext()?)) + } +} + +impl PacketBody { + pub async fn write_to(&self, writer: &mut W) -> io::Result<()> { + match self { + Self::SphinxPacket(data) => { + Self::write(writer, PacketBodyFlag::SphinxPacket, data).await + } + Self::Fragment(data) => Self::write(writer, PacketBodyFlag::Fragment, data).await, + } + } + + async fn write( + writer: &mut W, + flag: PacketBodyFlag, + data: &[u8], + ) -> io::Result<()> { + writer.write_all(&[flag as u8]).await?; + writer.write_all(&data.len().to_le_bytes()).await?; + writer.write_all(data).await?; + Ok(()) + } + + pub async fn read_from( + reader: &mut R, + ) -> io::Result> { + let mut flag = [0u8; 1]; + reader.read_exact(&mut flag).await?; + + let mut size = [0u8; std::mem::size_of::()]; + reader.read_exact(&mut size).await?; + + let mut data = vec![0u8; usize::from_le_bytes(size)]; + reader.read_exact(&mut data).await?; + + match PacketBodyFlag::try_from(flag[0]) { + Ok(PacketBodyFlag::SphinxPacket) => Ok(Ok(PacketBody::SphinxPacket(data))), + Ok(PacketBodyFlag::Fragment) => Ok(Ok(PacketBody::Fragment(data))), + Err(e) => Ok(Err(e)), + } + } + + pub(crate) fn process_sphinx_packet( + packet: &[u8], + private_key: &PrivateKey, + ) -> Result { + Packet::new(sphinx_packet::SphinxPacket::from_bytes(packet)?.process(private_key)?) + } +} + +#[repr(u8)] +enum PacketBodyFlag { + SphinxPacket, + Fragment, +} + +impl TryFrom for PacketBodyFlag { + type Error = MixnetError; + + fn try_from(value: u8) -> Result { + match value { + 0u8 => Ok(PacketBodyFlag::SphinxPacket), + 1u8 => Ok(PacketBodyFlag::Fragment), + _ => Err(MixnetError::InvalidPacketFlag), + } + } +} + +pub(crate) enum Message { + Real(Vec), + DropCover(Vec), +} + +impl Message { + fn bytes(self) -> Box<[u8]> { + match self { + Self::Real(msg) => Self::bytes_with_flag(MessageFlag::Real, msg), + Self::DropCover(msg) => Self::bytes_with_flag(MessageFlag::DropCover, msg), + } + } + + fn bytes_with_flag(flag: MessageFlag, mut msg: Vec) -> Box<[u8]> { + let mut out = Vec::with_capacity(1 + msg.len()); + out.push(flag as u8); + out.append(&mut msg); + out.into_boxed_slice() + } + + pub(crate) fn from_bytes(value: &[u8]) -> Result { + if value.is_empty() { + return Err(MixnetError::InvalidMessage); + } + + match MessageFlag::try_from(value[0])? { + MessageFlag::Real => Ok(Self::Real(value[1..].into())), + MessageFlag::DropCover => Ok(Self::DropCover(value[1..].into())), + } + } +} + +#[repr(u8)] +enum MessageFlag { + Real, + DropCover, +} + +impl TryFrom for MessageFlag { + type Error = MixnetError; + + fn try_from(value: u8) -> Result { + match value { + 0u8 => Ok(MessageFlag::Real), + 1u8 => Ok(MessageFlag::DropCover), + _ => Err(MixnetError::InvalidPacketFlag), + } + } +} diff --git a/mixnet/src/poisson.rs b/mixnet/src/poisson.rs new file mode 100644 index 00000000..e6d63bea --- /dev/null +++ b/mixnet/src/poisson.rs @@ -0,0 +1,94 @@ +use std::time::Duration; + +use rand::Rng; +use rand_distr::{Distribution, Exp}; + +use crate::error::MixnetError; + +#[allow(dead_code)] +pub struct Poisson(Exp); + +impl Poisson { + #[allow(dead_code)] + pub fn new(rate_per_min: f64) -> Result { + Ok(Self(Exp::new(rate_per_min)?)) + } + + /// Get a random interval between events that follow a Poisson distribution. + /// + /// If events occur in a Poisson distribution with rate_per_min, + /// the interval between events follow the exponential distribution with rate_per_min. + #[allow(dead_code)] + pub fn interval(&self, rng: &mut R) -> Duration { + // generate a random value from the distribution + let interval_min = self.0.sample(rng); + // convert minutes to seconds + Duration::from_secs_f64(interval_min * 60.0) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rand::rngs::OsRng; + use std::{collections::BTreeMap, time::Duration}; + + // Test the interval generation for a specific rate + #[test] + fn test_interval_generation() { + let interval = Poisson::new(1.0).unwrap().interval(&mut OsRng); + // Check if the interval is within a plausible range + // This is a basic check; in practice, you may want to perform a statistical test + assert!(interval > Duration::from_secs(0)); // Must be positive + } + + // Compute the empirical CDF + fn empirical_cdf(samples: &[Duration]) -> BTreeMap { + let mut map = BTreeMap::new(); + let n = samples.len() as f64; + + for &sample in samples { + *map.entry(sample).or_insert(0.0) += 1.0 / n; + } + + let mut acc = 0.0; + for value in map.values_mut() { + acc += *value; + *value = acc; + } + + map + } + + // Compare the empirical CDF to the theoretical CDF + #[test] + fn test_distribution_fit() { + let rate_per_min = 1.0; + let mut intervals = Vec::new(); + + // Generate 10,000 samples + let poisson = Poisson::new(rate_per_min).unwrap(); + for _ in 0..10_000 { + intervals.push(poisson.interval(&mut OsRng)); + } + + let empirical = empirical_cdf(&intervals); + + // theoretical CDF for exponential distribution + let rate_per_sec = rate_per_min / 60.0; + let theoretical_cdf = |x: f64| 1.0 - (-rate_per_sec * x).exp(); + + // Kolmogorov-Smirnov test + let ks_statistic: f64 = empirical + .iter() + .map(|(&k, &v)| { + let x = k.as_secs_f64(); + (theoretical_cdf(x) - v).abs() + }) + .fold(0.0, f64::max); + + println!("KS Statistic: {}", ks_statistic); + + assert!(ks_statistic < 0.05, "Distributions differ significantly."); + } +} diff --git a/mixnet/src/topology.rs b/mixnet/src/topology.rs new file mode 100644 index 00000000..f903cadb --- /dev/null +++ b/mixnet/src/topology.rs @@ -0,0 +1,199 @@ +use nomos_utils::fisheryates::FisherYatesShuffle; +use rand::Rng; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use sphinx_packet::{ + constants::IDENTIFIER_LENGTH, + crypto::{PublicKey, PUBLIC_KEY_SIZE}, + route::{DestinationAddressBytes, SURBIdentifier}, +}; + +use crate::{address::NodeAddress, error::MixnetError}; + +/// Defines Mixnet topology construction and route selection +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct MixnetTopology { + mixnode_candidates: Vec, + num_layers: usize, + num_mixnodes_per_layer: usize, +} + +impl MixnetTopology { + /// Generates [MixnetTopology] with random shuffling/sampling using a given entropy. + /// + /// # Errors + /// + /// This function will return an error if parameters are invalid. + pub fn new( + mut mixnode_candidates: Vec, + num_layers: usize, + num_mixnodes_per_layer: usize, + entropy: [u8; 32], + ) -> Result { + if mixnode_candidates.len() < num_layers * num_mixnodes_per_layer { + return Err(MixnetError::InvalidTopologySize); + } + + FisherYatesShuffle::shuffle(&mut mixnode_candidates, entropy); + Ok(Self { + mixnode_candidates, + num_layers, + num_mixnodes_per_layer, + }) + } + + /// Selects a mix destination randomly from the last mix layer + pub(crate) fn choose_destination(&self) -> sphinx_packet::route::Destination { + let idx_in_layer = rand::thread_rng().gen_range(0..self.num_mixnodes_per_layer); + let idx = self.num_mixnodes_per_layer * (self.num_layers - 1) + idx_in_layer; + self.mixnode_candidates[idx].clone().into() + } + + /// Selects a mix route randomly from all mix layers except the last layer + /// and append a mix destination to the end of the mix route. + /// + /// That is, the caller can generate multiple routes with one mix destination. + pub(crate) fn gen_route(&self) -> Vec { + let mut route = Vec::with_capacity(self.num_layers); + for layer in 0..self.num_layers - 1 { + let idx_in_layer = rand::thread_rng().gen_range(0..self.num_mixnodes_per_layer); + let idx = self.num_mixnodes_per_layer * layer + idx_in_layer; + route.push(self.mixnode_candidates[idx].clone().into()); + } + route + } +} + +/// Mix node information that is used for forwarding packets to the mix node +#[derive(Clone, Debug)] +pub struct MixNodeInfo(sphinx_packet::route::Node); + +impl MixNodeInfo { + /// Creates a [`MixNodeInfo`]. + pub fn new( + address: NodeAddress, + public_key: [u8; PUBLIC_KEY_SIZE], + ) -> Result { + Ok(Self(sphinx_packet::route::Node::new( + address.try_into()?, + PublicKey::from(public_key), + ))) + } +} + +impl From for sphinx_packet::route::Node { + fn from(info: MixNodeInfo) -> Self { + info.0 + } +} + +const DUMMY_SURB_IDENTIFIER: SURBIdentifier = [0u8; IDENTIFIER_LENGTH]; + +impl From for sphinx_packet::route::Destination { + fn from(info: MixNodeInfo) -> Self { + sphinx_packet::route::Destination::new( + DestinationAddressBytes::from_bytes(info.0.address.as_bytes()), + DUMMY_SURB_IDENTIFIER, + ) + } +} + +impl Serialize for MixNodeInfo { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + SerializableMixNodeInfo::try_from(self) + .map_err(serde::ser::Error::custom)? + .serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for MixNodeInfo { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + Self::try_from(SerializableMixNodeInfo::deserialize(deserializer)?) + .map_err(serde::de::Error::custom) + } +} + +// Only for serializing/deserializing [`MixNodeInfo`] since [`sphinx_packet::route::Node`] is not serializable. +#[derive(Serialize, Deserialize, Clone, Debug)] +struct SerializableMixNodeInfo { + address: NodeAddress, + public_key: [u8; PUBLIC_KEY_SIZE], +} + +impl TryFrom<&MixNodeInfo> for SerializableMixNodeInfo { + type Error = MixnetError; + + fn try_from(info: &MixNodeInfo) -> Result { + Ok(Self { + address: NodeAddress::try_from(info.0.address)?, + public_key: *info.0.pub_key.as_bytes(), + }) + } +} + +impl TryFrom for MixNodeInfo { + type Error = MixnetError; + + fn try_from(info: SerializableMixNodeInfo) -> Result { + Self::new(info.address, info.public_key) + } +} + +#[cfg(test)] +pub mod tests { + use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + + use rand::RngCore; + use sphinx_packet::crypto::{PrivateKey, PublicKey}; + + use crate::error::MixnetError; + + use super::{MixNodeInfo, MixnetTopology}; + + #[test] + fn shuffle() { + let candidates = gen_mixnodes(10); + let topology = MixnetTopology::new(candidates.clone(), 3, 2, gen_entropy()).unwrap(); + + assert_eq!(candidates.len(), topology.mixnode_candidates.len()); + } + + #[test] + fn route_and_destination() { + let topology = MixnetTopology::new(gen_mixnodes(10), 3, 2, gen_entropy()).unwrap(); + let _ = topology.choose_destination(); + assert_eq!(2, topology.gen_route().len()); // except a destination + } + + #[test] + fn invalid_topology_size() { + // if # of candidates is smaller than the topology size + assert!(matches!( + MixnetTopology::new(gen_mixnodes(5), 3, 2, gen_entropy()).err(), + Some(MixnetError::InvalidTopologySize), + )); + } + + pub fn gen_mixnodes(n: usize) -> Vec { + (0..n) + .map(|i| { + MixNodeInfo::new( + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), i as u16).into(), + *PublicKey::from(&PrivateKey::new()).as_bytes(), + ) + .unwrap() + }) + .collect() + } + + pub fn gen_entropy() -> [u8; 32] { + let mut entropy = [0u8; 32]; + rand::thread_rng().fill_bytes(&mut entropy); + entropy + } +} diff --git a/mixnet/topology/Cargo.toml b/mixnet/topology/Cargo.toml deleted file mode 100644 index 7171b17f..00000000 --- a/mixnet/topology/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "mixnet-topology" -version = "0.1.0" -edition = "2021" - -[dependencies] -hex = "0.4" -# Using an older version, since `nym-sphinx` depends on `rand` v0.7.3. -rand = "0.7.3" -serde = { version = "1.0", features = ["derive"] } -sphinx-packet = "0.1.0" -nym-sphinx = { package = "nym-sphinx", git = "https://github.com/nymtech/nym", tag = "v1.1.22" } -thiserror = "1" diff --git a/mixnet/topology/src/lib.rs b/mixnet/topology/src/lib.rs deleted file mode 100644 index 965a2c57..00000000 --- a/mixnet/topology/src/lib.rs +++ /dev/null @@ -1,127 +0,0 @@ -use std::net::SocketAddr; - -use nym_sphinx::addressing::nodes::{NymNodeRoutingAddress, NymNodeRoutingAddressError}; -use rand::{seq::IteratorRandom, Rng}; -use serde::{Deserialize, Serialize}; -use sphinx_packet::{crypto::PUBLIC_KEY_SIZE, route}; - -pub type MixnetNodeId = [u8; PUBLIC_KEY_SIZE]; - -pub type Result = core::result::Result; - -#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)] -pub struct MixnetTopology { - pub layers: Vec, -} - -#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] -pub struct Layer { - pub nodes: Vec, -} - -#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] -pub struct Node { - #[serde(with = "addr_serde")] - pub address: SocketAddr, - #[serde(with = "hex_serde")] - pub public_key: [u8; PUBLIC_KEY_SIZE], -} - -mod addr_serde { - use serde::{Deserialize, Deserializer, Serialize, Serializer}; - use std::net::{SocketAddr, ToSocketAddrs}; - - pub fn serialize(addr: &SocketAddr, serializer: S) -> Result { - addr.to_string().serialize(serializer) - } - - pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result { - let s = String::deserialize(deserializer)?; - - // Try to convert the string (which might be a domain name) to a SocketAddr. - let mut addrs = s.to_socket_addrs().map_err(serde::de::Error::custom)?; - - addrs - .next() - .ok_or_else(|| serde::de::Error::custom("Failed to resolve to a valid address")) - } -} - -mod hex_serde { - use super::PUBLIC_KEY_SIZE; - use serde::{Deserialize, Deserializer, Serialize, Serializer}; - - pub fn serialize( - pk: &[u8; PUBLIC_KEY_SIZE], - serializer: S, - ) -> Result { - if serializer.is_human_readable() { - hex::encode(pk).serialize(serializer) - } else { - serializer.serialize_bytes(pk) - } - } - - pub fn deserialize<'de, D: Deserializer<'de>>( - deserializer: D, - ) -> Result<[u8; PUBLIC_KEY_SIZE], D::Error> { - if deserializer.is_human_readable() { - let hex_str = String::deserialize(deserializer)?; - hex::decode(hex_str) - .map_err(serde::de::Error::custom) - .and_then(|v| v.as_slice().try_into().map_err(serde::de::Error::custom)) - } else { - <[u8; PUBLIC_KEY_SIZE]>::deserialize(deserializer) - } - } -} - -impl MixnetTopology { - pub fn random_route(&self, rng: &mut R) -> Result> { - let num_hops = self.layers.len(); - - let route: Vec = self - .layers - .iter() - .take(num_hops) - .map(|layer| { - layer - .random_node(rng) - .expect("layer is not empty") - .clone() - .try_into() - .unwrap() - }) - .collect(); - - Ok(route) - } - - // Choose a destination mixnet node randomly from the last layer. - pub fn random_destination(&self, rng: &mut R) -> Result { - self.layers - .last() - .expect("topology is not empty") - .random_node(rng) - .expect("layer is not empty") - .clone() - .try_into() - } -} - -impl Layer { - pub fn random_node(&self, rng: &mut R) -> Option<&Node> { - self.nodes.iter().choose(rng) - } -} - -impl TryInto for Node { - type Error = NymNodeRoutingAddressError; - - fn try_into(self) -> Result { - Ok(route::Node { - address: NymNodeRoutingAddress::from(self.address).try_into()?, - pub_key: self.public_key.into(), - }) - } -} diff --git a/mixnet/util/Cargo.toml b/mixnet/util/Cargo.toml deleted file mode 100644 index 66b323f0..00000000 --- a/mixnet/util/Cargo.toml +++ /dev/null @@ -1,8 +0,0 @@ -[package] -name = "mixnet-util" -version = "0.1.0" -edition = "2021" - -[dependencies] -tokio = { version = "1.32", default-features = false, features = ["sync", "net"] } -mixnet-protocol = { path = "../protocol" } diff --git a/mixnet/util/src/lib.rs b/mixnet/util/src/lib.rs deleted file mode 100644 index a8431eb2..00000000 --- a/mixnet/util/src/lib.rs +++ /dev/null @@ -1,35 +0,0 @@ -use std::{collections::HashMap, net::SocketAddr, sync::Arc}; - -use tokio::{net::TcpStream, sync::Mutex}; - -#[derive(Clone)] -pub struct ConnectionPool { - pool: Arc>>>>, -} - -impl ConnectionPool { - pub fn new(size: usize) -> Self { - Self { - pool: Arc::new(Mutex::new(HashMap::with_capacity(size))), - } - } - - pub async fn get_or_init( - &self, - addr: &SocketAddr, - ) -> mixnet_protocol::Result>> { - let mut pool = self.pool.lock().await; - match pool.get(addr).cloned() { - Some(tcp) => Ok(tcp), - None => { - let tcp = Arc::new(Mutex::new( - TcpStream::connect(addr) - .await - .map_err(mixnet_protocol::ProtocolError::IO)?, - )); - pool.insert(*addr, tcp.clone()); - Ok(tcp) - } - } - } -} diff --git a/nodes/mixnode/Cargo.toml b/nodes/mixnode/Cargo.toml deleted file mode 100644 index f3ae76d2..00000000 --- a/nodes/mixnode/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "mixnode" -version = "0.1.0" -edition = "2021" - -[dependencies] -async-trait = "0.1" -mixnet-node = { path = "../../mixnet/node" } -nomos-log = { path = "../../nomos-services/log" } -clap = { version = "4", features = ["derive"] } -color-eyre = "0.6.0" -overwatch-rs = { git = "https://github.com/logos-co/Overwatch", rev = "2f70806" } -overwatch-derive = { git = "https://github.com/logos-co/Overwatch", rev = "ac28d01" } -serde = "1" -serde_yaml = "0.9" -tracing = "0.1" -tracing-subscriber = "0.3" -tokio = { version = "1.33", features = ["macros"] } diff --git a/nodes/mixnode/README.md b/nodes/mixnode/README.md deleted file mode 100644 index a4f759ce..00000000 --- a/nodes/mixnode/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Mixnode - -A mixnode application that runs the mixnet [`node`](../../mixnet/node/) component, which will be deployed as a part of the mixnet. - -For the recommended architecture of mixnet, please see the [mixnet](../../mixnet/README.md) documentation. - -## Configurations - -A mixnode can be configured by [`config.yaml`](./config.yaml), for example. diff --git a/nodes/mixnode/config.yaml b/nodes/mixnode/config.yaml deleted file mode 100644 index a98c9a34..00000000 --- a/nodes/mixnode/config.yaml +++ /dev/null @@ -1,16 +0,0 @@ -mixnode: - # A listen address for other mixnodes in the mixnet and mixclients who want to send packets. - listen_address: 127.0.0.1:7777 - # A (internal) listen address only for a "single" mixclient who wants to receive packets - # from the last mixnet layer. - # For more details, see the documentation in the "mixnet" crate. - client_listen_address: 127.0.0.1:7778 - # A ed25519 private key for decrypting inbound Sphinx packets - # received from mixclients or mixnodes in the previous mixnet layer. - private_key: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] - # A max number of connections that will stay connected to mixnodes in the next layer. - connection_pool_size: 255 -log: - backend: "Stdout" - format: "Json" - level: "debug" diff --git a/nodes/mixnode/src/lib.rs b/nodes/mixnode/src/lib.rs deleted file mode 100644 index c3a9d85f..00000000 --- a/nodes/mixnode/src/lib.rs +++ /dev/null @@ -1,20 +0,0 @@ -mod services; - -use nomos_log::Logger; -use overwatch_derive::Services; -use overwatch_rs::services::handle::ServiceHandle; -use overwatch_rs::services::ServiceData; -use serde::{Deserialize, Serialize}; -use services::mixnet::MixnetNodeService; - -#[derive(Deserialize, Debug, Clone, Serialize)] -pub struct Config { - pub mixnode: ::Settings, - pub log: ::Settings, -} - -#[derive(Services)] -pub struct MixNode { - node: ServiceHandle, - logging: ServiceHandle, -} diff --git a/nodes/mixnode/src/main.rs b/nodes/mixnode/src/main.rs deleted file mode 100644 index 520b9e5c..00000000 --- a/nodes/mixnode/src/main.rs +++ /dev/null @@ -1,29 +0,0 @@ -mod services; - -use clap::Parser; -use color_eyre::eyre::Result; -use mixnode::{Config, MixNode, MixNodeServiceSettings}; -use overwatch_rs::overwatch::OverwatchRunner; -use overwatch_rs::DynError; - -#[derive(Parser, Debug)] -#[command(author, version, about, long_about = None)] -struct Args { - /// Path for a yaml-encoded mixnet-node config file - config: std::path::PathBuf, -} - -fn main() -> Result<(), DynError> { - let Args { config } = Args::parse(); - let config = serde_yaml::from_reader::<_, Config>(std::fs::File::open(config)?)?; - - let app = OverwatchRunner::::run( - MixNodeServiceSettings { - node: config.mixnode, - logging: config.log, - }, - None, - )?; - app.wait_finished(); - Ok(()) -} diff --git a/nodes/mixnode/src/services/mixnet.rs b/nodes/mixnode/src/services/mixnet.rs deleted file mode 100644 index cc7bdc42..00000000 --- a/nodes/mixnode/src/services/mixnet.rs +++ /dev/null @@ -1,31 +0,0 @@ -use mixnet_node::{MixnetNode, MixnetNodeConfig}; -use overwatch_rs::services::handle::ServiceStateHandle; -use overwatch_rs::services::relay::NoMessage; -use overwatch_rs::services::state::{NoOperator, NoState}; -use overwatch_rs::services::{ServiceCore, ServiceData, ServiceId}; -use overwatch_rs::DynError; - -pub struct MixnetNodeService(MixnetNode); - -impl ServiceData for MixnetNodeService { - const SERVICE_ID: ServiceId = "mixnet-node"; - type Settings = MixnetNodeConfig; - type State = NoState; - type StateOperator = NoOperator; - type Message = NoMessage; -} - -#[async_trait::async_trait] -impl ServiceCore for MixnetNodeService { - fn init(service_state: ServiceStateHandle) -> Result { - let settings: Self::Settings = service_state.settings_reader.get_updated_settings(); - Ok(Self(MixnetNode::new(settings))) - } - - async fn run(self) -> Result<(), DynError> { - if let Err(_e) = self.0.run().await { - todo!("Errors should match"); - } - Ok(()) - } -} diff --git a/nodes/mixnode/src/services/mod.rs b/nodes/mixnode/src/services/mod.rs deleted file mode 100644 index ae7f01c0..00000000 --- a/nodes/mixnode/src/services/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod mixnet; diff --git a/nodes/nomos-node/README.md b/nodes/nomos-node/README.md deleted file mode 100644 index d081648d..00000000 --- a/nodes/nomos-node/README.md +++ /dev/null @@ -1,91 +0,0 @@ -# Nomos Node - -Nomos blockchain node - - -## Network service - -Nomos node can be configured with one of the following network backends: -- [libp2p](../../nomos-services/backends/libp2p.rs) - -### Mixclient integration - -The [mixclient](../../mixnet/client/) is currently integrated as a part of the libp2p network backend. -To run a Nomos node with the libp2p network backend, the `mixnet_client` and `mixnet_delay` fields in the [`config.yaml`](./config.yaml) must be specified, so the Nomos node can send/receive packets to/from mixnodes. - -For more details about the mixnode/mixclient architecture, see the [mixnet documentation](../../mixnet/README.md). - -```mermaid -flowchart LR - - subgraph mixnet - direction LR - - subgraph layer-1 - direction TB - mixnode-1-1 - mixnode-1-2 - end - subgraph layer-2 - direction TB - mixnode-2-1 - mixnode-2-2 - end - subgraph layer-3 - direction TB - mixnode-3-1 - mixnode-3-2 - end - - mixnode-1-1 --> mixnode-2-1 - mixnode-1-1 --> mixnode-2-2 - mixnode-1-2 --> mixnode-2-1 - mixnode-1-2 --> mixnode-2-2 - mixnode-2-1 --> mixnode-3-1 - mixnode-2-1 --> mixnode-3-2 - mixnode-2-2 --> mixnode-3-1 - mixnode-2-2 --> mixnode-3-2 - end - - subgraph nomos-network - direction TB - - subgraph nomos-node-1 - libp2p-1[libp2p] --> mixclient-sender-1[mixclient-sender] - end - subgraph nomos-node-2 - libp2p-2[libp2p] --> mixclient-sender-2[mixclient-sender] - end - subgraph nomos-node-3 - libp2p-3[libp2p] <--> mixclient-senderreceiver - end - end - - mixclient-sender-1 --> mixnode-1-1 - mixclient-sender-1 --> mixnode-1-2 - mixclient-sender-2 --> mixnode-1-1 - mixclient-sender-2 --> mixnode-1-2 - mixclient-senderreceiver --> mixnode-1-1 - mixclient-senderreceiver --> mixnode-1-2 - mixnode-3-2 --> mixclient-senderreceiver -``` - -#### Sender mode - -If you are a node operator who wants to run only a Nomos node (not a mixnode), -you can configure the mixclient as the `Sender` mode (like `nomos-node-1` or `nomos-node-2` above). -Then, the Nomos node sends messages to the mixnet instead of broadcasting them directly through libp2p gossipsub. - -The mixclient in the `Sender` mode will split a message into multiple Sphinx packets by constructing mix routes based on the mixnet topology configured, and sends packets to the mixnode. - -#### SenderReceiver mode - -If you are a node operator who runs both a Nomos node and a mixnode, -you can configure the mixclient as the `SenderReceiver` mode by specifying the client listen address of your mixnode (like `nomos-node-3` and `mixnode-3-2` above). - -The Nomos node with the mixclient in the `SenderRecevier` mode will behave essentially the same as the one in the `Sender` mode. -In addition, the node will receive packets from the connected mixnode, reconstruct a message, and broadcast it through libp2p gossipsub, if the connected mixnode is part of the last mixnet layer. -In other words, at least one Nomos node in the entire network must have a mixclient in the `SenderReceiver` mode, so that reconstructed messages can be broadcasted to all other Nomos nodes. - - - diff --git a/nodes/nomos-node/config.yaml b/nodes/nomos-node/config.yaml index f63aca7c..65bdc75a 100644 --- a/nodes/nomos-node/config.yaml +++ b/nodes/nomos-node/config.yaml @@ -24,31 +24,6 @@ network: discV5BootstrapNodes: [] initial_peers: [] relayTopics: [] - # Mixclient configuration to communicate with mixnodes. - # The libp2p network backend always requires this mixclient configuration - # (cannot be disabled for now). - mixnet_client: - # A mixclient mode. For details, see the documentation of the "mixnet" crate. - # - Sender - # - !SenderReceiver [mixnode_client_listen_address] - mode: Sender - # A mixnet topology, which contains the information of all mixnodes in the mixnet. - # (The topology is static for now.) - topology: - # Each mixnet layer consists of a list of mixnodes. - layers: - - nodes: - - address: 127.0.0.1:7777 # A listen address of the mixnode - # A ed25519 public key for encrypting Sphinx packets for the mixnode - public_key: "0000000000000000000000000000000000000000000000000000000000000000" - # A max number of connections that will stay connected to mixnodes in the first mixnet layer. - connection_pool_size: 255 - # A range of total delay that will be set to each Sphinx packets - # sent to the mixnet for timing obfuscation. - # Panics if start > end. - mixnet_delay: - start: "0ms" - end: "0ms" http: backend_settings: diff --git a/nomos-cli/config.yaml b/nomos-cli/config.yaml index a1740c24..b9f11225 100644 --- a/nomos-cli/config.yaml +++ b/nomos-cli/config.yaml @@ -7,39 +7,3 @@ backend: discV5BootstrapNodes: [] initial_peers: ["/dns/testnet.nomos.tech/tcp/3000"] relayTopics: [] - # Mixclient configuration to communicate with mixnodes. - # The libp2p network backend always requires this mixclient configuration - # (cannot be disabled for now). - mixnet_client: - # A mixclient mode. For details, see the documentation of the "mixnet" crate. - # - Sender - # - !SenderReceiver [mixnode_client_listen_address] - mode: Sender - # A mixnet topology, which contains the information of all mixnodes in the mixnet. - # (The topology is static for now.) - topology: - # Each mixnet layer consists of a list of mixnodes. - layers: - - nodes: - - address: testnet.nomos.tech:7707 # A listen address of the mixnode - public_key: "fd3384e132ad02a56c78f45547ee40038dc79002b90d29ed90e08eee762ae715" - - nodes: - - address: testnet.nomos.tech:7717 # A listen address of the mixnode - public_key: "fd3384e132ad02a56c78f45547ee40038dc79002b90d29ed90e08eee762ae715" - - - nodes: - - address: testnet.nomos.tech:7727 # A listen address of the mixnode - public_key: "fd3384e132ad02a56c78f45547ee40038dc79002b90d29ed90e08eee762ae715" - - # A max number of connections that will stay connected to mixnodes in the first mixnet layer. - connection_pool_size: 255 - max_retries: 5 - retry_delay: - secs: 1 - nanos: 0 - # A range of total delay that will be set to each Sphinx packets - # sent to the mixnet for timing obfuscation. - # Panics if start > end. - mixnet_delay: - start: "0ms" - end: "0ms" diff --git a/nomos-libp2p/Cargo.toml b/nomos-libp2p/Cargo.toml index 4f798e7c..e5c2c594 100644 --- a/nomos-libp2p/Cargo.toml +++ b/nomos-libp2p/Cargo.toml @@ -7,17 +7,15 @@ edition = "2021" multiaddr = "0.18" tokio = { version = "1", features = ["sync", "macros"] } futures = "0.3" -libp2p = { version = "0.52.4", features = [ +libp2p = { version = "0.53.2", features = [ "dns", - "yamux", - "plaintext", "macros", "gossipsub", - "identify", - "tcp", "tokio", + "quic", "secp256k1", ] } +libp2p-stream = "0.1.0-alpha" blake2 = { version = "0.10" } serde = { version = "1.0.166", features = ["derive"] } hex = "0.4.3" @@ -26,5 +24,6 @@ thiserror = "1.0.40" tracing = "0.1" [dev-dependencies] +rand = "0.8.5" serde_json = "1.0.99" tokio = { version = "1", features = ["time"] } diff --git a/nomos-libp2p/src/lib.rs b/nomos-libp2p/src/lib.rs index faf4596e..d2bf9130 100644 --- a/nomos-libp2p/src/lib.rs +++ b/nomos-libp2p/src/lib.rs @@ -12,17 +12,16 @@ pub use libp2p; use blake2::digest::{consts::U32, Digest}; use blake2::Blake2b; use libp2p::gossipsub::{Message, MessageId, TopicHash}; -#[allow(deprecated)] +use libp2p::swarm::ConnectionId; pub use libp2p::{ core::upgrade, - dns, gossipsub::{self, PublishError, SubscriptionError}, identity::{self, secp256k1}, - plaintext::Config as PlainText2Config, - swarm::{dial_opts::DialOpts, DialError, NetworkBehaviour, SwarmEvent, THandlerErr}, - tcp, yamux, PeerId, SwarmBuilder, Transport, + swarm::{dial_opts::DialOpts, DialError, NetworkBehaviour, SwarmEvent}, + PeerId, SwarmBuilder, Transport, }; -use libp2p::{swarm::ConnectionId, tcp::tokio::Tcp}; +pub use libp2p_stream; +use libp2p_stream::Control; pub use multiaddr::{multiaddr, Multiaddr, Protocol}; /// Wraps [`libp2p::Swarm`], and config it for use within Nomos. @@ -33,57 +32,54 @@ pub struct Swarm { #[derive(NetworkBehaviour)] pub struct Behaviour { + stream: libp2p_stream::Behaviour, gossipsub: gossipsub::Behaviour, } +impl Behaviour { + fn new(peer_id: PeerId, gossipsub_config: gossipsub::Config) -> Result> { + let gossipsub = gossipsub::Behaviour::new( + gossipsub::MessageAuthenticity::Author(peer_id), + gossipsub::ConfigBuilder::from(gossipsub_config) + .validation_mode(gossipsub::ValidationMode::None) + .message_id_fn(compute_message_id) + .build()?, + )?; + Ok(Self { + stream: libp2p_stream::Behaviour::new(), + gossipsub, + }) + } +} + #[derive(thiserror::Error, Debug)] pub enum SwarmError { #[error("duplicate dialing")] DuplicateDialing, } -/// A timeout for the setup and protocol upgrade process for all in/outbound connections -const TRANSPORT_TIMEOUT: Duration = Duration::from_secs(20); +/// How long to keep a connection alive once it is idling. +const IDLE_CONN_TIMEOUT: Duration = Duration::from_secs(300); impl Swarm { /// Builds a [`Swarm`] configured for use with Nomos on top of a tokio executor. // // TODO: define error types pub fn build(config: &SwarmConfig) -> Result> { - let id_keys = identity::Keypair::from(secp256k1::Keypair::from(config.node_key.clone())); - let local_peer_id = PeerId::from(id_keys.public()); - log::info!("libp2p peer_id:{}", local_peer_id); + let keypair = + libp2p::identity::Keypair::from(secp256k1::Keypair::from(config.node_key.clone())); + let peer_id = PeerId::from(keypair.public()); + tracing::info!("libp2p peer_id:{}", peer_id); - // TODO: consider using noise authentication - let tcp_transport = tcp::Transport::::new(tcp::Config::default().nodelay(true)) - .upgrade(upgrade::Version::V1Lazy) - .authenticate(PlainText2Config::new(&id_keys)) - .multiplex(yamux::Config::default()) - .timeout(TRANSPORT_TIMEOUT) - .boxed(); + let mut swarm = libp2p::SwarmBuilder::with_existing_identity(keypair) + .with_tokio() + .with_quic() + .with_dns()? + .with_behaviour(|_| Behaviour::new(peer_id, config.gossipsub_config.clone()).unwrap())? + .with_swarm_config(|c| c.with_idle_connection_timeout(IDLE_CONN_TIMEOUT)) + .build(); - // Wrapping TCP transport into DNS transport to resolve hostnames. - let tcp_transport = dns::tokio::Transport::system(tcp_transport)?.boxed(); - - // TODO: consider using Signed or Anonymous. - // For Anonymous, a custom `message_id` function need to be set - // to prevent all messages from a peer being filtered as duplicates. - let gossipsub = gossipsub::Behaviour::new( - gossipsub::MessageAuthenticity::Author(local_peer_id), - gossipsub::ConfigBuilder::from(config.gossipsub_config.clone()) - .validation_mode(gossipsub::ValidationMode::None) - .message_id_fn(compute_message_id) - .build()?, - )?; - - let mut swarm = libp2p::Swarm::new( - tcp_transport, - Behaviour { gossipsub }, - local_peer_id, - libp2p::swarm::Config::with_tokio_executor(), - ); - - swarm.listen_on(multiaddr!(Ip4(config.host), Tcp(config.port)))?; + swarm.listen_on(Self::multiaddr(config.host, config.port))?; Ok(Swarm { swarm }) } @@ -148,11 +144,21 @@ impl Swarm { pub fn topic_hash(topic: &str) -> TopicHash { gossipsub::IdentTopic::new(topic).hash() } + + /// Returns a stream control that can be used to accept streams and establish streams to + /// other peers. + /// Stream controls can be cloned. + pub fn stream_control(&self) -> Control { + self.swarm.behaviour().stream.new_control() + } + + pub fn multiaddr(ip: std::net::Ipv4Addr, port: u16) -> Multiaddr { + multiaddr!(Ip4(ip), Udp(port), QuicV1) + } } impl futures::Stream for Swarm { - #[allow(deprecated)] - type Item = SwarmEvent>; + type Item = SwarmEvent; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { Pin::new(&mut self.swarm).poll_next(cx) @@ -164,3 +170,68 @@ fn compute_message_id(message: &Message) -> MessageId { hasher.update(&message.data); MessageId::from(hasher.finalize().to_vec()) } + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use futures::{AsyncReadExt, AsyncWriteExt, StreamExt}; + use libp2p::StreamProtocol; + use rand::Rng; + + use crate::{Swarm, SwarmConfig}; + + #[tokio::test] + async fn stream() { + // Init two swarms + let (config1, mut swarm1) = init_swarm(); + let (_, mut swarm2) = init_swarm(); + let swarm1_peer_id = *swarm1.swarm().local_peer_id(); + + // Dial to swarm1 + swarm2 + .connect(Swarm::multiaddr(config1.host, config1.port)) + .unwrap(); + + // Prepare stream controls + let mut stream_control1 = swarm1.stream_control(); + let mut stream_control2 = swarm2.stream_control(); + + // Poll swarms to make progress + tokio::spawn(async move { while (swarm1.next().await).is_some() {} }); + tokio::spawn(async move { while (swarm2.next().await).is_some() {} }); + + // Make swarm1 accept incoming streams + let protocol = StreamProtocol::new("/test"); + let mut incoming_streams = stream_control1.accept(protocol).unwrap(); + tokio::spawn(async move { + // If a new stream is established, write bytes and close the stream. + while let Some((_, mut stream)) = incoming_streams.next().await { + stream.write_all(&[1, 2, 3, 4]).await.unwrap(); + stream.close().await.unwrap(); + } + }); + + // Wait until the connection is established + tokio::time::sleep(Duration::from_secs(1)).await; + + // Establish a stream with swarm1 and read bytes + let mut stream = stream_control2 + .open_stream(swarm1_peer_id, StreamProtocol::new("/test")) + .await + .unwrap(); + let mut buf = [0u8; 4]; + stream.read_exact(&mut buf).await.unwrap(); + assert_eq!(buf, [1, 2, 3, 4]); + } + + fn init_swarm() -> (SwarmConfig, Swarm) { + let config = SwarmConfig { + host: std::net::Ipv4Addr::new(127, 0, 0, 1), + port: rand::thread_rng().gen_range(10000..30000), + ..Default::default() + }; + let swarm = Swarm::build(&config).unwrap(); + (config, swarm) + } +} diff --git a/nomos-services/network/Cargo.toml b/nomos-services/network/Cargo.toml index 1d14aabc..f02eb4b1 100644 --- a/nomos-services/network/Cargo.toml +++ b/nomos-services/network/Cargo.toml @@ -22,7 +22,7 @@ futures = "0.3" parking_lot = "0.12" nomos-core = { path = "../../nomos-core" } nomos-libp2p = { path = "../../nomos-libp2p", optional = true } -mixnet-client = { path = "../../mixnet/client" } +mixnet = { path = "../../mixnet" } utoipa = { version = "4.0", optional = true } serde_json = { version = "1", optional = true } @@ -34,4 +34,4 @@ tokio = { version = "1", features = ["full"] } default = [] libp2p = ["nomos-libp2p", "rand", "humantime-serde"] mock = ["rand", "chrono"] -openapi = ["dep:utoipa", "serde_json",] +openapi = ["dep:utoipa", "serde_json"] diff --git a/nomos-services/network/src/backends/libp2p/command.rs b/nomos-services/network/src/backends/libp2p/command.rs index 51108880..4417f574 100644 --- a/nomos-services/network/src/backends/libp2p/command.rs +++ b/nomos-services/network/src/backends/libp2p/command.rs @@ -1,4 +1,5 @@ -use nomos_libp2p::Multiaddr; +use mixnet::packet::PacketBody; +use nomos_libp2p::{libp2p::StreamProtocol, Multiaddr, PeerId}; use serde::{Deserialize, Serialize}; use tokio::sync::oneshot; @@ -16,18 +17,23 @@ pub enum Command { reply: oneshot::Sender, }, #[doc(hidden)] - // broadcast a message directly through gossipsub without mixnet - DirectBroadcastAndRetry { + RetryBroadcast { topic: Topic, message: Box<[u8]>, retry_count: usize, }, + StreamSend { + peer_id: PeerId, + protocol: StreamProtocol, + packet_body: PacketBody, + }, } -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct Dial { pub addr: Multiaddr, pub retry_count: usize, + pub result_sender: oneshot::Sender>, } pub type Topic = String; diff --git a/nomos-services/network/src/backends/libp2p/config.rs b/nomos-services/network/src/backends/libp2p/config.rs index e4baaa5b..a45d4c4b 100644 --- a/nomos-services/network/src/backends/libp2p/config.rs +++ b/nomos-services/network/src/backends/libp2p/config.rs @@ -1,6 +1,3 @@ -use std::{ops::Range, time::Duration}; - -use mixnet_client::MixnetClientConfig; use nomos_libp2p::{Multiaddr, SwarmConfig}; use serde::{Deserialize, Serialize}; @@ -11,47 +8,4 @@ pub struct Libp2pConfig { // Initial peers to connect to #[serde(default)] pub initial_peers: Vec, - pub mixnet_client: MixnetClientConfig, - #[serde(with = "humantime")] - pub mixnet_delay: Range, -} - -mod humantime { - use serde::{Deserialize, Deserializer, Serialize, Serializer}; - use std::{ops::Range, time::Duration}; - - #[derive(Serialize, Deserialize)] - struct DurationRangeHelper { - #[serde(with = "humantime_serde")] - start: Duration, - #[serde(with = "humantime_serde")] - end: Duration, - } - - pub fn serialize( - val: &Range, - serializer: S, - ) -> Result { - if serializer.is_human_readable() { - DurationRangeHelper { - start: val.start, - end: val.end, - } - .serialize(serializer) - } else { - val.serialize(serializer) - } - } - - pub fn deserialize<'de, D: Deserializer<'de>>( - deserializer: D, - ) -> Result, D::Error> { - if deserializer.is_human_readable() { - let DurationRangeHelper { start, end } = - DurationRangeHelper::deserialize(deserializer)?; - Ok(start..end) - } else { - Range::::deserialize(deserializer) - } - } } diff --git a/nomos-services/network/src/backends/libp2p/mixnet.rs b/nomos-services/network/src/backends/libp2p/mixnet.rs deleted file mode 100644 index 382f7059..00000000 --- a/nomos-services/network/src/backends/libp2p/mixnet.rs +++ /dev/null @@ -1,115 +0,0 @@ -use std::{ops::Range, time::Duration}; - -use mixnet_client::{MessageStream, MixnetClient}; -use nomos_core::wire; -use rand::{rngs::OsRng, thread_rng, Rng}; -use serde::{Deserialize, Serialize}; -use tokio::sync::mpsc; -use tokio_stream::StreamExt; - -use super::{command::Topic, Command, Libp2pConfig}; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MixnetMessage { - pub topic: Topic, - pub message: Box<[u8]>, -} - -impl MixnetMessage { - pub fn as_bytes(&self) -> Vec { - wire::serialize(self).expect("Couldn't serialize MixnetMessage") - } - pub fn from_bytes(data: &[u8]) -> Result { - wire::deserialize(data) - } -} - -pub fn random_delay(range: &Range) -> Duration { - if range.start == range.end { - return range.start; - } - thread_rng().gen_range(range.start, range.end) -} - -pub struct MixnetHandler { - client: MixnetClient, - commands_tx: mpsc::Sender, -} - -impl MixnetHandler { - pub fn new(config: &Libp2pConfig, commands_tx: mpsc::Sender) -> Self { - let client = MixnetClient::new(config.mixnet_client.clone(), OsRng); - - Self { - client, - commands_tx, - } - } - - pub async fn run(&mut self) { - const BASE_DELAY: Duration = Duration::from_secs(5); - // we need this loop to help us reestablish the connection in case - // the mixnet client fails for whatever reason - let mut backoff = 0; - loop { - match self.client.run().await { - Ok(stream) => { - backoff = 0; - self.handle_stream(stream).await; - } - Err(e) => { - tracing::error!("mixnet client error: {e}"); - backoff += 1; - tokio::time::sleep(BASE_DELAY * backoff).await; - } - } - } - } - - async fn handle_stream(&mut self, mut stream: MessageStream) { - while let Some(result) = stream.next().await { - match result { - Ok(msg) => { - tracing::debug!("receiving message from mixnet client"); - let Ok(MixnetMessage { topic, message }) = MixnetMessage::from_bytes(&msg) - else { - tracing::error!("failed to deserialize msg received from mixnet client"); - continue; - }; - - self.commands_tx - .send(Command::DirectBroadcastAndRetry { - topic, - message, - retry_count: 0, - }) - .await - .unwrap_or_else(|_| tracing::error!("could not schedule broadcast")); - } - Err(e) => { - tracing::error!("mixnet client stream error: {e}"); - return; - } - } - } - } -} - -#[cfg(test)] -mod tests { - use std::time::Duration; - - use super::random_delay; - - #[test] - fn test_random_delay() { - assert_eq!( - random_delay(&(Duration::ZERO..Duration::ZERO)), - Duration::ZERO - ); - - let range = Duration::from_millis(10)..Duration::from_millis(100); - let delay = random_delay(&range); - assert!(range.start <= delay && delay < range.end); - } -} diff --git a/nomos-services/network/src/backends/libp2p/mod.rs b/nomos-services/network/src/backends/libp2p/mod.rs index 63bd0a38..57de8b34 100644 --- a/nomos-services/network/src/backends/libp2p/mod.rs +++ b/nomos-services/network/src/backends/libp2p/mod.rs @@ -1,12 +1,10 @@ mod command; mod config; -mod mixnet; -mod swarm; +pub(crate) mod swarm; // std -pub use self::command::{Command, Libp2pInfo}; +pub use self::command::{Command, Dial, Libp2pInfo, Topic}; pub use self::config::Libp2pConfig; -use self::mixnet::MixnetHandler; use self::swarm::SwarmHandler; // internal @@ -46,11 +44,6 @@ impl NetworkBackend for Libp2p { let (commands_tx, commands_rx) = tokio::sync::mpsc::channel(BUFFER_SIZE); let (events_tx, _) = tokio::sync::broadcast::channel(BUFFER_SIZE); - let mut mixnet_handler = MixnetHandler::new(&config, commands_tx.clone()); - overwatch_handle.runtime().spawn(async move { - mixnet_handler.run().await; - }); - let mut swarm_handler = SwarmHandler::new(&config, commands_tx.clone(), commands_rx, events_tx.clone()); overwatch_handle.runtime().spawn(async move { diff --git a/nomos-services/network/src/backends/libp2p/swarm.rs b/nomos-services/network/src/backends/libp2p/swarm.rs index 14d16997..cd63804f 100644 --- a/nomos-services/network/src/backends/libp2p/swarm.rs +++ b/nomos-services/network/src/backends/libp2p/swarm.rs @@ -1,20 +1,19 @@ -use std::{collections::HashMap, ops::Range, time::Duration}; - -use mixnet_client::MixnetClient; -#[allow(deprecated)] -use nomos_libp2p::{ - gossipsub::{self, Message}, - libp2p::swarm::ConnectionId, - Behaviour, BehaviourEvent, Multiaddr, Swarm, SwarmEvent, THandlerErr, +use std::{ + collections::{hash_map::Entry, HashMap}, + time::Duration, }; -use rand::rngs::OsRng; -use tokio::sync::{broadcast, mpsc}; + +use futures::AsyncWriteExt; +use nomos_libp2p::{ + gossipsub, + libp2p::{swarm::ConnectionId, Stream, StreamProtocol}, + libp2p_stream::{Control, IncomingStreams, OpenStreamError}, + BehaviourEvent, Multiaddr, PeerId, Swarm, SwarmEvent, +}; +use tokio::sync::{broadcast, mpsc, oneshot}; use tokio_stream::StreamExt; -use crate::backends::libp2p::{ - mixnet::{random_delay, MixnetMessage}, - Libp2pInfo, -}; +use crate::backends::libp2p::Libp2pInfo; use super::{ command::{Command, Dial, Topic}, @@ -23,12 +22,12 @@ use super::{ pub struct SwarmHandler { pub swarm: Swarm, + stream_control: Control, + streams: HashMap, pub pending_dials: HashMap, pub commands_tx: mpsc::Sender, pub commands_rx: mpsc::Receiver, pub events_tx: broadcast::Sender, - pub mixnet_client: MixnetClient, - pub mixnet_delay: Range, } macro_rules! log_error { @@ -52,27 +51,29 @@ impl SwarmHandler { events_tx: broadcast::Sender, ) -> Self { let swarm = Swarm::build(&config.inner).unwrap(); - let mixnet_client = MixnetClient::new(config.mixnet_client.clone(), OsRng); + let stream_control = swarm.stream_control(); // Keep the dialing history since swarm.connect doesn't return the result synchronously let pending_dials = HashMap::::new(); Self { swarm, + stream_control, + streams: HashMap::new(), pending_dials, commands_tx, commands_rx, events_tx, - mixnet_client, - mixnet_delay: config.mixnet_delay.clone(), } } pub async fn run(&mut self, initial_peers: Vec) { for initial_peer in initial_peers { + let (tx, _) = oneshot::channel(); let dial = Dial { addr: initial_peer, retry_count: 0, + result_sender: tx, }; Self::schedule_connect(dial, self.commands_tx.clone()).await; } @@ -89,8 +90,7 @@ impl SwarmHandler { } } - #[allow(deprecated)] - fn handle_event(&mut self, event: SwarmEvent>) { + fn handle_event(&mut self, event: SwarmEvent) { match event { SwarmEvent::Behaviour(BehaviourEvent::Gossipsub(gossipsub::Event::Message { propagation_source: peer_id, @@ -108,7 +108,7 @@ impl SwarmHandler { } => { tracing::debug!("connected to peer:{peer_id}, connection_id:{connection_id:?}"); if endpoint.is_dialer() { - self.complete_connect(connection_id); + self.complete_connect(connection_id, peer_id); } } SwarmEvent::ConnectionClosed { @@ -142,10 +142,7 @@ impl SwarmHandler { self.connect(dial); } Command::Broadcast { topic, message } => { - tracing::debug!("sending message to mixnet client"); - let msg = MixnetMessage { topic, message }; - let delay = random_delay(&self.mixnet_delay); - log_error!(self.mixnet_client.send(msg.as_bytes(), delay)); + self.broadcast_and_retry(topic, message, 0).await; } Command::Subscribe(topic) => { tracing::debug!("subscribing to topic: {topic}"); @@ -167,13 +164,31 @@ impl SwarmHandler { }; log_error!(reply.send(info)); } - Command::DirectBroadcastAndRetry { + Command::RetryBroadcast { topic, message, retry_count, } => { self.broadcast_and_retry(topic, message, retry_count).await; } + Command::StreamSend { + peer_id, + protocol, + packet_body: packet, + } => { + tracing::debug!("StreamSend to {peer_id}"); + match self.open_stream(peer_id, protocol).await { + Ok(stream) => { + if let Err(e) = packet.write_to(stream).await { + tracing::error!("failed to write to the stream with ${peer_id}: {e}"); + self.close_stream(&peer_id).await; + } + } + Err(e) => { + tracing::error!("failed to open stream with {peer_id}: {e}"); + } + } + } } } @@ -197,12 +212,19 @@ impl SwarmHandler { "Failed to connect to {} with unretriable error: {e}", dial.addr ); + if let Err(err) = dial.result_sender.send(Err(e)) { + tracing::warn!("failed to send the Err result of dialing: {err:?}"); + } } } } - fn complete_connect(&mut self, connection_id: ConnectionId) { - self.pending_dials.remove(&connection_id); + fn complete_connect(&mut self, connection_id: ConnectionId, peer_id: PeerId) { + if let Some(dial) = self.pending_dials.remove(&connection_id) { + if let Err(e) = dial.result_sender.send(Ok(peer_id)) { + tracing::warn!("failed to send the Ok result of dialing: {e:?}"); + } + } } // TODO: Consider a common retry module for all use cases @@ -233,7 +255,7 @@ impl SwarmHandler { tracing::debug!("broadcasted message with id: {id} tp topic: {topic}"); // self-notification because libp2p doesn't do it if self.swarm.is_subscribed(&topic) { - log_error!(self.events_tx.send(Event::Message(Message { + log_error!(self.events_tx.send(Event::Message(gossipsub::Message { source: None, data: message.into(), sequence_number: None, @@ -249,7 +271,7 @@ impl SwarmHandler { tokio::spawn(async move { tokio::time::sleep(wait).await; commands_tx - .send(Command::DirectBroadcastAndRetry { + .send(Command::RetryBroadcast { topic, message, retry_count: retry_count + 1, @@ -267,4 +289,26 @@ impl SwarmHandler { fn exp_backoff(retry: usize) -> Duration { std::time::Duration::from_secs(BACKOFF.pow(retry as u32)) } + + pub fn incoming_streams(&mut self, protocol: StreamProtocol) -> IncomingStreams { + self.stream_control.accept(protocol).unwrap() + } + + async fn open_stream( + &mut self, + peer_id: PeerId, + protocol: StreamProtocol, + ) -> Result<&mut Stream, OpenStreamError> { + if let Entry::Vacant(entry) = self.streams.entry(peer_id) { + let stream = self.stream_control.open_stream(peer_id, protocol).await?; + entry.insert(stream); + } + Ok(self.streams.get_mut(&peer_id).unwrap()) + } + + async fn close_stream(&mut self, peer_id: &PeerId) { + if let Some(mut stream) = self.streams.remove(peer_id) { + let _ = stream.close().await; + } + } } diff --git a/nomos-services/network/src/backends/mixnet/mod.rs b/nomos-services/network/src/backends/mixnet/mod.rs new file mode 100644 index 00000000..59082803 --- /dev/null +++ b/nomos-services/network/src/backends/mixnet/mod.rs @@ -0,0 +1,291 @@ +use std::net::SocketAddr; + +// internal +use super::{ + libp2p::{self, swarm::SwarmHandler, Libp2pConfig, Topic}, + NetworkBackend, +}; +use futures::StreamExt; +use mixnet::{ + address::NodeAddress, + client::{MessageQueue, MixClient, MixClientConfig}, + node::{MixNode, MixNodeConfig, Output, PacketQueue}, + packet::PacketBody, +}; +use nomos_core::wire; +use nomos_libp2p::{ + libp2p::{Stream, StreamProtocol}, + libp2p_stream::IncomingStreams, + Multiaddr, Protocol, +}; +// crates +use overwatch_rs::{overwatch::handle::OverwatchHandle, services::state::NoState}; +use serde::{Deserialize, Serialize}; +use tokio::{ + runtime::Handle, + sync::{broadcast, mpsc, oneshot}, +}; + +/// A Mixnet network backend broadcasts messages to the network with mixing packets through mixnet, +/// and receives messages broadcasted from the network. +pub struct MixnetNetworkBackend { + libp2p_events_tx: broadcast::Sender, + libp2p_commands_tx: mpsc::Sender, + + mixclient_message_queue: MessageQueue, +} + +#[derive(Clone, Debug)] +pub struct MixnetConfig { + libp2p_config: Libp2pConfig, + mixclient_config: MixClientConfig, + mixnode_config: MixNodeConfig, +} + +const BUFFER_SIZE: usize = 64; +const STREAM_PROTOCOL: StreamProtocol = StreamProtocol::new("/mixnet"); + +#[async_trait::async_trait] +impl NetworkBackend for MixnetNetworkBackend { + type Settings = MixnetConfig; + type State = NoState; + type Message = libp2p::Command; + type EventKind = libp2p::EventKind; + type NetworkEvent = libp2p::Event; + + fn new(config: Self::Settings, overwatch_handle: OverwatchHandle) -> Self { + // TODO: One important task that should be spawned is + // subscribing NewEntropy events that will be emitted from the consensus service soon. + // so that new topology can be built internally. + // In the mixnet spec, the robustness layer is responsible for this task. + // We can implement the robustness layer in the mixnet-specific crate, + // that we're going to define at the root of the project. + + let (libp2p_commands_tx, libp2p_commands_rx) = tokio::sync::mpsc::channel(BUFFER_SIZE); + let (libp2p_events_tx, _) = tokio::sync::broadcast::channel(BUFFER_SIZE); + + let mut swarm_handler = SwarmHandler::new( + &config.libp2p_config, + libp2p_commands_tx.clone(), + libp2p_commands_rx, + libp2p_events_tx.clone(), + ); + + // Run mixnode + let (mixnode, packet_queue) = MixNode::new(config.mixnode_config).unwrap(); + let libp2p_cmd_tx = libp2p_commands_tx.clone(); + let queue = packet_queue.clone(); + overwatch_handle.runtime().spawn(async move { + Self::run_mixnode(mixnode, queue, libp2p_cmd_tx).await; + }); + let incoming_streams = swarm_handler.incoming_streams(STREAM_PROTOCOL); + let runtime_handle = overwatch_handle.runtime().clone(); + let queue = packet_queue.clone(); + overwatch_handle.runtime().spawn(async move { + Self::handle_incoming_streams(incoming_streams, queue, runtime_handle).await; + }); + + // Run mixclient + let (mixclient, message_queue) = MixClient::new(config.mixclient_config).unwrap(); + let libp2p_cmd_tx = libp2p_commands_tx.clone(); + overwatch_handle.runtime().spawn(async move { + Self::run_mixclient(mixclient, packet_queue, libp2p_cmd_tx).await; + }); + + // Run libp2p swarm to make progress + overwatch_handle.runtime().spawn(async move { + swarm_handler.run(config.libp2p_config.initial_peers).await; + }); + + Self { + libp2p_events_tx, + libp2p_commands_tx, + + mixclient_message_queue: message_queue, + } + } + + async fn process(&self, msg: Self::Message) { + match msg { + libp2p::Command::Broadcast { topic, message } => { + let msg = MixnetMessage { topic, message }; + if let Err(e) = self.mixclient_message_queue.send(msg.as_bytes()).await { + tracing::error!("failed to send messasge to mixclient: {e}"); + } + } + cmd => { + if let Err(e) = self.libp2p_commands_tx.send(cmd).await { + tracing::error!("failed to send command to libp2p swarm: {e:?}"); + } + } + } + } + + async fn subscribe( + &mut self, + kind: Self::EventKind, + ) -> broadcast::Receiver { + match kind { + libp2p::EventKind::Message => self.libp2p_events_tx.subscribe(), + } + } +} + +impl MixnetNetworkBackend { + async fn run_mixnode( + mut mixnode: MixNode, + packet_queue: PacketQueue, + swarm_commands_tx: mpsc::Sender, + ) { + while let Some(output) = mixnode.next().await { + match output { + Output::Forward(packet) => { + Self::stream_send( + packet.address(), + packet.body(), + &swarm_commands_tx, + &packet_queue, + ) + .await; + } + Output::ReconstructedMessage(message) => { + match MixnetMessage::from_bytes(&message) { + Ok(msg) => { + swarm_commands_tx + .send(libp2p::Command::Broadcast { + topic: msg.topic, + message: msg.message, + }) + .await + .unwrap(); + } + Err(e) => { + tracing::error!("failed to parse message received from mixnet: {e}"); + } + } + } + } + } + } + + async fn run_mixclient( + mut mixclient: MixClient, + packet_queue: PacketQueue, + swarm_commands_tx: mpsc::Sender, + ) { + while let Some(packet) = mixclient.next().await { + Self::stream_send( + packet.address(), + packet.body(), + &swarm_commands_tx, + &packet_queue, + ) + .await; + } + } + + async fn handle_incoming_streams( + mut incoming_streams: IncomingStreams, + packet_queue: PacketQueue, + runtime_handle: Handle, + ) { + while let Some((_, stream)) = incoming_streams.next().await { + let queue = packet_queue.clone(); + runtime_handle.spawn(async move { + if let Err(e) = Self::handle_stream(stream, queue).await { + tracing::warn!("stream closed: {e}"); + } + }); + } + } + + async fn handle_stream(mut stream: Stream, packet_queue: PacketQueue) -> std::io::Result<()> { + loop { + match PacketBody::read_from(&mut stream).await? { + Ok(packet_body) => { + packet_queue + .send(packet_body) + .await + .expect("The receiving half of packet queue should be always open"); + } + Err(e) => { + tracing::error!( + "failed to parse packet body. continuing reading the next packet: {e}" + ); + } + } + } + } + + async fn stream_send( + addr: NodeAddress, + packet_body: PacketBody, + swarm_commands_tx: &mpsc::Sender, + packet_queue: &PacketQueue, + ) { + let (tx, rx) = oneshot::channel(); + swarm_commands_tx + .send(libp2p::Command::Connect(libp2p::Dial { + addr: Self::multiaddr_from(addr), + retry_count: 3, + result_sender: tx, + })) + .await + .expect("Command receiver should be always open"); + + match rx.await { + Ok(Ok(peer_id)) => { + swarm_commands_tx + .send(libp2p::Command::StreamSend { + peer_id, + protocol: STREAM_PROTOCOL, + packet_body, + }) + .await + .expect("Command receiver should be always open"); + } + Ok(Err(e)) => match e { + nomos_libp2p::DialError::NoAddresses => { + tracing::debug!("Dialing failed because the peer is the local node. Sending msg directly to the queue"); + packet_queue + .send(packet_body) + .await + .expect("The receiving half of packet queue should be always open"); + } + _ => tracing::error!("failed to dial with unrecoverable error: {e}"), + }, + Err(e) => { + tracing::error!("channel closed before receiving: {e}"); + } + } + } + + fn multiaddr_from(addr: NodeAddress) -> Multiaddr { + match SocketAddr::from(addr) { + SocketAddr::V4(addr) => Multiaddr::empty() + .with(Protocol::Ip4(*addr.ip())) + .with(Protocol::Udp(addr.port())) + .with(Protocol::QuicV1), + SocketAddr::V6(addr) => Multiaddr::empty() + .with(Protocol::Ip6(*addr.ip())) + .with(Protocol::Udp(addr.port())) + .with(Protocol::QuicV1), + } + } +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct MixnetMessage { + pub topic: Topic, + pub message: Box<[u8]>, +} + +impl MixnetMessage { + pub fn as_bytes(&self) -> Vec { + wire::serialize(self).expect("Couldn't serialize MixnetMessage") + } + + pub fn from_bytes(data: &[u8]) -> Result { + wire::deserialize(data) + } +} diff --git a/nomos-services/network/src/backends/mod.rs b/nomos-services/network/src/backends/mod.rs index e3b451c2..c6fffe70 100644 --- a/nomos-services/network/src/backends/mod.rs +++ b/nomos-services/network/src/backends/mod.rs @@ -5,6 +5,8 @@ use tokio::sync::broadcast::Receiver; #[cfg(feature = "libp2p")] pub mod libp2p; +pub mod mixnet; + #[cfg(feature = "mock")] pub mod mock; diff --git a/nomos-utils/Cargo.toml b/nomos-utils/Cargo.toml index 3b5f223c..4bb75ce2 100644 --- a/nomos-utils/Cargo.toml +++ b/nomos-utils/Cargo.toml @@ -8,4 +8,7 @@ serde = ["dep:serde"] [dependencies] const-hex = "1" -serde = { version = "1.0", optional = true } \ No newline at end of file +serde = { version = "1.0", optional = true } +rand = "0.8" +rand_chacha = "0.3" + diff --git a/nomos-utils/src/fisheryates.rs b/nomos-utils/src/fisheryates.rs new file mode 100644 index 00000000..c0b427be --- /dev/null +++ b/nomos-utils/src/fisheryates.rs @@ -0,0 +1,23 @@ +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha20Rng; + +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub struct FisherYatesShuffle { + pub entropy: [u8; 32], +} + +impl FisherYatesShuffle { + pub fn new(entropy: [u8; 32]) -> Self { + Self { entropy } + } + + pub fn shuffle(elements: &mut [T], entropy: [u8; 32]) { + let mut rng = ChaCha20Rng::from_seed(entropy); + // Implementation of fisher yates shuffling + // https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle + for i in (1..elements.len()).rev() { + let j = rng.gen_range(0..=i); + elements.swap(i, j); + } + } +} diff --git a/nomos-utils/src/lib.rs b/nomos-utils/src/lib.rs index 01a2b3f5..0700712d 100644 --- a/nomos-utils/src/lib.rs +++ b/nomos-utils/src/lib.rs @@ -1,3 +1,5 @@ +pub mod fisheryates; + #[cfg(feature = "serde")] pub mod serde { fn serialize_human_readable_bytes_array( diff --git a/simulations/Cargo.toml b/simulations/Cargo.toml index ca2dcbed..9a64ab89 100644 --- a/simulations/Cargo.toml +++ b/simulations/Cargo.toml @@ -39,6 +39,7 @@ serde_json = "1.0" thiserror = "1" tracing = { version = "0.1", default-features = false, features = ["log", "attributes"] } tracing-subscriber = { version = "0.3", features = ["json", "env-filter", "tracing-log"]} +nomos-utils = { path = "../nomos-utils" } [target.'cfg(target_arch = "wasm32")'.dependencies] getrandom = { version = "0.2", features = ["js"] } diff --git a/simulations/src/bin/app/overlay_node.rs b/simulations/src/bin/app/overlay_node.rs index 7c296737..3f5af728 100644 --- a/simulations/src/bin/app/overlay_node.rs +++ b/simulations/src/bin/app/overlay_node.rs @@ -1,9 +1,10 @@ -use carnot_engine::overlay::{BranchOverlay, FisherYatesShuffle, RandomBeaconState}; +use carnot_engine::overlay::{BranchOverlay, RandomBeaconState}; use carnot_engine::Overlay; use carnot_engine::{ overlay::{FlatOverlay, FreezeMembership, RoundRobin, TreeOverlay}, NodeId, }; +use nomos_utils::fisheryates::FisherYatesShuffle; use rand::Rng; use simulations::overlay::overlay_info::{OverlayInfo, OverlayInfoExt}; use simulations::settings::OverlaySettings; diff --git a/simulations/src/overlay/overlay_info.rs b/simulations/src/overlay/overlay_info.rs index 5afc1e4b..80d8464e 100644 --- a/simulations/src/overlay/overlay_info.rs +++ b/simulations/src/overlay/overlay_info.rs @@ -60,11 +60,11 @@ impl OverlayInfoExt for T { mod tests { use carnot_engine::{ overlay::{ - BranchOverlay, BranchOverlaySettings, FisherYatesShuffle, RoundRobin, TreeOverlay, - TreeOverlaySettings, + BranchOverlay, BranchOverlaySettings, RoundRobin, TreeOverlay, TreeOverlaySettings, }, NodeId, Overlay, }; + use nomos_utils::fisheryates::FisherYatesShuffle; use super::*; const ENTROPY: [u8; 32] = [0; 32]; diff --git a/testnet/Dockerfile b/testnet/Dockerfile index 15d8231d..446a1d25 100644 --- a/testnet/Dockerfile +++ b/testnet/Dockerfile @@ -24,7 +24,6 @@ EXPOSE 3000 8080 9000 60000 COPY --from=builder /nomos/target/release/nomos-node /usr/bin/nomos-node COPY --from=builder /nomos/target/release/nomos-cli /usr/bin/nomos-cli -COPY --from=builder /nomos/target/release/mixnode /usr/bin/mixnode COPY --from=builder /usr/bin/etcdctl /usr/bin/etcdctl COPY nodes/nomos-node/config.yaml /etc/nomos/config.yaml diff --git a/testnet/README.md b/testnet/README.md index bbd0f42b..cb47982a 100644 --- a/testnet/README.md +++ b/testnet/README.md @@ -5,7 +5,6 @@ The Nomos Docker Compose Testnet contains four distinct service types: - **Bootstrap Node Service**: A singular Nomos node with its own service and a deterministic DNS address. Other nodes utilize this as their initial peer. - **Libp2p Node Services**: Multiple dynamically spawned Nomos nodes that announce their existence through etcd. - **Etcd Service**: A container running an etcd instance. -- **Mix-Node-{0,1,2}**: These are statically configured mixnet nodes. Every Libp2p node includes these in its topology configuration. ## Building @@ -42,7 +41,7 @@ docker compose up -d Followed by: ```bash -docker compose logs -f {bootstrap,libp2p-node,mixnode,etcd} +docker compose logs -f {bootstrap,libp2p-node,etcd} ``` ## Using testnet diff --git a/testnet/bootstrap_config.yaml b/testnet/bootstrap_config.yaml index 994cd330..fec85f37 100644 --- a/testnet/bootstrap_config.yaml +++ b/testnet/bootstrap_config.yaml @@ -25,43 +25,6 @@ network: discV5BootstrapNodes: [] initial_peers: [] relayTopics: [] - # Mixclient configuration to communicate with mixnodes. - # The libp2p network backend always requires this mixclient configuration - # (cannot be disabled for now). - mixnet_client: - # A mixclient mode. For details, see the documentation of the "mixnet" crate. - # - Sender - # - !SenderReceiver [mixnode_client_listen_address] - mode: !SenderReceiver mix-node-2:7778 - # A mixnet topology, which contains the information of all mixnodes in the mixnet. - # (The topology is static for now.) - topology: - # Each mixnet layer consists of a list of mixnodes. - layers: - - nodes: - - address: mix-node-0:7777 # A listen address of the mixnode - public_key: "fd3384e132ad02a56c78f45547ee40038dc79002b90d29ed90e08eee762ae715" - - nodes: - - address: mix-node-1:7777 # A listen address of the mixnode - public_key: "fd3384e132ad02a56c78f45547ee40038dc79002b90d29ed90e08eee762ae715" - - - nodes: - - address: mix-node-2:7777 # A listen address of the mixnode - public_key: "fd3384e132ad02a56c78f45547ee40038dc79002b90d29ed90e08eee762ae715" - - - # A max number of connections that will stay connected to mixnodes in the first mixnet layer. - connection_pool_size: 255 - max_retries: 5 - retry_delay: - secs: 1 - nanos: 0 - # A range of total delay that will be set to each Sphinx packets - # sent to the mixnet for timing obfuscation. - # Panics if start > end. - mixnet_delay: - start: "0ms" - end: "0ms" http: backend_settings: diff --git a/testnet/cli_config.yaml b/testnet/cli_config.yaml index 7f4eef37..6056fe93 100644 --- a/testnet/cli_config.yaml +++ b/testnet/cli_config.yaml @@ -4,41 +4,5 @@ backend: log_level: "fatal" node_key: "0000000000000000000000000000000000000000000000000000000000000667" discV5BootstrapNodes: [] - initial_peers: ["/dns/bootstrap/tcp/3000"] + initial_peers: ["/dns/bootstrap/udp/3000/quic-v1"] relayTopics: [] - # Mixclient configuration to communicate with mixnodes. - # The libp2p network backend always requires this mixclient configuration - # (cannot be disabled for now). - mixnet_client: - # A mixclient mode. For details, see the documentation of the "mixnet" crate. - # - Sender - # - !SenderReceiver [mixnode_client_listen_address] - mode: Sender - # A mixnet topology, which contains the information of all mixnodes in the mixnet. - # (The topology is static for now.) - topology: - # Each mixnet layer consists of a list of mixnodes. - layers: - - nodes: - - address: mix-node-0:7707 # A listen address of the mixnode - public_key: "fd3384e132ad02a56c78f45547ee40038dc79002b90d29ed90e08eee762ae715" - - nodes: - - address: mix-node-1:7717 # A listen address of the mixnode - public_key: "fd3384e132ad02a56c78f45547ee40038dc79002b90d29ed90e08eee762ae715" - - - nodes: - - address: mix-node-2:7727 # A listen address of the mixnode - public_key: "fd3384e132ad02a56c78f45547ee40038dc79002b90d29ed90e08eee762ae715" - - # A max number of connections that will stay connected to mixnodes in the first mixnet layer. - connection_pool_size: 255 - max_retries: 5 - retry_delay: - secs: 1 - nanos: 0 - # A range of total delay that will be set to each Sphinx packets - # sent to the mixnet for timing obfuscation. - # Panics if start > end. - mixnet_delay: - start: "0ms" - end: "0ms" diff --git a/testnet/libp2p_config.yaml b/testnet/libp2p_config.yaml index 1d8a3abf..f060adf9 100644 --- a/testnet/libp2p_config.yaml +++ b/testnet/libp2p_config.yaml @@ -23,45 +23,8 @@ network: log_level: "fatal" node_key: "0000000000000000000000000000000000000000000000000000000000000001" discV5BootstrapNodes: [] - initial_peers: ["/dns/bootstrap/tcp/3000"] + initial_peers: ["/dns/bootstrap/udp/3000/quic-v1"] relayTopics: [] - # Mixclient configuration to communicate with mixnodes. - # The libp2p network backend always requires this mixclient configuration - # (cannot be disabled for now). - mixnet_client: - # A mixclient mode. For details, see the documentation of the "mixnet" crate. - # - Sender - # - !SenderReceiver [mixnode_client_listen_address] - mode: Sender - # A mixnet topology, which contains the information of all mixnodes in the mixnet. - # (The topology is static for now.) - topology: - # Each mixnet layer consists of a list of mixnodes. - layers: - - nodes: - - address: mix-node-0:7777 # A listen address of the mixnode - public_key: "fd3384e132ad02a56c78f45547ee40038dc79002b90d29ed90e08eee762ae715" - - nodes: - - address: mix-node-1:7777 # A listen address of the mixnode - public_key: "fd3384e132ad02a56c78f45547ee40038dc79002b90d29ed90e08eee762ae715" - - - nodes: - - address: mix-node-2:7777 # A listen address of the mixnode - public_key: "fd3384e132ad02a56c78f45547ee40038dc79002b90d29ed90e08eee762ae715" - - - # A max number of connections that will stay connected to mixnodes in the first mixnet layer. - connection_pool_size: 255 - max_retries: 5 - retry_delay: - secs: 1 - nanos: 0 - # A range of total delay that will be set to each Sphinx packets - # sent to the mixnet for timing obfuscation. - # Panics if start > end. - mixnet_delay: - start: "0ms" - end: "0ms" http: backend_settings: diff --git a/testnet/mixnode_config.yaml b/testnet/mixnode_config.yaml deleted file mode 100644 index 22e3cf64..00000000 --- a/testnet/mixnode_config.yaml +++ /dev/null @@ -1,16 +0,0 @@ -mixnode: - # A listen address for other mixnodes in the mixnet and mixclients who want to send packets. - listen_address: 0.0.0.0:7777 - # A (internal) listen address only for a "single" mixclient who wants to receive packets - # from the last mixnet layer. - # For more details, see the documentation in the "mixnet" crate. - client_listen_address: 0.0.0.0:7778 - # A ed25519 private key for decrypting inbound Sphinx packets - # received from mixclients or mixnodes in the previous mixnet layer. - private_key: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] - # A max number of connections that will stay connected to mixnodes in the next layer. - connection_pool_size: 255 -log: - backend: "Stdout" - format: "Json" - level: "info" diff --git a/testnet/scripts/run_nomos_node.sh b/testnet/scripts/run_nomos_node.sh index ea462f95..83327afd 100755 --- a/testnet/scripts/run_nomos_node.sh +++ b/testnet/scripts/run_nomos_node.sh @@ -12,7 +12,7 @@ node_ids=$(etcdctl get "/node/" --prefix --keys-only) for node_id in $node_ids; do node_key=$(etcdctl get "/config${node_id}/key" --print-value-only) node_ip=$(etcdctl get "/config${node_id}/ip" --print-value-only) - node_multiaddr="/ip4/${node_ip}/tcp/3000" + node_multiaddr="/ip4/${node_ip}/udp/3000/quic-v1" if [ -z "$NET_INITIAL_PEERS" ]; then NET_INITIAL_PEERS=$node_multiaddr diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 4b4313db..e35a98bc 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -8,6 +8,7 @@ publish = false nomos-node = { path = "../nodes/nomos-node", default-features = false } carnot-consensus = { path = "../nomos-services/carnot-consensus" } nomos-network = { path = "../nomos-services/network", features = ["libp2p"]} +mixnet = { path = "../mixnet" } nomos-log = { path = "../nomos-services/log" } nomos-api = { path = "../nomos-services/api" } overwatch-rs = { git = "https://github.com/logos-co/Overwatch", rev = "2f70806" } @@ -16,12 +17,7 @@ carnot-engine = { path = "../consensus/carnot-engine", features = ["serde"] } nomos-mempool = { path = "../nomos-services/mempool", features = ["mock", "libp2p"] } nomos-da = { path = "../nomos-services/data-availability" } full-replication = { path = "../nomos-da/full-replication" } -mixnode = { path = "../nodes/mixnode" } -mixnet-node = { path = "../mixnet/node" } -mixnet-client = { path = "../mixnet/client" } -mixnet-topology = { path = "../mixnet/topology" } -# Using older versions, since `mixnet-*` crates depend on `rand` v0.7.3. -rand = "0.7.3" +rand = "0.8" once_cell = "1" secp256k1 = { version = "0.26", features = ["rand"] } reqwest = { version = "0.11", features = ["json"] } @@ -46,19 +42,10 @@ path = "src/tests/happy.rs" name = "test_consensus_unhappy_path" path = "src/tests/unhappy.rs" -[[test]] -name = "test_mixnet" -path = "src/tests/mixnet.rs" - [[test]] name = "test_cli" path = "src/tests/cli.rs" -[[bench]] -name = "mixnet" -path = "src/benches/mixnet.rs" -harness = false - [features] metrics = ["nomos-node/metrics"] diff --git a/tests/src/benches/mixnet.rs b/tests/src/benches/mixnet.rs deleted file mode 100644 index 5d56ca43..00000000 --- a/tests/src/benches/mixnet.rs +++ /dev/null @@ -1,82 +0,0 @@ -use std::time::Duration; - -use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use futures::StreamExt; -use mixnet_client::{MessageStream, MixnetClient, MixnetClientConfig, MixnetClientMode}; -use rand::{rngs::OsRng, Rng, RngCore}; -use tests::MixNode; -use tokio::time::Instant; - -pub fn mixnet(c: &mut Criterion) { - c.bench_function("mixnet", |b| { - b.to_async(tokio::runtime::Runtime::new().unwrap()) - .iter_custom(|iters| async move { - let (_mixnodes, mut sender_client, mut destination_stream, msg) = - setup(100 * 1024).await; - - let start = Instant::now(); - for _ in 0..iters { - black_box( - send_receive_message(&msg, &mut sender_client, &mut destination_stream) - .await, - ); - } - start.elapsed() - }) - }); -} - -async fn setup(msg_size: usize) -> (Vec, MixnetClient, MessageStream, Vec) { - let (mixnodes, node_configs, topology) = MixNode::spawn_nodes(3).await; - - let sender_client = MixnetClient::new( - MixnetClientConfig { - mode: MixnetClientMode::Sender, - topology: topology.clone(), - connection_pool_size: 255, - max_retries: 3, - retry_delay: Duration::from_secs(5), - }, - OsRng, - ); - let destination_client = MixnetClient::new( - MixnetClientConfig { - mode: MixnetClientMode::SenderReceiver( - // Connect with the MixnetNode in the exit layer - // According to the current implementation, - // one of mixnodes the exit layer always will be selected as a destination. - node_configs.last().unwrap().client_listen_address, - ), - topology, - connection_pool_size: 255, - max_retries: 3, - retry_delay: Duration::from_secs(5), - }, - OsRng, - ); - let destination_stream = destination_client.run().await.unwrap(); - - let mut msg = vec![0u8; msg_size]; - rand::thread_rng().fill_bytes(&mut msg); - - (mixnodes, sender_client, destination_stream, msg) -} - -async fn send_receive_message( - msg: &[u8], - sender_client: &mut MixnetClient, - destination_stream: &mut MessageStream, -) { - let res = sender_client.send(msg.to_vec(), Duration::ZERO); - assert!(res.is_ok()); - - let received = destination_stream.next().await.unwrap().unwrap(); - assert_eq!(msg, received.as_slice()); -} - -criterion_group!( - name = benches; - config = Criterion::default().sample_size(10).measurement_time(Duration::from_secs(180)); - targets = mixnet -); -criterion_main!(benches); diff --git a/tests/src/lib.rs b/tests/src/lib.rs index cbc6da5b..28f7288a 100644 --- a/tests/src/lib.rs +++ b/tests/src/lib.rs @@ -1,7 +1,4 @@ pub mod nodes; -use mixnet_node::MixnetNodeConfig; -use mixnet_topology::MixnetTopology; -pub use nodes::MixNode; pub use nodes::NomosNode; use once_cell::sync::Lazy; @@ -16,7 +13,7 @@ use std::{fmt::Debug, sync::Mutex}; use fraction::{Fraction, One}; use rand::{thread_rng, Rng}; -static NET_PORT: Lazy> = Lazy::new(|| Mutex::new(thread_rng().gen_range(8000, 10000))); +static NET_PORT: Lazy> = Lazy::new(|| Mutex::new(thread_rng().gen_range(8000..10000))); static IS_SLOW_TEST_ENV: Lazy = Lazy::new(|| env::var("SLOW_TEST_ENV").is_ok_and(|s| s == "true")); @@ -42,6 +39,7 @@ pub fn adjust_timeout(d: Duration) -> Duration { pub trait Node: Sized { type ConsensusInfo: Debug + Clone + PartialEq; async fn spawn_nodes(config: SpawnConfig) -> Vec; + fn node_configs(config: SpawnConfig) -> Vec; async fn consensus_info(&self) -> Self::ConsensusInfo; fn stop(&mut self); } @@ -49,20 +47,14 @@ pub trait Node: Sized { #[derive(Clone)] pub enum SpawnConfig { // Star topology: Every node is initially connected to a single node. - Star { - consensus: ConsensusConfig, - mixnet: MixnetConfig, - }, + Star { consensus: ConsensusConfig }, // Chain topology: Every node is chained to the node next to it. - Chain { - consensus: ConsensusConfig, - mixnet: MixnetConfig, - }, + Chain { consensus: ConsensusConfig }, } impl SpawnConfig { // Returns a SpawnConfig::Chain with proper configurations for happy-path tests - pub fn chain_happy(n_participants: usize, mixnet_config: MixnetConfig) -> Self { + pub fn chain_happy(n_participants: usize) -> Self { Self::Chain { consensus: ConsensusConfig { n_participants, @@ -73,7 +65,6 @@ impl SpawnConfig { // and it takes 1+ secs for each nomos-node to be started. timeout: adjust_timeout(Duration::from_millis(n_participants as u64 * 2500)), }, - mixnet: mixnet_config, } } } @@ -84,9 +75,3 @@ pub struct ConsensusConfig { pub threshold: Fraction, pub timeout: Duration, } - -#[derive(Clone)] -pub struct MixnetConfig { - pub node_configs: Vec, - pub topology: MixnetTopology, -} diff --git a/tests/src/nodes/mixnode.rs b/tests/src/nodes/mixnode.rs deleted file mode 100644 index 87213c1b..00000000 --- a/tests/src/nodes/mixnode.rs +++ /dev/null @@ -1,131 +0,0 @@ -use std::{ - net::{Ipv4Addr, SocketAddr, SocketAddrV4}, - process::{Child, Command, Stdio}, - time::Duration, -}; - -use super::{create_tempdir, persist_tempdir, LOGS_PREFIX}; -use mixnet_node::{MixnetNodeConfig, PRIVATE_KEY_SIZE}; -use mixnet_topology::{Layer, MixnetTopology, Node}; -use nomos_log::{LoggerBackend, LoggerFormat}; -use rand::{thread_rng, RngCore}; -use tempfile::NamedTempFile; - -use crate::{get_available_port, MixnetConfig}; - -const MIXNODE_BIN: &str = "../target/debug/mixnode"; - -pub struct MixNode { - _tempdir: tempfile::TempDir, - child: Child, -} - -impl Drop for MixNode { - fn drop(&mut self) { - if std::thread::panicking() { - if let Err(e) = persist_tempdir(&mut self._tempdir, "mixnode") { - println!("failed to persist tempdir: {e}"); - } - } - - if let Err(e) = self.child.kill() { - println!("failed to kill the child process: {e}"); - } - } -} - -impl MixNode { - pub async fn spawn(config: MixnetNodeConfig) -> Self { - let dir = create_tempdir().unwrap(); - - let mut config = mixnode::Config { - mixnode: config, - log: Default::default(), - }; - config.log.backend = LoggerBackend::File { - directory: dir.path().to_owned(), - prefix: Some(LOGS_PREFIX.into()), - }; - config.log.format = LoggerFormat::Json; - - let mut file = NamedTempFile::new().unwrap(); - let config_path = file.path().to_owned(); - serde_yaml::to_writer(&mut file, &config).unwrap(); - - let child = Command::new(std::env::current_dir().unwrap().join(MIXNODE_BIN)) - .arg(&config_path) - .stdout(Stdio::null()) - .spawn() - .unwrap(); - - //TODO: use a sophisticated way to wait until the node is ready - tokio::time::sleep(Duration::from_secs(1)).await; - - Self { - _tempdir: dir, - child, - } - } - - pub async fn spawn_nodes(num_nodes: usize) -> (Vec, MixnetConfig) { - let mut configs = Vec::::new(); - for _ in 0..num_nodes { - let mut private_key = [0u8; PRIVATE_KEY_SIZE]; - thread_rng().fill_bytes(&mut private_key); - - let config = MixnetNodeConfig { - listen_address: SocketAddr::V4(SocketAddrV4::new( - Ipv4Addr::new(127, 0, 0, 1), - get_available_port(), - )), - client_listen_address: SocketAddr::V4(SocketAddrV4::new( - Ipv4Addr::new(127, 0, 0, 1), - get_available_port(), - )), - private_key, - connection_pool_size: 255, - ..Default::default() - }; - configs.push(config); - } - - let mut nodes = Vec::::new(); - for config in &configs { - nodes.push(Self::spawn(*config).await); - } - - // We need to return configs as well, to configure mixclients accordingly - ( - nodes, - MixnetConfig { - node_configs: configs.clone(), - topology: Self::build_topology(configs), - }, - ) - } - - fn build_topology(configs: Vec) -> MixnetTopology { - // Build three empty layers first - let mut layers = vec![Layer { nodes: Vec::new() }; 3]; - let mut layer_id = 0; - - // Assign nodes to each layer in round-robin - for config in &configs { - let public_key = config.public_key(); - layers.get_mut(layer_id).unwrap().nodes.push(Node { - address: config.listen_address, - public_key, - }); - layer_id = (layer_id + 1) % layers.len(); - } - - // Exclude empty layers - MixnetTopology { - layers: layers - .iter() - .filter(|layer| !layer.nodes.is_empty()) - .cloned() - .collect(), - } - } -} diff --git a/tests/src/nodes/mod.rs b/tests/src/nodes/mod.rs index bc2a9e61..3c8f0ac8 100644 --- a/tests/src/nodes/mod.rs +++ b/tests/src/nodes/mod.rs @@ -1,7 +1,5 @@ -mod mixnode; pub mod nomos; -pub use self::mixnode::MixNode; pub use nomos::NomosNode; use tempfile::TempDir; diff --git a/tests/src/nodes/nomos.rs b/tests/src/nodes/nomos.rs index b99c0e61..5803dc9f 100644 --- a/tests/src/nodes/nomos.rs +++ b/tests/src/nodes/nomos.rs @@ -4,16 +4,13 @@ use std::process::{Child, Command, Stdio}; use std::time::Duration; // internal use super::{create_tempdir, persist_tempdir, LOGS_PREFIX}; -use crate::{adjust_timeout, get_available_port, ConsensusConfig, MixnetConfig, Node, SpawnConfig}; +use crate::{adjust_timeout, get_available_port, ConsensusConfig, Node, SpawnConfig}; use carnot_consensus::{CarnotInfo, CarnotSettings}; use carnot_engine::overlay::{RandomBeaconState, RoundRobin, TreeOverlay, TreeOverlaySettings}; use carnot_engine::{BlockId, NodeId, Overlay}; use full_replication::Certificate; -use mixnet_client::{MixnetClientConfig, MixnetClientMode}; -use mixnet_node::MixnetNodeConfig; -use mixnet_topology::MixnetTopology; use nomos_core::block::Block; -use nomos_libp2p::{multiaddr, Multiaddr}; +use nomos_libp2p::{Multiaddr, Swarm}; use nomos_log::{LoggerBackend, LoggerFormat}; use nomos_mempool::MempoolMetrics; use nomos_network::backends::libp2p::Libp2pConfig; @@ -201,35 +198,45 @@ impl NomosNode { impl Node for NomosNode { type ConsensusInfo = CarnotInfo; + /// Spawn nodes sequentially. + /// After one node is spawned successfully, the next node is spawned. async fn spawn_nodes(config: SpawnConfig) -> Vec { + let mut nodes = Vec::new(); + for conf in Self::node_configs(config) { + nodes.push(Self::spawn(conf).await); + } + nodes + } + + fn node_configs(config: SpawnConfig) -> Vec { match config { - SpawnConfig::Star { consensus, mixnet } => { - let (next_leader_config, configs) = create_node_configs(consensus, mixnet); + SpawnConfig::Star { consensus } => { + let (next_leader_config, configs) = create_node_configs(consensus); let first_node_addr = node_address(&next_leader_config); - let mut nodes = vec![Self::spawn(next_leader_config).await]; + let mut node_configs = vec![next_leader_config]; for mut conf in configs { conf.network .backend .initial_peers .push(first_node_addr.clone()); - nodes.push(Self::spawn(conf).await); + node_configs.push(conf); } - nodes + node_configs } - SpawnConfig::Chain { consensus, mixnet } => { - let (next_leader_config, configs) = create_node_configs(consensus, mixnet); + SpawnConfig::Chain { consensus } => { + let (next_leader_config, configs) = create_node_configs(consensus); let mut prev_node_addr = node_address(&next_leader_config); - let mut nodes = vec![Self::spawn(next_leader_config).await]; + let mut node_configs = vec![next_leader_config]; for mut conf in configs { conf.network.backend.initial_peers.push(prev_node_addr); prev_node_addr = node_address(&conf); - nodes.push(Self::spawn(conf).await); + node_configs.push(conf); } - nodes + node_configs } } } @@ -250,10 +257,7 @@ impl Node for NomosNode { /// so the leader can receive votes from all other nodes that will be subsequently spawned. /// If not, the leader will miss votes from nodes spawned before itself. /// This issue will be resolved by devising the block catch-up mechanism in the future. -fn create_node_configs( - consensus: ConsensusConfig, - mut mixnet: MixnetConfig, -) -> (Config, Vec) { +fn create_node_configs(consensus: ConsensusConfig) -> (Config, Vec) { let mut ids = vec![[0; 32]; consensus.n_participants]; for id in &mut ids { thread_rng().fill(id); @@ -267,8 +271,6 @@ fn create_node_configs( *id, consensus.threshold, consensus.timeout, - mixnet.node_configs.pop(), - mixnet.topology.clone(), ) }) .collect::>(); @@ -290,29 +292,12 @@ fn create_node_config( id: [u8; 32], threshold: Fraction, timeout: Duration, - mixnet_node_config: Option, - mixnet_topology: MixnetTopology, ) -> Config { - let mixnet_client_mode = match mixnet_node_config { - Some(node_config) => { - MixnetClientMode::SenderReceiver(node_config.client_listen_address.to_string()) - } - None => MixnetClientMode::Sender, - }; - let mut config = Config { network: NetworkConfig { backend: Libp2pConfig { inner: Default::default(), initial_peers: vec![], - mixnet_client: MixnetClientConfig { - mode: mixnet_client_mode, - topology: mixnet_topology, - connection_pool_size: 255, - max_retries: 3, - retry_delay: Duration::from_secs(5), - }, - mixnet_delay: Duration::ZERO..Duration::from_millis(10), }, }, consensus: CarnotSettings { @@ -359,7 +344,10 @@ fn create_node_config( } fn node_address(config: &Config) -> Multiaddr { - multiaddr!(Ip4([127, 0, 0, 1]), Tcp(config.network.backend.inner.port)) + Swarm::multiaddr( + std::net::Ipv4Addr::new(127, 0, 0, 1), + config.network.backend.inner.port, + ) } pub enum Pool { diff --git a/tests/src/tests/cli.rs b/tests/src/tests/cli.rs index 99b14012..294a5632 100644 --- a/tests/src/tests/cli.rs +++ b/tests/src/tests/cli.rs @@ -7,9 +7,7 @@ use nomos_cli::{ use nomos_core::da::{blob::Blob as _, DaProtocol}; use std::{io::Write, time::Duration}; use tempfile::NamedTempFile; -use tests::{ - adjust_timeout, get_available_port, nodes::nomos::Pool, MixNode, Node, NomosNode, SpawnConfig, -}; +use tests::{adjust_timeout, get_available_port, nodes::nomos::Pool, Node, NomosNode, SpawnConfig}; const CLI_BIN: &str = "../target/debug/nomos-cli"; @@ -35,13 +33,10 @@ fn run_disseminate(disseminate: &Disseminate) { } async fn disseminate(config: &mut Disseminate) { - let (_mixnodes, mixnet_config) = MixNode::spawn_nodes(2).await; - let mut nodes = NomosNode::spawn_nodes(SpawnConfig::chain_happy(2, mixnet_config)).await; + let node_configs = NomosNode::node_configs(SpawnConfig::chain_happy(2)); + let first_node = NomosNode::spawn(node_configs[0].clone()).await; - // kill the node so that we can reuse its network config - nodes[1].stop(); - - let mut network_config = nodes[1].config().network.clone(); + let mut network_config = node_configs[1].network.clone(); // use a new port because the old port is sometimes not closed immediately network_config.backend.inner.port = get_available_port(); @@ -68,7 +63,7 @@ async fn disseminate(config: &mut Disseminate) { config.node_addr = Some( format!( "http://{}", - nodes[0].config().http.backend_settings.address.clone() + first_node.config().http.backend_settings.address.clone() ) .parse() .unwrap(), @@ -79,7 +74,7 @@ async fn disseminate(config: &mut Disseminate) { tokio::time::timeout( adjust_timeout(Duration::from_secs(TIMEOUT_SECS)), - wait_for_cert_in_mempool(&nodes[0]), + wait_for_cert_in_mempool(&first_node), ) .await .unwrap(); @@ -93,7 +88,7 @@ async fn disseminate(config: &mut Disseminate) { }; assert_eq!( - get_blobs(&nodes[0].url(), vec![blob]).await.unwrap()[0].as_bytes(), + get_blobs(&first_node.url(), vec![blob]).await.unwrap()[0].as_bytes(), bytes.clone() ); } diff --git a/tests/src/tests/happy.rs b/tests/src/tests/happy.rs index 133d2b10..fa3e50c2 100644 --- a/tests/src/tests/happy.rs +++ b/tests/src/tests/happy.rs @@ -2,7 +2,7 @@ use carnot_engine::{Qc, View}; use futures::stream::{self, StreamExt}; use std::collections::HashSet; use std::time::Duration; -use tests::{adjust_timeout, MixNode, Node, NomosNode, SpawnConfig}; +use tests::{adjust_timeout, Node, NomosNode, SpawnConfig}; const TARGET_VIEW: View = View::new(20); @@ -71,22 +71,19 @@ async fn happy_test(nodes: &[NomosNode]) { #[tokio::test] async fn two_nodes_happy() { - let (_mixnodes, mixnet_config) = MixNode::spawn_nodes(2).await; - let nodes = NomosNode::spawn_nodes(SpawnConfig::chain_happy(2, mixnet_config)).await; + let nodes = NomosNode::spawn_nodes(SpawnConfig::chain_happy(2)).await; happy_test(&nodes).await; } #[tokio::test] async fn ten_nodes_happy() { - let (_mixnodes, mixnet_config) = MixNode::spawn_nodes(3).await; - let nodes = NomosNode::spawn_nodes(SpawnConfig::chain_happy(10, mixnet_config)).await; + let nodes = NomosNode::spawn_nodes(SpawnConfig::chain_happy(10)).await; happy_test(&nodes).await; } #[tokio::test] async fn test_get_block() { - let (_mixnodes, mixnet_config) = MixNode::spawn_nodes(3).await; - let nodes = NomosNode::spawn_nodes(SpawnConfig::chain_happy(2, mixnet_config)).await; + let nodes = NomosNode::spawn_nodes(SpawnConfig::chain_happy(2)).await; happy_test(&nodes).await; let id = nodes[0].consensus_info().await.last_committed_block.id; tokio::time::timeout(Duration::from_secs(10), async { diff --git a/tests/src/tests/mixnet.rs b/tests/src/tests/mixnet.rs deleted file mode 100644 index 53a0f1f5..00000000 --- a/tests/src/tests/mixnet.rs +++ /dev/null @@ -1,139 +0,0 @@ -use std::{ - net::{Ipv4Addr, SocketAddr, SocketAddrV4}, - time::Duration, -}; - -use futures::{Stream, StreamExt}; -use mixnet_client::{MixnetClient, MixnetClientConfig, MixnetClientError, MixnetClientMode}; -use mixnet_node::{MixnetNode, MixnetNodeConfig}; -use mixnet_topology::{Layer, MixnetTopology, Node}; -use rand::{rngs::OsRng, RngCore}; -use tests::get_available_port; - -#[tokio::test] -// Set timeout since the test won't stop even if mixnodes (spawned asynchronously) panic. -#[ntest::timeout(5000)] -async fn mixnet() { - let (topology, mut destination_stream) = run_nodes_and_destination_client().await; - - let mut msg = [0u8; 100 * 1024]; - rand::thread_rng().fill_bytes(&mut msg); - - let mut sender_client = MixnetClient::new( - MixnetClientConfig { - mode: MixnetClientMode::Sender, - topology: topology.clone(), - connection_pool_size: 255, - max_retries: 3, - retry_delay: Duration::from_secs(5), - }, - OsRng, - ); - - let res = sender_client.send(msg.to_vec(), Duration::from_millis(500)); - assert!(res.is_ok()); - - let received = destination_stream.next().await.unwrap().unwrap(); - assert_eq!(msg, received.as_slice()); -} - -async fn run_nodes_and_destination_client() -> ( - MixnetTopology, - impl Stream, MixnetClientError>> + Send, -) { - let config1 = MixnetNodeConfig { - listen_address: SocketAddr::V4(SocketAddrV4::new( - Ipv4Addr::new(127, 0, 0, 1), - get_available_port(), - )), - client_listen_address: SocketAddr::V4(SocketAddrV4::new( - Ipv4Addr::new(127, 0, 0, 1), - get_available_port(), - )), - ..Default::default() - }; - let config2 = MixnetNodeConfig { - listen_address: SocketAddr::V4(SocketAddrV4::new( - Ipv4Addr::new(127, 0, 0, 1), - get_available_port(), - )), - client_listen_address: SocketAddr::V4(SocketAddrV4::new( - Ipv4Addr::new(127, 0, 0, 1), - get_available_port(), - )), - ..Default::default() - }; - let config3 = MixnetNodeConfig { - listen_address: SocketAddr::V4(SocketAddrV4::new( - Ipv4Addr::new(127, 0, 0, 1), - get_available_port(), - )), - client_listen_address: SocketAddr::V4(SocketAddrV4::new( - Ipv4Addr::new(127, 0, 0, 1), - get_available_port(), - )), - ..Default::default() - }; - - let mixnode1 = MixnetNode::new(config1); - let mixnode2 = MixnetNode::new(config2); - let mixnode3 = MixnetNode::new(config3); - - let topology = MixnetTopology { - layers: vec![ - Layer { - nodes: vec![Node { - address: config1.listen_address, - public_key: mixnode1.public_key(), - }], - }, - Layer { - nodes: vec![Node { - address: config2.listen_address, - public_key: mixnode2.public_key(), - }], - }, - Layer { - nodes: vec![Node { - address: config3.listen_address, - public_key: mixnode3.public_key(), - }], - }, - ], - }; - - // Run all MixnetNodes - tokio::spawn(async move { - let res = mixnode1.run().await; - assert!(res.is_ok()); - }); - tokio::spawn(async move { - let res = mixnode2.run().await; - assert!(res.is_ok()); - }); - tokio::spawn(async move { - let res = mixnode3.run().await; - assert!(res.is_ok()); - }); - - // Wait until mixnodes are ready - // TODO: use a more sophisticated way - tokio::time::sleep(Duration::from_secs(1)).await; - - // Run a MixnetClient only for the MixnetNode in the exit layer. - // According to the current implementation, - // one of mixnodes the exit layer always will be selected as a destination. - let client = MixnetClient::new( - MixnetClientConfig { - mode: MixnetClientMode::SenderReceiver(config3.client_listen_address.to_string()), - topology: topology.clone(), - connection_pool_size: 255, - max_retries: 3, - retry_delay: Duration::from_secs(5), - }, - OsRng, - ); - let client_stream = client.run().await.unwrap(); - - (topology, client_stream) -} diff --git a/tests/src/tests/unhappy.rs b/tests/src/tests/unhappy.rs index 6e1a437a..fe35a47a 100644 --- a/tests/src/tests/unhappy.rs +++ b/tests/src/tests/unhappy.rs @@ -3,21 +3,19 @@ use carnot_engine::{Block, NodeId, TimeoutQc, View}; use fraction::Fraction; use futures::stream::{self, StreamExt}; use std::{collections::HashSet, time::Duration}; -use tests::{adjust_timeout, ConsensusConfig, MixNode, Node, NomosNode, SpawnConfig}; +use tests::{adjust_timeout, ConsensusConfig, Node, NomosNode, SpawnConfig}; const TARGET_VIEW: View = View::new(20); const DUMMY_NODE_ID: NodeId = NodeId::new([0u8; 32]); #[tokio::test] async fn ten_nodes_one_down() { - let (_mixnodes, mixnet_config) = MixNode::spawn_nodes(3).await; let mut nodes = NomosNode::spawn_nodes(SpawnConfig::Chain { consensus: ConsensusConfig { n_participants: 10, threshold: Fraction::new(9u32, 10u32), timeout: std::time::Duration::from_secs(5), }, - mixnet: mixnet_config, }) .await; let mut failed_node = nodes.pop().unwrap();