From 97c653efe3a9720dc085284f7c916e0990d2af6c Mon Sep 17 00:00:00 2001 From: gusto Date: Fri, 15 Sep 2023 17:27:38 +0300 Subject: [PATCH] Docker compose for small libp2p node network (#364) * Update rust build image version * Docker compose file for testnet * Wrap tcp into dns transport in order to resolve hostnames (#346) * Docker compose for small libp2p node network * Install etcdctl to node containers * Register libp2p nodes on etcd * Register IP address in KV store * Interconnect libp2p nodes * Use delimiter in cli and env variables * Use docker compose initial env config * Leave main Dockerfile as is, use new ones in testnet dir * Remove etcd installation script * run_mixnet.sh placeholder * Use .env file for docker compose config * Ignore local .env file * Wrap sh envvars used in strings * Remove mixnode placeholders * Use default values for envconfig * Update labels in Dockerfiles * Sanitize scripts via shellcheck * Export env for nomos node * Updated to latest libp2p config * Pass config to bootstrap node --- .env.example | 10 ++++++++ .gitignore | 1 + Dockerfile | 10 ++++---- compose.yml | 43 +++++++++++++++++++++++++++++++ nodes/nomos-node/src/config.rs | 4 +-- nomos-libp2p/Cargo.toml | 1 + nomos-libp2p/src/lib.rs | 7 +++++- testnet/Dockerfile | 38 ++++++++++++++++++++++++++++ testnet/config.yaml | 37 +++++++++++++++++++++++++++ testnet/configure_node.sh | 31 +++++++++++++++++++++++ testnet/register_node.sh | 46 ++++++++++++++++++++++++++++++++++ 11 files changed, 220 insertions(+), 8 deletions(-) create mode 100644 .env.example create mode 100644 compose.yml create mode 100644 testnet/Dockerfile create mode 100644 testnet/config.yaml create mode 100755 testnet/configure_node.sh create mode 100755 testnet/register_node.sh diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..e261cedf --- /dev/null +++ b/.env.example @@ -0,0 +1,10 @@ +# Environment variables for compose.yml file config. +DOCKER_COMPOSE_LIBP2P_REPLICAS=1 +DOCKER_COMPOSE_LIBP2P_NODE_KEY_MASK=2000000000000000000000000000000000000000000000000000000000000000 +DOCKER_COMPOSE_MIXNET_REPLICAS=1 +DOCKER_COMPOSE_MIXNET_NODE_KEY_MASK=3000000000000000000000000000000000000000000000000000000000000000 +DOCKER_COMPOSE_ETCDCTL_ENDPOINTS=etcd:2379 +DOCKER_COMPOSE_ETCDCTL_API=3 +DOCKER_COMPOSE_BOOSTRAP_NET_NODE_KEY=1000000000000000000000000000000000000000000000000000000000000000 +DOCKER_COMPOSE_OVERLAY_NODES=$DOCKER_COMPOSE_BOOSTRAP_NET_NODE_KEY +DOCKER_COMPOSE_NET_INITIAL_PEERS=/dns/bootstrap/tcp/3000 diff --git a/.gitignore b/.gitignore index e80810d3..3d3cc94b 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ config.yml store.* sim_config.json *.txt +.env diff --git a/Dockerfile b/Dockerfile index 2f4399a3..1acb8e13 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # BUILD IMAGE --------------------------------------------------------- -FROM rust:1.70.0-slim-bullseye AS builder +FROM rust:1.72.0-slim-bullseye AS builder # Using backports for go 1.19 RUN echo 'deb http://deb.debian.org/debian bullseye-backports main' \ @@ -16,15 +16,15 @@ RUN apt-get update && apt-get install -yq \ WORKDIR /nomos COPY . . -RUN cargo build --release -p nomos-node +RUN cargo build --release -p nomos-node --no-default-features --features libp2p # NODE IMAGE ---------------------------------------------------------- FROM bitnami/minideb:latest -LABEL maintainer="augustinas@status.im" -LABEL source="https://github.com/logos-co/nomos-research" -LABEL description="Nomos node image" +LABEL maintainer="augustinas@status.im" \ + source="https://github.com/logos-co/nomos-node" \ + description="Nomos node image" # nomos default ports EXPOSE 3000 8080 9000 60000 diff --git a/compose.yml b/compose.yml new file mode 100644 index 00000000..7803c4c0 --- /dev/null +++ b/compose.yml @@ -0,0 +1,43 @@ +services: + + bootstrap: + build: + context: . + dockerfile: testnet/Dockerfile + ports: + - "3000:3000/tcp" + - "8080:8080/tcp" + volumes: + - ./testnet:/etc/nomos + environment: + - NET_NODE_KEY=${DOCKER_COMPOSE_BOOSTRAP_NET_NODE_KEY:-1000000000000000000000000000000000000000000000000000000000000000} + command: /etc/nomos/config.yaml + + libp2p-node: + build: + context: . + dockerfile: testnet/Dockerfile + volumes: + - ./testnet:/etc/nomos + deploy: + replicas: ${DOCKER_COMPOSE_LIBP2P_REPLICAS:-1} + depends_on: + - bootstrap + - etcd + environment: + - DOCKER_REPLICAS=${DOCKER_COMPOSE_LIBP2P_REPLICAS:-1} + - ETCDCTL_ENDPOINTS=${DOCKER_COMPOSE_ETCDCTL_ENDPOINTS:-etcd:2379} + - ETCDCTL_API=${DOCKER_COMPOSE_ETCDCTL_API:-3} + - NODE_MASK=${DOCKER_COMPOSE_LIBP2P_NODE_KEY_MASK:-2000000000000000000000000000000000000000000000000000000000000000} + - OVERLAY_NODES=${DOCKER_COMPOSE_OVERLAY_NODES:-1000000000000000000000000000000000000000000000000000000000000000} + - NET_INITIAL_PEERS=${DOCKER_COMPOSE_NET_INITIAL_PEERS:-/dns/bootstrap/tcp/3000} + entrypoint: /etc/nomos/configure_node.sh + + etcd: + image: quay.io/coreos/etcd:v3.4.15 + ports: + - "2379:2379/tcp" + command: + - /usr/local/bin/etcd + - --advertise-client-urls=http://etcd:2379 + - --listen-client-urls=http://0.0.0.0:2379 diff --git a/nodes/nomos-node/src/config.rs b/nodes/nomos-node/src/config.rs index 060c3284..216f895b 100644 --- a/nodes/nomos-node/src/config.rs +++ b/nodes/nomos-node/src/config.rs @@ -70,7 +70,7 @@ pub struct NetworkArgs { #[clap(long = "net-node-key", env = "NET_NODE_KEY")] node_key: Option, - #[clap(long = "net-initial-peers", env = "NET_INITIAL_PEERS")] + #[clap(long = "net-initial-peers", env = "NET_INITIAL_PEERS", num_args = 1.., value_delimiter = ',')] pub initial_peers: Option>, } @@ -105,7 +105,7 @@ pub struct OverlayArgs { #[clap(long = "overlay-type", env = "OVERLAY_TYPE")] pub overlay_type: Option, - #[clap(long = "overlay-nodes", env = "OVERLAY_NODES")] + #[clap(long = "overlay-nodes", env = "OVERLAY_NODES", num_args = 1.., value_delimiter = ',')] pub overlay_nodes: Option>, #[clap(long = "overlay-leader", env = "OVERLAY_LEADER")] diff --git a/nomos-libp2p/Cargo.toml b/nomos-libp2p/Cargo.toml index f56cb46e..803b2b68 100644 --- a/nomos-libp2p/Cargo.toml +++ b/nomos-libp2p/Cargo.toml @@ -8,6 +8,7 @@ multiaddr = "0.18" tokio = { version = "1", features = ["sync", "macros"] } futures = "0.3" libp2p = { version = "0.52.1", features = [ + "dns", "yamux", "plaintext", "macros", diff --git a/nomos-libp2p/src/lib.rs b/nomos-libp2p/src/lib.rs index d9ae9dee..fb41f373 100644 --- a/nomos-libp2p/src/lib.rs +++ b/nomos-libp2p/src/lib.rs @@ -10,8 +10,10 @@ use blake2::digest::{consts::U32, Digest}; use blake2::Blake2b; use libp2p::gossipsub::{Message, MessageId, TopicHash}; +use libp2p::tcp::tokio::Tcp; pub use libp2p::{ core::upgrade, + dns, gossipsub::{self, PublishError, SubscriptionError}, identity::{self, secp256k1}, plaintext::PlainText2Config, @@ -76,7 +78,7 @@ impl Swarm { log::info!("libp2p peer_id:{}", local_peer_id); // TODO: consider using noise authentication - let tcp_transport = tcp::tokio::Transport::new(tcp::Config::default().nodelay(true)) + let tcp_transport = tcp::Transport::::new(tcp::Config::default().nodelay(true)) .upgrade(upgrade::Version::V1Lazy) .authenticate(PlainText2Config { local_public_key: id_keys.public(), @@ -85,6 +87,9 @@ impl Swarm { .timeout(TRANSPORT_TIMEOUT) .boxed(); + // Wrapping TCP transport into DNS transport to resolve hostnames. + let tcp_transport = dns::TokioDnsConfig::system(tcp_transport)?.boxed(); + // TODO: consider using Signed or Anonymous. // For Anonymous, a custom `message_id` function need to be set // to prevent all messages from a peer being filtered as duplicates. diff --git a/testnet/Dockerfile b/testnet/Dockerfile new file mode 100644 index 00000000..bd51b966 --- /dev/null +++ b/testnet/Dockerfile @@ -0,0 +1,38 @@ +# BUILD IMAGE --------------------------------------------------------- + +FROM rust:1.72.0-slim-bullseye AS builder + +# Using backports for go 1.19 +RUN echo 'deb http://deb.debian.org/debian bullseye-backports main' \ + >> /etc/apt/sources.list + +# Dependecies for publishing documentation and building waku-bindings. +RUN apt-get update && apt-get install -yq \ + git clang etcd-client \ + golang-src/bullseye-backports \ + golang-doc/bullseye-backports \ + golang/bullseye-backports + +WORKDIR /nomos +COPY . . + +RUN cargo build --release -p nomos-node --no-default-features --features libp2p +RUN cargo build --release -p mixnode + +# NODE IMAGE ---------------------------------------------------------- + +FROM bitnami/minideb:latest + +LABEL maintainer="augustinas@status.im" \ + source="https://github.com/logos-co/nomos-node" \ + description="Nomos testnet image" + +# nomos default ports +EXPOSE 3000 8080 9000 60000 + +COPY --from=builder /nomos/target/release/nomos-node /usr/bin/nomos-node +COPY --from=builder /nomos/target/release/mixnode /usr/bin/mixnode +COPY --from=builder /usr/bin/etcdctl /usr/bin/etcdctl +COPY nodes/nomos-node/config.yaml /etc/nomos/config.yaml + +ENTRYPOINT ["/usr/bin/nomos-node"] diff --git a/testnet/config.yaml b/testnet/config.yaml new file mode 100644 index 00000000..d60e6c86 --- /dev/null +++ b/testnet/config.yaml @@ -0,0 +1,37 @@ +log: + backend: "Stdout" + format: "Json" + level: "debug" +consensus: + private_key: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + fountain_settings: null + overlay_settings: + nodes: [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]] + leader_super_majority_threshold: 1 + leader: + cur: 0 +network: + backend: + host: 0.0.0.0 + port: 3000 + log_level: "fatal" + node_key: "0000000000000000000000000000000000000000000000000000000000000001" + discV5BootstrapNodes: [] + initial_peers: [] + relayTopics: [] + mixnet_client: + mode: Sender + topology: + layers: + - nodes: + - address: 127.0.0.1:7777 + public_key: "0000000000000000000000000000000000000000000000000000000000000000" + connection_pool_size: 255 + mixnet_delay: + start: "0ms" + end: "0ms" + +http: + backend: + address: 0.0.0.0:8080 + cors_origins: [] diff --git a/testnet/configure_node.sh b/testnet/configure_node.sh new file mode 100755 index 00000000..0ee22d37 --- /dev/null +++ b/testnet/configure_node.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +set -e + +# Set env variables for nomos-node. +NET_NODE_KEY=$(./etc/nomos/register_node.sh) +CONSENSUS_PRIV_KEY=$NET_NODE_KEY + +node_ids=$(etcdctl get "/node/" --prefix --keys-only) +for node_id in $node_ids; do + node_key=$(etcdctl get "/config${node_id}/key" --print-value-only) + node_ip=$(etcdctl get "/config${node_id}/ip" --print-value-only) + node_multiaddr="/ip4/${node_ip}/tcp/3000" + + if [ -z "$OVERLAY_NODES" ]; then + OVERLAY_NODES=$node_key + NET_INITIAL_PEERS=$node_multiaddr + else + OVERLAY_NODES="${OVERLAY_NODES},${node_key}" + NET_INITIAL_PEERS="${NET_INITIAL_PEERS},${node_multiaddr}" + fi +done + +export CONSENSUS_PRIV_KEY \ + OVERLAY_NODES \ + NET_NODE_KEY \ + NET_INITIAL_PEERS + +echo "I am a container ${HOSTNAME} node ${NET_NODE_KEY}" + +exec /usr/bin/nomos-node /etc/nomos/config.yaml diff --git a/testnet/register_node.sh b/testnet/register_node.sh new file mode 100755 index 00000000..387fa361 --- /dev/null +++ b/testnet/register_node.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +# NODE_MASK is set via compose.yml file. + +node_key_from_id() { + echo "${NODE_MASK}" | sed "s/.\{${#NODE_ID}\}$/${NODE_ID}/" +} + +END=$DOCKER_REPLICAS +NODE_ID=1 +NODE_IP=$(hostname -i) +NODE_KEY=$(node_key_from_id) + +register_node() { + ## Conditional transaction to set node config key if it doesn't exist. + ## Newlines in EOF block are important, more info here: + ## https://github.com/etcd-io/etcd/tree/main/etcdctl#examples-3 + etcdctl txn <