Docker compose for small libp2p node network (#364)

* Update rust build image version

* Docker compose file for testnet

* Wrap tcp into dns transport in order to resolve hostnames (#346)

* Docker compose for small libp2p node network

* Install etcdctl to node containers

* Register libp2p nodes on etcd

* Register IP address in KV store

* Interconnect libp2p nodes

* Use delimiter in cli and env variables

* Use docker compose initial env config

* Leave main Dockerfile as is, use new ones in testnet dir

* Remove etcd installation script

* run_mixnet.sh placeholder

* Use .env file for docker compose config

* Ignore local .env file

* Wrap sh envvars used in strings

* Remove mixnode placeholders

* Use default values for envconfig

* Update labels in Dockerfiles

* Sanitize scripts via shellcheck

* Export env for nomos node

* Updated to latest libp2p config

* Pass config to bootstrap node

.env.example

@@ -0,0 +1,10 @@
+# Environment variables for compose.yml file config.
+DOCKER_COMPOSE_LIBP2P_REPLICAS=1
+DOCKER_COMPOSE_LIBP2P_NODE_KEY_MASK=2000000000000000000000000000000000000000000000000000000000000000
+DOCKER_COMPOSE_MIXNET_REPLICAS=1
+DOCKER_COMPOSE_MIXNET_NODE_KEY_MASK=3000000000000000000000000000000000000000000000000000000000000000
+DOCKER_COMPOSE_ETCDCTL_ENDPOINTS=etcd:2379
+DOCKER_COMPOSE_ETCDCTL_API=3
+DOCKER_COMPOSE_BOOSTRAP_NET_NODE_KEY=1000000000000000000000000000000000000000000000000000000000000000
+DOCKER_COMPOSE_OVERLAY_NODES=$DOCKER_COMPOSE_BOOSTRAP_NET_NODE_KEY
+DOCKER_COMPOSE_NET_INITIAL_PEERS=/dns/bootstrap/tcp/3000

.gitignore

@@ -14,3 +14,4 @@ config.yml
 store.*
 sim_config.json
 *.txt
+.env

Dockerfile

@@ -1,6 +1,6 @@
 # BUILD IMAGE ---------------------------------------------------------
 
-FROM rust:1.70.0-slim-bullseye AS builder
+FROM rust:1.72.0-slim-bullseye AS builder
 
 # Using backports for go 1.19
 RUN echo 'deb http://deb.debian.org/debian bullseye-backports main' \
@@ -16,15 +16,15 @@ RUN apt-get update && apt-get install -yq \
 WORKDIR /nomos
 COPY . . 
 
-RUN cargo build --release -p nomos-node
+RUN cargo build --release -p nomos-node --no-default-features --features libp2p
 
 # NODE IMAGE ----------------------------------------------------------
 
 FROM bitnami/minideb:latest
 
-LABEL maintainer="augustinas@status.im"
-LABEL source="https://github.com/logos-co/nomos-research"
-LABEL description="Nomos node image"
+LABEL maintainer="augustinas@status.im" \
+      source="https://github.com/logos-co/nomos-node" \
+      description="Nomos node image"
 
 # nomos default ports
 EXPOSE 3000 8080 9000 60000	

compose.yml

@@ -0,0 +1,43 @@
+services:
+
+  bootstrap:
+    build:
+      context: .
+      dockerfile: testnet/Dockerfile
+    ports:
+      - "3000:3000/tcp"
+      - "8080:8080/tcp"
+    volumes:
+      - ./testnet:/etc/nomos
+    environment:
+      - NET_NODE_KEY=${DOCKER_COMPOSE_BOOSTRAP_NET_NODE_KEY:-1000000000000000000000000000000000000000000000000000000000000000}
+    command: /etc/nomos/config.yaml
+
+  libp2p-node:
+    build:
+      context: .
+      dockerfile: testnet/Dockerfile
+    volumes:
+      - ./testnet:/etc/nomos
+    deploy:
+      replicas: ${DOCKER_COMPOSE_LIBP2P_REPLICAS:-1} 
+    depends_on:
+      - bootstrap
+      - etcd
+    environment:
+      - DOCKER_REPLICAS=${DOCKER_COMPOSE_LIBP2P_REPLICAS:-1}
+      - ETCDCTL_ENDPOINTS=${DOCKER_COMPOSE_ETCDCTL_ENDPOINTS:-etcd:2379}
+      - ETCDCTL_API=${DOCKER_COMPOSE_ETCDCTL_API:-3}
+      - NODE_MASK=${DOCKER_COMPOSE_LIBP2P_NODE_KEY_MASK:-2000000000000000000000000000000000000000000000000000000000000000}
+      - OVERLAY_NODES=${DOCKER_COMPOSE_OVERLAY_NODES:-1000000000000000000000000000000000000000000000000000000000000000}
+      - NET_INITIAL_PEERS=${DOCKER_COMPOSE_NET_INITIAL_PEERS:-/dns/bootstrap/tcp/3000}
+    entrypoint: /etc/nomos/configure_node.sh
+
+  etcd:
+    image: quay.io/coreos/etcd:v3.4.15
+    ports:
+      - "2379:2379/tcp"
+    command:
+      - /usr/local/bin/etcd
+      - --advertise-client-urls=http://etcd:2379
+      - --listen-client-urls=http://0.0.0.0:2379

nodes/nomos-node/src/config.rs

@@ -70,7 +70,7 @@ pub struct NetworkArgs {
     #[clap(long = "net-node-key", env = "NET_NODE_KEY")]
     node_key: Option<String>,
 
-    #[clap(long = "net-initial-peers", env = "NET_INITIAL_PEERS")]
+    #[clap(long = "net-initial-peers", env = "NET_INITIAL_PEERS", num_args = 1.., value_delimiter = ',')]
     pub initial_peers: Option<Vec<Multiaddr>>,
 }
 
@@ -105,7 +105,7 @@ pub struct OverlayArgs {
     #[clap(long = "overlay-type", env = "OVERLAY_TYPE")]
     pub overlay_type: Option<OverlayType>,
 
-    #[clap(long = "overlay-nodes", env = "OVERLAY_NODES")]
+    #[clap(long = "overlay-nodes", env = "OVERLAY_NODES", num_args = 1.., value_delimiter = ',')]
     pub overlay_nodes: Option<Vec<String>>,
 
     #[clap(long = "overlay-leader", env = "OVERLAY_LEADER")]

nomos-libp2p/Cargo.toml

@@ -8,6 +8,7 @@ multiaddr = "0.18"
 tokio = { version = "1", features = ["sync", "macros"] }
 futures = "0.3"
 libp2p = { version = "0.52.1", features = [
+  "dns",
   "yamux",
   "plaintext",
   "macros",

nomos-libp2p/src/lib.rs

@@ -10,8 +10,10 @@ use blake2::digest::{consts::U32, Digest};
 use blake2::Blake2b;
 use libp2p::gossipsub::{Message, MessageId, TopicHash};
 
+use libp2p::tcp::tokio::Tcp;
 pub use libp2p::{
     core::upgrade,
+    dns,
     gossipsub::{self, PublishError, SubscriptionError},
     identity::{self, secp256k1},
     plaintext::PlainText2Config,
@@ -76,7 +78,7 @@ impl Swarm {
         log::info!("libp2p peer_id:{}", local_peer_id);
 
         // TODO: consider using noise authentication
-        let tcp_transport = tcp::tokio::Transport::new(tcp::Config::default().nodelay(true))
+        let tcp_transport = tcp::Transport::<Tcp>::new(tcp::Config::default().nodelay(true))
             .upgrade(upgrade::Version::V1Lazy)
             .authenticate(PlainText2Config {
                 local_public_key: id_keys.public(),
@@ -85,6 +87,9 @@ impl Swarm {
             .timeout(TRANSPORT_TIMEOUT)
             .boxed();
 
+        // Wrapping TCP transport into DNS transport to resolve hostnames.
+        let tcp_transport = dns::TokioDnsConfig::system(tcp_transport)?.boxed();
+
         // TODO: consider using Signed or Anonymous.
         //       For Anonymous, a custom `message_id` function need to be set
         //       to prevent all messages from a peer being filtered as duplicates.

testnet/Dockerfile

@@ -0,0 +1,38 @@
+# BUILD IMAGE ---------------------------------------------------------
+
+FROM rust:1.72.0-slim-bullseye AS builder
+
+# Using backports for go 1.19
+RUN echo 'deb http://deb.debian.org/debian bullseye-backports main' \
+    >> /etc/apt/sources.list
+
+# Dependecies for publishing documentation and building waku-bindings.
+RUN apt-get update && apt-get install -yq \
+    git clang etcd-client \
+    golang-src/bullseye-backports \
+    golang-doc/bullseye-backports \
+    golang/bullseye-backports
+
+WORKDIR /nomos
+COPY . . 
+
+RUN cargo build --release -p nomos-node --no-default-features --features libp2p
+RUN cargo build --release -p mixnode
+
+# NODE IMAGE ----------------------------------------------------------
+
+FROM bitnami/minideb:latest
+
+LABEL maintainer="augustinas@status.im" \
+      source="https://github.com/logos-co/nomos-node" \
+      description="Nomos testnet image"
+
+# nomos default ports
+EXPOSE 3000 8080 9000 60000	
+
+COPY --from=builder /nomos/target/release/nomos-node /usr/bin/nomos-node
+COPY --from=builder /nomos/target/release/mixnode /usr/bin/mixnode
+COPY --from=builder /usr/bin/etcdctl /usr/bin/etcdctl
+COPY nodes/nomos-node/config.yaml /etc/nomos/config.yaml
+
+ENTRYPOINT ["/usr/bin/nomos-node"]

testnet/config.yaml

@@ -0,0 +1,37 @@
+log:
+  backend: "Stdout"
+  format: "Json"
+  level: "debug"
+consensus:
+  private_key: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+  fountain_settings: null
+  overlay_settings:
+    nodes: [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]]
+    leader_super_majority_threshold: 1
+    leader:
+      cur: 0
+network:
+  backend:
+    host: 0.0.0.0
+    port: 3000
+    log_level: "fatal"
+    node_key: "0000000000000000000000000000000000000000000000000000000000000001"
+    discV5BootstrapNodes: []
+    initial_peers: []
+    relayTopics: []
+    mixnet_client:
+      mode: Sender
+      topology:
+        layers:
+         - nodes:
+            - address: 127.0.0.1:7777
+              public_key: "0000000000000000000000000000000000000000000000000000000000000000"
+      connection_pool_size: 255
+    mixnet_delay:
+      start: "0ms"
+      end: "0ms"
+
+http:
+  backend:
+    address: 0.0.0.0:8080
+    cors_origins: []

testnet/configure_node.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+set -e
+
+# Set env variables for nomos-node.
+NET_NODE_KEY=$(./etc/nomos/register_node.sh)
+CONSENSUS_PRIV_KEY=$NET_NODE_KEY
+
+node_ids=$(etcdctl get "/node/" --prefix --keys-only)
+for node_id in $node_ids; do
+	node_key=$(etcdctl get "/config${node_id}/key" --print-value-only)
+	node_ip=$(etcdctl get "/config${node_id}/ip" --print-value-only)
+	node_multiaddr="/ip4/${node_ip}/tcp/3000"
+
+	if [ -z "$OVERLAY_NODES" ]; then
+		OVERLAY_NODES=$node_key
+		NET_INITIAL_PEERS=$node_multiaddr
+	else
+		OVERLAY_NODES="${OVERLAY_NODES},${node_key}"
+		NET_INITIAL_PEERS="${NET_INITIAL_PEERS},${node_multiaddr}"
+	fi
+done
+
+export CONSENSUS_PRIV_KEY \
+       OVERLAY_NODES \
+       NET_NODE_KEY \
+       NET_INITIAL_PEERS
+
+echo "I am a container ${HOSTNAME} node ${NET_NODE_KEY}"
+
+exec /usr/bin/nomos-node /etc/nomos/config.yaml

testnet/register_node.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+
+# NODE_MASK is set via compose.yml file.
+
+node_key_from_id() {
+	echo "${NODE_MASK}" | sed "s/.\{${#NODE_ID}\}$/${NODE_ID}/"
+}
+
+END=$DOCKER_REPLICAS
+NODE_ID=1
+NODE_IP=$(hostname -i)
+NODE_KEY=$(node_key_from_id)
+
+register_node() {
+	## Conditional transaction to set node config key if it doesn't exist.
+	## Newlines in EOF block are important, more info here:
+	## https://github.com/etcd-io/etcd/tree/main/etcdctl#examples-3
+	etcdctl txn <<EOF
+mod("/node/${NODE_ID}") = "0"
+
+put /node/${NODE_ID} "${NODE_ID}"
+put /config/node/${NODE_ID}/key "${NODE_KEY}"
+put /config/node/${NODE_ID}/ip "${NODE_IP}"
+
+
+EOF
+}
+
+while [ "${NODE_ID}" -le "${END}" ]; do
+	result=$(register_node)
+
+	# Check if the key was registered or already exists
+	if [ "${result}" != "FAILURE" ]; then
+		break
+	else
+		NODE_ID=$((NODE_ID + 1))
+		NODE_KEY=$(node_key_from_id)
+	fi
+done
+
+if [ "${NODE_ID}" -gt "${END}" ]; then
+	echo "Reached the limit without registering a ${NODE_ID}."
+	return 1
+fi
+
+echo "${NODE_KEY}"