Static mixnet nodes in docker compose

2025-01-23 06:00:36 +00:00 · 2023-10-16 14:06:40 +03:00 · 2023-10-16 14:06:40 +03:00 · b521375ad4
commit b521375ad4
parent 474f5f7a19
10 changed files with 171 additions and 85 deletions
--- a/.env.example
+++ b/.env.example
@ -1,8 +1,6 @@
 # Environment variables for compose.yml file config.
 DOCKER_COMPOSE_LIBP2P_REPLICAS=1
 DOCKER_COMPOSE_LIBP2P_NODE_KEY_MASK=2000000000000000000000000000000000000000000000000000000000000000
-DOCKER_COMPOSE_MIXNET_REPLICAS=1
-DOCKER_COMPOSE_MIXNET_NODE_KEY_MASK=3000000000000000000000000000000000000000000000000000000000000000
 DOCKER_COMPOSE_ETCDCTL_ENDPOINTS=etcd:2379
 DOCKER_COMPOSE_ETCDCTL_API=3
 DOCKER_COMPOSE_BOOSTRAP_NET_NODE_KEY=1000000000000000000000000000000000000000000000000000000000000000
--- a/compose.yml
+++ b/compose.yml
@ -25,33 +25,40 @@ services:
      - bootstrap
      - etcd
    environment:
-      - DOCKER_REPLICAS=${DOCKER_COMPOSE_LIBP2P_REPLICAS:-1}
+      - LIBP2P_REPLICAS=${DOCKER_COMPOSE_LIBP2P_REPLICAS:-1}
      - ETCDCTL_ENDPOINTS=${DOCKER_COMPOSE_ETCDCTL_ENDPOINTS:-etcd:2379}
      - ETCDCTL_API=${DOCKER_COMPOSE_ETCDCTL_API:-3}
      - NODE_MASK=${DOCKER_COMPOSE_LIBP2P_NODE_KEY_MASK:-2000000000000000000000000000000000000000000000000000000000000000}
      - OVERLAY_NODES=${DOCKER_COMPOSE_OVERLAY_NODES:-1000000000000000000000000000000000000000000000000000000000000000}
      - NET_INITIAL_PEERS=${DOCKER_COMPOSE_NET_INITIAL_PEERS:-/dns/bootstrap/tcp/3000}
-    entrypoint: /etc/nomos/run_libp2p_node.sh
+    entrypoint: /etc/nomos/configure_node.sh

-  mix-node:
+  mix-node-0:
    build:
      context: .
      dockerfile: testnet/Dockerfile
    volumes:
      - ./testnet:/etc/nomos
-    deploy:
-      replicas: ${DOCKER_COMPOSE_MIXNET_REPLICAS:-1} 
-    depends_on:
-      - bootstrap
-      - etcd
-    environment:
-      - DOCKER_REPLICAS=${DOCKER_COMPOSE_MIXNET_REPLICAS:-1}
-      - ETCDCTL_ENDPOINTS=${DOCKER_COMPOSE_ETCDCTL_ENDPOINTS:-etcd:2379}
-      - ETCDCTL_API=${DOCKER_COMPOSE_ETCDCTL_API:-3}
-      - NODE_MASK=${DOCKER_COMPOSE_MIXNET_NODE_KEY_MASK:-2000000000000000000000000000000000000000000000000000000000000000}
-      - OVERLAY_NODES=${DOCKER_COMPOSE_OVERLAY_NODES:-1000000000000000000000000000000000000000000000000000000000000000}
-      - NET_INITIAL_PEERS=${DOCKER_COMPOSE_NET_INITIAL_PEERS:-/dns/bootstrap/tcp/3000}
-    entrypoint: /etc/nomos/run_mix_node.sh
+    entrypoint: /usr/bin/mixnode
+    command: mixnode_config.yaml
+
+  mix-node-1:
+    build:
+      context: .
+      dockerfile: testnet/Dockerfile
+    volumes:
+      - ./testnet:/etc/nomos
+    entrypoint: /usr/bin/mixnode
+    command: mixnode_config.yaml
+
+  mix-node-2:
+    build:
+      context: .
+      dockerfile: testnet/Dockerfile
+    volumes:
+      - ./testnet:/etc/nomos
+    entrypoint: /usr/bin/mixnode
+    command: mixnode_config.yaml

  etcd:
    image: quay.io/coreos/etcd:v3.4.15
--- a/testnet/Dockerfile
+++ b/testnet/Dockerfile
@ -8,10 +8,7 @@ RUN echo 'deb http://deb.debian.org/debian bullseye-backports main' \

 # Dependecies for publishing documentation.
 RUN apt-get update && apt-get install -yq \
-    git clang etcd-client libssl-dev \
-    golang-src/bullseye-backports \
-    golang-doc/bullseye-backports \
-    golang/bullseye-backports
+    git clang etcd-client libssl-dev pkg-config

 WORKDIR /nomos
 COPY . . 
@ -33,5 +30,6 @@ COPY --from=builder /nomos/target/release/nomos-node /usr/bin/nomos-node
 COPY --from=builder /nomos/target/release/mixnode /usr/bin/mixnode
 COPY --from=builder /usr/bin/etcdctl /usr/bin/etcdctl
 COPY nodes/nomos-node/config.yaml /etc/nomos/config.yaml
+RUN install_packages python3 python3-etcd3

 ENTRYPOINT ["/usr/bin/nomos-node"]
--- a/testnet/libp2p_config.yaml
+++ b/testnet/libp2p_config.yaml
@ -0,0 +1,73 @@
+log:
+  backend: "Stdout"
+  format: "Json"
+  level: "debug"
+consensus:
+  private_key: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+  fountain_settings: null
+  overlay_settings:
+    nodes: [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]]
+    number_of_committees: 1
+    current_leader: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+    leader:
+      cur: 0
+    committee_membership: !Sad
+      entropy: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+    leader_super_majority_threshold: 1
+    super_majority_threshold: 1
+
+network:
+  backend:
+    host: 0.0.0.0
+    port: 3000
+    log_level: "fatal"
+    node_key: "0000000000000000000000000000000000000000000000000000000000000001"
+    discV5BootstrapNodes: []
+    initial_peers: []
+    relayTopics: []
+    # Mixclient configuration to communicate with mixnodes.
+    # The libp2p network backend always requires this mixclient configuration
+    # (cannot be disabled for now).
+    mixnet_client:
+      # A mixclient mode. For details, see the documentation of the "mixnet" crate.
+      # - Sender
+      # - !SenderReceiver [mixnode_client_listen_address]
+      mode: Sender
+      # A mixnet topology, which contains the information of all mixnodes in the mixnet.
+      # (The topology is static for now.)
+      topology:
+        # Each mixnet layer consists of a list of mixnodes.
+        layers:
+         - nodes:
+            - address: mix-node-0:7777  # A listen address of the mixnode
+              public_key: "0000000000000000000000000000000000000000000000000000000000000000"
+            - address: mix-node-1:7777  # A listen address of the mixnode
+              public_key: "0000000000000000000000000000000000000000000000000000000000000000"
+            - address: mix-node-2:7777  # A listen address of the mixnode
+              public_key: "0000000000000000000000000000000000000000000000000000000000000000"
+      # A max number of connections that will stay connected to mixnodes in the first mixnet layer.
+      connection_pool_size: 255
+      max_retries: 5
+      retry_delay:
+        secs: 1
+        nanos: 0
+    # A range of total delay that will be set to each Sphinx packets
+    # sent to the mixnet for timing obfuscation.
+    # Panics if start > end.
+    mixnet_delay:
+      start: "0ms"
+      end: "0ms"
+
+http:
+  backend:
+    address: 0.0.0.0:8080
+    cors_origins: []
+
+da:
+  da_protocol:
+    num_attestations: 1
+  backend:
+    max_capacity: 10
+    evicting_period: 
+      secs: 3600
+      nanos: 0
--- a/testnet/mixnode_config.yaml
+++ b/testnet/mixnode_config.yaml
@ -0,0 +1,16 @@
+mixnode:
+  # A listen address for other mixnodes in the mixnet and mixclients who want to send packets.
+  listen_address: 127.0.0.1:7777
+  # A (internal) listen address only for a "single" mixclient who wants to receive packets
+  # from the last mixnet layer.
+  # For more details, see the documentation in the "mixnet" crate.
+  client_listen_address: 127.0.0.1:7778
+  # A ed25519 private key for decrypting inbound Sphinx packets
+  # received from mixclients or mixnodes in the previous mixnet layer.
+  private_key: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+  # A max number of connections that will stay connected to mixnodes in the next layer.
+  connection_pool_size: 255
+log:
+  backend: "Stdout"
+  format: "Json"
+  level: "debug"
--- a/testnet/register_node.sh
+++ b/testnet/register_node.sh
@ -6,7 +6,7 @@ node_key_from_id() {
 	echo "${NODE_MASK}" | sed "s/.\{${#NODE_ID}\}$/${NODE_ID}/"
 }

-END=$DOCKER_REPLICAS
+END=$LIBP2P_REPLICAS
 NODE_ID=1
 NODE_IP=$(hostname -i)
 NODE_KEY=$(node_key_from_id)
--- a/testnet/run_libp2p_node.py
+++ b/testnet/run_libp2p_node.py
@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+import os
+import time
+import subprocess
+import etcd3
+
+def node_key_from_id(node_id, node_mask):
+    return node_mask[:-len(str(node_id))] + str(node_id)
+
+def get_etcd_client():
+    etcd_endpoints = os.environ.get('ETCDCTL_ENDPOINTS', 'etcd:2379').split(',')
+    host, port = etcd_endpoints[0].split(':')
+    return etcd3.client(host=host, port=port)
+
+def register_etcd(client, node_id, node_key, node_ip) -> bool:
+    is_success, txn = client.transaction(
+        compare=[client.transactions.value(f"/node/{node_id}") == str(0)],
+        success=[
+            client.transactions.put(f"/node/{node_id}", str(node_id)),
+            client.transactions.put(f"/config/node/{node_id}/key", str(node_key)),
+            client.transactions.put(f"/config/node/{node_ip}/ip", str(node_ip))
+        ],
+        failure=[]
+    )
+    return is_success
+
+def register_and_run(node_mask, node_ip, replicas):
+    etcd_client = get_etcd_client()
+    node_id = 1
+    node_key = 0
+    time.sleep(2)
+
+    while node_id <= replicas:
+        if register_etcd(etcd_client, node_id, node_key, node_ip):
+            break
+        else:
+            node_id += 1
+            node_key = node_key_from_id(node_id, node_mask)
+
+    if node_id > replicas:
+        print("Reached the limit without registering a {}.".format(node_id))
+        return 1
+
+if __name__ == "__main__":
+    node_mask = os.environ.get("NODE_MASK", "")
+    node_ip = subprocess.check_output(['hostname', '-i']).decode().strip()
+    replicas = int(os.environ.get('DOCKER_REPLICAS', 0))
+
+    register_and_run(node_mask, node_ip, replicas)
--- a/testnet/run_libp2p_node.sh
+++ b/testnet/run_libp2p_node.sh
@ -1,31 +0,0 @@
-#!/bin/sh
-
-set -e
-
-# Set env variables for nomos-node.
-NET_NODE_KEY=$(./etc/nomos/register_node.sh)
-CONSENSUS_PRIV_KEY=$NET_NODE_KEY
-
-node_ids=$(etcdctl get "/node/" --prefix --keys-only)
-for node_id in $node_ids; do
-	node_key=$(etcdctl get "/config${node_id}/key" --print-value-only)
-	node_ip=$(etcdctl get "/config${node_id}/ip" --print-value-only)
-	node_multiaddr="/ip4/${node_ip}/tcp/3000"
-
-	if [ -z "$OVERLAY_NODES" ]; then
-		OVERLAY_NODES=$node_key
-		NET_INITIAL_PEERS=$node_multiaddr
-	else
-		OVERLAY_NODES="${OVERLAY_NODES},${node_key}"
-		NET_INITIAL_PEERS="${NET_INITIAL_PEERS},${node_multiaddr}"
-	fi
-done
-
-export CONSENSUS_PRIV_KEY \
-       OVERLAY_NODES \
-       NET_NODE_KEY \
-       NET_INITIAL_PEERS
-
-echo "I am a container ${HOSTNAME} node ${NET_NODE_KEY}"
-
-exec /usr/bin/nomos-node /etc/nomos/config.yaml
--- a/testnet/run_mix_node.sh
+++ b/testnet/run_mix_node.sh
@ -1,31 +0,0 @@
-#!/bin/sh
-
-set -e
-
-# Set env variables for nomos-node.
-NET_NODE_KEY=$(./etc/nomos/register_node.sh)
-CONSENSUS_PRIV_KEY=$NET_NODE_KEY
-
-node_ids=$(etcdctl get "/node/" --prefix --keys-only)
-for node_id in $node_ids; do
-	node_key=$(etcdctl get "/config${node_id}/key" --print-value-only)
-	node_ip=$(etcdctl get "/config${node_id}/ip" --print-value-only)
-	node_multiaddr="/ip4/${node_ip}/tcp/3000"
-
-	if [ -z "$OVERLAY_NODES" ]; then
-		OVERLAY_NODES=$node_key
-		NET_INITIAL_PEERS=$node_multiaddr
-	else
-		OVERLAY_NODES="${OVERLAY_NODES},${node_key}"
-		NET_INITIAL_PEERS="${NET_INITIAL_PEERS},${node_multiaddr}"
-	fi
-done
-
-export CONSENSUS_PRIV_KEY \
-       OVERLAY_NODES \
-       NET_NODE_KEY \
-       NET_INITIAL_PEERS
-
-echo "I am a container ${HOSTNAME} node ${NET_NODE_KEY}"
-
-exec /usr/bin/nomos-node /etc/nomos/config.yaml
--- a/testnet/run_mixnet_node.py
+++ b/testnet/run_mixnet_node.py
@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+import os
+import time
+import subprocess
+