diff --git a/.env.example b/.env.example index e261cedf..67293227 100644 --- a/.env.example +++ b/.env.example @@ -1,8 +1,6 @@ # Environment variables for compose.yml file config. DOCKER_COMPOSE_LIBP2P_REPLICAS=1 DOCKER_COMPOSE_LIBP2P_NODE_KEY_MASK=2000000000000000000000000000000000000000000000000000000000000000 -DOCKER_COMPOSE_MIXNET_REPLICAS=1 -DOCKER_COMPOSE_MIXNET_NODE_KEY_MASK=3000000000000000000000000000000000000000000000000000000000000000 DOCKER_COMPOSE_ETCDCTL_ENDPOINTS=etcd:2379 DOCKER_COMPOSE_ETCDCTL_API=3 DOCKER_COMPOSE_BOOSTRAP_NET_NODE_KEY=1000000000000000000000000000000000000000000000000000000000000000 diff --git a/compose.yml b/compose.yml index f35b5bbb..ab48d604 100644 --- a/compose.yml +++ b/compose.yml @@ -25,33 +25,40 @@ services: - bootstrap - etcd environment: - - DOCKER_REPLICAS=${DOCKER_COMPOSE_LIBP2P_REPLICAS:-1} + - LIBP2P_REPLICAS=${DOCKER_COMPOSE_LIBP2P_REPLICAS:-1} - ETCDCTL_ENDPOINTS=${DOCKER_COMPOSE_ETCDCTL_ENDPOINTS:-etcd:2379} - ETCDCTL_API=${DOCKER_COMPOSE_ETCDCTL_API:-3} - NODE_MASK=${DOCKER_COMPOSE_LIBP2P_NODE_KEY_MASK:-2000000000000000000000000000000000000000000000000000000000000000} - OVERLAY_NODES=${DOCKER_COMPOSE_OVERLAY_NODES:-1000000000000000000000000000000000000000000000000000000000000000} - NET_INITIAL_PEERS=${DOCKER_COMPOSE_NET_INITIAL_PEERS:-/dns/bootstrap/tcp/3000} - entrypoint: /etc/nomos/run_libp2p_node.sh + entrypoint: /etc/nomos/configure_node.sh - mix-node: + mix-node-0: build: context: . dockerfile: testnet/Dockerfile volumes: - ./testnet:/etc/nomos - deploy: - replicas: ${DOCKER_COMPOSE_MIXNET_REPLICAS:-1} - depends_on: - - bootstrap - - etcd - environment: - - DOCKER_REPLICAS=${DOCKER_COMPOSE_MIXNET_REPLICAS:-1} - - ETCDCTL_ENDPOINTS=${DOCKER_COMPOSE_ETCDCTL_ENDPOINTS:-etcd:2379} - - ETCDCTL_API=${DOCKER_COMPOSE_ETCDCTL_API:-3} - - NODE_MASK=${DOCKER_COMPOSE_MIXNET_NODE_KEY_MASK:-2000000000000000000000000000000000000000000000000000000000000000} - - OVERLAY_NODES=${DOCKER_COMPOSE_OVERLAY_NODES:-1000000000000000000000000000000000000000000000000000000000000000} - - NET_INITIAL_PEERS=${DOCKER_COMPOSE_NET_INITIAL_PEERS:-/dns/bootstrap/tcp/3000} - entrypoint: /etc/nomos/run_mix_node.sh + entrypoint: /usr/bin/mixnode + command: mixnode_config.yaml + + mix-node-1: + build: + context: . + dockerfile: testnet/Dockerfile + volumes: + - ./testnet:/etc/nomos + entrypoint: /usr/bin/mixnode + command: mixnode_config.yaml + + mix-node-2: + build: + context: . + dockerfile: testnet/Dockerfile + volumes: + - ./testnet:/etc/nomos + entrypoint: /usr/bin/mixnode + command: mixnode_config.yaml etcd: image: quay.io/coreos/etcd:v3.4.15 diff --git a/testnet/Dockerfile b/testnet/Dockerfile index 9c837a90..5398dd66 100644 --- a/testnet/Dockerfile +++ b/testnet/Dockerfile @@ -8,10 +8,7 @@ RUN echo 'deb http://deb.debian.org/debian bullseye-backports main' \ # Dependecies for publishing documentation. RUN apt-get update && apt-get install -yq \ - git clang etcd-client libssl-dev \ - golang-src/bullseye-backports \ - golang-doc/bullseye-backports \ - golang/bullseye-backports + git clang etcd-client libssl-dev pkg-config WORKDIR /nomos COPY . . @@ -33,5 +30,6 @@ COPY --from=builder /nomos/target/release/nomos-node /usr/bin/nomos-node COPY --from=builder /nomos/target/release/mixnode /usr/bin/mixnode COPY --from=builder /usr/bin/etcdctl /usr/bin/etcdctl COPY nodes/nomos-node/config.yaml /etc/nomos/config.yaml +RUN install_packages python3 python3-etcd3 ENTRYPOINT ["/usr/bin/nomos-node"] diff --git a/testnet/libp2p_config.yaml b/testnet/libp2p_config.yaml new file mode 100644 index 00000000..0ae24a6b --- /dev/null +++ b/testnet/libp2p_config.yaml @@ -0,0 +1,73 @@ +log: + backend: "Stdout" + format: "Json" + level: "debug" +consensus: + private_key: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + fountain_settings: null + overlay_settings: + nodes: [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]] + number_of_committees: 1 + current_leader: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + leader: + cur: 0 + committee_membership: !Sad + entropy: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + leader_super_majority_threshold: 1 + super_majority_threshold: 1 + +network: + backend: + host: 0.0.0.0 + port: 3000 + log_level: "fatal" + node_key: "0000000000000000000000000000000000000000000000000000000000000001" + discV5BootstrapNodes: [] + initial_peers: [] + relayTopics: [] + # Mixclient configuration to communicate with mixnodes. + # The libp2p network backend always requires this mixclient configuration + # (cannot be disabled for now). + mixnet_client: + # A mixclient mode. For details, see the documentation of the "mixnet" crate. + # - Sender + # - !SenderReceiver [mixnode_client_listen_address] + mode: Sender + # A mixnet topology, which contains the information of all mixnodes in the mixnet. + # (The topology is static for now.) + topology: + # Each mixnet layer consists of a list of mixnodes. + layers: + - nodes: + - address: mix-node-0:7777 # A listen address of the mixnode + public_key: "0000000000000000000000000000000000000000000000000000000000000000" + - address: mix-node-1:7777 # A listen address of the mixnode + public_key: "0000000000000000000000000000000000000000000000000000000000000000" + - address: mix-node-2:7777 # A listen address of the mixnode + public_key: "0000000000000000000000000000000000000000000000000000000000000000" + # A max number of connections that will stay connected to mixnodes in the first mixnet layer. + connection_pool_size: 255 + max_retries: 5 + retry_delay: + secs: 1 + nanos: 0 + # A range of total delay that will be set to each Sphinx packets + # sent to the mixnet for timing obfuscation. + # Panics if start > end. + mixnet_delay: + start: "0ms" + end: "0ms" + +http: + backend: + address: 0.0.0.0:8080 + cors_origins: [] + +da: + da_protocol: + num_attestations: 1 + backend: + max_capacity: 10 + evicting_period: + secs: 3600 + nanos: 0 diff --git a/testnet/mixnode_config.yaml b/testnet/mixnode_config.yaml new file mode 100644 index 00000000..a98c9a34 --- /dev/null +++ b/testnet/mixnode_config.yaml @@ -0,0 +1,16 @@ +mixnode: + # A listen address for other mixnodes in the mixnet and mixclients who want to send packets. + listen_address: 127.0.0.1:7777 + # A (internal) listen address only for a "single" mixclient who wants to receive packets + # from the last mixnet layer. + # For more details, see the documentation in the "mixnet" crate. + client_listen_address: 127.0.0.1:7778 + # A ed25519 private key for decrypting inbound Sphinx packets + # received from mixclients or mixnodes in the previous mixnet layer. + private_key: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + # A max number of connections that will stay connected to mixnodes in the next layer. + connection_pool_size: 255 +log: + backend: "Stdout" + format: "Json" + level: "debug" diff --git a/testnet/register_node.sh b/testnet/register_node.sh index 387fa361..a2ec94ec 100755 --- a/testnet/register_node.sh +++ b/testnet/register_node.sh @@ -6,7 +6,7 @@ node_key_from_id() { echo "${NODE_MASK}" | sed "s/.\{${#NODE_ID}\}$/${NODE_ID}/" } -END=$DOCKER_REPLICAS +END=$LIBP2P_REPLICAS NODE_ID=1 NODE_IP=$(hostname -i) NODE_KEY=$(node_key_from_id) diff --git a/testnet/run_libp2p_node.py b/testnet/run_libp2p_node.py new file mode 100755 index 00000000..51dfd83f --- /dev/null +++ b/testnet/run_libp2p_node.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +import os +import time +import subprocess +import etcd3 + +def node_key_from_id(node_id, node_mask): + return node_mask[:-len(str(node_id))] + str(node_id) + +def get_etcd_client(): + etcd_endpoints = os.environ.get('ETCDCTL_ENDPOINTS', 'etcd:2379').split(',') + host, port = etcd_endpoints[0].split(':') + return etcd3.client(host=host, port=port) + +def register_etcd(client, node_id, node_key, node_ip) -> bool: + is_success, txn = client.transaction( + compare=[client.transactions.value(f"/node/{node_id}") == str(0)], + success=[ + client.transactions.put(f"/node/{node_id}", str(node_id)), + client.transactions.put(f"/config/node/{node_id}/key", str(node_key)), + client.transactions.put(f"/config/node/{node_ip}/ip", str(node_ip)) + ], + failure=[] + ) + return is_success + +def register_and_run(node_mask, node_ip, replicas): + etcd_client = get_etcd_client() + node_id = 1 + node_key = 0 + time.sleep(2) + + while node_id <= replicas: + if register_etcd(etcd_client, node_id, node_key, node_ip): + break + else: + node_id += 1 + node_key = node_key_from_id(node_id, node_mask) + + if node_id > replicas: + print("Reached the limit without registering a {}.".format(node_id)) + return 1 + +if __name__ == "__main__": + node_mask = os.environ.get("NODE_MASK", "") + node_ip = subprocess.check_output(['hostname', '-i']).decode().strip() + replicas = int(os.environ.get('DOCKER_REPLICAS', 0)) + + register_and_run(node_mask, node_ip, replicas) diff --git a/testnet/run_libp2p_node.sh b/testnet/run_libp2p_node.sh deleted file mode 100755 index 0ee22d37..00000000 --- a/testnet/run_libp2p_node.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh - -set -e - -# Set env variables for nomos-node. -NET_NODE_KEY=$(./etc/nomos/register_node.sh) -CONSENSUS_PRIV_KEY=$NET_NODE_KEY - -node_ids=$(etcdctl get "/node/" --prefix --keys-only) -for node_id in $node_ids; do - node_key=$(etcdctl get "/config${node_id}/key" --print-value-only) - node_ip=$(etcdctl get "/config${node_id}/ip" --print-value-only) - node_multiaddr="/ip4/${node_ip}/tcp/3000" - - if [ -z "$OVERLAY_NODES" ]; then - OVERLAY_NODES=$node_key - NET_INITIAL_PEERS=$node_multiaddr - else - OVERLAY_NODES="${OVERLAY_NODES},${node_key}" - NET_INITIAL_PEERS="${NET_INITIAL_PEERS},${node_multiaddr}" - fi -done - -export CONSENSUS_PRIV_KEY \ - OVERLAY_NODES \ - NET_NODE_KEY \ - NET_INITIAL_PEERS - -echo "I am a container ${HOSTNAME} node ${NET_NODE_KEY}" - -exec /usr/bin/nomos-node /etc/nomos/config.yaml diff --git a/testnet/run_mix_node.sh b/testnet/run_mix_node.sh deleted file mode 100755 index 0ee22d37..00000000 --- a/testnet/run_mix_node.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh - -set -e - -# Set env variables for nomos-node. -NET_NODE_KEY=$(./etc/nomos/register_node.sh) -CONSENSUS_PRIV_KEY=$NET_NODE_KEY - -node_ids=$(etcdctl get "/node/" --prefix --keys-only) -for node_id in $node_ids; do - node_key=$(etcdctl get "/config${node_id}/key" --print-value-only) - node_ip=$(etcdctl get "/config${node_id}/ip" --print-value-only) - node_multiaddr="/ip4/${node_ip}/tcp/3000" - - if [ -z "$OVERLAY_NODES" ]; then - OVERLAY_NODES=$node_key - NET_INITIAL_PEERS=$node_multiaddr - else - OVERLAY_NODES="${OVERLAY_NODES},${node_key}" - NET_INITIAL_PEERS="${NET_INITIAL_PEERS},${node_multiaddr}" - fi -done - -export CONSENSUS_PRIV_KEY \ - OVERLAY_NODES \ - NET_NODE_KEY \ - NET_INITIAL_PEERS - -echo "I am a container ${HOSTNAME} node ${NET_NODE_KEY}" - -exec /usr/bin/nomos-node /etc/nomos/config.yaml diff --git a/testnet/run_mixnet_node.py b/testnet/run_mixnet_node.py new file mode 100755 index 00000000..e2c71676 --- /dev/null +++ b/testnet/run_mixnet_node.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 + +import os +import time +import subprocess +