diff --git a/scripts/localfleet/README.md b/scripts/localfleet/README.md new file mode 100644 index 0000000000..45f71ba13b --- /dev/null +++ b/scripts/localfleet/README.md @@ -0,0 +1,42 @@ +# Create a local fleet for tests + + +## Installation +The following software is required: +- `jq` +- `docker` +- `docker-compose` +- `qrencode` + +Be sure that NAT is configured in the router you're connected to, since this uses the external IP address to connect the peers + +## Usage +``` +go run main.go +``` +By default it will attempt to create 1 bootnode, 1 mailserver and 1 whisper node. You can control the number +of mailservers and whisper nodes by using the flags `--mailservers N` and `--whisper N`, where `N` is the +number of nodes to create. + +**WARNING** this will overwrite the following files: +- `fleets.json` +- `vendor/status-go/services/mailservers/fleet.go` + +The program will create the required nodes, and modify the fleet files. Afterwards you need to rebuild +`status-go` and `status-desktop`, so it uses the local fleet nodes instead of the default `eth.prod` nodes. + +Once you're done with the fleet, press `CTRL+C` to shutdown the fleet + +## Simulating network conditions +With https://github.com/tylertreat/comcast, you can test common network problems. Use the following command to setup some rules: + +``` +comcast --device=wlp2s0 --latency=250 --target-bw=1000 --default-bw=10000 --packet-loss=10% --target-addr=your_ip_address --target-proto=tcp,udp --target-port=30310:30320 +``` +`latency` is specified in milliseconds, `target-bw` and `default-bw` are in Kbit + + +Replace the `device` flag with a valid value from `ifconfig` and `target-addr` with your IP address. After you're done with testing, use: +``` +comcast --device=wlp2s0 --stop +``` \ No newline at end of file diff --git a/scripts/localfleet/docker-compose-host.yml b/scripts/localfleet/docker-compose-host.yml new file mode 100644 index 0000000000..7c451dd063 --- /dev/null +++ b/scripts/localfleet/docker-compose-host.yml @@ -0,0 +1,4 @@ +version: "3" +services: + mailserver: + network_mode: host \ No newline at end of file diff --git a/scripts/localfleet/fleet.go.template b/scripts/localfleet/fleet.go.template new file mode 100644 index 0000000000..26b7b40f0b --- /dev/null +++ b/scripts/localfleet/fleet.go.template @@ -0,0 +1,8 @@ +package mailservers + +func DefaultMailservers() []Mailserver { + + return []Mailserver{ +%MAILSERVER_LIST% + } +} diff --git a/scripts/localfleet/fleets.json.template b/scripts/localfleet/fleets.json.template new file mode 100644 index 0000000000..4b6a0d0e33 --- /dev/null +++ b/scripts/localfleet/fleets.json.template @@ -0,0 +1,23 @@ +{ + "fleets": { + "eth.prod": { + "boot": { + "test-bootnode": "%BOOTNODE%" + }, + "mail": { + %MAILSERVER_LIST% + }, + "rendezvous": { + "fake": "/ip4/127.0.0.1/tcp/12345/ethv4/16Uiu2HAmV8Hq9e3zm9TMVP4zrVHo3BjqW5D6bDVV6VQntQd687e4" + }, + "whisper": { + %WHISPER_LIST%, + } + } + }, + "meta": { + "hostname": "localhost", + "timestamp": "2022-08-25T00:00:23.181054", + "warning": "Local test fleet configuration" + } +} \ No newline at end of file diff --git a/scripts/localfleet/gen-config.sh b/scripts/localfleet/gen-config.sh new file mode 100755 index 0000000000..cf0b4bd529 --- /dev/null +++ b/scripts/localfleet/gen-config.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -euo pipefail + +GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel) + +# Settings & defaults +RPC_HOST="${RPC_HOST:-localhost}" +RPC_PORT="${RPC_PORT:-8545}" +LISTEN_PORT="${LISTEN_PORT:-30303}" +API_MODULES="${API_MODULES:-eth,web3,admin}" +MAX_PEERS="${MAX_PEERS:-50}" +DAYS_KEPT="${DAYS_KEPT-30}" +FLEET_NAME="${FLEET_NAME:-eth.prod}" +REGISTER_TOPIC="${REGISTER_TOPIC:-whispermail}" +MAIL_PASSWORD="${MAIL_PASSWORD:-status-offline-inbox}" +DATA_PATH="${DATA_PATH:-/var/tmp/status-go-mail}" +MAILSERVER_ENABLED="${MAILSERVER_ENABLED:-true}" +CONFIG_PATH="${CONFIG_PATH:-${DATA_PATH}/config.json}" + +if ! [[ -x $(command -v jq) ]]; then + echo "Cannot generate config. jq utility is not installed." >&2 + exit 1 +fi + +# Assemble the filter for changing the config JSON +JQ_FILTER_ARRAY=( + ".ListenAddr = \"0.0.0.0:${LISTEN_PORT}\"" + ".HTTPEnabled = true" + ".HTTPHost = \"${RPC_HOST}\"" + ".HTTPPort= ${RPC_PORT}" + ".MaxPeers = ${MAX_PEERS}" + ".DataDir = \"${DATA_PATH}\"" + ".APIModules = \"${API_MODULES}\"" + ".ClusterConfig.Fleet = \"${FLEET_NAME}\"" + ".ClusterConfig.BootNodes = [\"${BOOTNODE}\"]" + ".RegisterTopics = [\"${REGISTER_TOPIC}\"]" + ".WakuConfig.Enabled = true" + ".WakuConfig.EnableMailServer = ${MAILSERVER_ENABLED}" + ".WakuConfig.DataDir = \"${DATA_PATH}/waku\"" + ".WakuConfig.MailServerPassword = \"${MAIL_PASSWORD}\"" + ".WakuConfig.MailServerDataRetention = ${DAYS_KEPT}" +) + +JQ_FILTER=$(printf " | %s" "${JQ_FILTER_ARRAY[@]}") + +# make sure config destination exists +mkdir -p "${DATA_PATH}" + +echo "Generating config at: ${CONFIG_PATH}" + +cat "./config.json" \ + | jq "${JQ_FILTER:3}" > "${CONFIG_PATH}" diff --git a/scripts/localfleet/main.go b/scripts/localfleet/main.go new file mode 100644 index 0000000000..6c4dc275e0 --- /dev/null +++ b/scripts/localfleet/main.go @@ -0,0 +1,293 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "os/exec" + "os/signal" + "strings" + "syscall" +) + +type Node struct { + enr string + name string + rpcPort int +} + +type ClusterConfig struct { + Enabled bool + Fleet string + StaticNodes []string + BootNodes []string + TrustedMailServers []string + PushNotificationsServers []string +} + +type Mailserver struct { + ID string + Address string + Fleet string + Version int +} + +func main() { + // Only supports 1 boot node + mailserverNum := flag.Int("mailservers", 1, "number of mailservers") + whisperNum := flag.Int("whisper", 1, "number of whisper nodes") + flag.Parse() + + fmt.Println("Starting a bootnode...") + bootnodeENR, err := startBootnode() + if err != nil { + fmt.Println(err) + return + } + fmt.Println("Bootnode enr: ", bootnodeENR) + defer stopBootnode() + + nodeCnt := 0 + var mailservers []Node + for i := 0; i < *mailserverNum; i++ { + fmt.Println(fmt.Sprintf("Starting mailserver #%d...", i+1)) + node, err := startNode(nodeCnt, bootnodeENR, true) + if err != nil { + stopNodes(mailservers) + fmt.Println("Could not start node", err) + return + } + fmt.Println(fmt.Sprintf("Mailserver #%d enr: %s", i+1, node.enr)) + mailservers = append(mailservers, node) + nodeCnt++ + } + + var whisperNodes []Node + for i := 0; i < *whisperNum; i++ { + fmt.Println(fmt.Sprintf("Starting whisperNode #%d...", i+1)) + node, err := startNode(nodeCnt, bootnodeENR, false) + if err != nil { + stopNodes(whisperNodes) + return + } + fmt.Println(fmt.Sprintf("Whisper #%d enr: %s", i+1, node.enr)) + whisperNodes = append(whisperNodes, node) + nodeCnt++ + } + + for _, node := range append(mailservers, whisperNodes...) { + fmt.Println("Adding peers to ", node.name) + for _, peer := range append(mailservers, whisperNodes...) { + addPeer(peer.enr, node.rpcPort) + } + } + + // Output config + + cluster := ClusterConfig{ + Enabled: true, + Fleet: "eth.prod", + BootNodes: []string{bootnodeENR}, + } + for _, node := range mailservers { + cluster.TrustedMailServers = append(cluster.TrustedMailServers, node.enr) + } + for _, node := range whisperNodes { + cluster.StaticNodes = append(cluster.StaticNodes, node.enr) + } + + clusterJSON, _ := json.Marshal(cluster) + fmt.Println("\nNew cluster config:\n", string(clusterJSON)) + + // =============================================================================== + // Replacing status-go fleets.go file + statusGoMailserverFleet := "" + mailserverStrTemplate := ` Mailserver { + ID: "%s", + Address: "%s", + Fleet: "eth.prod", + Version: 1, + }, +` + for _, m := range mailservers { + statusGoMailserverFleet += fmt.Sprintf(mailserverStrTemplate, m.name, m.enr) + } + + b, _ := os.ReadFile("./fleet.go.template") + statusGoMailserverFleet = strings.Replace(string(b), "%MAILSERVER_LIST%", statusGoMailserverFleet, -1) + + err = os.WriteFile("../../vendor/status-go/services/mailservers/fleet.go", []byte(statusGoMailserverFleet), 0600) + if err != nil { + fmt.Println("Could not write fleet in status-go") + stopNodes(mailservers) + stopNodes(whisperNodes) + return + } + fmt.Println("\nvendor/status-go/services/mailservers/fleet.go was updated") + + // ===================================================================================================== + // Replacing status-desktop fleets.go file + b, _ = os.ReadFile("./fleets.json.template") + + fleetsJSON := strings.Replace(string(b), "%BOOTNODE%", bootnodeENR, -1) + + desktopMailserverFleet := "" + for _, m := range mailservers { + desktopMailserverFleet += fmt.Sprintf("\"%s\": \"%s\",", m.name, m.enr) + } + desktopMailserverFleet = strings.TrimSuffix(desktopMailserverFleet, ",") + fleetsJSON = strings.Replace(fleetsJSON, "%MAILSERVER_LIST%", desktopMailserverFleet, -1) + + desktopWhisperFleet := "" + for _, m := range whisperNodes { + desktopWhisperFleet += fmt.Sprintf("\"%s\": \"%s\",", m.name, m.enr) + } + desktopWhisperFleet = strings.TrimSuffix(desktopWhisperFleet, ",") + fleetsJSON = strings.Replace(fleetsJSON, "%WHISPER_LIST%", desktopWhisperFleet, -1) + + err = os.WriteFile("../../fleets.json", []byte(fleetsJSON), 0600) + if err != nil { + fmt.Println("Could not write fleet in status-desktop") + stopNodes(mailservers) + stopNodes(whisperNodes) + return + } + fmt.Println("fleets.json was updated") + + fmt.Println("\nDONE! rebuild status-go and desktop to use this new fleet") + + // Wait for a SIGINT or SIGTERM signal + fmt.Println("\n\nPress Crtl+C to shutdown nodes") + ch := make(chan os.Signal, 1) + signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM) + <-ch + fmt.Println("\nReceived signal, shutting down...") + + stopNodes(mailservers) + stopNodes(whisperNodes) +} + +func startBootnode() (string, error) { + envVars := os.Environ() + envVars = append(envVars, "CONTAINER_NAME=test-bootnode") + envVars = append(envVars, "COMPOSE_UP_FLAGS=-f ./docker-compose.yml -f ../../../../../scripts/localfleet/docker-compose-host.yml") + + e := exec.Command("make", "-C", "../../vendor/status-go", "run-bootnode-docker") + var stderr bytes.Buffer + e.Stderr = &stderr + e.Env = envVars + if err := e.Run(); err != nil { + return "", fmt.Errorf("could not start bootnode: %w, %s", err, stderr.String()) + } + + e = exec.Command("make", "-s", "-C", "../../vendor/status-go/_assets/compose/bootnode", "enode") + e.Env = envVars + var out bytes.Buffer + e.Stdout = &out + err := e.Run() + if err != nil { + return "", fmt.Errorf("could not obtain bootnode enr: %w", err) + } + + return strings.TrimSpace(out.String()), nil +} + +func stopBootnode() { + fmt.Println("Stopping bootnode...") + envVars := os.Environ() + envVars = append(envVars, "CONTAINER_NAME=test-bootnode") + e := exec.Command("make", "-C", "../../vendor/status-go/_assets/compose/bootnode", "stop") + e.Env = envVars + err := e.Run() + if err != nil { + fmt.Println(fmt.Errorf("could not stop bootnode: %w", err)) + } +} + +func startNode(i int, bootnodeENR string, mailserver bool) (Node, error) { + envVars := os.Environ() + + name := fmt.Sprintf("%s-%d", "test-mailserver", i) + if !mailserver { + name = fmt.Sprintf("%s-%d", "test-whisper", i) + envVars = append(envVars, "MAILSERVER_ENABLED=false") + } + + rpcPort := 8656 + i + + envVars = append(envVars, fmt.Sprintf("RPC_HOST=%s", "0.0.0.0")) + envVars = append(envVars, fmt.Sprintf("LISTEN_PORT=%d", 30310+i)) + envVars = append(envVars, fmt.Sprintf("METRICS_PORT=%d", 9191+i)) + envVars = append(envVars, fmt.Sprintf("RPC_PORT=%d", rpcPort)) + envVars = append(envVars, fmt.Sprintf("CONTAINER_NAME=%s", name)) + envVars = append(envVars, fmt.Sprintf("DATA_PATH=/var/tmp/%s", name)) + envVars = append(envVars, fmt.Sprintf("BOOTNODE=%s", strings.TrimSpace(bootnodeENR))) + envVars = append(envVars, "CONTAINER_IMG=statusteam/status-go") + envVars = append(envVars, "LOG_LEVEL=DEBUG") + envVars = append(envVars, "CONTAINER_TAG=v0.84.0") + envVars = append(envVars, "API_MODULES=eth,web3,admin,waku,wakuext") + envVars = append(envVars, "REGISTER_TOPIC=whispermail") + envVars = append(envVars, "MAIL_PASSWORD=status-offline-inbox") + + e := exec.Command("./gen-config.sh") + e.Env = envVars + if err := e.Run(); err != nil { + return Node{}, fmt.Errorf("could not generate config: %w", err) + } + + e = exec.Command("docker-compose", "-p", name, "-f", "../../vendor/status-go/_assets/compose/mailserver/docker-compose.yml", "-f", "./docker-compose-host.yml", "up", "-d") + e.Env = envVars + var stderr bytes.Buffer + e.Stderr = &stderr + + if err := e.Run(); err != nil { + return Node{}, fmt.Errorf("could not start mailserver: %w", errors.New(stderr.String())) + } + + e = exec.Command("make", "-s", "-C", "../../vendor/status-go/_assets/compose/mailserver", "enode") + e.Env = envVars + var out bytes.Buffer + e.Stdout = &out + err := e.Run() + if err != nil { + return Node{}, fmt.Errorf("could not obtain mailserver #%d enr: %w", i, err) + } + + return Node{ + enr: strings.Replace(strings.TrimSpace(out.String()), ":status-offline-inbox", "", -1), + name: name, + rpcPort: 8656 + i, + }, nil +} + +func stopNodes(nodes []Node) { + for _, node := range nodes { + fmt.Println(fmt.Sprintf("Stopping node %s...", node.name)) + envVars := os.Environ() + envVars = append(envVars, fmt.Sprintf("CONTAINER_NAME=%s", node.name)) + e := exec.Command("docker-compose", "-p", node.name, "down") + e.Env = envVars + if err := e.Run(); err != nil { + fmt.Println(fmt.Errorf("could not stop node #%s: %w", node.name, err)) + } + } +} + +func addPeer(peerENR string, port int) { + envVars := os.Environ() + + envVars = append(envVars, fmt.Sprintf("RPC_HOST=%s", "0.0.0.0")) + envVars = append(envVars, fmt.Sprintf("RPC_PORT=%d", port)) + + e := exec.Command("../../vendor/status-go/_assets/scripts/rpc.sh", "admin_addPeer", peerENR) + e.Env = envVars + var out bytes.Buffer + e.Stdout = &out + err := e.Run() + if err != nil { + fmt.Println("could not add peer: ", err) + } +}