Fix wrong copy in Dockerfile

.gitignore

@@ -1,2 +1,5 @@
 nimbledeps/*
 *.exe
+stew/*
+nim-stew-master/*
+.idea/

.gitmodules

@@ -0,0 +1,70 @@
+[submodule "vendor/nimbus-build-system"]
+	path = vendor/nimbus-build-system
+	url = https://github.com/status-im/nimbus-build-system.git
+	branch = master
+	ignore = dirty
+[submodule "vendor/nim-libp2p"]
+	path = vendor/nim-libp2p
+	url = https://github.com/vacp2p/nim-libp2p
+	branch = master
+	ignore = dirty
+[submodule "vendor/nim-stew"]
+	path = vendor/nim-stew
+	url = https://github.com/status-im/nim-stew
+	branch = master
+	ignore = dirty
+[submodule "vendor/nim-results"]
+	path = vendor/nim-results
+	url = https://github.com/arnetheduck/nim-results
+	branch = master
+	ignore = dirty
+[submodule "vendor/nim-chronos"]
+	path = vendor/nim-chronos
+	url = https://github.com/status-im/nim-chronos
+	branch = master
+	ignore = dirty
+[submodule "vendor/nim-chronicles"]
+	path = vendor/nim-chronicles
+	url = https://github.com/status-im/nim-chronicles
+	branch = master
+	ignore = dirty
+[submodule "vendor/nim-faststreams"]
+	path = vendor/nim-faststreams
+	url = https://github.com/status-im/nim-faststreams
+	branch = master
+	ignore = dirty
+[submodule "vendor/nim-json-serialization"]
+	path = vendor/nim-json-serialization
+	url = https://github.com/status-im/nim-json-serialization
+	branch = master
+	ignore = dirty
+[submodule "vendor/nim-serialization"]
+	path = vendor/nim-serialization
+	url = https://github.com/status-im/nim-serialization
+	branch = master
+	ignore = dirty
+[submodule "vendor/nim-metrics"]
+	path = vendor/nim-metrics
+	url = https://github.com/status-im/nim-metrics
+	branch = master
+	ignore = dirty
+[submodule "vendor/nimcrypto"]
+	path = vendor/nimcrypto
+	url = https://github.com/status-im/nimcrypto
+	branch = master
+	ignore = dirty
+[submodule "vendor/nim-bearssl"]
+	path = vendor/nim-bearssl
+	url = https://github.com/status-im/nim-bearssl
+	branch = master
+	ignore = dirty
+[submodule "vendor/nim-secp256k1"]
+	path = vendor/nim-secp256k1
+	url = https://github.com/status-im/nim-secp256k1
+	branch = master
+	ignore = dirty
+[submodule "vendor/nim-http-utils"]
+	path = vendor/nim-http-utils
+	url = https://github.com/status-im/nim-http-utils
+	branch = master
+	ignore = dirty

Dockerfile

@@ -0,0 +1,51 @@
+# BUILD NIM APP ----------------------------------------------------------------
+FROM rust:1.77.1-alpine3.18  AS nim-build
+
+ARG NIMFLAGS
+ARG MAKE_TARGET
+ARG NIM_COMMIT
+
+# Get build tools and required header files
+RUN apk add --no-cache bash git build-base pcre-dev linux-headers curl jq
+
+WORKDIR /app
+COPY . .
+
+# workaround for alpine issue: https://github.com/alpinelinux/docker-alpine/issues/383
+RUN apk update && apk upgrade
+
+# Ran separately from 'make' to avoid re-doing
+RUN git submodule update --init --recursive
+
+# Slowest build step for the sake of caching layers
+RUN make -j$(nproc) deps QUICK_AND_DIRTY_COMPILER=1
+
+# Build the final node binary
+RUN make -j$(nproc) $MAKE_TARGET NIMFLAGS="${NIMFLAGS}"
+
+# PRODUCTION IMAGE -------------------------------------------------------------
+
+FROM alpine:3.18 as prod
+
+LABEL maintainer="asoutullo@status.im"
+LABEL source="https://github.com/vacp2p/dst-gossipsub-test-node/tree/dockerized"
+
+# LibP2P, Metrics ports
+EXPOSE 5000 8008
+
+# Referenced in the binary
+RUN apk add --no-cache busybox-suid \
+    && apk add --no-cache --update busybox-extras \
+    && apk add --no-cache bash
+
+# Copy to separate location to accomodate different MAKE_TARGET values
+COPY --from=nim-build /app/build/main /node/main
+
+WORKDIR /node
+
+COPY cron_runner.sh .
+
+RUN chmod +x /node/main
+RUN chmod +x cron_runner.sh
+
+ENTRYPOINT ["./cron_runner.sh"]

Makefile

@@ -0,0 +1,68 @@
+# Copyright (c) 2022 Status Research & Development GmbH. Licensed under
+# either of:
+# - Apache License, version 2.0
+# - MIT license
+# at your option. This file may not be copied, modified, or distributed except
+# according to those terms.
+BUILD_SYSTEM_DIR := vendor/nimbus-build-system
+
+# we don't want an error here, so we can handle things later, in the ".DEFAULT" target
+-include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk
+
+ifeq ($(NIM_PARAMS),)
+# "variables.mk" was not included, so we update the submodules.
+GIT_SUBMODULE_UPDATE := git submodule update --init --recursive
+.DEFAULT:
+	+@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \
+		$(GIT_SUBMODULE_UPDATE); \
+		echo
+# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself:
+# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles
+#
+# After restarting, it will execute its original goal, so we don't have to start a child Make here
+# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great?
+
+else # "variables.mk" was included. Business as usual until the end of this file.
+
+# must be included after the default target
+-include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
+
+.PHONY: deps dstnode
+
+dstnode.nims:
+	ln -s dstnode.nimble $@
+
+update: | update-common
+	rm -rf dstnode.nims && \
+        $(MAKE) dstnode.nims $(HANDLE_OUTPUT)
+
+deps: | deps-common dstnode.nims
+
+dstnode: | build deps
+	echo -e $(BUILD_MSG) "build/$@" && \
+	    $(ENV_SCRIPT) nim dstnode $(NIM_PARAMS) dstnode.nims
+
+clean: | clean-common
+	rm -rf build/dstnode
+
+#####################
+## Container image ##
+#####################
+# -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics
+# -d:chronicles_colors:none - Necessary to disable colors in logs for Docker
+DOCKER_IMAGE_NIMFLAGS := -d:chronicles_colors:none -d:insecure --threads:on -d:metrics -d:libp2p_network_protocols_metrics -d:release
+# build a docker image
+docker-image: MAKE_TARGET ?= dstnode
+docker-image: DOCKER_IMAGE_TAG ?= soutullostatus/dst-test-node:v1.2
+docker-image:
+	docker build \
+		--build-arg="MAKE_TARGET=$(MAKE_TARGET)" \
+		--build-arg="NIMFLAGS=$(DOCKER_IMAGE_NIMFLAGS)" \
+		--target prod \
+		--tag $(DOCKER_IMAGE_TAG) . \
+		--progress=plain
+
+docker-push:
+	docker push $(DOCKER_IMAGE_TAG)
+
+endif

README.md

@@ -8,13 +8,11 @@
 ```sh
 nimble install -dy
 cd shadow
-./run.sh x n
-# The run.sh file runs the simulation "x" number of times and every simulation run uses "n" number of nodes
-# The number of nodes is maintained in the shadow.yaml file, and automatically updated by run.sh.
-# The output files latencies(x), stats(x) and shadowlog(x) carries the outputs for each simulation run.
-# The summary_dontwant.awk, summary_latency.awk, and summary_shadowlog.awk parse the output files.
-# The run.sh script automatically calls these files to display the output
-# a temperary data.shadow folder is created for each simulation and removed by the run.sh after the simulation is over
+# the default shadow.yml will start 5k nodes, you might want to change that by removing
+# lines and setting PEERS to the number of instances
+./run.sh
+# the output is a "latencies" file, or you can find each host output in the
+# data.shadow folder
 
 # you can use the plotter tool to extract useful metrics & generate a graph
 cd ../tools

add_submodule.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2018-2020 Status Research & Development GmbH. Licensed under
+# either of:
+# - Apache License, version 2.0
+# - MIT license
+# at your option. This file may not be copied, modified, or distributed except
+# according to those terms.
+
+[ -z "$1" -o `echo "$1" | tr '/' '\n' | wc -l` != 2 ] && \
+	{ echo "Usage: `basename $0` some/repo [destdir] # 'destdir' defaults to 'vendor/repo'"; exit 1; }
+REPO="$1"
+
+DEST="vendor/${REPO#*/}"
+[ -n "$2" ] && DEST="$2"
+
+git submodule add --force https://github.com/${REPO}.git "$DEST"
+git config -f .gitmodules submodule.${DEST}.ignore untracked
+git config -f .gitmodules submodule.${DEST}.branch master
+

cron_runner.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+if [ "$#" -ne 2 ]; then
+    echo "Usage: $0 <minutes> <hours>"
+    exit 1
+fi
+
+minutes="$1"
+hours="$2"
+
+cron_expression="$minutes $hours * * *"
+
+cron_job_file="/etc/cron.d/my-cron-job"
+mkdir -p /etc/cron.d
+
+echo -e "$cron_expression /node/main  > /proc/1/fd/1 2>&1 \n" > "$cron_job_file"
+
+echo "Cron job file created at $cron_job_file"
+
+env >> /etc/environment
+
+crontab /etc/cron.d/my-cron-job
+
+crond -f

dstnode.nimble

@@ -0,0 +1,31 @@
+mode = ScriptMode.Verbose
+
+### Package
+version       = "0.1.0"
+author        = "Distributed Systems Testing"
+description   = "DstNode - Libp2p version: V1.2 (c4da9be32cc01efa2de066c396fe9ef1c7769aa1)"
+license       = "MIT or Apache License 2.0"
+#bin           = @["build/waku"]
+
+### Dependencies
+requires "nim >= 1.6.0",
+          "libp2p",
+          "stew",
+          "chronos",
+          "chronicles"
+
+
+### Helper functions
+proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") =
+  if not dirExists "build":
+    mkDir "build"
+  # allow something like "nim nimbus --verbosity:0 --hints:off nimbus.nims"
+  var extra_params = params
+  for i in 2..<paramCount():
+    extra_params &= " " & paramStr(i)
+  exec "nim " & lang & " --out:build/" & name & " " & extra_params & " " & srcDir & name & ".nim"
+
+### DstNode tasks
+task dstnode, "Build DstNode":
+  let name = "main"
+  buildBinary name

main.nim

@@ -1,8 +1,8 @@
 import stew/endians2, stew/byteutils, tables, strutils, os
-import ../../nim-libp2p/libp2p, ../../nim-libp2p/libp2p/protocols/pubsub/rpc/messages
-import ../../nim-libp2p/libp2p/muxers/mplex/lpchannel, ../../nim-libp2p/libp2p/protocols/ping
+import libp2p, libp2p/protocols/pubsub/rpc/messages
+import libp2p/muxers/mplex/lpchannel, libp2p/protocols/ping
 import chronos
-import sequtils, hashes, math, metrics
+import sequtils, hashes, math, metrics, metrics/chronos_httpserver
 from times import getTime, toUnix, fromUnix, `-`, initTime, `$`, inMilliseconds
 from nativesockets import getHostname
 
@@ -11,20 +11,44 @@ const chunks = 1
 proc msgIdProvider(m: Message): Result[MessageId, ValidationResult] =
   return ok(($m.data.hash).toBytes())
 
+proc startMetricsServer(
+    serverIp: IpAddress, serverPort: Port
+): Result[MetricsHttpServerRef, string] =
+  info "Starting metrics HTTP server", serverIp = $serverIp, serverPort = $serverPort
+
+  let metricsServerRes = MetricsHttpServerRef.new($serverIp, serverPort)
+  if metricsServerRes.isErr():
+    return err("metrics HTTP server start failed: " & $metricsServerRes.error)
+
+  let server = metricsServerRes.value
+  try:
+    waitFor server.start()
+  except CatchableError:
+    return err("metrics HTTP server start failed: " & getCurrentExceptionMsg())
+
+  info "Metrics HTTP server started", serverIp = $serverIp, serverPort = $serverPort
+  ok(metricsServerRes.value)
+
 proc main {.async.} =
   let
     hostname = getHostname()
-    myId = parseInt(hostname[4..^1])
+    myId = parseInt(getEnv("PEERNUMBER"))
+    msg_rate = parseInt(getEnv("MSGRATE"))
+    msg_size = parseInt(getEnv("MSGSIZE"))
+
     #publisherCount = client.param(int, "publisher_count")
-    publisherCount = 10
+    publisherCount = parseInt(getEnv("PEERS"))
     isPublisher = myId <= publisherCount
     #isAttacker = (not isPublisher) and myId - publisherCount <= client.param(int, "attacker_count")
     isAttacker = false
     rng = libp2p.newRng()
     #randCountry = rng.rand(distribCumSummed[^1])
     #country = distribCumSummed.find(distribCumSummed.filterIt(it >= randCountry)[0])
+  echo "Hostname: ", hostname
   let
-    address = initTAddress("0.0.0.0:5000")
+    myport = 5000 + parseInt(getEnv("PEERNUMBER"))
+    myaddress = "0.0.0.0:" & $myport
+    address = initTAddress(myaddress)
     switch =
       SwitchBuilder
         .new()
@@ -32,7 +56,7 @@ proc main {.async.} =
         .withRng(rng)
         #.withYamux()
         .withMplex()
-        .withMaxConnections(10000)
+        .withMaxConnections(250)
         .withTcpTransport(flags = {ServerFlags.TcpNoDelay})
         #.withPlainText()
         .withNoise()
@@ -45,16 +69,20 @@ proc main {.async.} =
       anonymize = true,
       )
     pingProtocol = Ping.new(rng=rng)
-  gossipSub.parameters.floodPublish = false
+  # Metrics
+  echo "Starting metrics HTTP server"
+  let metricsServer = startMetricsServer(parseIpAddress("0.0.0.0"), Port(8008))
+
+  gossipSub.parameters.floodPublish = true
   #gossipSub.parameters.lazyPushThreshold = 1_000_000_000
   #gossipSub.parameters.lazyPushThreshold = 0
   gossipSub.parameters.opportunisticGraftThreshold = -10000
-  gossipSub.parameters.heartbeatInterval = 700.milliseconds
-  gossipSub.parameters.pruneBackoff = 3.seconds
-  gossipSub.parameters.gossipFactor = 0.05
-  gossipSub.parameters.d = 8
-  gossipSub.parameters.dLow = 6
-  gossipSub.parameters.dHigh = 12
+  gossipSub.parameters.heartbeatInterval = 1.seconds
+  gossipSub.parameters.pruneBackoff = 60.seconds
+  gossipSub.parameters.gossipFactor = 0.25
+  gossipSub.parameters.d = 6
+  gossipSub.parameters.dLow = 4
+  gossipSub.parameters.dHigh = 8
   gossipSub.parameters.dScore = 6
   gossipSub.parameters.dOut = 6 div 2
   gossipSub.parameters.dLazy = 6
@@ -96,14 +124,12 @@ proc main {.async.} =
   switch.mount(gossipSub)
   switch.mount(pingProtocol)
   await switch.start()
-  #TODO
   #defer: await switch.stop()
 
   echo "Listening on ", switch.peerInfo.addrs
   echo myId, ", ", isPublisher, ", ", switch.peerInfo.peerId
-
-  var peersInfo = toSeq(1..parseInt(getEnv("PEERS")))
-  rng.shuffle(peersInfo)
+  echo "Waiting 60 seconds for node building..."
+  await sleepAsync(60.seconds)
 
   proc pinger(peerId: PeerId) {.async.} =
     try:
@@ -120,48 +146,58 @@ proc main {.async.} =
 
   let connectTo = parseInt(getEnv("CONNECTTO"))
   var connected = 0
-  for peerInfo in peersInfo:
-    if connected >= connectTo: break
-    let tAddress = "peer" & $peerInfo & ":5000"
-    echo tAddress
-    let addrs = resolveTAddress(tAddress).mapIt(MultiAddress.init(it).tryGet())
+  let tAddress = "nimp2p-service:5000"
+  var addrs: seq[MultiAddress]
+
+  echo "Trying to resolve ", tAddress
+  while true:
     try:
-      let peerId = await switch.connect(addrs[0], allowUnknownPeerId=true).wait(5.seconds)
-      #asyncSpawn pinger(peerId)
-      connected.inc()
+      addrs = resolveTAddress(tAddress).mapIt(MultiAddress.init(it).tryGet())
+      echo tAddress, " resolved: ", addrs
+      break  # Break out of the loop on successful resolution
     except CatchableError as exc:
-      echo "Failed to dial", exc.msg
+      echo "Failed to resolve address:", exc.msg
+      echo "Waiting 15 seconds..."
+      await sleepAsync(15.seconds)
+
+  rng.shuffle(addrs)
+  var index = 0
+  while true:
+    if connected >= connectTo: break
+    while true:
+      try:
+        echo "Trying to connect to ", addrs[index]
+        let peerId = await switch.connect(addrs[index], allowUnknownPeerId=true).wait(5.seconds)
+        #asyncSpawn pinger(peerId)
+        connected.inc()
+        index.inc()
+        echo "Connected!"
+        break
+      except CatchableError as exc:
+        echo "Failed to dial", exc.msg
+        echo "Waiting 15 seconds..."
+        await sleepAsync(15.seconds)
 
   #let
   #  maxMessageDelay = client.param(int, "max_message_delay")
   #  warmupMessages = client.param(int, "warmup_messages")
   #startOfTest = Moment.now() + milliseconds(warmupMessages * maxMessageDelay div 2)
 
-  await sleepAsync(10.seconds)
   echo "Mesh size: ", gossipSub.mesh.getOrDefault("test").len
 
-  for msg in 0 ..< 10:#client.param(int, "message_count"):
-    await sleepAsync(12.seconds)
-    if msg mod publisherCount == myId - 1:
-    #if myId == 1:
+  let turnToPublish = parseInt(getHostname()[4..^1])
+  echo "Publishing turn is: ", turnToPublish
+  for msg in 0 ..< 10000:#client.param(int, "message_count"):
+    await sleepAsync(msg_rate)
+    if msg mod publisherCount == turnToPublish:
+      echo "Sending message at: " ,times.getTime()
       let
         now = getTime()
         nowInt = seconds(now.toUnix()) + nanoseconds(times.nanosecond(now))
-      #var nowBytes = @(toBytesLE(uint64(nowInt.nanoseconds))) & newSeq[byte](500_000 div chunks)
-      var nowBytes = @(toBytesLE(uint64(nowInt.nanoseconds))) & newSeq[byte](50)
-      #echo "sending ", uint64(nowInt.nanoseconds)
-      for chunk in 0..<chunks:
-        nowBytes[10] = byte(chunk)
-        doAssert((await gossipSub.publish("test", nowBytes)) > 0)
+      var nowBytes = @(toBytesLE(uint64(nowInt.nanoseconds))) & newSeq[byte](msg_size)
+      doAssert((await gossipSub.publish("test", nowBytes)) > 0)
 
   #echo "BW: ", libp2p_protocols_bytes.value(labelValues=["/meshsub/1.1.0", "in"]) + libp2p_protocols_bytes.value(labelValues=["/meshsub/1.1.0", "out"])
   #echo "DUPS: ", libp2p_gossipsub_duplicate.value(), " / ", libp2p_gossipsub_received.value()
-  #requires exporting counters from GossipSub.nim
-  echo "statcounters: dup_during_validation ", libp2p_gossipsub_duplicate_during_validation.value(),
-       "\tidontwant_saves ", libp2p_gossipsub_idontwant_saved_messages.value(),
-       #"gossip optimization saves ", libp2p_gossipsub_saved_bytes.value(),
-       "\tdup_received ", libp2p_gossipsub_duplicate.value(),
-       "\tUnique_msg_received ", libp2p_gossipsub_received.value(),
-       "\tStaggered_Saves ", libp2p_gossipsub_staggerSave.value(),
-       "\tDontWant_IN_Stagger ", libp2p_gossipsub_staggerDontWantSave.value()
-waitFor(main())
+
+waitFor(main())

nimbus-build-system.paths

@@ -0,0 +1,3 @@
+--noNimblePath
+--path:"/mnt/d/Projects/status/dst-gossipsub-test-node/vendor/nimbus-build-system"
+--path:"/mnt/d/Projects/status/dst-gossipsub-test-node/vendor/nim-libp2p"

runner.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+custom_network_name="my_custom_network"
+num_peers=10
+
+if ! docker network inspect "$custom_network_name" >/dev/null 2>&1; then
+  docker network create "$custom_network_name"
+  docker network create --attachable --driver bridge "$custom_network_name"
+fi
+
+for ((i = 0; i < num_peers; i++)); do
+    # Construct the hostname (e.g., peer1, peer2, ...)
+    hostname="pod-$i"
+
+    # Run the Docker container with the current hostname
+    docker run -e PEERSPERPOD="1" -e PEERS="10" -e CONNECTTO="5" -e MSGRATE="1000" -e MSGSIZE="1000" -e PEERNUMBER="0" --hostname="$hostname" --network="$custom_network_name" dst-test-node &
+
+done

shadow/run.sh

@@ -1,36 +0,0 @@
-#!/bin/sh
-
-if [ $# -ne 2 ]; then
-    echo "Usage: $0 <runs> <nodes>"
-    exit 1
-fi
-
-runs="$1"			#number of simulation runs
-nodes="$2"			#number of nodes to simulate
-shadow_file="shadow.yaml"	
-sed -i '/environment:/q' "$shadow_file"
-sed -E -i "s/\"PEERS\": \"[0-9]+\"/\"PEERS\": \"$nodes\"/" "$shadow_file"
-
-counter=2
-while [ $counter -le $nodes ]; do
-  echo "  peer$counter: *client_host" >> "$shadow_file"
-  counter=$((counter + 1))
-done
-
-rm -f shadowlog* latencies* stats* main && rm -rf shadow.data/
-nim c -d:chronicles_colors=None --threads:on -d:metrics -d:libp2p_network_protocols_metrics -d:release main 
-
-for i in $(seq $runs); do
-    echo "Running for turn "$i
-    shadow shadow.yaml > shadowlog$i && 
-        grep -rne 'milliseconds\|BW' shadow.data/ > latencies$i && 
-        grep -rne 'statcounters:' shadow.data/ > stats$i
-    rm -rf shadow.data/
-done
-
-for i in $(seq $runs); do
-    echo "Summary for turn "$i
-    awk -f summary_latency.awk latencies$i
-    awk -f summary_shadowlog.awk shadowlog$i
-    awk -f summary_dontwant.awk stats$i    
-done

shadow/summary_dontwant.awk

@@ -1,30 +0,0 @@
-BEGIN {
-	FS = " ";		#default column separator
-	idontwant_saves = min_idontwant = max_idontwant = 0;
-    dup_received = min_dup = max_dup = 0;
-    unique_msg_received        = 0;
-    stagger_saves           = 0;
-    stagger_DontWantSaves   = 0;
-}
-
-{
-	#print $5, $7, $9
-    idontwant_saves += $5
-    if ($5 < min_idontwant || min_idontwant == 0)  min_idontwant  = $5
-    if ($5 > max_idontwant) max_idontwant = $5
-
-    dup_received += $7
-    if ($7 < min_dup || min_dup == 0)  min_dup  = $7
-    if ($7 > max_dup) max_dup = $7
-
-    unique_msg_received     += $9
-    stagger_saves           += $11
-    stagger_DontWantSaves   += $13
-
-}
-
-END {
-    print "idontwant_saves min, max, avg, total : ", min_idontwant, "\t", max_idontwant, "\t", idontwant_saves/NR, "\t", idontwant_saves 
-    print "dup_received    min, max, avg, total : ", min_dup, "\t", max_dup, "\t", dup_received/NR, "\t", dup_received
-    print "Unique_msg_received: ", unique_msg_received, "\tStagger Saves : ", stagger_saves, "\tStaggerDontWantSaves", stagger_DontWantSaves
-}

shadow/summary_latency.awk

@@ -1,49 +0,0 @@
-# we parse the latencies(x) file produced by run.sh to receive results summary (Max/Avg Latency --> per packet, overall)
-# runs $awk -f result_summary.awk latencies(x)
-
-BEGIN {
-	FS = " ";		#default column separator
-	network_size = 0
-	max_nw_lat = sum_nw_lat = 0
-	hop_lat = 100	#should be consistent with shadow.yaml
-}
-
-{
-	clean_int = $3
-	gsub(/[^0-9]/, "", clean_int); 
-	if ($3 == clean_int){	#get rid of unwanted rows
-		sum_nw_lat += $NF 
-		if (max_nw_lat < $NF) {max_nw_lat = $NF} 
-		if (split($1, arr, "peer|/main|:.*:")) {
-				#$3 = rx_latency,	arr[4] = publish_time,	arr[2] = peerID
-				lat_arr[arr[4], $3]++;	
-				msg_arr[arr[4]] = 1;	#we maintain set of messages identified by their publish time
-				if (network_size < arr[2]) {network_size = arr[2]}
-		}
-	}
-}
-
-END {
-
-	print "Total Nodes : ", network_size, "Total Messages Published : ", length(msg_arr), 
-            "Network Latency\t MAX : ", max_nw_lat, "\tAverage : ", sum_nw_lat/NR
-	print "   Message ID \t       Avg Latency \t Messages Received"
-	for (value in msg_arr) {
-		sum_rx_msgs = 0;
-        latency = 0;
-		for (key in lat_arr) {
-			split(key, parts, SUBSEP);
-			if (parts[1] == value) {
-				sum_rx_msgs = sum_rx_msgs + lat_arr[key]; 					#total receives / message
-				latency = latency + (lat_arr[key] * parts[2]) 
-				spread[ int((parts[2]) / hop_lat) ] = lat_arr[key]           #hop-by-hop spread count of messages
-	    	}
-	    }
-
-		print value, "\t", latency/sum_rx_msgs, "\t  ", sum_rx_msgs, "spread is", 
-                spread[1], spread[2], spread[3], spread[4], spread[5], spread[6], spread[7]   
-        delete spread
-    }
-}
-
-

shadow/summary_shadowlog.awk

@@ -1,145 +0,0 @@
-BEGIN {
-	FS = " ";			#column separator
-	fg_index = 7		#flags start index in $10
-	flag_size  = 12		#size of flags
-	local_in   = 0		#inbound-localhost-counters
-	local_out  = 1		#outbound-localhost-counters
-	remote_in  = 2		#inbound-remote-counters
-	remote_out = 3		#outbound-remote-counters
-}
-
-{
-    if ($9 == "[node]") {    
-        #$5: peer info,      $10: traffic stats, we need to split
-        peerlist[$5] = 1     #list for all peers
-
-        if (split($10, arr, ",|;")) {
-        #arr[2]: received bytes,    arr[3]: transferred bytes    
-            if (arr[2] > 0) {sum_rx[$5] += arr[2]} #bytes received
-            if (arr[3] > 0) {sum_tx[$5] += arr[3]} #bytes transferred
-
-            #inbound-localhost-counters
-            idx = fg_index + (flag_size * local_in)
-            #if (arr[idx] > 0) {
-                local_in_pkt[$5]				+= arr[idx]
-                local_in_bytes[$5]				+= arr[idx+1]
-                local_in_ctrl_pkt[$5]			+= arr[idx+2]
-                local_in_ctrl_hdr_bytes[$5]	    += arr[idx+3]
-                local_in_data_pkt[$5]			+= arr[idx+6]
-                local_in_data_hdr_bytes[$5]	    += arr[idx+7]
-                local_in_data_bytes[$5]		    += arr[idx+8]
-            #}
-            #outbound-localhost-counters
-            idx = fg_index + (flag_size * local_out)
-            #if (arr[idx] > 0) {
-                local_out_pkt[$5]				+= arr[idx]
-                local_out_bytes[$5]				+= arr[idx+1]
-                local_out_ctrl_pkt[$5]			+= arr[idx+2]
-                local_out_ctrl_hdr_bytes[$5]	+= arr[idx+3]
-                local_out_data_pkt[$5]			+= arr[idx+6]
-                local_out_data_hdr_bytes[$5]	+= arr[idx+7]
-                local_out_data_bytes[$5]		+= arr[idx+8]
-            #}
-            #inbound-remote-counters
-            idx = fg_index + (flag_size * remote_in)
-            #if (arr[idx] > 0) {
-                remote_in_pkt[$5]				+= arr[idx]
-                remote_in_bytes[$5]				+= arr[idx+1]
-                remote_in_ctrl_pkt[$5]			+= arr[idx+2]
-                remote_in_ctrl_hdr_bytes[$5]	+= arr[idx+3]
-                remote_in_data_pkt[$5]			+= arr[idx+6]
-                remote_in_data_hdr_bytes[$5]	+= arr[idx+7]
-                remote_in_data_bytes[$5]		+= arr[idx+8]
-            #}
-            #outbound-remote-counters
-            idx = fg_index + (flag_size * remote_out)
-            #if (arr[idx] > 0) {
-                remote_out_pkt[$5]				+= arr[idx]
-                remote_out_bytes[$5]			+= arr[idx+1]
-                remote_out_ctrl_pkt[$5]			+= arr[idx+2]
-                remote_out_ctrl_hdr_bytes[$5]	+= arr[idx+3]
-                remote_out_data_pkt[$5]			+= arr[idx+6]
-                remote_out_data_hdr_bytes[$5]	+= arr[idx+7]
-                remote_out_data_bytes[$5]		+= arr[idx+8]
-            #}
-
-        }		
-    } 
-}
-
-END {
-        nw_size = length(peerlist)
-        min_in  = max_in = min_out = max_out = sum_in = sum_out = avg_in = avg_out = 0
-        for (value in peerlist) {               #node specific tx/rx stats (bytes)
-            sum_in  += sum_rx[value]
-            sum_out += sum_tx[value]
-
-            if (sum_rx[value] < min_in  || min_in == 0)  min_in  = sum_rx[value]
-            if (sum_tx[value] < min_out || min_out == 0) min_out = sum_tx[value] 
-            if (sum_rx[value] > max_in)  max_in  = sum_rx[value]
-            if (sum_tx[value] > max_out) max_out = sum_tx[value] 
-        }
-        avg_in  = sum_in/nw_size
-        avg_out = sum_out/nw_size
-
-
-
-        for (value in peerlist) {
-
-            sum_sq_in  += (sum_rx[value] - avg_in) ^ 2      #for stddev
-            sum_sq_out += (sum_tx[value] - avg_out) ^ 2
-
-                sum_local_in_pkt               += local_in_pkt[value]
-                sum_local_in_bytes             += local_in_bytes[value]
-                sum_local_in_ctrl_pkt          += local_in_ctrl_pkt[value]
-                sum_local_in_ctrl_hdr_bytes    += local_in_ctrl_hdr_bytes[value]
-                sum_local_in_data_pkt          += local_in_data_pkt[value]
-                sum_local_in_data_hdr_bytes    += local_in_data_hdr_bytes[value]
-                sum_local_in_data_bytes        += local_in_data_bytes[value]
-
-                sum_local_out_pkt               += local_out_pkt[value]
-                sum_local_out_bytes             += local_out_bytes[value]
-                sum_local_out_ctrl_pkt          += local_out_ctrl_pkt[value]
-                sum_local_out_ctrl_hdr_bytes    += local_out_ctrl_hdr_bytes[value]
-                sum_local_out_data_pkt          += local_out_data_pkt[value]
-                sum_local_out_data_hdr_bytes    += local_out_data_hdr_bytes[value]
-                sum_local_out_data_bytes        += local_out_data_bytes[value]
-
-                sum_remote_in_pkt               += remote_in_pkt[value]
-                sum_romote_in_bytes             += remote_in_bytes[value]
-                sum_remote_in_ctrl_pkt          += remote_in_ctrl_pkt[value]
-                sum_remote_in_ctrl_hdr_bytes    += remote_in_ctrl_hdr_bytes[value]
-                sum_remote_in_data_pkt          += remote_in_data_pkt[value]
-                sum_remote_in_data_hdr_bytes    += remote_in_data_hdr_bytes[value]
-                sum_remote_in_data_bytes        +=remote_in_data_bytes[value]
-
-                sum_remote_out_pkt              +=remote_out_pkt[value]
-                sum_remote_out_bytes            +=remote_out_bytes[value]
-                sum_remote_out_ctrl_pkt         +=remote_out_ctrl_pkt[value]
-                sum_remote_out_ctrl_hdr_bytes   +=remote_out_ctrl_hdr_bytes[value] 
-                sum_remote_out_data_pkt         +=remote_out_data_pkt[value]
-                sum_remote_out_data_hdr_bytes   +=remote_out_data_hdr_bytes[value]
-                sum_remote_out_data_bytes       +=remote_out_data_bytes[value]
-
-            #}
-        }
-
-    print "\nTotal Bytes Received : ", sum_in, "Total Bytes Transferred : ", sum_out
-    print "Per Node Pkt Receives : min, max, avg, stddev = ", min_in, max_in, avg_in, sqrt(sum_sq_in/nw_size)
-    print "Per Node Pkt Transfers: min, max, avg, stddev = ", min_out, max_out, avg_out, sqrt(sum_sq_out/nw_size)    
-    
-
-    print "Details..."
-    #print "Local IN pkt: ", sum_local_in_pkt, "Bytes : ", sum_local_in_bytes, "ctrlPkt: ", sum_local_in_ctrl_pkt, "ctrlHdrBytes: ", sum_local_in_ctrl_hdr_bytes, 
-    #        "DataPkt: ", sum_local_in_data_pkt, "DataHdrBytes: ", sum_local_in_data_hdr_bytes, "DataBytes", sum_local_in_data_bytes
-    #print "Local OUT pkt: ", sum_local_out_pkt, "Bytes : ", sum_local_out_bytes, "ctrlPkt: ", sum_local_out_ctrl_pkt, "ctrlHdrBytes: ", sum_local_out_ctrl_hdr_bytes, 
-    #        "DataPkt: ", sum_local_out_data_pkt, "DataHdrBytes: ", sum_local_out_data_hdr_bytes, "DataBytes", sum_local_out_data_bytes
-    print "Remote IN pkt: ", sum_remote_in_pkt, "Bytes : ", sum_romote_in_bytes, "ctrlPkt: ", sum_remote_in_ctrl_pkt, "ctrlHdrBytes: ", sum_remote_in_ctrl_hdr_bytes, 
-            "DataPkt: ", sum_remote_in_data_pkt, "DataHdrBytes: ", sum_remote_in_data_hdr_bytes, "DataBytes", sum_remote_in_data_bytes
-    print "Remote OUT pkt: ", sum_remote_out_pkt, "Bytes : ", sum_romote_out_bytes, "ctrlPkt: ", sum_remote_out_ctrl_pkt, "ctrlHdrBytes: ", sum_remote_out_ctrl_hdr_bytes, 
-            "DataPkt: ", sum_remote_out_data_pkt, "DataHdrBytes: ", sum_remote_out_data_hdr_bytes, "DataBytes", sum_remote_out_data_bytes
-
-
-}
-
-

test_node.nimble

@@ -1,12 +0,0 @@
-mode = ScriptMode.Verbose
-
-packageName   = "test_node"
-version       = "0.1.0"
-author        = "Status Research & Development GmbH"
-description   = "A test node for gossipsub"
-license       = "MIT"
-skipDirs      = @[]
-
-requires "nim >= 1.6.0",
-         "libp2p",
-         "ggplotnim"