Local sim now includes a prometheus server, a Ganache instance and a simulation dashboard server (in Tmux mode)

.gitmodules

@@ -153,3 +153,28 @@
 	url = https://github.com/status-im/nim-rocksdb.git
 	ignore = dirty
 	branch = master
+[submodule "vendor/asynctools"]
+	path = vendor/asynctools
+	url = https://github.com/cheatfate/asynctools.git
+	ignore = dirty
+	branch = master
+[submodule "vendor/karax"]
+	path = vendor/karax
+	url = https://github.com/pragmagic/karax.git
+	ignore = dirty
+	branch = master
+[submodule "vendor/jswebsockets"]
+	path = vendor/jswebsockets
+	url = https://github.com/stisa/jswebsockets.git
+	ignore = dirty
+	branch = master
+[submodule "vendor/websocket.nim"]
+	path = vendor/websocket.nim
+	url = https://github.com/niv/websocket.nim.git
+	ignore = dirty
+	branch = master
+[submodule "vendor/nim-chronicles-tail"]
+	path = vendor/nim-chronicles-tail
+	url = https://github.com/status-im/nim-chronicles-tail.git
+	ignore = dirty
+	branch = master

.nvmrc

@@ -0,0 +1 @@
+v13.12.0

config.nims

@@ -41,6 +41,8 @@ else:
 # for heap-usage-by-instance-type metrics and object base-type strings
 --define:nimTypeNames
 
+# switch("define", "snappy_implementation=libp2p")
+
 switch("import", "testutils/moduletests")
 
 const useLibStackTrace = not defined(macosx) and

env.sh

@@ -4,5 +4,22 @@
 # and we fall back to a Zsh-specific special var to also support Zsh.
 REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})"
 ABS_PATH="$(cd ${REL_PATH}; pwd)"
+
+# Activate nvm only when this file is sourced without arguments:
+if [ -z "$*" ]; then
+  if command -v nvm > /dev/null; then
+    nvm use
+    command -v ganache-cli > /dev/null || { npm install -g ganache-cli; }
+  else
+    echo <<EOF
+  In order to use Ganache (a development ETH1 chain), please install NVM with:
+  curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash
+
+  For more info:
+  https://github.com/nvm-sh/nvm
+EOF
+  fi
+fi
+
 source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh
 

tests/fuzzing/nim.cfg

@@ -1,2 +1,3 @@
 -d:ssz_testing
+-d:"const_preset=mainnet"
 

tests/simulation/dashboard/chronicles_tail_p2p_plugin.nim

@@ -0,0 +1,189 @@
+import
+  strformat, jsconsole, jsffi,
+  karax/[karax, kdom, karaxdsl, vdom],
+  chronicles_tail/jsplugins
+
+# Make sure that the Karax instance in the plugin is the same one
+# as the Karax instance in the enclosing chronicle-tail page.
+kxi = getKarax()
+
+type EventsTable = ref object of VComponent
+
+proc renderNetworkEvents(page: VComponent): VNode =
+  result = buildHtml:
+    table:
+      tr:
+        th: text "Time"
+        th: text "Nodes"
+
+const
+  columnWidth = 320
+  timestampsHeight = 50
+  eventsMargin = 10
+
+var
+  eventsTable = newComponent(EventsTable, renderNetworkEvents)
+  protocolMessages = newJsAssoc[cstring, JsAssoc[cstring, cstring]]()
+
+  pendingEvents = newSeq[TailEvent]()
+  freedColumns = newSeq[int]()
+  columnBottoms = newSeq[int]()
+  peerToColumnTable = newJsAssoc[cstring, int]()
+  lastTimestampBottom = timestampsHeight
+
+proc startsWith*(a, b: cstring): bool {.importcpp: "startsWith", nodecl.}
+
+proc getMsgName(protocol: cstring, msgId: int): cstring =
+  protocolMessages[protocol][cast[cstring](msgId)]
+
+proc renderEvent(ev: TailEvent): cstring =
+  var res = newStringOfCap(1024)
+  let eventType = ev.msg
+
+  res.add &"""<div class="event {eventType}">"""
+
+  template addField(class, value) =
+    res.add "<div class=\"" & class & "\">"
+    res.addEscaped $value
+    res.add "</div>"
+
+  if eventType.startsWith(cstring("peer_")):
+    addField "peer", ev.peer
+    addField "port", ev.port
+  else:
+    addField "msgName", getMsgName(ev.protocol, ev.msgId)
+    res.addAsHtml ev.data
+
+  res.add """</div>"""
+  return cstring(res)
+
+proc selectColumn(ev: TailEvent): int =
+  let key = cast[cstring](ev.port)# & ev.peer
+  kout ev.msg, key
+
+  if ev.msg in [cstring"peer_accepted", "peer_connected"]:
+    if freedColumns.len > 0:
+      result = freedColumns.pop()
+    else:
+      result = columnBottoms.len
+      columnBottoms.add(timestampsHeight)
+    peerToColumnTable[key] = result
+
+  elif ev.msg == cstring("peer_disconnected"):
+    result = peerToColumnTable[key]
+    discard jsDelete peerToColumnTable[key]
+    freedColumns.add result
+
+  else:
+    result = peerToColumnTable[key]
+
+template pixels(n: int): cstring =
+  cast[cstring](n) & "px"
+
+proc addEvent(ev: TailEvent) =
+  var
+    row = document.createElement("tr")
+    timeElem = document.createElement("td")
+    eventElem = document.createElement("td")
+    eventsTable = eventsTable.dom
+    eventsCount = eventsTable.children.len
+    lastEventRow = eventsTable.children[eventsCount - 1]
+
+  row.class = if eventsCount mod 2 == 0: "even" else: "odd"
+
+  # Hide the element initially, so we can safely measure its size.
+  # It has to be added to the DOM before it can be measured.
+  row.style.visibility = "hidden"
+  row.appendChild(timeElem)
+  row.appendChild(eventElem)
+
+  timeElem.innerHtml = ev.ts
+  timeElem.class = "time"
+
+  eventElem.innerHTML = renderEvent(ev)
+
+  eventsTable.appendChild(row)
+  let rowHeight = row.offsetHeight
+  let eventColumn = selectColumn(ev)
+  let timestampOffset = max(lastTimestampBottom, columnBottoms[eventColumn])
+  let prevTimestampOffset = lastTimestampBottom - timestampsHeight
+
+  lastTimestampBottom = timestampOffset + timestampsHeight
+  columnBottoms[eventColumn] += rowHeight + eventsMargin
+
+  # Make sure the event data is in the right column and that it
+  # can overflow past the row height:
+  eventElem.style.paddingLeft = pixels(eventColumn * columnWidth)
+
+  # Position the row in its right place and show it:
+  lastEventRow.style.height = pixels(timestampOffset - prevTimestampOffset)
+  row.style.top = pixels(timestampOffset)
+  row.style.visibility = ""
+
+proc networkSectionContent: VNode =
+  result = buildHtml(tdiv(id = "network")):
+    text "Network section"
+    eventsTable
+
+proc tailEventFilter(ev: TailEvent): bool =
+  if ev.topics != "p2pdump":
+    return false
+
+  if ev.msg == "p2p_protocols":
+    protocolMessages = cast[type(protocolMessages)](ev.data)
+  else:
+    if eventsTable.dom == nil:
+      pendingEvents.add ev
+    else:
+      addEvent ev
+
+  return true
+
+proc addPending =
+  if eventsTable.dom != nil and pendingEvents.len > 0:
+    defer: pendingEvents.setLen(0)
+    for ev in pendingEvents:
+      addEvent ev
+
+let interval = window.setInterval(addPending, 1000)
+
+proc addStyles(styles: cstring) =
+  var s = document.createElement("style")
+  s.appendChild document.createTextNode(styles)
+  document.head.appendChild(s)
+
+once:
+  addStyles cstring"""
+    #network > table {
+      position: relative;
+    }
+
+    #network .event {
+      border: 1px solid blue;
+    }
+
+    #network .event table {
+      width: 100%;
+    }
+
+    #network > table > tr {
+      position: absolute;
+      display: flex;
+      flex-direction: row;
+      border-left: 1px solid red;
+    }
+
+    #network .time {
+      width: 160px;
+    }
+
+    #network .event {
+      width: 320px;
+    }
+  """
+
+  addSection("Network", networkSectionContent)
+  addEventFilter(tailEventFilter)
+
+kxi.redraw()
+

tests/simulation/start.sh

@@ -43,18 +43,18 @@ if [[ "$USE_MULTITAIL" != "no" ]]; then
   type "$MULTITAIL" &>/dev/null || { echo "${MULTITAIL}" is missing; USE_MULTITAIL="no"; }
 fi
 
-USE_TMUX="${USE_TMUX:-no}"
-if [[ "$USE_TMUX" != "no" ]]; then
+USE_TMUX="${USE_TMUX:-yes}"
+if [[ "$USE_TMUX" == "yes" ]]; then
   type "$TMUX" &>/dev/null || { echo "${TMUX}" is missing; USE_TMUX="no"; }
 fi
 
-USE_GANACHE="${USE_GANACHE:-no}"
-if [[ "$USE_GANACHE" != "no" ]]; then
+USE_GANACHE="${USE_GANACHE:-yes}"
+if [[ "$USE_GANACHE" == "yes" ]]; then
   type "$GANACHE" &>/dev/null || { echo $GANACHE is missing; USE_GANACHE="no"; }
 fi
 
-USE_PROMETHEUS="${USE_PROMETHEUS:-no}"
-if [[ "$USE_PROMETHEUS" != "no" ]]; then
+USE_PROMETHEUS="${USE_PROMETHEUS:-yes}"
+if [[ "$USE_PROMETHEUS" == "yes" ]]; then
   type "$PROMETHEUS" &>/dev/null || { echo $PROMETHEUS is missing; USE_PROMETHEUS="no"; }
 fi
 
@@ -83,10 +83,9 @@ fi
 if [[ "$USE_GANACHE" != "no" ]]; then
   if [[ "$USE_TMUX" != "no" ]]; then
     $TMUX new-window -d -t $TMUX_SESSION_NAME -n "$GANACHE" "$GANACHE"
-  elif [[ "$USE_MULTITAIL" != "no" ]]; then
-    COMMANDS+=( " -cT ansi -t '$GANACHE'" )
   else
-    $GANACHE &
+    echo NOTICE: $GANACHE will be started automatically only with USE_TMUX=1
+    USE_GANACHE="no"
   fi
 fi
 
@@ -94,7 +93,8 @@ if [[ "$USE_PROMETHEUS" != "no" ]]; then
   if [[ "$USE_TMUX" != "no" ]]; then
     $TMUX new-window -d -t $TMUX_SESSION_NAME -n "$PROMETHEUS" "cd '$METRICS_DIR' && $PROMETHEUS"
   else
-    echo "$PROMETHEUS can be used currently only with USE_TMUX=1"
+    echo NOTICE: $PROMETHEUS will be started automatically only with USE_TMUX=1
+    USE_PROMETHEUS="no"
   fi
 fi
 

vendor/asynctools

@@ -0,0 +1 @@
+Subproject commit c478bb74268c65e7400e81f49d26f82d20f1f57a

vendor/jswebsockets

@@ -0,0 +1 @@
+Subproject commit ff0ceecc4c071fa8aecbf223d678ac54af120f8d

vendor/karax

@@ -0,0 +1 @@
+Subproject commit f1cd9e606d117bf387a1ddf9e71cafd02d10a50f

vendor/nim-chronicles-tail

@@ -0,0 +1 @@
+Subproject commit 66e95a2ec6353dcb98d08aabdc46f6f0d1ea120b

vendor/websocket.nim

@@ -0,0 +1 @@
+Subproject commit 28cc44c8defc0b248b3abbc8205759b69a98f7f6