Dht debug (#446)

* Support for building docker images with local modifications for the purpose of testing and debugging

* exposes peer information via debug/info

* api-names slightly kinder to json serializers

* Moves debug image-building

* fixes misalignment of debug peer info array

* Changes switchPeers source from KeyBook to AddressBook (filed ticket in libp2p, discussed with Tanguy)

* Limited success with dist-test peer discovery tests

* Removes unnecessary random-timer

* bumps dht repo. Adds peerId to formatNode

* Removes unused prints

* bumps libp2p-dht

* Exposes node address on debug api

* Adds traces

* review comments by me

* Hides debug/peers api behind compile flag

* Waiting for nim-libp2p-dht PR merge

* bumps libp2p-dht back to main after PRs were merged there.

* Cleanup

codex/conf.nim

@@ -35,6 +35,9 @@ import ./stores
 
 export DefaultCacheSizeMiB, net, DefaultQuotaBytes, DefaultBlockTtl, DefaultBlockMaintenanceInterval, DefaultNumberOfBlocksToMaintainPerInterval
 
+const
+  codex_enable_api_debug_peers* {.booldefine.} = false
+
 type
   StartUpCommand* {.pure.} = enum
     noCommand,

codex/discovery.nim

@@ -35,7 +35,7 @@ logScope:
 
 type
   Discovery* = ref object of RootObj
-    protocol: discv5.Protocol           # dht protocol
+    protocol*: discv5.Protocol           # dht protocol
     key: PrivateKey                     # private key
     peerId: PeerId                      # the peer id of the local node
     announceAddrs: seq[MultiAddress]    # addresses announced as part of the provider records
@@ -58,13 +58,16 @@ proc toNodeId*(host: ca.Address): NodeId =
 proc findPeer*(
   d: Discovery,
   peerId: PeerId): Future[?PeerRecord] {.async.} =
+  trace "protocol.resolve..."
   let
     node = await d.protocol.resolve(toNodeId(peerId))
 
   return
     if node.isSome():
+      trace "protocol.resolve some data"
       node.get().record.data.some
     else:
+      trace "protocol.resolve none"
       PeerRecord.none
 
 method find*(
@@ -127,16 +130,10 @@ method provide*(d: Discovery, host: ca.Address) {.async, base.} =
     trace "Provided to nodes", nodes = nodes.len
 
 method removeProvider*(d: Discovery, peerId: PeerId): Future[void] {.base.} =
-  ## Remove provider from providers table
-  ##
-
   trace "Removing provider", peerId
   d.protocol.removeProvidersLocal(peerId)
 
 proc updateAnnounceRecord*(d: Discovery, addrs: openArray[MultiAddress]) =
-  ## Update providers record
-  ##
-
   d.announceAddrs = @addrs
 
   trace "Updating announce record", addrs = d.announceAddrs
@@ -149,9 +146,6 @@ proc updateAnnounceRecord*(d: Discovery, addrs: openArray[MultiAddress]) =
       .expect("Should update SPR")
 
 proc updateDhtRecord*(d: Discovery, ip: ValidIpAddress, port: Port) =
-  ## Update providers record
-  ##
-
   trace "Updating Dht record", ip, port = $port
   d.dhtRecord = SignedPeerRecord.init(
     d.key, PeerRecord.init(d.peerId, @[
@@ -160,6 +154,10 @@ proc updateDhtRecord*(d: Discovery, ip: ValidIpAddress, port: Port) =
         IpTransportProtocol.udpProtocol,
         port)])).expect("Should construct signed record").some
 
+  if not d.protocol.isNil:
+    d.protocol.updateRecord(d.dhtRecord)
+      .expect("Should update SPR")
+
 proc start*(d: Discovery) {.async.} =
   d.protocol.open()
   await d.protocol.start()

codex/rest/api.nim

@@ -24,8 +24,11 @@ import pkg/stew/base10
 import pkg/stew/byteutils
 import pkg/confutils
 
+import pkg/libp2p
 import pkg/libp2p/routing_record
 import pkg/libp2pdht/discv5/spr as spr
+import pkg/libp2pdht/discv5/routing_table as rt
+import pkg/libp2pdht/discv5/node as dn
 
 import ../node
 import ../blocktype
@@ -45,6 +48,47 @@ proc validate(
   {.gcsafe, raises: [Defect].} =
   0
 
+proc formatAddress(address: Option[dn.Address]): string =
+  if address.isSome():
+    return $address.get()
+  return "<none>"
+
+proc formatNode(node: dn.Node): JsonNode =
+  let jobj = %*{
+    "nodeId": $node.id,
+    "peerId": $node.record.data.peerId,
+    "record": $node.record,
+    "address": formatAddress(node.address),
+    "seen": $node.seen
+  }
+  return jobj
+
+proc formatTable(routingTable: rt.RoutingTable): JsonNode =
+  let jarray = newJArray()
+  for bucket in routingTable.buckets:
+    for node in bucket.nodes:
+      jarray.add(formatNode(node))
+
+  let jobj = %*{
+    "localNode": formatNode(routingTable.localNode),
+    "nodes": jarray
+  }
+  return jobj
+
+proc formatPeerRecord(peerRecord: PeerRecord): JsonNode =
+  let jarray = newJArray()
+  for maddr in peerRecord.addresses:
+    jarray.add(%*{
+      "address": $maddr.address
+    })
+
+  let jobj = %*{
+    "peerId": $peerRecord.peerId,
+    "seqNo": $peerRecord.seqNo,
+    "addresses": jarray
+  }
+  return jobj
+
 proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter =
   var router = RestRouter.init(validate)
   router.api(
@@ -244,6 +288,7 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter =
               node.discovery.dhtRecord.get.toURI
             else:
               "",
+          "table": formatTable(node.discovery.protocol.routingTable),
           "codex": {
             "version": $codexVersion,
             "revision": $codexRevision
@@ -252,6 +297,22 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter =
 
       return RestApiResponse.response($json, contentType="application/json")
 
+  when codex_enable_api_debug_peers:
+    router.api(
+      MethodGet,
+      "/api/codex/v1/debug/peer/{peerId}") do (peerId: PeerId) -> RestApiResponse:
+
+        trace "debug/peer start"
+        without peerRecord =? (await node.findPeer(peerId.get())):
+          trace "debug/peer peer not found!"
+          return RestApiResponse.error(
+            Http400,
+            "Unable to find Peer!")
+
+        let json = formatPeerRecord(peerRecord)
+        trace "debug/peer returning peer record"
+        return RestApiResponse.response($json)
+
   router.api(
     MethodGet,
     "/api/codex/v1/sales/availability") do () -> RestApiResponse:
@@ -313,5 +374,4 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter =
 
       return RestApiResponse.response($json, contentType="application/json")
 
-
   return router

docker/codex.Dockerfile

@@ -3,8 +3,8 @@ WORKDIR /src
 RUN apk update && apk add git cmake curl make git bash linux-headers
 COPY . .
 RUN make clean
-RUN make update
+RUN make -j4 update
-RUN make NIM_PARAMS="-d:disableMarchNative"
+RUN make -j4 NIM_PARAMS="-d:disableMarchNative -d:codex_enable_api_debug_peers=true"
 
 FROM alpine:3.17.2
 WORKDIR /root/

docker/prometheus/alert.yml

@@ -1,6 +0,0 @@
-groups:
-  - name: DemoAlerts
-    rules:
-      - alert: InstanceDown
-        expr: up{job="services"} < 1
-        for: 5m

docker/prometheus/prometheus.yml

@@ -1,15 +0,0 @@
-global:
-  scrape_interval: 30s
-  scrape_timeout: 10s
-
-rule_files:
-  - alert.yml
-
-scrape_configs:
-  - job_name: services
-    metrics_path: /metrics
-    static_configs:
-      - targets:
-          - 'prometheus:9090'
-          - 'codex-node1:9090'
-          - 'codex-node2:9090'

docker/startCodex.sh

@@ -2,6 +2,10 @@ echo "Starting Codex..."
 
 args=""
 
+## Using local ip as NAT
+nat_addr=$(ifconfig eth0 | awk '/inet addr/ {gsub("addr:", "", $2); print $2}')
+echo "Local IP: $nat_addr"
+
 # Required arguments
 if [ -n "$LISTEN_ADDRS" ]; then
   echo "Listen address: $LISTEN_ADDRS"
@@ -40,10 +44,8 @@ if [ -n "$METRICS_ADDR" ] && [ -n "$METRICS_PORT" ]; then
 fi
 
 # NAT
-if [ -n "$NAT_IP" ]; then
+echo "NAT: $nat_addr"
-  echo "NAT: $NAT_IP"
+args="$args --nat=$nat_addr"
-  args="$args --nat=$NAT_IP"
-fi
 
 # Discovery IP
 if [ -n "$DISC_IP" ]; then
@@ -108,7 +110,7 @@ fi
 # Ethereum persistence
 if [ -n "$ETH_PROVIDER" ] && [ -n "$ETH_ACCOUNT" ] && [ -n "$ETH_MARKETPLACE_ADDRESS" ]; then
     echo "Persistence enabled"
-    args="$args --persistence=true"
+    args="$args --persistence"
     args="$args --eth-provider=$ETH_PROVIDER"
     args="$args --eth-account=$ETH_ACCOUNT"
     # args="$args --validator"

vendor/nim-libp2p-dht

@@ -1 +1 @@
-Subproject commit 4375b9229815c332a3b1a9d0091d5cf5a74adb2e
+Subproject commit bd517f0e8da38a1b5da15f7deb2d5c652ca389f1