Add `libp2p_dump` and `libp2p_dump_dir` compiler time options. (#359)

* Add `libp2p_dump` and `libp2p_dump_dir` compiler time options.

* Add tools/pbcap_parser.

* Add mplex control messages decoding.

libp2p.nimble

@@ -5,7 +5,7 @@ version       = "0.0.2"
 author        = "Status Research & Development GmbH"
 description   = "LibP2P implementation"
 license       = "MIT"
-skipDirs      = @["tests", "examples", "Nim"]
+skipDirs      = @["tests", "examples", "Nim", "tools", "scripts", "docs"]
 
 requires "nim >= 1.2.0",
          "nimcrypto >= 0.4.1",

libp2p/debugutils.nim

@@ -0,0 +1,280 @@
+## Nim-LibP2P
+## Copyright (c) 2018 Status Research & Development GmbH
+## Licensed under either of
+##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+## at your option.
+## This file may not be copied, modified, or distributed except according to
+## those terms.
+
+## To enable dump of all incoming and outgoing unencrypted messages you need
+## to compile project with ``-d:libp2p_dump`` compile-time option. When this
+## option enabled ``nim-libp2p`` will create dumps of unencrypted messages for
+## every peer libp2p communicates.
+##
+## Every file is created with name "<PeerID>.pbcap". One file represents
+## all the communication with peer which identified by ``PeerID``.
+##
+## File can have multiple protobuf encoded messages of this format:
+##
+## Protobuf's message fields:
+##   1. SeqID: optional uint64
+##   2. Timestamp: required uint64
+##   3. Type: optional uint64
+##   4. Direction: required uint64
+##   5. LocalAddress: optional bytes
+##   6. RemoteAddress: optional bytes
+##   7. Message: required bytes
+import os, options
+import nimcrypto/utils, stew/endians2
+import protobuf/minprotobuf, stream/connection, protocols/secure/secure,
+       multiaddress, peerid, varint, muxers/mplex/types
+
+from times import getTime, toUnix, fromUnix, nanosecond, format, Time,
+                  NanosecondRange, initTime
+from strutils import toHex, repeat
+export peerid, options, multiaddress
+
+type
+  FlowDirection* = enum
+    Outgoing, Incoming
+
+  ProtoMessage* = object
+    timestamp*: uint64
+    direction*: FlowDirection
+    message*: seq[byte]
+    seqID*: Option[uint64]
+    mtype*: Option[uint64]
+    local*: Option[MultiAddress]
+    remote*: Option[MultiAddress]
+
+const
+  libp2p_dump_dir* {.strdefine.} = "nim-libp2p"
+    ## default directory where all the dumps will be stored, if the path
+    ## relative it will be created in home directory. You can overload this path
+    ## using ``-d:libp2p_dump_dir=<otherpath>``.
+
+proc getTimestamp(): uint64 =
+  ## This procedure is present because `stdlib.times` missing it.
+  let time = getTime()
+  uint64(time.toUnix() * 1_000_000_000 + time.nanosecond())
+
+proc getTimedate(value: uint64): string =
+  ## This procedure is workaround for `stdlib.times` to just convert
+  ## nanoseconds' timestamp to DateTime object.
+  let time = initTime(int64(value div 1_000_000_000), value mod 1_000_000_000)
+  time.format("yyyy-MM-dd HH:mm:ss'.'fffzzz")
+
+proc dumpMessage*(conn: SecureConn, direction: FlowDirection,
+                  data: openArray[byte]) =
+  ## Store unencrypted message ``data`` to dump file, all the metadata will be
+  ## extracted from ``conn`` instance.
+  var pb = initProtoBuffer(options = {WithVarintLength})
+  pb.write(2, getTimestamp())
+  pb.write(4, uint64(direction))
+  if len(conn.peerInfo.addrs) > 0:
+    pb.write(6, conn.peerInfo.addrs[0])
+  pb.write(7, data)
+  pb.finish()
+
+  let dirName =
+    if isAbsolute(libp2p_dump_dir):
+      libp2p_dump_dir
+    else:
+      getHomeDir() / libp2p_dump_dir
+
+  let fileName = $(conn.peerInfo.peerId) & ".pbcap"
+
+  # This is debugging procedure so it should not generate any exceptions,
+  # and we going to return at every possible OS error.
+  if not(dirExists(dirName)):
+    try:
+      createDir(dirName)
+    except CatchableError:
+      return
+
+  let pathName = dirName / fileName
+  var handle: File
+  try:
+    if open(handle, pathName, fmAppend):
+      discard writeBuffer(handle, addr pb.buffer[pb.offset], pb.getLen())
+  finally:
+    close(handle)
+
+proc decodeDumpMessage*(data: openArray[byte]): Option[ProtoMessage] =
+  ## Decode protobuf's message ProtoMessage from array of bytes ``data``.
+  var
+    pb = initProtoBuffer(data)
+    value: uint64
+    ma1, ma2: MultiAddress
+    pmsg: ProtoMessage
+
+  let res2 = pb.getField(2, pmsg.timestamp)
+  if res2.isErr() or not(res2.get()):
+    return none[ProtoMessage]()
+
+  let res4 = pb.getField(4, value)
+  if res4.isErr() or not(res4.get()):
+    return none[ProtoMessage]()
+
+  # `case` statement could not work here with an error "selector must be of an
+  # ordinal type, float or string"
+  pmsg.direction =
+    if value == uint64(Outgoing):
+      Outgoing
+    elif value == uint64(Incoming):
+      Incoming
+    else:
+      return none[ProtoMessage]()
+
+  let res7 = pb.getField(7, pmsg.message)
+  if res7.isErr() or not(res7.get()):
+    return none[ProtoMessage]()
+
+  value = 0'u64
+  let res1 = pb.getField(1, value)
+  if res1.isOk() and res1.get():
+    pmsg.seqID = some(value)
+  value = 0'u64
+  let res3 = pb.getField(3, value)
+  if res3.isOk() and res3.get():
+    pmsg.mtype = some(value)
+  let res5 = pb.getField(5, ma1)
+  if res5.isOk() and res5.get():
+    pmsg.local = some(ma1)
+  let res6 = pb.getField(6, ma2)
+  if res6.isOk() and res6.get():
+    pmsg.remote = some(ma2)
+
+  some(pmsg)
+
+iterator messages*(data: seq[byte]): Option[ProtoMessage] =
+  ## Iterate over sequence of bytes and decode all the ``ProtoMessage``
+  ## messages we found.
+  var value: uint64
+  var size: int
+  var offset = 0
+  while offset < len(data):
+    value = 0
+    size = 0
+    let res = PB.getUVarint(data.toOpenArray(offset, len(data) - 1),
+                            size, value)
+    if res.isOk():
+      if (value > 0'u64) and (value < uint64(len(data) - offset)):
+        offset += size
+        yield decodeDumpMessage(data.toOpenArray(offset,
+                                                 offset + int(value) - 1))
+        # value is previously checked to be less then len(data) which is `int`.
+        offset += int(value)
+      else:
+        break
+    else:
+      break
+
+proc dumpHex*(pbytes: openarray[byte], groupBy = 1, ascii = true): string =
+  ## Get hexadecimal dump of memory for array ``pbytes``.
+  var res = ""
+  var offset = 0
+  var ascii = ""
+
+  while offset < len(pbytes):
+    if (offset mod 16) == 0:
+      res = res & toHex(uint64(offset)) & ":  "
+
+    for k in 0 ..< groupBy:
+      let ch = pbytes[offset + k]
+      ascii.add(if ord(ch) > 31 and ord(ch) < 127: char(ch) else: '.')
+
+    let item =
+      case groupBy:
+      of 1:
+        toHex(pbytes[offset])
+      of 2:
+        toHex(uint16.fromBytes(pbytes.toOpenArray(offset, len(pbytes) - 1)))
+      of 4:
+        toHex(uint32.fromBytes(pbytes.toOpenArray(offset, len(pbytes) - 1)))
+      of 8:
+        toHex(uint64.fromBytes(pbytes.toOpenArray(offset, len(pbytes) - 1)))
+      else:
+        ""
+    res.add(item)
+    res.add(" ")
+    offset = offset + groupBy
+
+    if (offset mod 16) == 0:
+      res.add(" ")
+      res.add(ascii)
+      ascii.setLen(0)
+      res.add("\p")
+
+  if (offset mod 16) != 0:
+    let spacesCount = ((16 - (offset mod 16)) div groupBy) *
+                        (groupBy * 2 + 1) + 1
+    res = res & repeat(' ', spacesCount)
+    res = res & ascii
+
+  res.add("\p")
+  res
+
+proc mplexMessage*(data: seq[byte]): string =
+  var value = 0'u64
+  var size = 0
+  let res = PB.getUVarint(data, size, value)
+  if res.isOk():
+    if size < len(data) and data[size] == 0x00'u8:
+      let header = cast[MessageType](value and 0x07'u64)
+      let ident = (value shr 3)
+      "mplex: [" & $header & ", ident = " & $ident & "] "
+    else:
+      ""
+  else:
+    ""
+
+proc toString*(msg: ProtoMessage, dump = true): string =
+  ## Convert message ``msg`` to its string representation.
+  ## If ``dump`` is ``true`` (default) full hexadecimal dump with ascii will be
+  ## used, otherwise just a simple hexadecimal string will be printed.
+  var res = getTimedate(msg.timestamp)
+  let direction =
+    case msg.direction
+    of Incoming:
+      " << "
+    of Outgoing:
+      " >> "
+  let address =
+    block:
+      let local =
+        if msg.local.isSome():
+          "[" & $(msg.local.get()) & "]"
+        else:
+          "[LOCAL]"
+      let remote =
+        if msg.remote.isSome():
+          "[" & $(msg.remote.get()) & "]"
+        else:
+          "[REMOTE]"
+      local & direction & remote
+  let seqid =
+    if msg.seqID.isSome():
+      "seqID = " & $(msg.seqID.get()) & " "
+    else:
+      ""
+  let mtype =
+    if msg.mtype.isSome():
+      "type = " & $(msg.mtype.get()) & " "
+    else:
+      ""
+  res.add(" ")
+  res.add(address)
+  res.add(" ")
+  res.add(mtype)
+  res.add(seqid)
+  res.add(mplexMessage(msg.message))
+  res.add(" ")
+  res.add("\p")
+  if dump:
+    res.add(dumpHex(msg.message))
+  else:
+    res.add(utils.toHex(msg.message))
+    res.add("\p")
+  res

libp2p/protobuf/minprotobuf.nim

@@ -168,7 +168,7 @@ proc initProtoBuffer*(data: openarray[byte], offset = 0,
 
 proc initProtoBuffer*(options: set[ProtoFlags] = {}): ProtoBuffer =
   ## Initialize ProtoBuffer with new sequence of capacity ``cap``.
-  result.buffer = newSeqOfCap[byte](128)
+  result.buffer = newSeq[byte]()
   result.options = options
   if WithVarintLength in options:
     # Our buffer will start from position 10, so we can store length of buffer

libp2p/protocols/secure/noise.nim

@@ -21,6 +21,9 @@ import ../../utility
 import secure,
        ../../crypto/[crypto, chacha20poly1305, curve25519, hkdf]
 
+when defined(libp2p_dump):
+  import ../../debugutils
+
 logScope:
   topics = "noise"
 
@@ -403,8 +406,15 @@ method readMessage*(sconn: NoiseConnection): Future[seq[byte]] {.async.} =
     if size > 0:
       var buffer = newSeq[byte](size)
       await sconn.stream.readExactly(addr buffer[0], buffer.len)
-      return sconn.readCs.decryptWithAd([], buffer)
+      when defined(libp2p_dump):
+        let res = sconn.readCs.decryptWithAd([], buffer)
+        dumpMessage(sconn, FlowDirection.Incoming, res)
+        return res
+      else:
+        return sconn.readCs.decryptWithAd([], buffer)
     else:
+      when defined(libp2p_dump):
+        dumpMessage(sconn, FlowDirection.Incoming, [])
       trace "Received 0-length message", sconn
 
 method write*(sconn: NoiseConnection, message: seq[byte]): Future[void] {.async.} =
@@ -429,6 +439,10 @@ method write*(sconn: NoiseConnection, message: seq[byte]): Future[void] {.async.
     outbuf &= besize
     outbuf &= cipher
     await sconn.stream.write(outbuf)
+
+    when defined(libp2p_dump):
+      dumpMessage(sconn, FlowDirection.Outgoing, message)
+
     sconn.activity = true
 
 method handshake*(p: Noise, conn: Connection, initiator: bool): Future[SecureConn] {.async.} =

tools/pbcap_parser.nim

@@ -0,0 +1,93 @@
+## Nim-LibP2P
+## Copyright (c) 2018 Status Research & Development GmbH
+## Licensed under either of
+##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+## at your option.
+## This file may not be copied, modified, or distributed except according to
+## those terms.
+import std/[os, strutils, times]
+import chronicles
+import ../libp2p/debugutils
+
+const
+  PBCapParserName* = "PBCapParser"
+  ## project name string
+
+  PBCapParserMajor*: int = 0
+  ## is the major number of PBCapParser' version.
+
+  PBCapParserMinor*: int = 0
+  ## is the minor number of PBCapParser' version.
+
+  PBCapParserPatch*: int = 1
+  ## is the patch number of PBCapParser' version.
+
+  PBCapParserVersion* = $PBCapParserMajor & "." & $PBCapParserMinor & "." &
+                        $PBCapParserPatch
+  ## is the version of Nimbus as a string.
+
+  GitRevision = staticExec("git rev-parse --short HEAD").replace("\n") # remove CR
+
+  PBCapParserUsage = """
+
+USAGE:
+  pbcap_parser <command> <filename>
+
+COMMANDS:
+  dump        Show full hexadecimal dump with ascii symbols of packets
+  hex         Show only hexadecimal string of message
+"""
+
+let
+  PBCapParserCopyright* = "Copyright (c) 2020-" & $(now().utc.year) &
+                          " Status Research & Development GmbH"
+  PBCapParserHeader* = "$# Version $# [$#: $#, $#]\p$#\p" %
+    [PBCapParserName, PBCapParserVersion, hostOS, hostCPU, GitRevision,
+     PBCapParserCopyright]
+
+proc parseFile*(pathName: string, dump: bool): string =
+  ## Parse `pbcap` file ``pathName``, and return decoded output as string dump.
+  ##
+  ## If ``pathName`` is relative path, then
+  ## ``getHomeDir() / libp2p_dump_dir / pathName`` will be used.
+  var res = ""
+  let path =
+    if isAbsolute(pathName):
+      pathName
+    else:
+      getHomeDir() / libp2p_dump_dir / pathName
+  var sdata = string(readFile(path))
+  var buffer = cast[seq[byte]](sdata)
+  for item in messages(buffer):
+    if item.isNone():
+      break
+    res = res & toString(item.get(), dump)
+  res
+
+when isMainModule:
+  echo PBCapParserHeader
+  if paramCount() < 2:
+    echo PBCapParserUsage
+    quit 0
+  else:
+    let cmd = toLowerAscii(paramStr(1))
+    if cmd notin ["dump", "hex"]:
+      fatal "Unrecognized command found", command = paramStr(1)
+      quit 1
+
+    let path =
+      if isAbsolute(paramStr(2)):
+        paramStr(2)
+      else:
+        getHomeDir() / libp2p_dump_dir / paramStr(2)
+
+    if not(fileExists(path)):
+      fatal "Could not find pbcap file", filename = path
+      quit 1
+
+    let dump = if cmd == "dump": true else: false
+    try:
+      echo parseFile(paramStr(2), dump)
+    except:
+      fatal "Could not read pbcap file", filename = path