From e2d46c6b532d899f994d07e8c0c57df3799c2c05 Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Tue, 15 Sep 2020 12:16:43 +0300 Subject: [PATCH] Add `libp2p_dump` and `libp2p_dump_dir` compiler time options. (#359) * Add `libp2p_dump` and `libp2p_dump_dir` compiler time options. * Add tools/pbcap_parser. * Add mplex control messages decoding. --- libp2p.nimble | 2 +- libp2p/debugutils.nim | 280 ++++++++++++++++++++++++++++++ libp2p/protobuf/minprotobuf.nim | 2 +- libp2p/protocols/secure/noise.nim | 16 +- tools/pbcap_parser.nim | 93 ++++++++++ 5 files changed, 390 insertions(+), 3 deletions(-) create mode 100644 libp2p/debugutils.nim create mode 100644 tools/pbcap_parser.nim diff --git a/libp2p.nimble b/libp2p.nimble index e1d4a08e3..4b424271a 100644 --- a/libp2p.nimble +++ b/libp2p.nimble @@ -5,7 +5,7 @@ version = "0.0.2" author = "Status Research & Development GmbH" description = "LibP2P implementation" license = "MIT" -skipDirs = @["tests", "examples", "Nim"] +skipDirs = @["tests", "examples", "Nim", "tools", "scripts", "docs"] requires "nim >= 1.2.0", "nimcrypto >= 0.4.1", diff --git a/libp2p/debugutils.nim b/libp2p/debugutils.nim new file mode 100644 index 000000000..cf0e5dff0 --- /dev/null +++ b/libp2p/debugutils.nim @@ -0,0 +1,280 @@ +## Nim-LibP2P +## Copyright (c) 2018 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. + +## To enable dump of all incoming and outgoing unencrypted messages you need +## to compile project with ``-d:libp2p_dump`` compile-time option. When this +## option enabled ``nim-libp2p`` will create dumps of unencrypted messages for +## every peer libp2p communicates. +## +## Every file is created with name ".pbcap". One file represents +## all the communication with peer which identified by ``PeerID``. +## +## File can have multiple protobuf encoded messages of this format: +## +## Protobuf's message fields: +## 1. SeqID: optional uint64 +## 2. Timestamp: required uint64 +## 3. Type: optional uint64 +## 4. Direction: required uint64 +## 5. LocalAddress: optional bytes +## 6. RemoteAddress: optional bytes +## 7. Message: required bytes +import os, options +import nimcrypto/utils, stew/endians2 +import protobuf/minprotobuf, stream/connection, protocols/secure/secure, + multiaddress, peerid, varint, muxers/mplex/types + +from times import getTime, toUnix, fromUnix, nanosecond, format, Time, + NanosecondRange, initTime +from strutils import toHex, repeat +export peerid, options, multiaddress + +type + FlowDirection* = enum + Outgoing, Incoming + + ProtoMessage* = object + timestamp*: uint64 + direction*: FlowDirection + message*: seq[byte] + seqID*: Option[uint64] + mtype*: Option[uint64] + local*: Option[MultiAddress] + remote*: Option[MultiAddress] + +const + libp2p_dump_dir* {.strdefine.} = "nim-libp2p" + ## default directory where all the dumps will be stored, if the path + ## relative it will be created in home directory. You can overload this path + ## using ``-d:libp2p_dump_dir=``. + +proc getTimestamp(): uint64 = + ## This procedure is present because `stdlib.times` missing it. + let time = getTime() + uint64(time.toUnix() * 1_000_000_000 + time.nanosecond()) + +proc getTimedate(value: uint64): string = + ## This procedure is workaround for `stdlib.times` to just convert + ## nanoseconds' timestamp to DateTime object. + let time = initTime(int64(value div 1_000_000_000), value mod 1_000_000_000) + time.format("yyyy-MM-dd HH:mm:ss'.'fffzzz") + +proc dumpMessage*(conn: SecureConn, direction: FlowDirection, + data: openArray[byte]) = + ## Store unencrypted message ``data`` to dump file, all the metadata will be + ## extracted from ``conn`` instance. + var pb = initProtoBuffer(options = {WithVarintLength}) + pb.write(2, getTimestamp()) + pb.write(4, uint64(direction)) + if len(conn.peerInfo.addrs) > 0: + pb.write(6, conn.peerInfo.addrs[0]) + pb.write(7, data) + pb.finish() + + let dirName = + if isAbsolute(libp2p_dump_dir): + libp2p_dump_dir + else: + getHomeDir() / libp2p_dump_dir + + let fileName = $(conn.peerInfo.peerId) & ".pbcap" + + # This is debugging procedure so it should not generate any exceptions, + # and we going to return at every possible OS error. + if not(dirExists(dirName)): + try: + createDir(dirName) + except CatchableError: + return + + let pathName = dirName / fileName + var handle: File + try: + if open(handle, pathName, fmAppend): + discard writeBuffer(handle, addr pb.buffer[pb.offset], pb.getLen()) + finally: + close(handle) + +proc decodeDumpMessage*(data: openArray[byte]): Option[ProtoMessage] = + ## Decode protobuf's message ProtoMessage from array of bytes ``data``. + var + pb = initProtoBuffer(data) + value: uint64 + ma1, ma2: MultiAddress + pmsg: ProtoMessage + + let res2 = pb.getField(2, pmsg.timestamp) + if res2.isErr() or not(res2.get()): + return none[ProtoMessage]() + + let res4 = pb.getField(4, value) + if res4.isErr() or not(res4.get()): + return none[ProtoMessage]() + + # `case` statement could not work here with an error "selector must be of an + # ordinal type, float or string" + pmsg.direction = + if value == uint64(Outgoing): + Outgoing + elif value == uint64(Incoming): + Incoming + else: + return none[ProtoMessage]() + + let res7 = pb.getField(7, pmsg.message) + if res7.isErr() or not(res7.get()): + return none[ProtoMessage]() + + value = 0'u64 + let res1 = pb.getField(1, value) + if res1.isOk() and res1.get(): + pmsg.seqID = some(value) + value = 0'u64 + let res3 = pb.getField(3, value) + if res3.isOk() and res3.get(): + pmsg.mtype = some(value) + let res5 = pb.getField(5, ma1) + if res5.isOk() and res5.get(): + pmsg.local = some(ma1) + let res6 = pb.getField(6, ma2) + if res6.isOk() and res6.get(): + pmsg.remote = some(ma2) + + some(pmsg) + +iterator messages*(data: seq[byte]): Option[ProtoMessage] = + ## Iterate over sequence of bytes and decode all the ``ProtoMessage`` + ## messages we found. + var value: uint64 + var size: int + var offset = 0 + while offset < len(data): + value = 0 + size = 0 + let res = PB.getUVarint(data.toOpenArray(offset, len(data) - 1), + size, value) + if res.isOk(): + if (value > 0'u64) and (value < uint64(len(data) - offset)): + offset += size + yield decodeDumpMessage(data.toOpenArray(offset, + offset + int(value) - 1)) + # value is previously checked to be less then len(data) which is `int`. + offset += int(value) + else: + break + else: + break + +proc dumpHex*(pbytes: openarray[byte], groupBy = 1, ascii = true): string = + ## Get hexadecimal dump of memory for array ``pbytes``. + var res = "" + var offset = 0 + var ascii = "" + + while offset < len(pbytes): + if (offset mod 16) == 0: + res = res & toHex(uint64(offset)) & ": " + + for k in 0 ..< groupBy: + let ch = pbytes[offset + k] + ascii.add(if ord(ch) > 31 and ord(ch) < 127: char(ch) else: '.') + + let item = + case groupBy: + of 1: + toHex(pbytes[offset]) + of 2: + toHex(uint16.fromBytes(pbytes.toOpenArray(offset, len(pbytes) - 1))) + of 4: + toHex(uint32.fromBytes(pbytes.toOpenArray(offset, len(pbytes) - 1))) + of 8: + toHex(uint64.fromBytes(pbytes.toOpenArray(offset, len(pbytes) - 1))) + else: + "" + res.add(item) + res.add(" ") + offset = offset + groupBy + + if (offset mod 16) == 0: + res.add(" ") + res.add(ascii) + ascii.setLen(0) + res.add("\p") + + if (offset mod 16) != 0: + let spacesCount = ((16 - (offset mod 16)) div groupBy) * + (groupBy * 2 + 1) + 1 + res = res & repeat(' ', spacesCount) + res = res & ascii + + res.add("\p") + res + +proc mplexMessage*(data: seq[byte]): string = + var value = 0'u64 + var size = 0 + let res = PB.getUVarint(data, size, value) + if res.isOk(): + if size < len(data) and data[size] == 0x00'u8: + let header = cast[MessageType](value and 0x07'u64) + let ident = (value shr 3) + "mplex: [" & $header & ", ident = " & $ident & "] " + else: + "" + else: + "" + +proc toString*(msg: ProtoMessage, dump = true): string = + ## Convert message ``msg`` to its string representation. + ## If ``dump`` is ``true`` (default) full hexadecimal dump with ascii will be + ## used, otherwise just a simple hexadecimal string will be printed. + var res = getTimedate(msg.timestamp) + let direction = + case msg.direction + of Incoming: + " << " + of Outgoing: + " >> " + let address = + block: + let local = + if msg.local.isSome(): + "[" & $(msg.local.get()) & "]" + else: + "[LOCAL]" + let remote = + if msg.remote.isSome(): + "[" & $(msg.remote.get()) & "]" + else: + "[REMOTE]" + local & direction & remote + let seqid = + if msg.seqID.isSome(): + "seqID = " & $(msg.seqID.get()) & " " + else: + "" + let mtype = + if msg.mtype.isSome(): + "type = " & $(msg.mtype.get()) & " " + else: + "" + res.add(" ") + res.add(address) + res.add(" ") + res.add(mtype) + res.add(seqid) + res.add(mplexMessage(msg.message)) + res.add(" ") + res.add("\p") + if dump: + res.add(dumpHex(msg.message)) + else: + res.add(utils.toHex(msg.message)) + res.add("\p") + res diff --git a/libp2p/protobuf/minprotobuf.nim b/libp2p/protobuf/minprotobuf.nim index 4476d0732..02ea60bbb 100644 --- a/libp2p/protobuf/minprotobuf.nim +++ b/libp2p/protobuf/minprotobuf.nim @@ -168,7 +168,7 @@ proc initProtoBuffer*(data: openarray[byte], offset = 0, proc initProtoBuffer*(options: set[ProtoFlags] = {}): ProtoBuffer = ## Initialize ProtoBuffer with new sequence of capacity ``cap``. - result.buffer = newSeqOfCap[byte](128) + result.buffer = newSeq[byte]() result.options = options if WithVarintLength in options: # Our buffer will start from position 10, so we can store length of buffer diff --git a/libp2p/protocols/secure/noise.nim b/libp2p/protocols/secure/noise.nim index c48ef5456..0ff11fdd1 100644 --- a/libp2p/protocols/secure/noise.nim +++ b/libp2p/protocols/secure/noise.nim @@ -21,6 +21,9 @@ import ../../utility import secure, ../../crypto/[crypto, chacha20poly1305, curve25519, hkdf] +when defined(libp2p_dump): + import ../../debugutils + logScope: topics = "noise" @@ -403,8 +406,15 @@ method readMessage*(sconn: NoiseConnection): Future[seq[byte]] {.async.} = if size > 0: var buffer = newSeq[byte](size) await sconn.stream.readExactly(addr buffer[0], buffer.len) - return sconn.readCs.decryptWithAd([], buffer) + when defined(libp2p_dump): + let res = sconn.readCs.decryptWithAd([], buffer) + dumpMessage(sconn, FlowDirection.Incoming, res) + return res + else: + return sconn.readCs.decryptWithAd([], buffer) else: + when defined(libp2p_dump): + dumpMessage(sconn, FlowDirection.Incoming, []) trace "Received 0-length message", sconn method write*(sconn: NoiseConnection, message: seq[byte]): Future[void] {.async.} = @@ -429,6 +439,10 @@ method write*(sconn: NoiseConnection, message: seq[byte]): Future[void] {.async. outbuf &= besize outbuf &= cipher await sconn.stream.write(outbuf) + + when defined(libp2p_dump): + dumpMessage(sconn, FlowDirection.Outgoing, message) + sconn.activity = true method handshake*(p: Noise, conn: Connection, initiator: bool): Future[SecureConn] {.async.} = diff --git a/tools/pbcap_parser.nim b/tools/pbcap_parser.nim new file mode 100644 index 000000000..af55e700e --- /dev/null +++ b/tools/pbcap_parser.nim @@ -0,0 +1,93 @@ +## Nim-LibP2P +## Copyright (c) 2018 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. +import std/[os, strutils, times] +import chronicles +import ../libp2p/debugutils + +const + PBCapParserName* = "PBCapParser" + ## project name string + + PBCapParserMajor*: int = 0 + ## is the major number of PBCapParser' version. + + PBCapParserMinor*: int = 0 + ## is the minor number of PBCapParser' version. + + PBCapParserPatch*: int = 1 + ## is the patch number of PBCapParser' version. + + PBCapParserVersion* = $PBCapParserMajor & "." & $PBCapParserMinor & "." & + $PBCapParserPatch + ## is the version of Nimbus as a string. + + GitRevision = staticExec("git rev-parse --short HEAD").replace("\n") # remove CR + + PBCapParserUsage = """ + +USAGE: + pbcap_parser + +COMMANDS: + dump Show full hexadecimal dump with ascii symbols of packets + hex Show only hexadecimal string of message +""" + +let + PBCapParserCopyright* = "Copyright (c) 2020-" & $(now().utc.year) & + " Status Research & Development GmbH" + PBCapParserHeader* = "$# Version $# [$#: $#, $#]\p$#\p" % + [PBCapParserName, PBCapParserVersion, hostOS, hostCPU, GitRevision, + PBCapParserCopyright] + +proc parseFile*(pathName: string, dump: bool): string = + ## Parse `pbcap` file ``pathName``, and return decoded output as string dump. + ## + ## If ``pathName`` is relative path, then + ## ``getHomeDir() / libp2p_dump_dir / pathName`` will be used. + var res = "" + let path = + if isAbsolute(pathName): + pathName + else: + getHomeDir() / libp2p_dump_dir / pathName + var sdata = string(readFile(path)) + var buffer = cast[seq[byte]](sdata) + for item in messages(buffer): + if item.isNone(): + break + res = res & toString(item.get(), dump) + res + +when isMainModule: + echo PBCapParserHeader + if paramCount() < 2: + echo PBCapParserUsage + quit 0 + else: + let cmd = toLowerAscii(paramStr(1)) + if cmd notin ["dump", "hex"]: + fatal "Unrecognized command found", command = paramStr(1) + quit 1 + + let path = + if isAbsolute(paramStr(2)): + paramStr(2) + else: + getHomeDir() / libp2p_dump_dir / paramStr(2) + + if not(fileExists(path)): + fatal "Could not find pbcap file", filename = path + quit 1 + + let dump = if cmd == "dump": true else: false + try: + echo parseFile(paramStr(2), dump) + except: + fatal "Could not read pbcap file", filename = path