protobuf encoding for all message types

README.md

@@ -1,6 +1,10 @@
 # A DHT implementation for Dagger
 
-![GitHub CI](https://github.com/status-im/nim-libp2p-dht/actions/workflows/ci.yml/badge.svg) ![codecov](https://codecov.io/gh/status-im/nim-libp2p-dht/branch/main/graph/badge.svg?token=tlmMJgU4l7)](https://codecov.io/gh/status-im/nim-libp2p-dht)
+[![License: Apache](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+[![Stability: experimental](https://img.shields.io/badge/stability-experimental-orange.svg)](#stability)
+[![CI (GitHub Actions)](https://github.com/status-im/nim-libp2p-dht/workflows/CI/badge.svg?branch=main)](https://github.com/status-im/nim-libp2p-dht/actions?query=workflow%3ACI+branch%3Amain)
+[![codecov](https://codecov.io/gh/status-im/nim-libp2p-dht/branch/main/graph/badge.svg?token=tlmMJgU4l7)](https://codecov.io/gh/status-im/nim-libp2p-dht)
 
 This DHT implementation is aiming to provide a DHT for Dagger with the following properties
 * flexible transport usage with
@@ -12,5 +16,6 @@ This DHT implementation is aiming to provide a DHT for Dagger with the following
   * roughly follow the libp2p-dht specifications from https://github.com/libp2p/specs/tree/master/kad-dht
   * provide compatibility mode with the above specs
 
-Current implementation is based on nim-eth's Discovery v5 implementation. Base files were copied
+Current implementation is based on nim-eth's Discovery v5 implementation.
-from nim-eth@779d767b024175a51cf74c79ec7513301ebe2f46
+
+Base files were copied from [`status-im/nim-eth@779d767b024175a51cf74c79ec7513301ebe2f46`](https://github.com/status-im/nim-eth/commit/779d767b024175a51cf74c79ec7513301ebe2f46)

libp2pdht/dht/providers_encoding.nim

@@ -121,4 +121,3 @@ proc encode*(msg: ProvidersMessage): seq[byte] =
 
   pb.finish()
   pb.buffer
-

libp2pdht/private/eth/p2p/discoveryv5/encoding.nim

@@ -15,7 +15,6 @@
 
 import
   std/[hashes, net, options, sugar, tables],
-  eth/rlp,
   bearssl,
   chronicles,
   stew/[results, byteutils],
@@ -519,7 +518,7 @@ proc decodeHandshakePacket(c: var Codec, fromAddr: Address, nonce: AESGCMNonce,
                       ("Invalid bytes for SignedPeerRecord: " & $e).cstring
                   )
       record = some(decoded)
-    except RlpError, ValueError:
+    except ValueError:
       return err("Invalid encoded SPR")
 
   var pubkey: PublicKey

libp2pdht/private/eth/p2p/discoveryv5/messages.nim

@@ -7,14 +7,14 @@
 #
 ## Discovery v5 Protocol Messages as specified at
 ## https://github.com/ethereum/devp2p/blob/master/discv5/discv5-wire.md#protocol-messages
-## These messages get RLP encoded.
+## These messages get protobuf encoded, while in the spec they get RLP encoded.
 ##
 
 {.push raises: [Defect].}
 
 import
   std/[hashes, net],
-  eth/[keys],
+  bearssl,
   ./spr,
   ./node,
   ../../../../dht/providers_messages

libp2pdht/private/eth/p2p/discoveryv5/messages_encoding.nim

@@ -10,8 +10,6 @@
 
 import
   std/net,
-  stew/arrayops,
-  eth/[rlp],
   chronicles,
   libp2p/routing_record,
   libp2p/signed_envelope,
@@ -23,83 +21,312 @@ from stew/objects import checkedEnumAssign
 type
   DecodeResult*[T] = Result[T, cstring]
 
-proc read*(rlp: var Rlp, T: type RequestId): T
+  Distances = seq[uint16]
-    {.raises: [ValueError, RlpError, Defect].} =
-  mixin read
-  var reqId: RequestId
-  reqId.id = rlp.toBytes()
-  if reqId.id.len > 8:
-    raise newException(ValueError, "RequestId is > 8 bytes")
-  rlp.skipElem()
 
-  reqId
+  EncodedMessage = seq[byte]
 
-proc append*(writer: var RlpWriter, value: RequestId) =
+  IPv4 = array[0..3, uint8]
-  writer.append(value.id)
 
-proc read*(rlp: var Rlp, T: type IpAddress): T
+  IPv6 = array[0..15, uint8]
-    {.raises: [RlpError, Defect].} =
-  let ipBytes = rlp.toBytes()
-  rlp.skipElem()
 
-  if ipBytes.len == 4:
+  Port = uint16
-    var ip: array[4, byte]
+
-    discard copyFrom(ip, ipBytes)
+proc getField*(pb: ProtoBuffer, field: int,
-    IpAddress(family: IPv4, address_v4: ip)
+               reqId: var RequestId): ProtoResult[bool] {.inline.} =
-  elif ipBytes.len == 16:
+  ## Read ``RequestId`` from ProtoBuf's message and validate it
-    var ip: array[16, byte]
+  var buffer: seq[byte]
-    discard copyFrom(ip, ipBytes)
+  let res = ? pb.getField(field, buffer)
-    IpAddress(family: IPv6, address_v6: ip)
+  if not(res):
+    ok(false)
+  elif buffer.len > 8:
+    ok(false) # RequestId must not be more than 8 bytes
   else:
-    raise newException(RlpTypeMismatch,
+    reqId = RequestId(id: buffer)
-      "Amount of bytes for IP address is different from 4 or 16")
+    ok(true)
+
+proc write*(pb: var ProtoBuffer, field: int, reqId: RequestId) =
+  ## Write RequestId value ``reqId`` to object ``pb`` using ProtoBuf's encoding.
+  let encoded = reqId.id
+  write(pb, field, encoded)
+
+proc decode*(
+  T: typedesc[PingMessage],
+  buffer: openArray[byte]): Result[PingMessage, ProtoError] =
+
+  let pb = initProtoBuffer(buffer)
+  var msg = PingMessage()
+
+  ? pb.getRequiredField(1, msg.sprSeq)
+
+  ok(msg)
+
+proc encode*(msg: PingMessage): seq[byte] =
+  var pb = initProtoBuffer()
+
+  pb.write(1, msg.sprSeq)
+
+  pb.finish()
+  pb.buffer
+
+proc getField*(pb: ProtoBuffer, field: int,
+               ipv4: var IPv4): ProtoResult[bool] {.inline.} =
+  ## Read ``IPv4`` from ProtoBuf's message and validate it
+  var buffer: seq[byte]
+  let res = ? pb.getField(field, buffer)
+  if not(res):
+    ok(false)
+  else:
+    for i in 0..<ipv4.len: ipv4[i] = buffer[i]
+    ok(true)
+
+proc getField*(pb: ProtoBuffer, field: int,
+               ipv6: var IPv6): ProtoResult[bool] {.inline.} =
+  ## Read ``IPv6`` from ProtoBuf's message and validate it
+  var buffer: seq[byte]
+  let res = ? pb.getField(field, buffer)
+  if not(res):
+    ok(false)
+  else:
+    for i in 0..<ipv6.len: ipv6[i] = buffer[i]
+    ok(true)
+
+proc getField*(pb: ProtoBuffer, field: int,
+               family: var IpAddressFamily): ProtoResult[bool] {.inline.} =
+  ## Read ``IpAddressFamily`` from ProtoBuf's message and validate it
+  var buffer: seq[byte]
+  let res = ? pb.getField(field, buffer)
+  if not(res):
+    ok(false)
+  else:
+    family = uint8.fromBytesBE(buffer).IpAddressFamily
+    ok(true)
+
+proc write*(pb: var ProtoBuffer, field: int, family: IpAddressFamily) =
+  ## Write IpAddressFamily value ``family`` to object ``pb`` using ProtoBuf's encoding.
+  let encoded = family.uint8.toBytesBe()
+  write(pb, field, encoded)
+
+proc decode*(
+  T: typedesc[IpAddress],
+  buffer: openArray[byte]): Result[IpAddress, ProtoError] =
+
+  let pb = initProtoBuffer(buffer)
+  var family: IpAddressFamily
+
+  ? pb.getRequiredField(1, family)
+
+  var ip = IpAddress(family: family)
 
-proc append*(writer: var RlpWriter, ip: IpAddress) =
   case ip.family:
+  of IpAddressFamily.IPv6:
+    ? pb.getRequiredField(2, ip.address_v6)
   of IpAddressFamily.IPv4:
-    writer.append(ip.address_v4)
+    ? pb.getRequiredField(2, ip.address_v4)
-  of IpAddressFamily.IPv6: writer.append(ip.address_v6)
 
-proc read*(rlp: var Rlp, T: type NodeId): T
+  ok(ip)
-    {.raises: [ValueError, RlpError, Defect].} =
-  mixin read
-  let nodeId = NodeId.fromBytesBE(rlp.toBytes())
-  rlp.skipElem()
-  nodeId
 
-proc append*(writer: var RlpWriter, value: NodeId) =
+proc encode*(ip: IpAddress): seq[byte] =
-  writer.append(value.toBytesBE)
+  var pb = initProtoBuffer()
 
-proc numFields(T: typedesc): int =
+  pb.write(1, ip.family)
-  for k, v in fieldPairs(default(T)): inc result
+  case ip.family:
+  of IpAddressFamily.IPv6:
+    pb.write(2, ip.address_v6)
+  of IpAddressFamily.IPv4:
+    pb.write(2, ip.address_v4)
+
+  pb.finish()
+  pb.buffer
+
+proc getField*(pb: ProtoBuffer, field: int,
+               ip: var IpAddress): ProtoResult[bool] {.inline.} =
+  ## Read ``IpAddress`` from ProtoBuf's message and validate it
+  var buffer: seq[byte]
+  let res = ? pb.getField(field, buffer)
+  if not(res):
+    ok(false)
+  else:
+    let res2 = IpAddress.decode(buffer)
+    if res2.isOk():
+      ip = res2.get()
+      ok(true)
+    else:
+      err(ProtoError.IncorrectBlob)
+
+proc write*(pb: var ProtoBuffer, field: int, ip: IpAddress) =
+  ## Write IpAddress value ``ip`` to object ``pb`` using ProtoBuf's encoding.
+  let encoded = ip.encode()
+  write(pb, field, encoded)
+
+proc getField*(pb: ProtoBuffer, field: int,
+               port: var Port): ProtoResult[bool] {.inline.} =
+  ## Read ``Port`` from ProtoBuf's message and validate it
+  var buffer: seq[byte]
+  let res = ? pb.getField(field, buffer)
+  if not(res):
+    ok(false)
+  else:
+    port = uint16.fromBytesBE(buffer)
+    ok(true)
+
+proc write*(pb: var ProtoBuffer, field: int, port: Port) =
+  ## Write Port value ``port`` to object ``pb`` using ProtoBuf's encoding.
+  write(pb, field, port.toBytesBE())
+
+proc decode*(
+  T: typedesc[PongMessage],
+  buffer: openArray[byte]): Result[PongMessage, ProtoError] =
+
+  let pb = initProtoBuffer(buffer)
+  var msg = PongMessage()
+
+  ? pb.getRequiredField(1, msg.sprSeq)
+  ? pb.getRequiredField(2, msg.ip)
+  ? pb.getRequiredField(3, msg.port)
+
+  ok(msg)
+
+proc encode*(msg: PongMessage): seq[byte] =
+  var pb = initProtoBuffer()
+
+  pb.write(1, msg.sprSeq)
+  pb.write(2, msg.ip)
+  pb.write(3, msg.port)
+
+  pb.finish()
+  pb.buffer
+
+proc getRepeatedField*(pb: ProtoBuffer, field: int,
+                       distances: var Distances): ProtoResult[bool] {.inline.} =
+  ## Read ``Distances`` from ProtoBuf's message and validate it
+  var buffers: seq[seq[byte]]
+  distances.setLen(0)
+  let res = ? pb.getRepeatedField(field, buffers)
+  if not(res):
+    ok(false)
+  else:
+    for b in buffers:
+      distances.add(uint16.fromBytesBE(b))
+    ok(true)
+
+proc decode*(
+  T: typedesc[FindNodeMessage],
+  buffer: openArray[byte]): Result[FindNodeMessage, ProtoError] =
+
+  let pb = initProtoBuffer(buffer)
+  var msg = FindNodeMessage()
+
+  ? pb.getRequiredRepeatedField(1, msg.distances)
+
+  ok(msg)
+
+proc encode*(msg: FindNodeMessage): seq[byte] =
+  var pb = initProtoBuffer()
+
+  for d in msg.distances:
+    pb.write(1, d.toBytesBE())
+
+  pb.finish()
+  pb.buffer
+
+proc decode*(
+  T: typedesc[FindNodeFastMessage],
+  buffer: openArray[byte]): Result[FindNodeFastMessage, ProtoError] =
+
+  let pb = initProtoBuffer(buffer)
+  var msg = FindNodeFastMessage()
+
+  ? pb.getRequiredField(1, msg.target)
+
+  ok(msg)
+
+proc encode*(msg: FindNodeFastMessage): seq[byte] =
+  var pb = initProtoBuffer()
+
+  pb.write(1, msg.target)
+
+  pb.finish()
+  pb.buffer
+
+proc decode*(
+  T: typedesc[NodesMessage],
+  buffer: openArray[byte]): Result[NodesMessage, ProtoError] =
+
+  let pb = initProtoBuffer(buffer)
+  var msg = NodesMessage()
+
+  ? pb.getRequiredField(1, msg.total)
+  discard ? pb.getRepeatedField(2, msg.sprs)
+
+  ok(msg)
+
+proc encode*(msg: NodesMessage): seq[byte] =
+  var pb = initProtoBuffer()
+
+  pb.write(1, msg.total)
+  for r in msg.sprs:
+    pb.write(2, r)
+
+  pb.finish()
+  pb.buffer
+
+proc decode*(
+  T: typedesc[TalkReqMessage],
+  buffer: openArray[byte]): Result[TalkReqMessage, ProtoError] =
+
+  let pb = initProtoBuffer(buffer)
+  var msg = TalkReqMessage()
+
+  ? pb.getRequiredField(1, msg.protocol)
+  ? pb.getRequiredField(2, msg.request)
+
+  ok(msg)
+
+proc encode*(msg: TalkReqMessage): seq[byte] =
+  var pb = initProtoBuffer()
+
+  pb.write(1, msg.protocol)
+  pb.write(2, msg.request)
+
+  pb.finish()
+  pb.buffer
+
+proc decode*(
+  T: typedesc[TalkRespMessage],
+  buffer: openArray[byte]): Result[TalkRespMessage, ProtoError] =
+
+  let pb = initProtoBuffer(buffer)
+  var msg = TalkRespMessage()
+
+  ? pb.getRequiredField(2, msg.response)
+
+  ok(msg)
+
+proc encode*(msg: TalkRespMessage): seq[byte] =
+  var pb = initProtoBuffer()
+
+  pb.write(2, msg.response)
+
+  pb.finish()
+  pb.buffer
 
 proc encodeMessage*[T: SomeMessage](p: T, reqId: RequestId): seq[byte] =
-  # TODO: Remove all RLP encoding in favour of Protobufs
   result = newSeqOfCap[byte](64)
   result.add(messageKind(T).ord)
 
-  const
+  let encoded =
-    usePbs = T is AddProviderMessage | GetProvidersMessage | ProvidersMessage
+    try: p.encode()
-    sz = if usePbs: 1 else: numFields(T)
+    except ResultError[CryptoError] as e:
-
+      error "Failed to encode protobuf message", typ = $T, msg = e.msg
-  var writer = initRlpList(sz + 1)
+      @[]
-  writer.append(reqId)
+  var pb = initProtoBuffer()
-
+  pb.write(1, reqId)
-  when usePbs:
+  pb.write(2, encoded)
-    let encoded =
+  pb.finish()
-      try: p.encode()
+  result.add(pb.buffer)
-      except ResultError[CryptoError] as e:
+  trace "Encoded protobuf message", typ = $T, encoded
-        error "Failed to encode protobuf message", typ = $T, msg = e.msg
-        @[]
-    writer.append(encoded)
-    trace "Encoded protobuf message", typ = $T, encoded
-  else:
-    for k, v in fieldPairs(p):
-      writer.append(v)
-  result.add(writer.finish())
 
 proc decodeMessage*(body: openArray[byte]): DecodeResult[Message] =
-  # TODO: Remove all RLP decoding in favour of Protobufs
   ## Decodes to the specific `Message` type.
   if body.len < 1:
     return err("No message data")
@@ -109,56 +336,107 @@ proc decodeMessage*(body: openArray[byte]): DecodeResult[Message] =
     return err("Invalid message type")
 
   var message = Message(kind: kind)
-  var rlp = rlpFromBytes(body.toOpenArray(1, body.high))
-  if rlp.enterList:
-    try:
-      message.reqId = rlp.read(RequestId)
-    except RlpError, ValueError:
-      return err("Invalid request-id")
 
-    proc decode[T](rlp: var Rlp, v: var T)
+  let pb = initProtoBuffer(body[1..body.high])
-        {.nimcall, raises:[RlpError, ValueError, Defect].} =
-      for k, v in v.fieldPairs:
-        v = rlp.read(typeof(v))
 
-    try:
+  var
-      case kind
+    reqId: RequestId
-      of unused: return err("Invalid message type")
+    encoded: EncodedMessage
-      of ping: rlp.decode(message.ping)
-      of pong: rlp.decode(message.pong)
-      of findNode: rlp.decode(message.findNode)
-      of findNodeFast: rlp.decode(message.findNodeFast)
-      of nodes: rlp.decode(message.nodes)
-      of talkReq: rlp.decode(message.talkReq)
-      of talkResp: rlp.decode(message.talkResp)
-      of addProvider:
-        let res = AddProviderMessage.decode(rlp.toBytes)
-        if res.isOk:
-          message.addProvider = res.get
-        else:
-          return err "Unable to decode AddProviderMessage"
-      of getProviders:
-        let res = GetProvidersMessage.decode(rlp.toBytes)
-        if res.isOk:
-          message.getProviders = res.get
-        else:
-          return err "Unable to decode GetProvidersMessage"
-      of providers:
-        let res = ProvidersMessage.decode(rlp.toBytes)
-        if res.isOk:
-          message.provs = res.get
-        else:
-          return err "Unable to decode ProvidersMessage"
-      of regTopic, ticket, regConfirmation, topicQuery:
-        # We just pass the empty type of this message without attempting to
-        # decode, so that the protocol knows what was received.
-        # But we ignore the message as per specification as "the content and
-        # semantics of this message are not final".
-        discard
-    except RlpError, ValueError:
-      return err("Invalid message encoding")
 
-    ok(message)
+  if pb.getRequiredField(1, reqId).isErr:
-  else:
+    return err("Invalid request-id")
-    err("Invalid message encoding: no rlp list")
 
+  message.reqId = reqId
+
+  if pb.getRequiredField(2, encoded).isErr:
+    return err("Invalid message encoding")
+
+  case kind
+  of unused: return err("Invalid message type")
+
+  of ping:
+    let res = PingMessage.decode(encoded)
+    if res.isOk:
+      message.ping = res.get
+      return ok(message)
+    else:
+      return err("Unable to decode PingMessage")
+
+  of pong:
+    let res = PongMessage.decode(encoded)
+    if res.isOk:
+      message.pong = res.get
+      return ok(message)
+    else:
+      return err("Unable to decode PongMessage")
+
+  of findNode:
+    let res = FindNodeMessage.decode(encoded)
+    if res.isOk:
+      message.findNode = res.get
+      return ok(message)
+    else:
+      return err("Unable to decode FindNodeMessage")
+
+  of nodes:
+    let res = NodesMessage.decode(encoded)
+    if res.isOk:
+      message.nodes = res.get
+      return ok(message)
+    else:
+      return err("Unable to decode NodesMessage")
+
+  of talkReq:
+    let res = TalkReqMessage.decode(encoded)
+    if res.isOk:
+      message.talkReq = res.get
+      return ok(message)
+    else:
+      return err("Unable to decode TalkReqMessage")
+
+  of talkResp:
+    let res = TalkRespMessage.decode(encoded)
+    if res.isOk:
+      message.talkResp = res.get
+      return ok(message)
+    else:
+      return err("Unable to decode TalkRespMessage")
+
+  of findNodeFast:
+    let res = FindNodeFastMessage.decode(encoded)
+    if res.isOk:
+      message.findNodeFast = res.get
+      return ok(message)
+    else:
+      return err("Unable to decode FindNodeFastMessage")
+
+  of addProvider:
+    let res = AddProviderMessage.decode(encoded)
+    if res.isOk:
+      message.addProvider = res.get
+      return ok(message)
+    else:
+      return err "Unable to decode AddProviderMessage"
+
+  of getProviders:
+    let res = GetProvidersMessage.decode(encoded)
+    if res.isOk:
+      message.getProviders = res.get
+      return ok(message)
+    else:
+      return err("Unable to decode GetProvidersMessage")
+
+  of providers:
+    let res = ProvidersMessage.decode(encoded)
+    if res.isOk:
+      message.provs = res.get
+      return ok(message)
+    else:
+      return err("Unable to decode ProvidersMessage")
+
+  of regTopic, ticket, regConfirmation, topicQuery:
+    # We just pass the empty type of this message without attempting to
+    # decode, so that the protocol knows what was received.
+    # But we ignore the message as per specification as "the content and
+    # semantics of this message are not final".
+    discard

libp2pdht/private/eth/p2p/discoveryv5/protocol.nim

@@ -77,7 +77,7 @@ import
   std/[tables, sets, options, math, sequtils, algorithm],
   stew/shims/net as stewNet, json_serialization/std/net,
   stew/[base64, endians2, results], chronicles, chronos, chronos/timer, stint, bearssl,
-  metrics, eth/[rlp, async_utils],
+  metrics, eth/async_utils,
   libp2p/[crypto/crypto, routing_record],
   "."/[transport, messages, messages_encoding, node, routing_table, spr, random2, ip_vote, nodes_verification]
 
@@ -296,7 +296,7 @@ proc handleFindNodeFast(d: Protocol, fromId: NodeId, fromAddr: Address,
     fnf: FindNodeFastMessage, reqId: RequestId) =
   d.sendNodes(fromId, fromAddr, reqId,
     d.routingTable.neighbours(fnf.target, seenOnly = true))
-  # TODO: if known, maybe we should add exact target even if not yet "seen" 
+  # TODO: if known, maybe we should add exact target even if not yet "seen"
 
 proc handleTalkReq(d: Protocol, fromId: NodeId, fromAddr: Address,
     talkreq: TalkReqMessage, reqId: RequestId) =
@@ -593,7 +593,7 @@ proc lookup*(d: Protocol, target: NodeId, fast: bool = false): Future[seq[Node]]
     while i < closestNodes.len and pendingQueries.len < alpha:
       let n = closestNodes[i]
       if not asked.containsOrIncl(n.id):
-        if fast: 
+        if fast:
           pendingQueries.add(d.lookupWorkerFast(n, target))
         else:
           pendingQueries.add(d.lookupWorker(n, target))

libp2pdht/private/eth/p2p/discoveryv5/spr.nim

@@ -6,12 +6,11 @@
 #
 import
   chronicles,
-  std/[options, sequtils, strutils, sugar],
+  std/[options, strutils, sugar],
   pkg/stew/[results, byteutils, arrayops],
   stew/endians2,
   stew/shims/net,
   stew/base64,
-  eth/rlp,
   libp2p/crypto/crypto,
   libp2p/crypto/secp,
   libp2p/routing_record,
@@ -31,14 +30,6 @@ type
 proc seqNum*(r: SignedPeerRecord): uint64 =
     r.data.seqNo
 
-#proc encode
-proc append*(rlpWriter: var RlpWriter, value: SignedPeerRecord) =
-  # echo "encoding to:" & $value.signedPeerRecord.encode.get
-  var encoded = value.encode
-  if encoded.isErr:
-    error "Error encoding SignedPeerRecord for RLP", error = encoded.error
-  rlpWriter.append encoded.get(@[])
-
 proc fromBytes(r: var SignedPeerRecord, s: openArray[byte]): bool =
 
   let decoded = SignedPeerRecord.decode(@s)
@@ -49,16 +40,6 @@ proc fromBytes(r: var SignedPeerRecord, s: openArray[byte]): bool =
   r = decoded.get
   return true
 
-proc read*(rlp: var Rlp, T: typedesc[SignedPeerRecord]):
-    T {.raises: [RlpError, ValueError, Defect].} =
-    # echo "read:" & $rlp.rawData
-    ## code directly borrowed from spr.nim
-    if not rlp.hasData() or not result.fromBytes(rlp.toBytes):
-        # TODO: This could also just be an invalid signature, would be cleaner to
-        # split of RLP deserialisation errors from this.
-        raise newException(ValueError, "Could not deserialize")
-    rlp.skipElem()
-
 proc get*(r: SignedPeerRecord, T: type PublicKey): Option[T] =
   ## Get the `PublicKey` from provided `Record`. Return `none` when there is
   ## no `PublicKey` in the record.
@@ -100,8 +81,6 @@ proc update*(r: var SignedPeerRecord, pk: crypto.PrivateKey,
   let
     sprPubKey = r.get(PublicKey)
     pubKey = pk.getPublicKey
-    # keysPubKey = pubkey.get.pkToPk.get # remove when move away from eth/keys
-    # keysPrivKey = pk.pkToPk.get
   if sprPubKey.isNone or pubKey.isErr or sprPubKey.get != pubKey.get:
     return err("Public key does not correspond with given private key")
 
@@ -220,7 +199,7 @@ proc udp*(r: SignedPeerRecord): Option[int] =
         return some(p.int)
 
 proc fromBase64*(r: var SignedPeerRecord, s: string): bool =
-  ## Loads SPR from base64-encoded rlp-encoded bytes, and validates the
+  ## Loads SPR from base64-encoded protobuf-encoded bytes, and validates the
   ## signature.
   let bytes = Base64Url.decode(s)
   r.fromBytes(bytes)

libp2pdht/private/eth/p2p/discoveryv5/transport.nim

@@ -110,7 +110,7 @@ proc receive*(t: Transport, a: Address, packet: openArray[byte]) =
       if packet.messageOpt.isSome():
         let message = packet.messageOpt.get()
         trace "Received message packet", srcId = packet.srcId, address = a,
-          kind = message.kind
+          kind = message.kind, packet
         t.client.handleMessage(packet.srcId, a, message)
       else:
         trace "Not decryptable message packet received",

tests/discv5/test_discoveryv5_encoding.nim

@@ -5,7 +5,6 @@ import
   asynctest/unittest2,
   bearssl,
   chronos,
-  eth/rlp,
   libp2p/crypto/secp,
   libp2pdht/discv5/[messages, messages_encoding, encoding, spr, node, sessions],
   libp2pdht/discv5/crypto,
@@ -24,7 +23,7 @@ suite "Discovery v5.1 Protocol Message Encodings":
       reqId = RequestId(id: @[1.byte])
 
     let encoded = encodeMessage(p, reqId)
-    check byteutils.toHex(encoded) == "01c20101"
+    check byteutils.toHex(encoded) == "010a010112020801"
 
     let decoded = decodeMessage(encoded)
     check decoded.isOk()
@@ -44,7 +43,7 @@ suite "Discovery v5.1 Protocol Message Encodings":
       reqId = RequestId(id: @[1.byte])
 
     let encoded = encodeMessage(p, reqId)
-    check byteutils.toHex(encoded) == "02ca0101847f000001821388"
+    check byteutils.toHex(encoded) == "020a01011211080112090a010112047f0000011a021388"
 
     let decoded = decodeMessage(encoded)
     check decoded.isOk()
@@ -64,7 +63,7 @@ suite "Discovery v5.1 Protocol Message Encodings":
       reqId = RequestId(id: @[1.byte])
 
     let encoded = encodeMessage(fn, reqId)
-    check byteutils.toHex(encoded) == "03c501c3820100"
+    check byteutils.toHex(encoded) == "030a010112040a020100"
 
     let decoded = decodeMessage(encoded)
     check decoded.isOk()
@@ -82,7 +81,7 @@ suite "Discovery v5.1 Protocol Message Encodings":
       reqId = RequestId(id: @[1.byte])
 
     let encoded = encodeMessage(n, reqId)
-    check byteutils.toHex(encoded) == "04c30101c0"
+    check byteutils.toHex(encoded) == "040a010112020801"
 
     let decoded = decodeMessage(encoded)
     check decoded.isOk()
@@ -104,7 +103,7 @@ suite "Discovery v5.1 Protocol Message Encodings":
       reqId = RequestId(id: @[1.byte])
 
     let encoded = encodeMessage(n, reqId)
-    check byteutils.toHex(encoded) == "04f901a00101f9019bb8cc0a250802122102339d487ed237392d83791950dc891f0636de698c1fa051ea01ae3fa58bd78580120203011a560a2700250802122102339d487ed237392d83791950dc891f0636de698c1fa051ea01ae3fa58bd78580109cfd8992061a0b0a090400000000910200011a0b0a090400000000910200021a0b0a090400000000910200032a4730450221008cc77fd265e33c955174b9f49628048b2d72a6395acb30f0ba9d90536fa1a5d502207fa8e5bab8e8ddee9884a8e244b0990228e3546b5a9b6848632abd924796e576b8cb0a2508021221026beda5cfddf1cd89130e7b5bb6092bac23db4a044bf847328aa0310dd123a445120203011a560a27002508021221026beda5cfddf1cd89130e7b5bb6092bac23db4a044bf847328aa0310dd123a445109cfd8992061a0b0a090400000000910200011a0b0a090400000000910200021a0b0a090400000000910200032a46304402203d41b1a78c5e6d98c9b4f3fcb213dc16ae4de50a1c8715ab29c516afe6488b4e02205841d09e92b3d2f1ad72c7bc066e561dab57320886f3fbbf272d2cf1732ca259"
+    check byteutils.toHex(encoded) == "040a0101129f03080112cc010a250802122102339d487ed237392d83791950dc891f0636de698c1fa051ea01ae3fa58bd78580120203011a560a2700250802122102339d487ed237392d83791950dc891f0636de698c1fa051ea01ae3fa58bd78580109cfd8992061a0b0a090400000000910200011a0b0a090400000000910200021a0b0a090400000000910200032a4730450221008cc77fd265e33c955174b9f49628048b2d72a6395acb30f0ba9d90536fa1a5d502207fa8e5bab8e8ddee9884a8e244b0990228e3546b5a9b6848632abd924796e57612cb010a2508021221026beda5cfddf1cd89130e7b5bb6092bac23db4a044bf847328aa0310dd123a445120203011a560a27002508021221026beda5cfddf1cd89130e7b5bb6092bac23db4a044bf847328aa0310dd123a445109cfd8992061a0b0a090400000000910200011a0b0a090400000000910200021a0b0a090400000000910200032a46304402203d41b1a78c5e6d98c9b4f3fcb213dc16ae4de50a1c8715ab29c516afe6488b4e02205841d09e92b3d2f1ad72c7bc066e561dab57320886f3fbbf272d2cf1732ca259"
 
     let decoded = decodeMessage(encoded)
     check decoded.isOk()
@@ -124,7 +123,7 @@ suite "Discovery v5.1 Protocol Message Encodings":
       reqId = RequestId(id: @[1.byte])
 
     let encoded = encodeMessage(tr, reqId)
-    check byteutils.toHex(encoded) == "05c901846563686f826869"
+    check byteutils.toHex(encoded) == "050a0101120a0a046563686f12026869"
 
     let decoded = decodeMessage(encoded)
     check decoded.isOk()
@@ -142,7 +141,7 @@ suite "Discovery v5.1 Protocol Message Encodings":
       reqId = RequestId(id: @[1.byte])
 
     let encoded = encodeMessage(tr, reqId)
-    check byteutils.toHex(encoded) == "06c401826869"
+    check byteutils.toHex(encoded) == "060a0101120412026869"
 
     let decoded = decodeMessage(encoded)
     check decoded.isOk()
@@ -160,7 +159,7 @@ suite "Discovery v5.1 Protocol Message Encodings":
       # 1 byte too large
       reqId = RequestId(id: @[0.byte, 1, 2, 3, 4, 5, 6, 7, 8])
     let encoded = encodeMessage(p, reqId)
-    check byteutils.toHex(encoded) == "01cb8900010203040506070801"
+    check byteutils.toHex(encoded) == "010a0900010203040506070812020801"
 
     let decoded = decodeMessage(encoded)
     check decoded.isErr()
@@ -299,7 +298,8 @@ suite "Discovery v5.1 Packet Encodings Test Vectors":
       encodedPacket =
         "000000000000000000000000000000003788c1e1079e89374c4beac74d76364d" &
         "bd9e8cd1847adc2f49fbacc6862425583586c023b19b6fdd1d836777ee39fee8" &
-        "7afd279a5fe4ffdded6d1a6d388217da82d38761b60b0c6e9dd94a8713bc5d"
+        "7afd279a5fe4ffdd21ed1a6d388207f00b48115b8ee4e8eaf675b0865821e126" &
+        "36bc"
 
     let dummyKey = "0x00000000000000000000000000000001" # of no importance
     codecA.sessions.store(nodeB.id, nodeB.address.get(),
@@ -356,11 +356,11 @@ suite "Discovery v5.1 Packet Encodings Test Vectors":
       encodedPacket =
         "000000000000000000000000000000003788c1e1079e89374e4beac74d76364d" &
         "bd9e8cd1847a712f49fbacc6862425583586c023b19b6fdd1d836777ee39fee8" &
-        "7afd279a5fe4ff441af3b17ec968350f37edbda9e0ba8ac0fd2617ef67a1e362" &
+        "7afd279a5fe4ff441af3b17ec840104be4fae12da6828a752f1f7df615c4c757" &
-        "5ea8eb284a3ca85f7ef976ccf2e87932ffeada775849d7aca378033b7a75dbe8" &
+        "9558b0b537760549eb0e5ada50545b066734abb4b7bddbd503fad873e981b485" &
-        "7cc1767123bb7d7e5d96b5d6ad7c26cb55f6160b250d042ef1b9e6000191ce4e" &
+        "dc9f8eee725661ebf766ce027edc9dcb55f6160b06d9c764481fb520b7581678" &
-        "a93234ca3de051518684902e70e6a47eb8f0c2efeca8e42d2ea7f5bd1f27c12d" &
+        "e5fce2ab0d5fa2edaa92f252db6d6ea30d52a78e4114fb9dcf26fb1690e65ca0" &
-        "ae3c579ddcef630659089c99"
+        "079386331f311bfc0aef4a771155ed"
 
     let
       whoareyouData = WhoareyouData(
@@ -403,17 +403,17 @@ suite "Discovery v5.1 Packet Encodings Test Vectors":
 
       encodedPacket =
         "000000000000000000000000000000003788c1e1079e89374e4beac74d76364d" &
-        "bd9e8cd1847bc02f49fbacc6862425583586c023b19b6fdd1d836777ee39fee8" &
+        "bd9e8cd1847bc72f49fbacc6862425583586c023b19b6fdd1d836777ee39fee8" &
-        "7afd279a5fe4ff451af3b07ec8407cedec19c57a8460e08d3d8a908f78261170" &
+        "7afd279a5fe4ff441af3b17ec96b7e0586467988bd0c4784575e02a32d0e7594" &
-        "68196e7df56279e7493fbb2076025b395dde6ffeecc45daa59def06c9be97b1f" &
+        "b65e939a8490bd71813894c9f4e8796f8c3d3fce29171fc3c568ace28d89ea80" &
-        "95636fb8f16887cf13b4a8cca0bcaf805fe62529ad86c59204e73917cf183d19" &
+        "a9707693d2ee9388e74f89a5a62bc3cb55f6160bae865f0779b75c989805232d" &
-        "847617448722cc8c0eea80b68653e858eff5d250abbd55315db21fac1485db8f" &
+        "55b73645bbd797a7d550db7233300b3d1de011fbde92783b4d813ddabff9c96a" &
-        "deaadba582d43c88f0b25512a5fd8395bd2f9519362d29cceb29028de04e0076" &
+        "a0361139ec10df16cf2c136caf44d064194cc6777202952dc24c6fccf04f026d" &
-        "4f6aece318e26e2d123888e484cb1c0ce37ecfee42ced9a811966bae40f40e9d" &
+        "6d58c1c43dcf643d211afe4ff8d9f9727fb45380862d47978fd015a4f9a7ff39" &
-        "4b46e27c388330304409a405b6455547661361d2129aa7bed4ff26f68d53532d" &
+        "28158c38173fd1192164e714ad5454446b681ed2130a349fdffd4b81f814700c" &
-        "cb6bae00506a7c5161b0652afcbf2416e97116bdcf9a7a548d6d8b5b0ab2ed0e" &
+        "5b04551e621b65432d3e1beca05dc8e801a4c21da44d4cb29b1eddb92417aedd" &
-        "b7a737afc0dbf65f32fd22c27cb17ebfe3c0d43e9bf45cfd24170c9fea348b10" &
+        "de855ba1016e93eb25d55879a0d27d170dc7904c6dfa06ce813b0455b3258c66" &
-        "1207010ad51e28040b46770c1e96e22e7c552a6f1a62b4e29f8c99"
+        "1de55dcd349397bc5b8eb8e8ef5b4ed2115e968197effe44a0f5fb2b7d"
 
     let
       whoareyouData = WhoareyouData(

tests/testAll.nim

@@ -1,3 +1,3 @@
 import
   ./dht/test_providers,
-  ./discv5/[test_discoveryv5, test_discoveryv5_encoding]
+  ./discv5/[test_discoveryv5, test_discoveryv5_encoding]