Further implement Portal Nodes and FoundContent messages (#757)

fluffy/content.nim

@@ -47,6 +47,8 @@ template toSszType*(x: ContentType): uint8 =
   uint8(x)
 
 template toSszType*(x: auto): auto =
+  mixin toSszType
+
   x
 
 func fromSszBytes*(T: type ContentType, data: openArray[byte]):
@@ -61,4 +63,13 @@ func fromSszBytes*(T: type ContentType, data: openArray[byte]):
   contentType
 
 func toContentId*(contentKey: ContentKey): ContentId =
+  # TODO: Hash function to be defined, sha256 used now, might be confusing
+  # with keccak256 that is used for the actual nodes:
+  # https://github.com/ethereum/stateless-ethereum-specs/blob/master/state-network.md#content
   sha2.sha_256.digest(SSZ.encode(contentKey))
+
+type
+  ContentStorage* = object
+
+func getContent*(storage: ContentStorage, key: ContentKey): Option[seq[byte]] =
+  discard

fluffy/network/messages.nim

@@ -12,13 +12,12 @@
 
 import
   stint, stew/[results, objects],
-  eth/ssz/ssz_serialization
+  eth/ssz/ssz_serialization,
+  ../content
 
-export ssz_serialization, stint
+export ssz_serialization, stint, content
 
 type
-  ByteList* = List[byte, 2048]
-
   MessageKind* = enum
     unused = 0x00
 
@@ -48,7 +47,7 @@ type
     # also be limited to 300 bytes instead of 2048
 
   FindContentMessage* = object
-    contentKey*: ByteList
+    contentKey*: ContentKey
 
   FoundContentMessage* = object
     enrs*: List[ByteList, 32]
@@ -104,9 +103,6 @@ template messageKind*(T: typedesc[SomeMessage]): MessageKind =
 template toSszType*(x: UInt256): array[32, byte] =
   toBytesLE(x)
 
-template toSszType*(x: auto): auto =
-  x
-
 func fromSszBytes*(T: type UInt256, data: openArray[byte]):
     T {.raises: [MalformedSszError, Defect].} =
   if data.len != sizeof(result):

fluffy/network/portal_protocol.nim

@@ -8,8 +8,10 @@
 {.push raises: [Defect].}
 
 import
+  std/sequtils,
   stew/[results, byteutils], chronicles,
-  eth/rlp, eth/p2p/discoveryv5/[protocol, node],
+  eth/rlp, eth/p2p/discoveryv5/[protocol, node, enr],
+  ../content,
   ./messages
 
 export messages
@@ -24,6 +26,7 @@ type
   PortalProtocol* = ref object of TalkProtocol
     baseProtocol*: protocol.Protocol
     dataRadius*: UInt256
+    contentStorage*: ContentStorage
 
 proc handlePing(p: PortalProtocol, ping: PingMessage):
     seq[byte] =
@@ -41,18 +44,45 @@ proc handleFindNode(p: PortalProtocol, fn: FindNodeMessage): seq[byte] =
     let enr = ByteList(rlp.encode(p.baseProtocol.localNode.record))
     encodeMessage(NodesMessage(total: 1, enrs: List[ByteList, 32](@[enr])))
   else:
-    # TODO: Not implemented for now, sending empty back.
-    let enrs = List[ByteList, 32](@[])
-    encodeMessage(NodesMessage(total: 1, enrs: enrs))
+    let distances = fn.distances.asSeq()
+    if distances.all(proc (x: uint16): bool = return x <= 256):
+      let
+        nodes = p.baseProtocol.neighboursAtDistances(distances, seenOnly = true)
+        enrs = nodes.map(proc(x: Node): ByteList = ByteList(x.record.raw))
 
-proc handleFindContent(p: PortalProtocol, ping: FindContentMessage): seq[byte] =
-  # TODO: Neither payload nor enrs implemented, sending empty back.
+      # TODO: Fixed here to total message of 1 for now, as else we would need to
+      # either move the send of the talkresp messages here, or allow for
+      # returning multiple messages.
+      # On the long run, it might just be better to use a stream in these cases?
+      encodeMessage(
+        NodesMessage(total: 1, enrs: List[ByteList, 32](List(enrs))))
+    else:
+      # invalid request, send empty back
+      let enrs = List[ByteList, 32](@[])
+      encodeMessage(NodesMessage(total: 1, enrs: enrs))
+
+proc handleFindContent(p: PortalProtocol, fc: FindContentMessage): seq[byte] =
+  # TODO: Need to check networkId, type, trie path
   let
-    enrs = List[ByteList, 32](@[])
-    payload = ByteList(@[])
-  encodeMessage(FoundContentMessage(enrs: enrs, payload: payload))
+    # TODO: Should we first do a simple check on ContentId versus Radius?
+    contentId = toContentId(fc.contentKey)
+    content = p.contentStorage.getContent(fc.contentKey)
+  if content.isSome():
+    let enrs = List[ByteList, 32](@[]) # Empty enrs when payload is send
+    encodeMessage(FoundContentMessage(
+      enrs: enrs, payload: ByteList(content.get())))
+  else:
+    let
+      closestNodes = p.baseProtocol.neighbours(
+        NodeId(readUintBE[256](contentId.data)), seenOnly = true)
+      payload = ByteList(@[]) # Empty payload when enrs are send
+      enrs =
+        closestNodes.map(proc(x: Node): ByteList = ByteList(x.record.raw))
 
-proc handleAdvertise(p: PortalProtocol, ping: AdvertiseMessage): seq[byte] =
+    encodeMessage(FoundContentMessage(
+      enrs: List[ByteList, 32](List(enrs)), payload: payload))
+
+proc handleAdvertise(p: PortalProtocol, a: AdvertiseMessage): seq[byte] =
   # TODO: Not implemented
   let
     connectionId = List[byte, 4](@[])
@@ -141,7 +171,7 @@ proc findNode*(p: PortalProtocol, dst: Node, distances: List[uint16, 256]):
   else:
     return err(talkresp.error)
 
-proc findContent*(p: PortalProtocol, dst: Node, contentKey: ByteList):
+proc findContent*(p: PortalProtocol, dst: Node, contentKey: ContentKey):
     Future[DiscResult[FoundContentMessage]] {.async.} =
   let fc = FindContentMessage(contentKey: contentKey)
 

fluffy/network/portalcli.nim

@@ -200,8 +200,10 @@ proc run(config: DiscoveryConf) =
 
       key
 
-    # For now just random content keys
-    let contentKey = ByteList(@(UInt256.random(rng[]).toBytes()))
+    # For now just random content node hash
+    let contentKey = ContentKey(networkId: 0'u16,
+      contentType: messages.ContentType.Account,
+      nodeHash: List[byte, 32](@(UInt256.random(rng[]).toBytes())))
     let foundContent = waitFor portal.findContent(config.findContentTarget,
       contentKey)
 

fluffy/tests/test_portal.nim

@@ -14,11 +14,11 @@ import
   eth/p2p/discoveryv5/protocol as discv5_protocol,
   ../network/portal_protocol
 
-proc localAddress(port: int): Address =
-  Address(ip: ValidIpAddress.init("127.0.0.1"), port: Port(port))
+proc localAddress(port: int): node.Address =
+  node.Address(ip: ValidIpAddress.init("127.0.0.1"), port: Port(port))
 
 proc initDiscoveryNode(rng: ref BrHmacDrbgContext, privKey: PrivateKey,
-                        address: Address,
+                        address: node.Address,
                         bootstrapRecords: openarray[Record] = [],
                         localEnrFields: openarray[(string, seq[byte])] = [],
                         previousRecord = none[enr.Record]()):
@@ -96,13 +96,23 @@ procSuite "Portal Tests":
         nodes.get().enrs.len() == 0
 
     block: # Find for distance
-      # TODO: Add test when implemented
-      discard
+      # ping in one direction to add, ping in the other to update as seen.
+      check (await node1.ping(node2.localNode)).isOk()
+      check (await node2.ping(node1.localNode)).isOk()
+
+      let distance = logDist(node1.localNode.id, node2.localNode.id)
+      let nodes = await proto1.findNode(proto2.baseProtocol.localNode,
+        List[uint16, 256](@[distance]))
+
+      check:
+        nodes.isOk()
+        nodes.get().total == 1'u8
+        nodes.get().enrs.len() == 1
 
     await node1.closeWait()
     await node2.closeWait()
 
-  asyncTest "Portal FindContent/FoundContent":
+  asyncTest "Portal FindContent/FoundContent - send enrs":
     let
       node1 = initDiscoveryNode(
         rng, PrivateKey.random(rng[]), localAddress(20302))
@@ -112,15 +122,22 @@ procSuite "Portal Tests":
       proto1 = PortalProtocol.new(node1)
       proto2 = PortalProtocol.new(node2)
 
-    let contentKey = ByteList(@(UInt256.random(rng[]).toBytes()))
+    # ping in one direction to add, ping in the other to update as seen.
+    check (await node1.ping(node2.localNode)).isOk()
+    check (await node2.ping(node1.localNode)).isOk()
 
+    let contentKey = ContentKey(networkId: 0'u16,
+      contentType: ContentType.Account,
+      nodeHash: List[byte, 32](@(UInt256.random(rng[]).toBytes())))
+
+    # content does not exist so this should provide us with the closest nodes
+    # to the content, which is the only node in the routing table.
     let foundContent = await proto1.findContent(proto2.baseProtocol.localNode,
       contentKey)
 
     check:
       foundContent.isOk()
-      # TODO: adjust when implemented
-      foundContent.get().enrs.len() == 0
+      foundContent.get().enrs.len() == 1
       foundContent.get().payload.len() == 0
 
     await node1.closeWait()

fluffy/tests/test_portal_encoding.nim

@@ -83,12 +83,16 @@ suite "Portal Protocol Message Encodings":
       message.nodes.enrs.len() == 0
 
   test "FindContent Request":
+    var nodeHash: List[byte, 32]
     let
-      contentKey = ByteList(@[byte 0x01, 0x02, 0x03])
+      contentKey = ContentKey(
+        networkId: 0'u16,
+        contentType: ContentType.Account,
+        nodeHash: nodeHash)
       fn = FindContentMessage(contentKey: contentKey)
 
     let encoded = encodeMessage(fn)
-    check encoded.toHex == "0504000000010203"
+    check encoded.toHex == "050400000000000107000000"
 
     let decoded = decodeMessage(encoded)
     check decoded.isOk()

vendor/nim-eth

@@ -1 +1 @@
-Subproject commit 41127eaee8c31b58505754c03a3178eeeb040af2
+Subproject commit 79911ed5d8c951d5dcab9f2b20e1db8d2f59cdd1