Adjust Portal JSON-RPC Offer call and add Gossip call (#1418)

fluffy/network/wire/portal_protocol.nim

@@ -711,7 +711,7 @@ func getMaxOfferedContentKeys*(protocolIdLen: uint32, maxKeySize: uint32): int =
   )
 
 proc offer(p: PortalProtocol, o: OfferRequest):
-  Future[PortalResult[void]] {.async.} =
+  Future[PortalResult[ContentKeysBitList]] {.async.} =
   ## Offer triggers offer-accept interaction with one peer
   ## Whole flow has two phases:
   ## 1. Come to an agreement on what content to transfer, by using offer and
@@ -762,7 +762,7 @@ proc offer(p: PortalProtocol, o: OfferRequest):
     if acceptedKeysAmount == 0:
       debug "No content accepted"
       # Don't open an uTP stream if no content was requested
-      return ok()
+      return ok(m.contentKeys)
 
     let nodeAddress = NodeAddress.init(o.dst)
     if nodeAddress.isNone():
@@ -837,27 +837,25 @@ proc offer(p: PortalProtocol, o: OfferRequest):
     debug "Content successfully offered"
 
     await socket.closeWait()
-    return ok()
+    return ok(m.contentKeys)
   else:
     warn "Offer failed due to accept request failure ",
       error = acceptMessageResponse.error
     return err("No accept response")
 
 proc offer*(p: PortalProtocol, dst: Node, contentKeys: ContentKeysList):
-    Future[PortalResult[void]] {.async.} =
-    let req = OfferRequest(dst: dst, kind: Database, contentKeys: contentKeys)
-    let res = await p.offer(req)
-    return res
+    Future[PortalResult[ContentKeysBitList]] {.async.} =
+  let req = OfferRequest(dst: dst, kind: Database, contentKeys: contentKeys)
+  return await p.offer(req)
 
 proc offer*(p: PortalProtocol, dst: Node, content: seq[ContentInfo]):
-    Future[PortalResult[void]] {.async.} =
-    if len(content) > contentKeysLimit:
-      return err("Cannot offer more than 64 content items")
+    Future[PortalResult[ContentKeysBitList]] {.async.} =
+  if len(content) > contentKeysLimit:
+    return err("Cannot offer more than 64 content items")
 
-    let contentList = List[ContentInfo, contentKeysLimit].init(content)
-    let req = OfferRequest(dst: dst, kind: Direct, contentList: contentList)
-    let res = await p.offer(req)
-    return res
+  let contentList = List[ContentInfo, contentKeysLimit].init(content)
+  let req = OfferRequest(dst: dst, kind: Direct, contentList: contentList)
+  return await p.offer(req)
 
 proc offerWorker(p: PortalProtocol) {.async.} =
   while true:
@@ -1157,7 +1155,7 @@ proc neighborhoodGossip*(
   # 1. Select the closest neighbours in the routing table
   # 2. Check if the radius is known for these these nodes and whether they are
   # in range of the content to be offered.
-  # 3. If more than n (= 4) nodes are in range, offer these nodes the content
+  # 3. If more than n (= 8) nodes are in range, offer these nodes the content
   # (max nodes set at 8).
   # 4. If less than n nodes are in range, do a node lookup, and offer the nodes
   # returned from the lookup the content (max nodes set at 8)

fluffy/rpc/rpc_calls/rpc_portal_calls.nim

@@ -16,13 +16,13 @@ proc portal_stateFindContent(enr: Record, contentKey: string): tuple[
 proc portal_stateFindContentFull(enr: Record, contentKey: string): tuple[
   content: Option[string],
   enrs: Option[seq[Record]]]
-proc portal_stateOfferReal(
-  enr: Record, contentKey: string, contentValue: string): bool
-proc portal_stateOffer(contentKey: string, contentValue: string): int
+proc portal_stateOffer(
+  enr: Record, contentKey: string, contentValue: string): string
 proc portal_stateRecursiveFindNodes(nodeId: NodeId): seq[Record]
 proc portal_stateRecursiveFindContent(contentKey: string): string
 proc portal_stateStore(contentKey: string, contentValue: string): bool
 proc portal_stateLocalContent(contentKey: string): string
+proc portal_stateGossip(contentKey: string, contentValue: string): int
 
 ## Portal History Network json-rpc calls
 proc portal_historyNodeInfo(): NodeInfo
@@ -42,10 +42,10 @@ proc portal_historyFindContent(enr: Record, contentKey: string): tuple[
 proc portal_historyFindContentFull(enr: Record, contentKey: string): tuple[
   content: Option[string],
   enrs: Option[seq[Record]]]
-proc portal_historyOfferReal(
-  enr: Record, contentKey: string, contentValue: string): bool
-proc portal_historyOffer(contentKey: string, contentValue: string): int
+proc portal_historyOffer(
+  enr: Record, contentKey: string, contentValue: string): string
 proc portal_historyRecursiveFindNodes(nodeId: NodeId): seq[Record]
 proc portal_historyRecursiveFindContent(contentKey: string): string
 proc portal_historyStore(contentKey: string, contentValue: string): bool
 proc portal_historyLocalContent(contentKey: string): string
+proc portal_historyGossip(contentKey: string, contentValue: string): int

fluffy/rpc/rpc_portal_api.nim

@@ -16,6 +16,9 @@ import
 
 export rpcserver
 
+# Portal Network JSON-RPC impelentation as per specification:
+# https://github.com/ethereum/portal-network-specs/tree/master/jsonrpc
+
 # Note:
 # Using a string for the network parameter will give an error in the rpc macro:
 # Error: Invalid node kind nnkInfix for macros.`$`
@@ -25,9 +28,6 @@ export rpcserver
 proc installPortalApiHandlers*(
     rpcServer: RpcServer|RpcProxy, p: PortalProtocol, network: static string)
     {.raises: [Defect, CatchableError].} =
-  ## Portal routing table and portal wire json-rpc API is not yet defined but
-  ## will look something similar as what exists here now:
-  ## https://github.com/ethereum/portal-network-specs/pull/88
 
   rpcServer.rpc("portal_" & network & "NodeInfo") do() -> NodeInfo:
     return p.routingTable.getNodeInfo()
@@ -73,7 +73,7 @@ proc installPortalApiHandlers*(
   rpcServer.rpc("portal_" & network & "LookupEnr") do(nodeId: NodeId) -> Record:
     # TODO: Not fully according to spec, missing optional enrSeq
     # Can add `enrSeq: Option[uint64]` as parameter but Option appears to be
-    # not implemented as an option parameter in nim-json-rpc?
+    # not implemented as an optional parameter in nim-json-rpc?
     let lookup = await p.resolve(nodeId)
     if lookup.isSome():
       return lookup.get().record
@@ -168,10 +168,8 @@ proc installPortalApiHandlers*(
           none(string),
           some(foundContent.nodes.map(proc(n: Node): Record = n.record)))
 
-  rpcServer.rpc("portal_" & network & "OfferReal") do(
-      enr: Record, contentKey: string, contentValue: string) -> bool:
-    # Note: unspecified RPC, but the spec took over the Offer call to actually
-    # do gossip. This should be adjusted.
+  rpcServer.rpc("portal_" & network & "Offer") do(
+      enr: Record, contentKey: string, contentValue: string) -> string:
     let
       node = toNodeWithAddress(enr)
       key = hexToSeqByte(contentKey)
@@ -180,20 +178,10 @@ proc installPortalApiHandlers*(
       res = await p.offer(node, @[contentInfo])
 
     if res.isOk():
-      return true
+      return "0x" & SSZ.encode(res.get()).toHex()
     else:
       raise newException(ValueError, $res.error)
 
-  rpcServer.rpc("portal_" & network & "Offer") do(
-      contentKey: string, contentValue: string) -> int:
-    let
-      key = hexToSeqByte(contentKey)
-      content = hexToSeqByte(contentValue)
-      contentKeys = ContentKeysList(@[ByteList.init(key)])
-      numberOfPeers = await p.neighborhoodGossip(contentKeys, @[content])
-
-    return numberOfPeers
-
   rpcServer.rpc("portal_" & network & "RecursiveFindNodes") do(
       nodeId: NodeId) -> seq[Record]:
     let discovered = await p.lookup(nodeId)
@@ -234,3 +222,13 @@ proc installPortalApiHandlers*(
       return contentResult.get().toHex()
     else:
       return "0x0"
+
+  rpcServer.rpc("portal_" & network & "Gossip") do(
+      contentKey: string, contentValue: string) -> int:
+    let
+      key = hexToSeqByte(contentKey)
+      content = hexToSeqByte(contentValue)
+      contentKeys = ContentKeysList(@[ByteList.init(key)])
+      numberOfPeers = await p.neighborhoodGossip(contentKeys, @[content])
+
+    return numberOfPeers

fluffy/scripts/test_portal_testnet.nim

@@ -269,7 +269,7 @@ procSuite "Portal testnet tests":
     # Gossiping all block headers with proof first, as bodies and receipts
     # require them for validation.
     for (content, contentKey) in blockHeadersWithProof:
-      discard (await clients[0].portal_history_offer(
+      discard (await clients[0].portal_history_gossip(
         content.toHex(), contentKey.toHex()))
 
     # This will fill the first node its db with blocks from the data file. Next,