Fix Portal history/state JSON-RPC API and add missing calls (#1380)

fluffy/network/wire/portal_protocol.nim

@@ -222,6 +222,9 @@ proc addNode*(p: PortalProtocol, r: Record): bool =
   else:
     false
 
+proc getNode*(p: PortalProtocol, id: NodeId): Option[Node] =
+  p.routingTable.getNode(id)
+
 func localNode*(p: PortalProtocol): Node = p.baseProtocol.localNode
 
 func neighbours*(p: PortalProtocol, id: NodeId, seenOnly = false): seq[Node] =

fluffy/rpc/rpc_calls/rpc_portal_calls.nim

@@ -1,36 +1,45 @@
 ## Portal State Network json-rpc calls
 proc portal_stateNodeInfo(): NodeInfo
 proc portal_stateRoutingTableInfo(): RoutingTableInfo
-proc portal_stateLookupEnr(nodeId: NodeId): Record
+proc portal_stateAddEnr(enr: Record): bool
 proc portal_stateAddEnrs(enrs: seq[Record]): bool
+proc portal_stateGetEnr(nodeId: NodeId): Record
+proc portal_stateDeleteEnr(nodeId: NodeId): bool
+proc portal_stateLookupEnr(nodeId: NodeId): Record
 proc portal_statePing(enr: Record): tuple[
-  seqNum: uint64, customPayload: string]
+  enrSeq: uint64, customPayload: string]
 proc portal_stateFindNodes(enr: Record): seq[Record]
-proc portal_stateFindContentRaw(enr: Record, contentKey: string): tuple[
+proc portal_stateFindContent(enr: Record, contentKey: string): tuple[
   connectionId: Option[string],
   content: Option[string],
   enrs: Option[seq[Record]]]
-proc portal_stateFindContent(enr: Record, contentKey: string): tuple[
+proc portal_stateFindContentFull(enr: Record, contentKey: string): tuple[
   content: Option[string],
   enrs: Option[seq[Record]]]
-proc portal_stateOffer(enr: Record, contentKey: string): bool
+proc portal_stateOffer(contentKey: string, contentValue: string): int
-proc portal_stateRecursiveFindNodes(): seq[Record]
+proc portal_stateRecursiveFindNodes(nodeId: NodeId): seq[Record]
+proc portal_stateRecursiveFindContent(contentKey: string): string
+proc portal_stateStore(contentKey: string, contentValue: string): bool
 
 ## Portal History Network json-rpc calls
 proc portal_historyNodeInfo(): NodeInfo
 proc portal_historyRoutingTableInfo(): RoutingTableInfo
-proc portal_historyLookupEnr(nodeId: NodeId): Record
+proc portal_historyAddEnr(enr: Record): bool
 proc portal_historyAddEnrs(enrs: seq[Record]): bool
+proc portal_historyGetEnr(nodeId: NodeId): Record
+proc portal_historyDeleteEnr(nodeId: NodeId): bool
+proc portal_historyLookupEnr(nodeId: NodeId): Record
 proc portal_historyPing(enr: Record): tuple[
-  seqNum: uint64, customPayload: string]
+  enrSeq: uint64, customPayload: string]
 proc portal_historyFindNodes(enr: Record): seq[Record]
-proc portal_historyFindContentRaw(enr: Record, contentKey: string): tuple[
+proc portal_historyFindContent(enr: Record, contentKey: string): tuple[
   connectionId: Option[string],
   content: Option[string],
   enrs: Option[seq[Record]]]
-proc portal_historyFindContent(enr: Record, contentKey: string): tuple[
+proc portal_historyFindContentFull(enr: Record, contentKey: string): tuple[
   content: Option[string],
   enrs: Option[seq[Record]]]
-proc portal_historyOffer(contentKey: string, content: string): int
+proc portal_historyOffer(contentKey: string, contentValue: string): int
-proc portal_historyRecursiveFindNodes(): seq[Record]
+proc portal_historyRecursiveFindNodes(nodeId: NodeId): seq[Record]
-proc portal_historyStore(contentKey: string, content: string): bool
+proc portal_historyRecursiveFindContent(contentKey: string): string
+proc portal_historyStore(contentKey: string, contentValue: string): bool

fluffy/rpc/rpc_discovery_api.nim

@@ -89,8 +89,8 @@ proc installDiscoveryApiHandlers*(rpcServer: RpcServer|RpcProxy,
 
   rpcServer.rpc("discv5_lookupEnr") do(
       nodeId: NodeId) -> Record:
-    # TODO: Not according to spec, missing optional seqNum
+    # TODO: Not according to spec, missing optional enrSeq
-    # Can add `seqNum: Option[uint64]` as parameter but Option appears to be
+    # Can add `enrSeq: Option[uint64]` as parameter but Option appears to be
     # not implemented as an option parameter in nim-json-rpc?
     let lookup = await d.resolve(nodeId)
     if lookup.isSome():

fluffy/rpc/rpc_portal_api.nim

@@ -35,14 +35,16 @@ proc installPortalApiHandlers*(
   rpcServer.rpc("portal_" & network & "RoutingTableInfo") do() -> RoutingTableInfo:
     return getRoutingTableInfo(p.routingTable)
 
-  rpcServer.rpc("portal_" & network & "LookupEnr") do(nodeId: NodeId) -> Record:
+  rpcServer.rpc("portal_" & network & "AddEnr") do(enr: Record) -> bool:
-    let lookup = await p.resolve(nodeId)
+    let node = newNode(enr).valueOr:
-    if lookup.isSome():
+      raise newException(ValueError, "Failed creating Node from ENR")
-      return lookup.get().record
+
-    else:
+    let addResult = p.addNode(node)
-      raise newException(ValueError, "Record not found in DHT lookup.")
+    p.routingTable.setJustSeen(node)
+    return addResult == Added
 
   rpcServer.rpc("portal_" & network & "AddEnrs") do(enrs: seq[Record]) -> bool:
+    # Note: unspecified RPC, but useful for our local testnet test
     for enr in enrs:
       let nodeRes = newNode(enr)
       if nodeRes.isOk():
@@ -52,8 +54,37 @@ proc installPortalApiHandlers*(
 
     return true
 
+  rpcServer.rpc("portal_" & network & "GetEnr") do(nodeId: NodeId) -> Record:
+    let node = p.getNode(nodeId)
+    if node.isSome():
+      return node.get().record
+    else:
+      raise newException(ValueError, "Record not in local routing table.")
+
+  rpcServer.rpc("portal_" & network & "DeleteEnr") do(nodeId: NodeId) -> bool:
+    # TODO: Adjust `removeNode` to accept NodeId as param and to return bool.
+    let node = p.getNode(nodeId)
+    if node.isSome():
+      p.routingTable.removeNode(node.get())
+      return true
+    else:
+      raise newException(ValueError, "Record not in local routing table.")
+
+  rpcServer.rpc("portal_" & network & "LookupEnr") do(nodeId: NodeId) -> Record:
+    # TODO: Not fully according to spec, missing optional enrSeq
+    # Can add `enrSeq: Option[uint64]` as parameter but Option appears to be
+    # not implemented as an option parameter in nim-json-rpc?
+    let lookup = await p.resolve(nodeId)
+    if lookup.isSome():
+      return lookup.get().record
+    else:
+      raise newException(ValueError, "Record not found in DHT lookup.")
+
   rpcServer.rpc("portal_" & network & "Ping") do(
-      enr: Record) -> tuple[seqNum: uint64, customPayload: string]:
+      enr: Record) -> tuple[enrSeq: uint64, customPayload: string]:
+    # TODO: Not fully according to spec:
+    # - missing optional dataRadius
+    # - customPayload instead of dataRadius returned
     let
       node = toNodeWithAddress(enr)
       pong = await p.ping(node)
@@ -76,49 +107,48 @@ proc installPortalApiHandlers*(
 
   # TODO: This returns null values for the `none`s. Not sure what it should be
   # according to spec, no k:v pair at all?
-  # Note: `*_findContentRaw` is actually `*_findContent` call according to
+  rpcServer.rpc("portal_" & network & "FindContent") do(
-  # WIP Portal JSON-RPC API specification. Not sure about the best naming here.
-  rpcServer.rpc("portal_" & network & "FindContentRaw") do(
       enr: Record, contentKey: string) -> tuple[
         connectionId: Option[string],
         content: Option[string],
         enrs: Option[seq[Record]]]:
     let
       node = toNodeWithAddress(enr)
-      content = await p.findContentImpl(
+      res = await p.findContentImpl(
         node, ByteList.init(hexToSeqByte(contentKey)))
 
-    if content.isErr():
+    if res.isErr():
-      raise newException(ValueError, $content.error)
+      raise newException(ValueError, $res.error)
-    else:
-      let contentMessage = content.get()
-      case contentMessage.contentMessageType:
-      of connectionIdType:
-        return (
-          some("0x" & contentMessage.connectionId.toHex()),
-          none(string),
-          none(seq[Record]))
-      of contentType:
-        return (
-          none(string),
-          some("0x" & contentMessage.content.asSeq().toHex()),
-          none(seq[Record]))
-      of enrsType:
-        let records = recordsFromBytes(contentMessage.enrs)
-        if records.isErr():
-          raise newException(ValueError, $records.error)
-        else:
-          return (
-            none(string),
-            none(string),
-            # Note: Could also pass not verified nodes
-            some(verifyNodesRecords(
-              records.get(), node, enrsResultLimit).map(
-                proc(n: Node): Record = n.record)))
 
-  rpcServer.rpc("portal_" & network & "FindContent") do(
+    let contentMessage = res.get()
+    case contentMessage.contentMessageType:
+    of connectionIdType:
+      return (
+        some("0x" & contentMessage.connectionId.toHex()),
+        none(string),
+        none(seq[Record]))
+    of contentType:
+      return (
+        none(string),
+        some("0x" & contentMessage.content.asSeq().toHex()),
+        none(seq[Record]))
+    of enrsType:
+      let records = recordsFromBytes(contentMessage.enrs)
+      if records.isErr():
+        raise newException(ValueError, $records.error)
+      else:
+        return (
+          none(string),
+          none(string),
+          # Note: Could also pass not verified nodes
+          some(verifyNodesRecords(
+            records.get(), node, enrsResultLimit).map(
+              proc(n: Node): Record = n.record)))
+
+  rpcServer.rpc("portal_" & network & "FindContentFull") do(
       enr: Record, contentKey: string) -> tuple[
         content: Option[string], enrs: Option[seq[Record]]]:
+    # Note: unspecified RPC, but useful as we can get content from uTP also
     let
       node = toNodeWithAddress(enr)
       foundContentResult = await p.findContent(
@@ -139,26 +169,39 @@ proc installPortalApiHandlers*(
           some(foundContent.nodes.map(proc(n: Node): Record = n.record)))
 
   rpcServer.rpc("portal_" & network & "Offer") do(
-      contentKey: string, content: string) -> int:
+      contentKey: string, contentValue: string) -> int:
     let
-      ck = hexToSeqByte(contentKey)
+      key = hexToSeqByte(contentKey)
-      ct = hexToSeqByte(content)
+      content = hexToSeqByte(contentValue)
-      contentKeys = ContentKeysList(@[ByteList.init(ck)])
+      contentKeys = ContentKeysList(@[ByteList.init(key)])
-      numberOfPeers = await p.neighborhoodGossip(contentKeys, @[ct])
+      numberOfPeers = await p.neighborhoodGossip(contentKeys, @[content])
 
     return numberOfPeers
 
-  rpcServer.rpc("portal_" & network & "RecursiveFindNodes") do() -> seq[Record]:
+  rpcServer.rpc("portal_" & network & "RecursiveFindNodes") do(
-    let discovered = await p.queryRandom()
+      nodeId: NodeId) -> seq[Record]:
+    let discovered = await p.lookup(nodeId)
     return discovered.map(proc(n: Node): Record = n.record)
 
+  rpcServer.rpc("portal_" & network & "RecursiveFindContent") do(
+      contentKey: string) -> string:
+    let
+      key = ByteList.init(hexToSeqByte(contentKey))
+      contentId = p.toContentId(key).valueOr:
+        raise newException(ValueError, "Invalid content key")
+
+      contentResult = (await p.contentLookup(key, contentId)).valueOr:
+        raise newException(ValueError, "Content not found")
+
+    return contentResult.content.toHex()
+
   rpcServer.rpc("portal_" & network & "Store") do(
-      contentKey: string, content: string) -> bool:
+      contentKey: string, contentValue: string) -> bool:
     let key = ByteList.init(hexToSeqByte(contentKey))
     let contentId = p.toContentId(key)
 
     if contentId.isSome():
-      p.storeContent(key, contentId.get(), hexToSeqByte(content))
+      p.storeContent(key, contentId.get(), hexToSeqByte(contentValue))
       return true
     else:
       raise newException(ValueError, "Invalid content key")