Fix Portal Hive fails by correcting Portal history JSON RPC API (#1365)

* Fix Portal Hive fails by correcting Portal history JSON RPC API

- Field naming in discv5_nodeInfo
- Call naming of portal_historyStore
- Other: some proc to func adjustements

fluffy/rpc/rpc_calls/rpc_portal_calls.nim

@@ -33,3 +33,4 @@ proc portal_historyFindContent(enr: Record, contentKey: string): tuple[
   enrs: Option[seq[Record]]]
 proc portal_historyOffer(contentKey: string, content: string): int
 proc portal_historyRecursiveFindNodes(): seq[Record]
+proc portal_historyStore(contentKey: string, content: string): bool

fluffy/rpc/rpc_calls/rpc_portal_debug_calls.nim

@@ -1,5 +1,4 @@
 ## Portal History Network json-rpc debug & testing calls
-proc portal_history_store(contentKey: string, content: string): bool
 proc portal_history_storeContent(dataFile: string): bool
 proc portal_history_propagate(dataFile: string): bool
 proc portal_history_propagateHeaders(dataFile: string): bool

fluffy/rpc/rpc_portal_api.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2021 Status Research & Development GmbH
+# Copyright (c) 2021-2022 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -151,3 +151,14 @@ proc installPortalApiHandlers*(
   rpcServer.rpc("portal_" & network & "RecursiveFindNodes") do() -> seq[Record]:
     let discovered = await p.queryRandom()
     return discovered.map(proc(n: Node): Record = n.record)
+
+  rpcServer.rpc("portal_" & network & "Store") do(
+      contentKey: string, content: string) -> bool:
+    let key = ByteList.init(hexToSeqByte(contentKey))
+    let contentId = p.toContentId(key)
+
+    if contentId.isSome():
+      p.storeContent(key, contentId.get(), hexToSeqByte(content))
+      return true
+    else:
+      raise newException(ValueError, "Invalid content key")

fluffy/rpc/rpc_portal_debug_api.nim

@@ -15,23 +15,11 @@ import
 
 export rpcserver
 
-# Some RPCs that are (currently) useful for testing & debugging
+# Non-spec-RPCs that are (currently) useful for testing & debugging
 proc installPortalDebugApiHandlers*(
     rpcServer: RpcServer|RpcProxy, p: PortalProtocol, network: static string)
     {.raises: [Defect, CatchableError].} =
 
-  rpcServer.rpc("portal_" & network & "_store") do(
-      contentKey: string, content: string) -> bool:
-    let key = ByteList.init(hexToSeqByte(contentKey))
-    let contentId = p.toContentId(key)
-
-    if contentId.isSome():
-      p.storeContent(key, contentId.get(), hexToSeqByte(content))
-
-      return true
-    else:
-      raise newException(ValueError, "Invalid content key")
-
   rpcServer.rpc("portal_" & network & "_storeContent") do(
       dataFile: string) -> bool:
     let res = p.historyStore(dataFile)

fluffy/rpc/rpc_types.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2021 Status Research & Development GmbH
+# Copyright (c) 2021-2022 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -16,17 +16,17 @@ export jsonmarshal, routing_table, enr, node
 
 type
   NodeInfo* = object
+    enr*: Record
     nodeId*: NodeId
-    nodeENR*: Record
 
   RoutingTableInfo* = object
     localKey*: NodeId
     buckets*: seq[seq[NodeId]]
 
-proc getNodeInfo*(r: RoutingTable): NodeInfo =
-  NodeInfo(nodeId: r.localNode.id, nodeENR: r.localNode.record)
+func getNodeInfo*(r: RoutingTable): NodeInfo =
+  NodeInfo(enr: r.localNode.record, nodeId: r.localNode.id)
 
-proc getRoutingTableInfo*(r: RoutingTable): RoutingTableInfo =
+func getRoutingTableInfo*(r: RoutingTable): RoutingTableInfo =
   var info: RoutingTableInfo
   for b in r.buckets:
     var bucket: seq[NodeId]
@@ -39,7 +39,7 @@ proc getRoutingTableInfo*(r: RoutingTable): RoutingTableInfo =
 
   info
 
-proc toNodeWithAddress*(enr: Record): Node {.raises: [Defect, ValueError].} =
+func toNodeWithAddress*(enr: Record): Node {.raises: [Defect, ValueError].} =
   let nodeRes = newNode(enr)
   if nodeRes.isErr():
     raise newException(ValueError, $nodeRes.error)
@@ -50,19 +50,19 @@ proc toNodeWithAddress*(enr: Record): Node {.raises: [Defect, ValueError].} =
   else:
     node
 
-proc `%`*(value: Record): JsonNode =
+func `%`*(value: Record): JsonNode =
   newJString(value.toURI())
 
-proc fromJson*(n: JsonNode, argName: string, result: var Record)
+func fromJson*(n: JsonNode, argName: string, result: var Record)
     {.raises: [Defect, ValueError].} =
   n.kind.expect(JString, argName)
   if not fromURI(result, n.getStr()):
     raise newException(ValueError, "Invalid ENR")
 
-proc `%`*(value: NodeId): JsonNode =
+func `%`*(value: NodeId): JsonNode =
   %("0x" & value.toHex())
 
-proc fromJson*(n: JsonNode, argName: string, result: var NodeId)
+func fromJson*(n: JsonNode, argName: string, result: var NodeId)
     {.raises: [Defect, ValueError].} =
   n.kind.expect(JString, argName)
 
@@ -72,7 +72,7 @@ proc fromJson*(n: JsonNode, argName: string, result: var NodeId)
 
 # TODO: This one should go to nim-json-rpc but before we can do that we will
 # have to update the vendor module to the current latest.
-proc fromJson*(n: JsonNode, argName: string, result: var uint16)
+func fromJson*(n: JsonNode, argName: string, result: var uint16)
     {.raises: [Defect, ValueError].} =
   n.kind.expect(JInt, argName)
   let asInt = n.getBiggestInt()

fluffy/scripts/test_portal_testnet.nim

@@ -157,7 +157,7 @@ procSuite "Portal testnet tests":
     # error prone
     for client in clients:
       discard await client.discv5_addEnrs(nodeInfos.map(
-        proc(x: NodeInfo): Record = x.nodeENR))
+        proc(x: NodeInfo): Record = x.enr))
       await client.close()
 
     for client in clients:
@@ -174,7 +174,7 @@ procSuite "Portal testnet tests":
     for client in clients:
       var enr: Record
       enr = await client.discv5_lookupEnr(randomNodeInfo.nodeId)
-      check enr == randomNodeInfo.nodeENR
+      check enr == randomNodeInfo.enr
       await client.close()
 
   asyncTest "Portal State - Random node lookup from each node":
@@ -188,7 +188,7 @@ procSuite "Portal testnet tests":
 
     for client in clients:
       discard await client.portal_state_addEnrs(nodeInfos.map(
-        proc(x: NodeInfo): Record = x.nodeENR))
+        proc(x: NodeInfo): Record = x.enr))
       await client.close()
 
     for client in clients:
@@ -211,7 +211,7 @@ procSuite "Portal testnet tests":
       # doesn't converge to the target always with this distance function. To be
       # further investigated.
       skip()
-      # check enr == randomNodeInfo.nodeENR
+      # check enr == randomNodeInfo.enr
       await client.close()
 
   asyncTest "Portal History - Random node lookup from each node":
@@ -225,7 +225,7 @@ procSuite "Portal testnet tests":
 
     for client in clients:
       discard await client.portal_history_addEnrs(nodeInfos.map(
-        proc(x: NodeInfo): Record = x.nodeENR))
+        proc(x: NodeInfo): Record = x.enr))
       await client.close()
 
     for client in clients:
@@ -241,7 +241,7 @@ procSuite "Portal testnet tests":
       var enr: Record
       enr = await client.portal_history_lookupEnr(randomNodeInfo.nodeId)
       await client.close()
-      check enr == randomNodeInfo.nodeENR
+      check enr == randomNodeInfo.enr
 
   asyncTest "Portal History - Propagate blocks and do content lookups":
     const

fluffy/tests/test_discovery_rpc.nim

@@ -43,7 +43,7 @@ proc stop(testCase: TestCase) {.async.} =
   await testCase.server.closeWait()
   await testCase.localDiscovery.closeWait()
 
-procSuite "Discovery Rpc":
+procSuite "Discovery RPC":
   let rng = newRng()
 
   asyncTest "Get local node info":
@@ -53,11 +53,11 @@ procSuite "Discovery Rpc":
     check:
       resp.contains("nodeId")
       resp["nodeId"].kind == JString
-      resp.contains("nodeENR")
-      resp["nodeENR"].kind == JString
+      resp.contains("enr")
+      resp["enr"].kind == JString
 
     let nodeId = resp["nodeId"].getStr()
-    let nodeEnr = resp["nodeENR"].getStr()
+    let nodeEnr = resp["enr"].getStr()
 
     check:
       nodeEnr == tc.localDiscovery.localNode.record.toURI()

fluffy/tools/utp_testing/utp_rpc_types.nim

@@ -7,7 +7,7 @@
 {.push raises: [Defect].}
 
 
-import 
+import
   std/hashes,
   json_rpc/jsonmarshal,
   stew/byteutils,

fluffy/tools/utp_testing/utp_test.nim

@@ -77,7 +77,7 @@ procSuite "Utp integration tests":
     let serverInfo = await repeatTillSuccess(() => server.discv5_nodeInfo(), 10)
 
     # nodes need to have established session before the utp try
-    discard await repeatTillSuccess(() => client.discv5_ping(serverInfo.nodeEnr))
+    discard await repeatTillSuccess(() => client.discv5_ping(serverInfo.enr))
 
     return (client, clientInfo, server, serverInfo)
 
@@ -85,7 +85,7 @@ procSuite "Utp integration tests":
     let (client, clientInfo, server, serverInfo) = await setupTest()
     let numOfBytes = 100000
     let
-      clientConnectionKey = await repeatTillSuccess(() => client.utp_connect(serverInfo.nodeEnr))
+      clientConnectionKey = await repeatTillSuccess(() => client.utp_connect(serverInfo.enr))
       serverConnections = await repeatTillSuccess(() => server.utp_get_connections())
       maybeServerConnectionKey = serverConnections.findServerConnection(clientInfo.nodeId, clientConnectionKey.id)
 
@@ -110,7 +110,7 @@ procSuite "Utp integration tests":
     let (client, clientInfo, server, serverInfo) = await setupTest()
     let numOfBytes = 100000
     let
-      clientConnectionKey = await repeatTillSuccess(() => client.utp_connect(serverInfo.nodeEnr))
+      clientConnectionKey = await repeatTillSuccess(() => client.utp_connect(serverInfo.enr))
       serverConnections = await repeatTillSuccess(() => server.utp_get_connections())
       maybeServerConnectionKey = serverConnections.findServerConnection(clientInfo.nodeId, clientConnectionKey.id)
 
@@ -132,7 +132,7 @@ procSuite "Utp integration tests":
     let (client, clientInfo, server, serverInfo) = await setupTest()
     let numOfBytes = 10000
     let
-      clientConnectionKey = await repeatTillSuccess(() => client.utp_connect(serverInfo.nodeEnr))
+      clientConnectionKey = await repeatTillSuccess(() => client.utp_connect(serverInfo.enr))
       serverConnections = await repeatTillSuccess(() => server.utp_get_connections())
       maybeServerConnectionKey = serverConnections.findServerConnection(clientInfo.nodeId, clientConnectionKey.id)
 
@@ -162,9 +162,9 @@ procSuite "Utp integration tests":
     let (client, clientInfo, server, serverInfo) = await setupTest()
     let numOfBytes = 10000
     let
-      clientConnectionKey1 = await repeatTillSuccess(() => client.utp_connect(serverInfo.nodeEnr))
-      clientConnectionKey2 = await repeatTillSuccess(() => client.utp_connect(serverInfo.nodeEnr))
-      clientConnectionKey3 = await repeatTillSuccess(() => client.utp_connect(serverInfo.nodeEnr))
+      clientConnectionKey1 = await repeatTillSuccess(() => client.utp_connect(serverInfo.enr))
+      clientConnectionKey2 = await repeatTillSuccess(() => client.utp_connect(serverInfo.enr))
+      clientConnectionKey3 = await repeatTillSuccess(() => client.utp_connect(serverInfo.enr))
       serverConnections = await repeatTillSuccess(() => server.utp_get_connections())
 
       maybeServerConnectionKey1 = serverConnections.findServerConnection(clientInfo.nodeId, clientConnectionKey1.id)

fluffy/tools/utp_testing/utp_test_app.nim

@@ -42,7 +42,7 @@ type AppConf* = object
     defaultValue: defaultListenAddress
     desc: "RPC listening address"
     name: "rpc-listen-address" .}: ValidIpAddress
- 
+
 proc `%`*(value: enr.Record): JsonNode =
   newJString(value.toURI())
 
@@ -89,14 +89,14 @@ func toSKey(k: UtpSocketKey[NodeAddress]): SKey =
 
 proc installUtpHandlers(
   srv: RpcHttpServer,
-  d: protocol.Protocol, 
-  s: UtpDiscv5Protocol, 
+  d: protocol.Protocol,
+  s: UtpDiscv5Protocol,
   t: ref Table[SKey, UtpSocket[NodeAddress]]) {.raises: [Defect, CatchableError].} =
 
   srv.rpc("utp_connect") do(r: enr.Record) -> SKey:
     let
       nodeRes = newNode(r)
-      
+
     if nodeRes.isOk():
       let node = nodeRes.get()
       let nodeAddress = NodeAddress.init(node).unsafeGet()
@@ -131,7 +131,7 @@ proc installUtpHandlers(
 
     for k in t.keys:
       keys.add(k)
-    
+
     return keys
 
   srv.rpc("utp_read") do(k: SKey, n: int) -> string:
@@ -151,7 +151,7 @@ proc installUtpHandlers(
     else:
       raise newException(ValueError, "Socket with provided key is missing")
 
-proc buildAcceptConnection(t: ref Table[SKey, UtpSocket[NodeAddress]]): AcceptConnectionCallback[NodeAddress] = 
+proc buildAcceptConnection(t: ref Table[SKey, UtpSocket[NodeAddress]]): AcceptConnectionCallback[NodeAddress] =
   return (
     proc (server: UtpRouter[NodeAddress], client: UtpSocket[NodeAddress]): Future[void] =
       let fut = newFuture[void]()
@@ -170,7 +170,7 @@ when isMainModule:
   let conf = AppConf.load()
   {.push raises: [Defect].}
 
-  let 
+  let
     protName = "test-utp".toBytes()
     la = initTAddress(conf.rpcListenAddress, conf.rpcPort)
     key = PrivateKey.random(rng[])
@@ -189,7 +189,7 @@ when isMainModule:
 
   d.open()
 
-  let 
+  let
     cfg = SocketConfig.init(incomingSocketReceiveTimeout = none[Duration]())
     utp = UtpDiscv5Protocol.new(d, protName, buildAcceptConnection(table), socketConfig = cfg)