Rename Fluffy debug rpc methods (#2710)

* Rename Fluffy debug rpc methods.

* Organize portal json-rpc errors.

fluffy/docs/the_fluffy_book/docs/history-content-bridging.md

@@ -143,11 +143,11 @@ This will store blocks 1 to 10 into a json file located at
 `./user_data_dir/eth-history-data.json`.
 
 3. Run Fluffy and trigger the propagation of data with the
-`portal_history_propagate` JSON-RPC API call:
+`portal_debug_history_propagate` JSON-RPC API call:
 
 ```bash
 ./build/fluffy --rpc --rpc-api:portal,portal_debug
 
 # From another shell
-curl -s -X POST -H 'Content-Type: application/json' -d '{"jsonrpc":"2.0","id":"1","method":"portal_history_propagate","params":["./user_data_dir/eth-history-data.json"]}' http://localhost:8545 | jq
+curl -s -X POST -H 'Content-Type: application/json' -d '{"jsonrpc":"2.0","id":"1","method":"portal_debug_history_propagate","params":["./user_data_dir/eth-history-data.json"]}' http://localhost:8545 | jq
 ```

fluffy/rpc/rpc_calls/rpc_portal_debug_calls.nim

@@ -15,10 +15,13 @@ Opt[string].useDefaultSerializationIn JrpcConv
 
 createRpcSigsFromNim(RpcClient):
   ## Portal History Network json-rpc debug & custom calls
-  proc portal_historyGossipHeaders(era1File: string, epochRecordFile: Opt[string]): bool
-  proc portal_historyGossipHeaders(era1File: string): bool
-  proc portal_historyGossipBlockContent(era1File: string): bool
-  proc portal_history_storeContent(dataFile: string): bool
-  proc portal_history_propagate(dataFile: string): bool
-  proc portal_history_propagateHeaders(dataFile: string): bool
-  proc portal_history_propagateBlock(dataFile: string, blockHash: string): bool
+  proc portal_debug_historyGossipHeaders(
+    era1File: string, epochRecordFile: Opt[string]
+  ): bool
+
+  proc portal_debug_historyGossipHeaders(era1File: string): bool
+  proc portal_debug_historyGossipBlockContent(era1File: string): bool
+  proc portal_debug_history_storeContent(dataFile: string): bool
+  proc portal_debug_history_propagate(dataFile: string): bool
+  proc portal_debug_history_propagateHeaders(dataFile: string): bool
+  proc portal_debug_history_propagateBlock(dataFile: string, blockHash: string): bool

fluffy/rpc/rpc_portal_beacon_api.nim

@@ -23,14 +23,6 @@ export tables
 # Portal Network JSON-RPC implementation as per specification:
 # https://github.com/ethereum/portal-network-specs/tree/master/jsonrpc
 
-const
-  ContentNotFoundError = (code: -39001, msg: "Content not found")
-  ContentNotFoundErrorWithTrace = (code: -39002, msg: "Content not found")
-
-type ContentInfo = object
-  content: string
-  utpTransfer: bool
-
 ContentInfo.useDefaultSerializationIn JrpcConv
 TraceContentLookupResult.useDefaultSerializationIn JrpcConv
 TraceObject.useDefaultSerializationIn JrpcConv
@@ -38,12 +30,6 @@ NodeMetadata.useDefaultSerializationIn JrpcConv
 TraceResponse.useDefaultSerializationIn JrpcConv
 
 proc installPortalBeaconApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) =
-  let
-    invalidKeyErr =
-      (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key")
-    invalidValueErr =
-      (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content value")
-
   rpcServer.rpc("portal_beaconFindContent") do(
     enr: Record, contentKey: string
   ) -> JsonString:
@@ -96,12 +82,10 @@ proc installPortalBeaconApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) =
     let
       key = ContentKeyByteList.init(hexToSeqByte(contentKey))
       contentId = p.toContentId(key).valueOr:
-        raise (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key")
+        raise invalidKeyErr()
 
       contentResult = (await p.contentLookup(key, contentId)).valueOr:
-        raise (ref ApplicationError)(
-          code: ContentNotFoundError.code, msg: ContentNotFoundError.msg
-        )
+        raise contentNotFoundErr()
 
     return ContentInfo(
       content: contentResult.content.to0xHex(), utpTransfer: contentResult.utpTransfer
@@ -113,7 +97,7 @@ proc installPortalBeaconApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) =
     let
       key = ContentKeyByteList.init(hexToSeqByte(contentKey))
       contentId = p.toContentId(key).valueOr:
-        raise (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key")
+        raise invalidKeyErr()
 
       res = await p.traceContentLookup(key, contentId)
 
@@ -123,11 +107,7 @@ proc installPortalBeaconApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) =
       return res
     else:
       let data = Opt.some(JrpcConv.encode(res.trace).JsonString)
-      raise (ref ApplicationError)(
-        code: ContentNotFoundErrorWithTrace.code,
-        msg: ContentNotFoundErrorWithTrace.msg,
-        data: data,
-      )
+      raise contentNotFoundErrWithTrace(data)
 
   rpcServer.rpc("portal_beaconStore") do(
     contentKey: string, contentValue: string
@@ -141,18 +121,16 @@ proc installPortalBeaconApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) =
       p.storeContent(key, contentId.get(), contentValueBytes)
       return true
     else:
-      raise invalidKeyErr
+      raise invalidKeyErr()
 
   rpcServer.rpc("portal_beaconLocalContent") do(contentKey: string) -> string:
     let
       key = ContentKeyByteList.init(hexToSeqByte(contentKey))
       contentId = p.toContentId(key).valueOr:
-        raise (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key")
+        raise invalidKeyErr()
 
       contentResult = p.dbGet(key, contentId).valueOr:
-        raise (ref ApplicationError)(
-          code: ContentNotFoundError.code, msg: ContentNotFoundError.msg
-        )
+        raise contentNotFoundErr()
 
     return contentResult.to0xHex()
 

fluffy/rpc/rpc_portal_debug_history_api.nim

@@ -15,13 +15,11 @@ import
 
 export rpcserver
 
-# TODO: perhaps these endpoints should be named differently staring with "portal_debug_"?
-
 # Non-spec-RPCs that are used for testing, debugging and seeding data without a
 # bridge.
 proc installPortalDebugHistoryApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) =
   ## Portal debug API calls related to storage and seeding from Era1 files.
-  rpcServer.rpc("portal_historyGossipHeaders") do(
+  rpcServer.rpc("portal_debug_historyGossipHeaders") do(
     era1File: string, epochRecordFile: Opt[string]
   ) -> bool:
     let res = await p.historyGossipHeadersWithProof(era1File, epochRecordFile)
@@ -30,7 +28,7 @@ proc installPortalDebugHistoryApiHandlers*(rpcServer: RpcServer, p: PortalProtoc
     else:
       raise newException(ValueError, $res.error)
 
-  rpcServer.rpc("portal_historyGossipBlockContent") do(era1File: string) -> bool:
+  rpcServer.rpc("portal_debug_historyGossipBlockContent") do(era1File: string) -> bool:
     let res = await p.historyGossipBlockContent(era1File)
     if res.isOk():
       return true
@@ -39,28 +37,28 @@ proc installPortalDebugHistoryApiHandlers*(rpcServer: RpcServer, p: PortalProtoc
 
   ## Portal debug API calls related to storage and seeding
   ## TODO: To be removed/replaced with the Era1 versions where applicable.
-  rpcServer.rpc("portal_history_storeContent") do(dataFile: string) -> bool:
+  rpcServer.rpc("portal_debug_history_storeContent") do(dataFile: string) -> bool:
     let res = p.historyStore(dataFile)
     if res.isOk():
       return true
     else:
       raise newException(ValueError, $res.error)
 
-  rpcServer.rpc("portal_history_propagate") do(dataFile: string) -> bool:
+  rpcServer.rpc("portal_debug_history_propagate") do(dataFile: string) -> bool:
     let res = await p.historyPropagate(dataFile)
     if res.isOk():
       return true
     else:
       raise newException(ValueError, $res.error)
 
-  rpcServer.rpc("portal_history_propagateHeaders") do(dataDir: string) -> bool:
+  rpcServer.rpc("portal_debug_history_propagateHeaders") do(dataDir: string) -> bool:
     let res = await p.historyPropagateHeadersWithProof(dataDir)
     if res.isOk():
       return true
     else:
       raise newException(ValueError, $res.error)
 
-  rpcServer.rpc("portal_history_propagateHeaders") do(
+  rpcServer.rpc("portal_debug_history_propagateHeaders") do(
     epochHeadersFile: string, epochRecordFile: string
   ) -> bool:
     let res =
@@ -70,7 +68,7 @@ proc installPortalDebugHistoryApiHandlers*(rpcServer: RpcServer, p: PortalProtoc
     else:
       raise newException(ValueError, $res.error)
 
-  rpcServer.rpc("portal_history_propagateBlock") do(
+  rpcServer.rpc("portal_debug_history_propagateBlock") do(
     dataFile: string, blockHash: string
   ) -> bool:
     let res = await p.historyPropagateBlock(dataFile, blockHash)

fluffy/rpc/rpc_portal_history_api.nim

@@ -23,14 +23,6 @@ export tables
 # Portal Network JSON-RPC implementation as per specification:
 # https://github.com/ethereum/portal-network-specs/tree/master/jsonrpc
 
-const
-  ContentNotFoundError = (code: -39001, msg: "Content not found")
-  ContentNotFoundErrorWithTrace = (code: -39002, msg: "Content not found")
-
-type ContentInfo = object
-  content: string
-  utpTransfer: bool
-
 ContentInfo.useDefaultSerializationIn JrpcConv
 TraceContentLookupResult.useDefaultSerializationIn JrpcConv
 TraceObject.useDefaultSerializationIn JrpcConv
@@ -38,12 +30,6 @@ NodeMetadata.useDefaultSerializationIn JrpcConv
 TraceResponse.useDefaultSerializationIn JrpcConv
 
 proc installPortalHistoryApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) =
-  let
-    invalidKeyErr =
-      (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key")
-    invalidValueErr =
-      (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content value")
-
   rpcServer.rpc("portal_historyFindContent") do(
     enr: Record, contentKey: string
   ) -> JsonString:
@@ -96,12 +82,10 @@ proc installPortalHistoryApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) =
     let
       key = ContentKeyByteList.init(hexToSeqByte(contentKey))
       contentId = p.toContentId(key).valueOr:
-        raise (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key")
+        raise invalidKeyErr()
 
       contentResult = (await p.contentLookup(key, contentId)).valueOr:
-        raise (ref ApplicationError)(
-          code: ContentNotFoundError.code, msg: ContentNotFoundError.msg
-        )
+        raise contentNotFoundErr()
 
     return ContentInfo(
       content: contentResult.content.to0xHex(), utpTransfer: contentResult.utpTransfer
@@ -113,7 +97,7 @@ proc installPortalHistoryApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) =
     let
       key = ContentKeyByteList.init(hexToSeqByte(contentKey))
       contentId = p.toContentId(key).valueOr:
-        raise (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key")
+        raise invalidKeyErr()
 
       res = await p.traceContentLookup(key, contentId)
 
@@ -123,11 +107,7 @@ proc installPortalHistoryApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) =
       return res
     else:
       let data = Opt.some(JrpcConv.encode(res.trace).JsonString)
-      raise (ref ApplicationError)(
-        code: ContentNotFoundErrorWithTrace.code,
-        msg: ContentNotFoundErrorWithTrace.msg,
-        data: data,
-      )
+      raise contentNotFoundErrWithTrace(data)
 
   rpcServer.rpc("portal_historyStore") do(
     contentKey: string, contentValue: string
@@ -141,18 +121,16 @@ proc installPortalHistoryApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) =
       p.storeContent(key, contentId.get(), contentValueBytes)
       return true
     else:
-      raise invalidKeyErr
+      raise invalidKeyErr()
 
   rpcServer.rpc("portal_historyLocalContent") do(contentKey: string) -> string:
     let
       key = ContentKeyByteList.init(hexToSeqByte(contentKey))
       contentId = p.toContentId(key).valueOr:
-        raise (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key")
+        raise invalidKeyErr()
 
       contentResult = p.dbGet(key, contentId).valueOr:
-        raise (ref ApplicationError)(
-          code: ContentNotFoundError.code, msg: ContentNotFoundError.msg
-        )
+        raise contentNotFoundErr()
 
     return contentResult.to0xHex()
 

fluffy/rpc/rpc_portal_state_api.nim

@@ -24,14 +24,6 @@ export tables
 # Portal Network JSON-RPC implementation as per specification:
 # https://github.com/ethereum/portal-network-specs/tree/master/jsonrpc
 
-const
-  ContentNotFoundError = (code: -39001, msg: "Content not found")
-  ContentNotFoundErrorWithTrace = (code: -39002, msg: "Content not found")
-
-type ContentInfo = object
-  content: string
-  utpTransfer: bool
-
 ContentInfo.useDefaultSerializationIn JrpcConv
 TraceContentLookupResult.useDefaultSerializationIn JrpcConv
 TraceObject.useDefaultSerializationIn JrpcConv
@@ -39,12 +31,6 @@ NodeMetadata.useDefaultSerializationIn JrpcConv
 TraceResponse.useDefaultSerializationIn JrpcConv
 
 proc installPortalStateApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) =
-  let
-    invalidKeyErr =
-      (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key")
-    invalidValueErr =
-      (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content value")
-
   rpcServer.rpc("portal_stateFindContent") do(
     enr: Record, contentKey: string
   ) -> JsonString:
@@ -97,12 +83,10 @@ proc installPortalStateApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) =
     let
       key = ContentKeyByteList.init(hexToSeqByte(contentKey))
       contentId = p.toContentId(key).valueOr:
-        raise (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key")
+        raise invalidKeyErr()
 
       contentResult = (await p.contentLookup(key, contentId)).valueOr:
-        raise (ref ApplicationError)(
-          code: ContentNotFoundError.code, msg: ContentNotFoundError.msg
-        )
+        raise contentNotFoundErr()
 
     return ContentInfo(
       content: contentResult.content.to0xHex(), utpTransfer: contentResult.utpTransfer
@@ -114,7 +98,7 @@ proc installPortalStateApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) =
     let
       key = ContentKeyByteList.init(hexToSeqByte(contentKey))
       contentId = p.toContentId(key).valueOr:
-        raise (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key")
+        raise invalidKeyErr()
 
       res = await p.traceContentLookup(key, contentId)
 
@@ -124,11 +108,7 @@ proc installPortalStateApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) =
       return res
     else:
       let data = Opt.some(JrpcConv.encode(res.trace).JsonString)
-      raise (ref ApplicationError)(
-        code: ContentNotFoundErrorWithTrace.code,
-        msg: ContentNotFoundErrorWithTrace.msg,
-        data: data,
-      )
+      raise contentNotFoundErrWithTrace(data)
 
   rpcServer.rpc("portal_stateStore") do(
     contentKey: string, contentValue: string
@@ -137,11 +117,11 @@ proc installPortalStateApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) =
       key = ContentKeyByteList.init(hexToSeqByte(contentKey))
       contentValueBytes = hexToSeqByte(contentValue)
       decodedKey = ContentKey.decode(key).valueOr:
-        raise invalidKeyErr
+        raise invalidKeyErr()
       valueToStore =
         case decodedKey.contentType
         of unused:
-          raise invalidKeyErr
+          raise invalidKeyErr()
         of accountTrieNode:
           let offerValue = AccountTrieNodeOffer.decode(contentValueBytes).valueOr:
             raise invalidValueErr
@@ -160,18 +140,16 @@ proc installPortalStateApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) =
       p.storeContent(key, contentId.get(), valueToStore)
       return true
     else:
-      raise invalidKeyErr
+      raise invalidKeyErr()
 
   rpcServer.rpc("portal_stateLocalContent") do(contentKey: string) -> string:
     let
       key = ContentKeyByteList.init(hexToSeqByte(contentKey))
       contentId = p.toContentId(key).valueOr:
-        raise (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key")
+        raise invalidKeyErr()
 
       contentResult = p.dbGet(key, contentId).valueOr:
-        raise (ref ApplicationError)(
-          code: ContentNotFoundError.code, msg: ContentNotFoundError.msg
-        )
+        raise contentNotFoundErr()
 
     return contentResult.to0xHex()
 

fluffy/rpc/rpc_types.nim

@@ -9,13 +9,42 @@
 
 import
   stint,
-  json_rpc/jsonmarshal,
-  stew/[byteutils],
+  json_rpc/[jsonmarshal, errors],
+  stew/byteutils,
   results,
   eth/p2p/discoveryv5/[routing_table, enr, node]
 
 export jsonmarshal, routing_table, enr, node
 
+# Portal Network JSON-RPC errors
+const
+  # These errors are defined in the portal jsonrpc spec: https://github.com/ethereum/portal-network-specs/tree/master/jsonrpc
+  ContentNotFoundError* = (code: -39001, msg: "Content not found")
+  ContentNotFoundErrorWithTrace* = (code: -39002, msg: "Content not found")
+  # These errors are used by Fluffy but are not yet in the spec
+  InvalidContentKeyError* = (code: -32602, msg: "Invalid content key")
+  InvalidContentValueError* = (code: -32602, msg: "Invalid content value")
+
+template contentNotFoundErr*(): auto =
+  (ref ApplicationError)(code: ContentNotFoundError.code, msg: ContentNotFoundError.msg)
+
+template contentNotFoundErrWithTrace*(data: typed): auto =
+  (ref ApplicationError)(
+    code: ContentNotFoundErrorWithTrace.code,
+    msg: ContentNotFoundErrorWithTrace.msg,
+    data: data,
+  )
+
+template invalidKeyErr*(): auto =
+  (ref errors.InvalidRequest)(
+    code: InvalidContentKeyError.code, msg: InvalidContentKeyError.msg
+  )
+
+template invalidValueErr*(): auto =
+  (ref errors.InvalidRequest)(
+    code: InvalidContentValueError.code, msg: InvalidContentValueError.msg
+  )
+
 type
   NodeInfo* = object
     enr*: Record

fluffy/scripts/test_portal_testnet.nim

@@ -267,7 +267,7 @@ procSuite "Portal testnet tests":
 
     # This will fill the first node its db with blocks from the data file. Next,
     # this node wil offer all these blocks their headers one by one.
-    check (await clients[0].portal_history_propagate(blockDataFile))
+    check (await clients[0].portal_debug_history_propagate(blockDataFile))
     await clients[0].close()
 
     for i, client in clients:

fluffy/tools/portal_bridge/portal_bridge_history.nim

@@ -363,18 +363,19 @@ proc runBackfillLoop(
       info "Gossip headers from era1 file", eraFile
       let headerRes =
         try:
-          await portalClient.portal_historyGossipHeaders(eraFile)
+          await portalClient.portal_debug_historyGossipHeaders(eraFile)
         except CatchableError as e:
-          error "JSON-RPC portal_historyGossipHeaders failed", error = e.msg
+          error "JSON-RPC portal_debug_historyGossipHeaders failed", error = e.msg
           false
 
       if headerRes:
         info "Gossip block content from era1 file", eraFile
         let res =
           try:
-            await portalClient.portal_historyGossipBlockContent(eraFile)
+            await portalClient.portal_debug_historyGossipBlockContent(eraFile)
           except CatchableError as e:
-            error "JSON-RPC portal_historyGossipBlockContent failed", error = e.msg
+            error "JSON-RPC portal_debug_historyGossipBlockContent failed",
+              error = e.msg
             false
         if res:
           error "Failed to gossip block content from era1 file", eraFile