diff --git a/fluffy/docs/the_fluffy_book/docs/history-content-bridging.md b/fluffy/docs/the_fluffy_book/docs/history-content-bridging.md index a180bdf32..f0248633b 100644 --- a/fluffy/docs/the_fluffy_book/docs/history-content-bridging.md +++ b/fluffy/docs/the_fluffy_book/docs/history-content-bridging.md @@ -143,11 +143,11 @@ This will store blocks 1 to 10 into a json file located at `./user_data_dir/eth-history-data.json`. 3. Run Fluffy and trigger the propagation of data with the -`portal_history_propagate` JSON-RPC API call: +`portal_debug_history_propagate` JSON-RPC API call: ```bash ./build/fluffy --rpc --rpc-api:portal,portal_debug # From another shell -curl -s -X POST -H 'Content-Type: application/json' -d '{"jsonrpc":"2.0","id":"1","method":"portal_history_propagate","params":["./user_data_dir/eth-history-data.json"]}' http://localhost:8545 | jq +curl -s -X POST -H 'Content-Type: application/json' -d '{"jsonrpc":"2.0","id":"1","method":"portal_debug_history_propagate","params":["./user_data_dir/eth-history-data.json"]}' http://localhost:8545 | jq ``` diff --git a/fluffy/rpc/rpc_calls/rpc_portal_debug_calls.nim b/fluffy/rpc/rpc_calls/rpc_portal_debug_calls.nim index db93c3a27..282c903c0 100644 --- a/fluffy/rpc/rpc_calls/rpc_portal_debug_calls.nim +++ b/fluffy/rpc/rpc_calls/rpc_portal_debug_calls.nim @@ -15,10 +15,13 @@ Opt[string].useDefaultSerializationIn JrpcConv createRpcSigsFromNim(RpcClient): ## Portal History Network json-rpc debug & custom calls - proc portal_historyGossipHeaders(era1File: string, epochRecordFile: Opt[string]): bool - proc portal_historyGossipHeaders(era1File: string): bool - proc portal_historyGossipBlockContent(era1File: string): bool - proc portal_history_storeContent(dataFile: string): bool - proc portal_history_propagate(dataFile: string): bool - proc portal_history_propagateHeaders(dataFile: string): bool - proc portal_history_propagateBlock(dataFile: string, blockHash: string): bool + proc portal_debug_historyGossipHeaders( + era1File: string, epochRecordFile: Opt[string] + ): bool + + proc portal_debug_historyGossipHeaders(era1File: string): bool + proc portal_debug_historyGossipBlockContent(era1File: string): bool + proc portal_debug_history_storeContent(dataFile: string): bool + proc portal_debug_history_propagate(dataFile: string): bool + proc portal_debug_history_propagateHeaders(dataFile: string): bool + proc portal_debug_history_propagateBlock(dataFile: string, blockHash: string): bool diff --git a/fluffy/rpc/rpc_portal_beacon_api.nim b/fluffy/rpc/rpc_portal_beacon_api.nim index 8654299f1..32714f3e6 100644 --- a/fluffy/rpc/rpc_portal_beacon_api.nim +++ b/fluffy/rpc/rpc_portal_beacon_api.nim @@ -23,14 +23,6 @@ export tables # Portal Network JSON-RPC implementation as per specification: # https://github.com/ethereum/portal-network-specs/tree/master/jsonrpc -const - ContentNotFoundError = (code: -39001, msg: "Content not found") - ContentNotFoundErrorWithTrace = (code: -39002, msg: "Content not found") - -type ContentInfo = object - content: string - utpTransfer: bool - ContentInfo.useDefaultSerializationIn JrpcConv TraceContentLookupResult.useDefaultSerializationIn JrpcConv TraceObject.useDefaultSerializationIn JrpcConv @@ -38,12 +30,6 @@ NodeMetadata.useDefaultSerializationIn JrpcConv TraceResponse.useDefaultSerializationIn JrpcConv proc installPortalBeaconApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) = - let - invalidKeyErr = - (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key") - invalidValueErr = - (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content value") - rpcServer.rpc("portal_beaconFindContent") do( enr: Record, contentKey: string ) -> JsonString: @@ -96,12 +82,10 @@ proc installPortalBeaconApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) = let key = ContentKeyByteList.init(hexToSeqByte(contentKey)) contentId = p.toContentId(key).valueOr: - raise (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key") + raise invalidKeyErr() contentResult = (await p.contentLookup(key, contentId)).valueOr: - raise (ref ApplicationError)( - code: ContentNotFoundError.code, msg: ContentNotFoundError.msg - ) + raise contentNotFoundErr() return ContentInfo( content: contentResult.content.to0xHex(), utpTransfer: contentResult.utpTransfer @@ -113,7 +97,7 @@ proc installPortalBeaconApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) = let key = ContentKeyByteList.init(hexToSeqByte(contentKey)) contentId = p.toContentId(key).valueOr: - raise (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key") + raise invalidKeyErr() res = await p.traceContentLookup(key, contentId) @@ -123,11 +107,7 @@ proc installPortalBeaconApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) = return res else: let data = Opt.some(JrpcConv.encode(res.trace).JsonString) - raise (ref ApplicationError)( - code: ContentNotFoundErrorWithTrace.code, - msg: ContentNotFoundErrorWithTrace.msg, - data: data, - ) + raise contentNotFoundErrWithTrace(data) rpcServer.rpc("portal_beaconStore") do( contentKey: string, contentValue: string @@ -141,18 +121,16 @@ proc installPortalBeaconApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) = p.storeContent(key, contentId.get(), contentValueBytes) return true else: - raise invalidKeyErr + raise invalidKeyErr() rpcServer.rpc("portal_beaconLocalContent") do(contentKey: string) -> string: let key = ContentKeyByteList.init(hexToSeqByte(contentKey)) contentId = p.toContentId(key).valueOr: - raise (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key") + raise invalidKeyErr() contentResult = p.dbGet(key, contentId).valueOr: - raise (ref ApplicationError)( - code: ContentNotFoundError.code, msg: ContentNotFoundError.msg - ) + raise contentNotFoundErr() return contentResult.to0xHex() diff --git a/fluffy/rpc/rpc_portal_debug_history_api.nim b/fluffy/rpc/rpc_portal_debug_history_api.nim index dbfbaa1fb..368b0d781 100644 --- a/fluffy/rpc/rpc_portal_debug_history_api.nim +++ b/fluffy/rpc/rpc_portal_debug_history_api.nim @@ -15,13 +15,11 @@ import export rpcserver -# TODO: perhaps these endpoints should be named differently staring with "portal_debug_"? - # Non-spec-RPCs that are used for testing, debugging and seeding data without a # bridge. proc installPortalDebugHistoryApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) = ## Portal debug API calls related to storage and seeding from Era1 files. - rpcServer.rpc("portal_historyGossipHeaders") do( + rpcServer.rpc("portal_debug_historyGossipHeaders") do( era1File: string, epochRecordFile: Opt[string] ) -> bool: let res = await p.historyGossipHeadersWithProof(era1File, epochRecordFile) @@ -30,7 +28,7 @@ proc installPortalDebugHistoryApiHandlers*(rpcServer: RpcServer, p: PortalProtoc else: raise newException(ValueError, $res.error) - rpcServer.rpc("portal_historyGossipBlockContent") do(era1File: string) -> bool: + rpcServer.rpc("portal_debug_historyGossipBlockContent") do(era1File: string) -> bool: let res = await p.historyGossipBlockContent(era1File) if res.isOk(): return true @@ -39,28 +37,28 @@ proc installPortalDebugHistoryApiHandlers*(rpcServer: RpcServer, p: PortalProtoc ## Portal debug API calls related to storage and seeding ## TODO: To be removed/replaced with the Era1 versions where applicable. - rpcServer.rpc("portal_history_storeContent") do(dataFile: string) -> bool: + rpcServer.rpc("portal_debug_history_storeContent") do(dataFile: string) -> bool: let res = p.historyStore(dataFile) if res.isOk(): return true else: raise newException(ValueError, $res.error) - rpcServer.rpc("portal_history_propagate") do(dataFile: string) -> bool: + rpcServer.rpc("portal_debug_history_propagate") do(dataFile: string) -> bool: let res = await p.historyPropagate(dataFile) if res.isOk(): return true else: raise newException(ValueError, $res.error) - rpcServer.rpc("portal_history_propagateHeaders") do(dataDir: string) -> bool: + rpcServer.rpc("portal_debug_history_propagateHeaders") do(dataDir: string) -> bool: let res = await p.historyPropagateHeadersWithProof(dataDir) if res.isOk(): return true else: raise newException(ValueError, $res.error) - rpcServer.rpc("portal_history_propagateHeaders") do( + rpcServer.rpc("portal_debug_history_propagateHeaders") do( epochHeadersFile: string, epochRecordFile: string ) -> bool: let res = @@ -70,7 +68,7 @@ proc installPortalDebugHistoryApiHandlers*(rpcServer: RpcServer, p: PortalProtoc else: raise newException(ValueError, $res.error) - rpcServer.rpc("portal_history_propagateBlock") do( + rpcServer.rpc("portal_debug_history_propagateBlock") do( dataFile: string, blockHash: string ) -> bool: let res = await p.historyPropagateBlock(dataFile, blockHash) diff --git a/fluffy/rpc/rpc_portal_history_api.nim b/fluffy/rpc/rpc_portal_history_api.nim index e17a0ee94..bcfbbb2ca 100644 --- a/fluffy/rpc/rpc_portal_history_api.nim +++ b/fluffy/rpc/rpc_portal_history_api.nim @@ -23,14 +23,6 @@ export tables # Portal Network JSON-RPC implementation as per specification: # https://github.com/ethereum/portal-network-specs/tree/master/jsonrpc -const - ContentNotFoundError = (code: -39001, msg: "Content not found") - ContentNotFoundErrorWithTrace = (code: -39002, msg: "Content not found") - -type ContentInfo = object - content: string - utpTransfer: bool - ContentInfo.useDefaultSerializationIn JrpcConv TraceContentLookupResult.useDefaultSerializationIn JrpcConv TraceObject.useDefaultSerializationIn JrpcConv @@ -38,12 +30,6 @@ NodeMetadata.useDefaultSerializationIn JrpcConv TraceResponse.useDefaultSerializationIn JrpcConv proc installPortalHistoryApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) = - let - invalidKeyErr = - (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key") - invalidValueErr = - (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content value") - rpcServer.rpc("portal_historyFindContent") do( enr: Record, contentKey: string ) -> JsonString: @@ -96,12 +82,10 @@ proc installPortalHistoryApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) = let key = ContentKeyByteList.init(hexToSeqByte(contentKey)) contentId = p.toContentId(key).valueOr: - raise (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key") + raise invalidKeyErr() contentResult = (await p.contentLookup(key, contentId)).valueOr: - raise (ref ApplicationError)( - code: ContentNotFoundError.code, msg: ContentNotFoundError.msg - ) + raise contentNotFoundErr() return ContentInfo( content: contentResult.content.to0xHex(), utpTransfer: contentResult.utpTransfer @@ -113,7 +97,7 @@ proc installPortalHistoryApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) = let key = ContentKeyByteList.init(hexToSeqByte(contentKey)) contentId = p.toContentId(key).valueOr: - raise (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key") + raise invalidKeyErr() res = await p.traceContentLookup(key, contentId) @@ -123,11 +107,7 @@ proc installPortalHistoryApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) = return res else: let data = Opt.some(JrpcConv.encode(res.trace).JsonString) - raise (ref ApplicationError)( - code: ContentNotFoundErrorWithTrace.code, - msg: ContentNotFoundErrorWithTrace.msg, - data: data, - ) + raise contentNotFoundErrWithTrace(data) rpcServer.rpc("portal_historyStore") do( contentKey: string, contentValue: string @@ -141,18 +121,16 @@ proc installPortalHistoryApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) = p.storeContent(key, contentId.get(), contentValueBytes) return true else: - raise invalidKeyErr + raise invalidKeyErr() rpcServer.rpc("portal_historyLocalContent") do(contentKey: string) -> string: let key = ContentKeyByteList.init(hexToSeqByte(contentKey)) contentId = p.toContentId(key).valueOr: - raise (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key") + raise invalidKeyErr() contentResult = p.dbGet(key, contentId).valueOr: - raise (ref ApplicationError)( - code: ContentNotFoundError.code, msg: ContentNotFoundError.msg - ) + raise contentNotFoundErr() return contentResult.to0xHex() diff --git a/fluffy/rpc/rpc_portal_state_api.nim b/fluffy/rpc/rpc_portal_state_api.nim index 9fc76ea7d..3a42ea376 100644 --- a/fluffy/rpc/rpc_portal_state_api.nim +++ b/fluffy/rpc/rpc_portal_state_api.nim @@ -24,14 +24,6 @@ export tables # Portal Network JSON-RPC implementation as per specification: # https://github.com/ethereum/portal-network-specs/tree/master/jsonrpc -const - ContentNotFoundError = (code: -39001, msg: "Content not found") - ContentNotFoundErrorWithTrace = (code: -39002, msg: "Content not found") - -type ContentInfo = object - content: string - utpTransfer: bool - ContentInfo.useDefaultSerializationIn JrpcConv TraceContentLookupResult.useDefaultSerializationIn JrpcConv TraceObject.useDefaultSerializationIn JrpcConv @@ -39,12 +31,6 @@ NodeMetadata.useDefaultSerializationIn JrpcConv TraceResponse.useDefaultSerializationIn JrpcConv proc installPortalStateApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) = - let - invalidKeyErr = - (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key") - invalidValueErr = - (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content value") - rpcServer.rpc("portal_stateFindContent") do( enr: Record, contentKey: string ) -> JsonString: @@ -97,12 +83,10 @@ proc installPortalStateApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) = let key = ContentKeyByteList.init(hexToSeqByte(contentKey)) contentId = p.toContentId(key).valueOr: - raise (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key") + raise invalidKeyErr() contentResult = (await p.contentLookup(key, contentId)).valueOr: - raise (ref ApplicationError)( - code: ContentNotFoundError.code, msg: ContentNotFoundError.msg - ) + raise contentNotFoundErr() return ContentInfo( content: contentResult.content.to0xHex(), utpTransfer: contentResult.utpTransfer @@ -114,7 +98,7 @@ proc installPortalStateApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) = let key = ContentKeyByteList.init(hexToSeqByte(contentKey)) contentId = p.toContentId(key).valueOr: - raise (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key") + raise invalidKeyErr() res = await p.traceContentLookup(key, contentId) @@ -124,11 +108,7 @@ proc installPortalStateApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) = return res else: let data = Opt.some(JrpcConv.encode(res.trace).JsonString) - raise (ref ApplicationError)( - code: ContentNotFoundErrorWithTrace.code, - msg: ContentNotFoundErrorWithTrace.msg, - data: data, - ) + raise contentNotFoundErrWithTrace(data) rpcServer.rpc("portal_stateStore") do( contentKey: string, contentValue: string @@ -137,11 +117,11 @@ proc installPortalStateApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) = key = ContentKeyByteList.init(hexToSeqByte(contentKey)) contentValueBytes = hexToSeqByte(contentValue) decodedKey = ContentKey.decode(key).valueOr: - raise invalidKeyErr + raise invalidKeyErr() valueToStore = case decodedKey.contentType of unused: - raise invalidKeyErr + raise invalidKeyErr() of accountTrieNode: let offerValue = AccountTrieNodeOffer.decode(contentValueBytes).valueOr: raise invalidValueErr @@ -160,18 +140,16 @@ proc installPortalStateApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) = p.storeContent(key, contentId.get(), valueToStore) return true else: - raise invalidKeyErr + raise invalidKeyErr() rpcServer.rpc("portal_stateLocalContent") do(contentKey: string) -> string: let key = ContentKeyByteList.init(hexToSeqByte(contentKey)) contentId = p.toContentId(key).valueOr: - raise (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key") + raise invalidKeyErr() contentResult = p.dbGet(key, contentId).valueOr: - raise (ref ApplicationError)( - code: ContentNotFoundError.code, msg: ContentNotFoundError.msg - ) + raise contentNotFoundErr() return contentResult.to0xHex() diff --git a/fluffy/rpc/rpc_types.nim b/fluffy/rpc/rpc_types.nim index ff99ec8c2..7bc682bb7 100644 --- a/fluffy/rpc/rpc_types.nim +++ b/fluffy/rpc/rpc_types.nim @@ -9,13 +9,42 @@ import stint, - json_rpc/jsonmarshal, - stew/[byteutils], + json_rpc/[jsonmarshal, errors], + stew/byteutils, results, eth/p2p/discoveryv5/[routing_table, enr, node] export jsonmarshal, routing_table, enr, node +# Portal Network JSON-RPC errors +const + # These errors are defined in the portal jsonrpc spec: https://github.com/ethereum/portal-network-specs/tree/master/jsonrpc + ContentNotFoundError* = (code: -39001, msg: "Content not found") + ContentNotFoundErrorWithTrace* = (code: -39002, msg: "Content not found") + # These errors are used by Fluffy but are not yet in the spec + InvalidContentKeyError* = (code: -32602, msg: "Invalid content key") + InvalidContentValueError* = (code: -32602, msg: "Invalid content value") + +template contentNotFoundErr*(): auto = + (ref ApplicationError)(code: ContentNotFoundError.code, msg: ContentNotFoundError.msg) + +template contentNotFoundErrWithTrace*(data: typed): auto = + (ref ApplicationError)( + code: ContentNotFoundErrorWithTrace.code, + msg: ContentNotFoundErrorWithTrace.msg, + data: data, + ) + +template invalidKeyErr*(): auto = + (ref errors.InvalidRequest)( + code: InvalidContentKeyError.code, msg: InvalidContentKeyError.msg + ) + +template invalidValueErr*(): auto = + (ref errors.InvalidRequest)( + code: InvalidContentValueError.code, msg: InvalidContentValueError.msg + ) + type NodeInfo* = object enr*: Record diff --git a/fluffy/scripts/test_portal_testnet.nim b/fluffy/scripts/test_portal_testnet.nim index 449696f87..d5d5addc2 100644 --- a/fluffy/scripts/test_portal_testnet.nim +++ b/fluffy/scripts/test_portal_testnet.nim @@ -267,7 +267,7 @@ procSuite "Portal testnet tests": # This will fill the first node its db with blocks from the data file. Next, # this node wil offer all these blocks their headers one by one. - check (await clients[0].portal_history_propagate(blockDataFile)) + check (await clients[0].portal_debug_history_propagate(blockDataFile)) await clients[0].close() for i, client in clients: diff --git a/fluffy/tools/portal_bridge/portal_bridge_history.nim b/fluffy/tools/portal_bridge/portal_bridge_history.nim index fed19bc1d..0473a7db2 100644 --- a/fluffy/tools/portal_bridge/portal_bridge_history.nim +++ b/fluffy/tools/portal_bridge/portal_bridge_history.nim @@ -363,18 +363,19 @@ proc runBackfillLoop( info "Gossip headers from era1 file", eraFile let headerRes = try: - await portalClient.portal_historyGossipHeaders(eraFile) + await portalClient.portal_debug_historyGossipHeaders(eraFile) except CatchableError as e: - error "JSON-RPC portal_historyGossipHeaders failed", error = e.msg + error "JSON-RPC portal_debug_historyGossipHeaders failed", error = e.msg false if headerRes: info "Gossip block content from era1 file", eraFile let res = try: - await portalClient.portal_historyGossipBlockContent(eraFile) + await portalClient.portal_debug_historyGossipBlockContent(eraFile) except CatchableError as e: - error "JSON-RPC portal_historyGossipBlockContent failed", error = e.msg + error "JSON-RPC portal_debug_historyGossipBlockContent failed", + error = e.msg false if res: error "Failed to gossip block content from era1 file", eraFile