diff --git a/fluffy/README.md b/fluffy/README.md index eb3a0cfcd..0471e19ff 100644 --- a/fluffy/README.md +++ b/fluffy/README.md @@ -22,6 +22,9 @@ Current status of specifications can be found in the To keep up to date with changes and development progress, follow the [Nimbus blog](https://our.status.im/tag/nimbus/). +Monthly development updates are shared +[here](https://hackmd.io/jRpxY4WBQJ-hnsKaPDYqTw). + ## How to Build & Run ### Prerequisites @@ -36,8 +39,8 @@ make fluffy # See available command line options ./build/fluffy --help -# Example command: Run the client and connect to a bootnode. -./build/fluffy --log-level:debug --bootnode:enr: +# Example command: Run the client and connect to a bootstrap node. +./build/fluffy --bootstrap-node:enr: ``` ### Update and rebuild fluffy client @@ -56,6 +59,11 @@ make fluffy make fluffy-test ``` +### Run fluffy local testnet +```bash +./fluffy/scripts/launch_local_testnet.sh +``` + ### Windows support Follow the steps outlined [here](../README.md#windows) to build fluffy on Windows. @@ -80,6 +88,9 @@ can be found on the general nimbus-eth1 readme. The code follows the [Status Nim Style Guide](https://status-im.github.io/nim-style-guide/). +Detailed document showing commands to +[test client protocol interoperability](./docs/protocol_interop.md). + ## License Licensed and distributed under either of diff --git a/fluffy/conf.nim b/fluffy/conf.nim index b3471cd28..bb4bd30b8 100644 --- a/fluffy/conf.nim +++ b/fluffy/conf.nim @@ -82,11 +82,11 @@ type "This option allows to enable/disable this functionality" name: "enr-auto-update" .}: bool - nodeKey* {. - desc: "P2P node private key as hex", + networkKey* {. + desc: "Private key (secp256k1) for the p2p network, hex encoded. Safer keyfile support to be added.", defaultValue: PrivateKey.random(keys.newRng()[]) defaultValueDesc: "random" - name: "nodekey" .}: PrivateKey + name: "network-key-unsafe" .}: PrivateKey dataDir* {. desc: "The directory where fluffy will store the content data" @@ -116,7 +116,7 @@ type name: "rpc" }: bool rpcPort* {. - desc: "HTTP port for the JSON-RPC service" + desc: "HTTP port for the JSON-RPC server" defaultValue: 8545 name: "rpc-port" }: Port diff --git a/fluffy/docs/protocol_interop.md b/fluffy/docs/protocol_interop.md new file mode 100644 index 000000000..f837aed57 --- /dev/null +++ b/fluffy/docs/protocol_interop.md @@ -0,0 +1,111 @@ +# Testing Client Protocol Interoperability +This document shows some commands that can be used to test the individual +protocol messages per network (Discovery v5 and Portal networks). + +Two ways are explained, the first by keeping a node running and interacting +with it through the JSON-RPC service. The second by running cli applications +that attempt to send 1 specific message and then shutdown. + +The first is more powerful and complete, the second might be easier to do some +quick testing. + +## Run Fluffy and test protocol messages launched via JSON-RPC API + +First build Fluffy as explained [here](../README.md#build-fluffy-client). + +Next run it with the JSON-RPC server enabled: +```bash +./build/fluffy --rpc --bootstrap-node:enr: +``` + +### Testing Discovery v5 Layer +Testing the Discovery v5 protocol messages: + +```bash +# Ping / Pong +curl -s -X POST -H 'Content-Type: application/json' -d '{"jsonrpc":"2.0","id":"1","method":"discv5_ping","params":["enr:"]}' http://localhost:8545 | jq + +# FindNode / Nodes +# Extra parameter is an array of requested logarithmic distances +curl -s -X POST -H 'Content-Type: application/json' -d '{"jsonrpc":"2.0","id":"1","method":"discv5_findNode","params":["enr:", [254, 255, 256]]}' http://localhost:8545 | jq + +# TalkReq / TalkResp +# Extra parameters are the protocol id and the request byte string, hex encoded. +curl -s -X POST -H 'Content-Type: application/json' -d '{"jsonrpc":"2.0","id":"1","method":"discv5_talkReq","params":["enr:", "", ""]}' http://localhost:8545 | jq + +# Read out the discover v5 routing table contents +curl -s -X POST -H 'Content-Type: application/json' -d '{"jsonrpc":"2.0","id":"1","method":"discv5_routingTableInfo","params":[]}' http://localhost:8545 | jq +``` + +### Testing Portal Networks Layer +Testing the Portal wire protocol messages: + +```bash +# Ping / Pong +curl -s -X POST -H 'Content-Type: application/json' -d '{"jsonrpc":"2.0","id":"1","method":"portal_state_ping","params":["enr:"]}' http://localhost:8545 | jq + +# FindNode / Nodes +# Extra parameter is an array of requested logarithmic distances +curl -s -X POST -H 'Content-Type: application/json' -d '{"jsonrpc":"2.0","id":"1","method":"portal_state_findNodes","params":["enr:", [254, 255, 256]]}' http://localhost:8545 | jq + +# Read out the Portal state network routing table contents +curl -s -X POST -H 'Content-Type: application/json' -d '{"jsonrpc":"2.0","id":"1","method":"portal_state_routingTableInfo","params":[]}' http://localhost:8545 | jq +``` + +> The `portal_state_` prefix can be replaced for testing other networks such as +`portal_history_`. + +## Test Discovery and Portal Wire protocol messages through cli tools + +### Testing Discovery v5 Layer: dcli + +```bash +# Build dcli from nim-eth vendor module +(cd vendor/nim-eth/; ../../env.sh nimble build_dcli) +``` + +With the `dcli` tool you can test the individual Discovery v5 protocol messages, +e.g.: + +```bash +# Test Discovery Ping, should print the content of ping message +./vendor/nim-eth/eth/p2p/discoveryv5/dcli ping enr: + +# Test Discovery FindNode, should print the content of the returned ENRs +# Default a distance of 256 is requested, change this with --distance argument +./vendor/nim-eth/eth/p2p/discoveryv5/dcli findnode enr: + +# Test Discovery TalkReq, should print the TalkResp content +./vendor/nim-eth/eth/p2p/discoveryv5/dcli talkreq enr: +``` + +> Each `dcli` run will default generate a new network key and thus a new node id +and ENR. + +### Testing Portal Networks Layer: portalcli + +```bash +# Build portalcli +make fluffy-tools +``` + +With the `portalcli` tool you can test the individual Portal wire protocol +messages, e.g.: + +```bash +# Test Portal wire Ping, should print the content of ping message +./build/portalcli ping enr: + +# Test Portal wire FindNode, should print the content of the returned ENRs +# Default a distance of 256 is requested, change this with --distance argument +./build/portalcli findnodes enr: + +# Test Portal wire FindContent, should print the returned content +./build/portalcli findcontent enr: + +# Default the State network is tested, but you can provide another protocol id +./build/portalcli ping enr: --protocol-id:0x500B +``` + +> Each `portalcli` run will default generate a new network key and thus a new +node id and ENR. diff --git a/fluffy/fluffy.nim b/fluffy/fluffy.nim index 90131f8a7..b79a6db4e 100644 --- a/fluffy/fluffy.nim +++ b/fluffy/fluffy.nim @@ -53,7 +53,7 @@ proc run(config: PortalConf) {.raises: [CatchableError, Defect].} = bootstrapRecords.add(config.bootstrapNodes) let d = newProtocol( - config.nodeKey, + config.networkKey, extIp, none(Port), extUdpPort, bootstrapRecords = bootstrapRecords, bindIp = bindIp, bindPort = udpPort, diff --git a/fluffy/network/wire/README.md b/fluffy/network/wire/README.md index f7aa821fd..1878c57b8 100644 --- a/fluffy/network/wire/README.md +++ b/fluffy/network/wire/README.md @@ -44,11 +44,12 @@ git clone git@github.com:status-im/nimbus-eth1.git cd nimbus-eth1 # Build the fluffy tools -make tools-fluffy +make fluffy-tools # See all options ./build/portalcli --help # Example command: Ping another node ./build/portalcli ping enr: -# Example command: Run discovery + portal node -./build/portalcli --log-level:debug --bootnode:enr: +# Example command: Run a discovery + portal node +./build/portalcli --log-level:debug --bootstrap-node:enr: +``` diff --git a/fluffy/rpc/rpc_discovery_api.nim b/fluffy/rpc/rpc_discovery_api.nim index 9f1e83fc6..67e571bb9 100644 --- a/fluffy/rpc/rpc_discovery_api.nim +++ b/fluffy/rpc/rpc_discovery_api.nim @@ -93,7 +93,7 @@ proc installDiscoveryApiHandlers*(rpcServer: RpcServer|RpcProxy, recipientPort: p.port ) - rpcServer.rpc("discv5_findNodes") do( + rpcServer.rpc("discv5_findNode") do( enr: Record, distances: seq[uint16]) -> seq[Record]: let node = toNodeWithAddress(enr) @@ -103,7 +103,7 @@ proc installDiscoveryApiHandlers*(rpcServer: RpcServer|RpcProxy, else: return nodes.get().map(proc(n: Node): Record = n.record) - rpcServer.rpc("discv5_talk") do(enr: Record, protocol, payload: string) -> string: + rpcServer.rpc("discv5_talkReq") do(enr: Record, protocol, payload: string) -> string: let node = toNodeWithAddress(enr) talkresp = await d.talkreq( diff --git a/fluffy/rpc/rpc_portal_api.nim b/fluffy/rpc/rpc_portal_api.nim index 72de24e51..d69708c94 100644 --- a/fluffy/rpc/rpc_portal_api.nim +++ b/fluffy/rpc/rpc_portal_api.nim @@ -9,7 +9,7 @@ import std/sequtils, - json_rpc/[rpcproxy, rpcserver], + json_rpc/[rpcproxy, rpcserver], stew/byteutils, ../network/wire/portal_protocol, ./rpc_types @@ -41,6 +41,28 @@ proc installPortalApiHandlers*( else: raise newException(ValueError, "Record not found in DHT lookup.") + rpcServer.rpc("portal_" & network & "_ping") do( + enr: Record) -> tuple[seqNum: uint64, customPayload: string]: + let + node = toNodeWithAddress(enr) + pong = await p.ping(node) + + if pong.isErr(): + raise newException(ValueError, $pong.error) + else: + let p = pong.get() + return (p.enrSeq, p.customPayload.asSeq().toHex()) + + rpcServer.rpc("portal_" & network & "_findNodes") do( + enr: Record, distances: seq[uint16]) -> seq[Record]: + let + node = toNodeWithAddress(enr) + nodes = await p.findNodesVerified(node, distances) + if nodes.isErr(): + raise newException(ValueError, $nodes.error) + else: + return nodes.get().map(proc(n: Node): Record = n.record) + rpcServer.rpc("portal_" & network & "_recursiveFindNodes") do() -> seq[Record]: let discovered = await p.queryRandom() return discovered.map(proc(n: Node): Record = n.record) diff --git a/fluffy/tools/portalcli.nim b/fluffy/tools/portalcli.nim index 36f72e3ba..765a19d55 100644 --- a/fluffy/tools/portalcli.nim +++ b/fluffy/tools/portalcli.nim @@ -15,7 +15,7 @@ import eth/p2p/discoveryv5/protocol as discv5_protocol, ../common/common_utils, ../network/wire/[messages, portal_protocol], - ../network/state/state_content + ../network/state/[state_content, state_network] const defaultListenAddress* = (static ValidIpAddress.init("0.0.0.0")) @@ -75,11 +75,11 @@ type "This option allows to enable/disable this functionality" name: "enr-auto-update" .}: bool - nodeKey* {. - desc: "P2P node private key as hex", + networkKey* {. + desc: "Private key (secp256k1) for the p2p network, hex encoded.", defaultValue: PrivateKey.random(keys.newRng()[]) defaultValueDesc: "random" - name: "nodekey" .}: PrivateKey + name: "network-key" .}: PrivateKey metricsEnabled* {. defaultValue: false @@ -97,6 +97,11 @@ type desc: "Listening HTTP port of the metrics server" name: "metrics-port" .}: Port + protocolId* {. + defaultValue: stateProtocolId + desc: "Portal wire protocol id for the network to connect to" + name: "protocol-id" .}: PortalProtocolId + case cmd* {. command defaultValue: noCommand }: PortalCmd @@ -157,13 +162,23 @@ proc parseCmdArg*(T: type PrivateKey, p: TaintedString): T = proc completeCmdArg*(T: type PrivateKey, val: TaintedString): seq[string] = return @[] +proc parseCmdArg*(T: type PortalProtocolId, p: TaintedString): T = + try: + result = byteutils.hexToByteArray(string(p), 2) + except ValueError: + raise newException(ConfigurationError, + "Invalid protocol id, not a valid hex value") + +proc completeCmdArg*(T: type PortalProtocolId, val: TaintedString): seq[string] = + return @[] + proc discover(d: discv5_protocol.Protocol) {.async.} = while true: let discovered = await d.queryRandom() info "Lookup finished", nodes = discovered.len await sleepAsync(30.seconds) -proc testHandler(contentKey: state_content.ByteList): ContentResult = +proc testHandler(contentKey: ByteList): ContentResult = # Note: We don't incorperate storage in this tool so we always return # missing content. For now we are using the state network derivation but it # could be selective based on the network the tool is used for. @@ -181,14 +196,14 @@ proc run(config: PortalCliConf) = udpPort = Port(config.udpPort) # TODO: allow for no TCP port mapping! (extIp, _, extUdpPort) = setupAddress(config.nat, - config.listenAddress, udpPort, udpPort, "dcli") + config.listenAddress, udpPort, udpPort, "portalcli") var bootstrapRecords: seq[Record] loadBootstrapFile(string config.bootstrapNodesFile, bootstrapRecords) bootstrapRecords.add(config.bootstrapNodes) let d = newProtocol( - config.nodeKey, + config.networkKey, extIp, none(Port), extUdpPort, bootstrapRecords = bootstrapRecords, bindIp = bindIp, bindPort = udpPort, @@ -197,8 +212,7 @@ proc run(config: PortalCliConf) = d.open() - # TODO: Configurable protocol id - let portal = PortalProtocol.new(d, [byte 0x50, 0x0A], testHandler, + let portal = PortalProtocol.new(d, config.protocolId, testHandler, bootstrapRecords = bootstrapRecords) if config.metricsEnabled: