Initial implementation of a JSON-RPC service

beacon_chain/beacon_node.nim

@@ -1,13 +1,13 @@
 import
   # Standard library
-  os, net, tables, random, strutils, times, sequtils,
+  os, tables, random, strutils, times, sequtils,
 
   # Nimble packages
   stew/[objects, bitseqs, byteutils],
-  chronos, chronicles, confutils, metrics,
-  json_serialization/std/[options, sets], serialization/errors,
+  chronos, chronicles, confutils, metrics, json_rpc/[rpcserver, jsonmarshal],
+  json_serialization/std/[options, sets, net], serialization/errors,
   kvstore, kvstore_sqlite3,
-  eth/p2p/enode, eth/[keys, async_utils], eth/p2p/discoveryv5/enr,
+  eth/p2p/enode, eth/[keys, async_utils], eth/p2p/discoveryv5/[protocol, enr],
 
   # Local modules
   spec/[datatypes, digest, crypto, beaconstate, helpers, validator, network,
@@ -24,6 +24,10 @@ const
 
 type
   KeyPair = eth2_network.KeyPair
+  RpcServer = RpcHttpServer
+
+template init(T: type RpcHttpServer, ip: IpAddress, port: Port): T =
+  newRpcHttpServer([initTAddress(ip, port)])
 
 # https://github.com/ethereum/eth2.0-metrics/blob/master/metrics.md#interop-metrics
 declareGauge beacon_slot,
@@ -61,6 +65,7 @@ type
     attestationPool: AttestationPool
     mainchainMonitor: MainchainMonitor
     beaconClock: BeaconClock
+    rpcServer: RpcServer
 
 proc onBeaconBlock*(node: BeaconNode, signedBlock: SignedBeaconBlock) {.gcsafe.}
 proc updateHead(node: BeaconNode): BlockRef
@@ -203,6 +208,11 @@ proc init*(T: type BeaconNode, conf: BeaconNodeConf): Future[BeaconNode] {.async
     addressFile = string(conf.dataDir) / "beacon_node.address"
   network.saveConnectionAddressFile(addressFile)
 
+  let rpcServer = if conf.rpcEnabled:
+    RpcServer.init(conf.rpcAddress, conf.rpcPort)
+  else:
+    nil
+
   var res = BeaconNode(
     nickname: nickname,
     network: network,
@@ -218,6 +228,7 @@ proc init*(T: type BeaconNode, conf: BeaconNodeConf): Future[BeaconNode] {.async
     attestationPool: AttestationPool.init(blockPool),
     mainchainMonitor: mainchainMonitor,
     beaconClock: BeaconClock.init(blockPool.headState.data.data),
+    rpcServer: rpcServer,
   )
 
   # TODO sync is called when a remote peer is connected - is that the right
@@ -845,7 +856,107 @@ proc onSecond(node: BeaconNode, moment: Moment) {.async.} =
   addTimer(nextSecond) do (p: pointer):
     asyncCheck node.onSecond(nextSecond)
 
+# TODO: Should we move these to other modules?
+# This would require moving around other type definitions
+proc installValidatorApiHandlers(rpcServer: RpcServer, node: BeaconNode) =
+  discard
+
+func slotOrZero(time: BeaconTime): Slot =
+  let exSlot = time.toSlot
+  if exSlot.afterGenesis: exSlot.slot
+  else: Slot(0)
+
+func currentSlot(node: BeaconNode): Slot =
+  node.beaconClock.now.slotOrZero
+
+proc connectedPeersCount(node: BeaconNode): int =
+  libp2p_peers.value.int
+
+proc fromJson(n: JsonNode; argName: string; result: var Slot) =
+  var i: int
+  fromJson(n, argName, i)
+  result = Slot(i)
+
+proc installBeaconApiHandlers(rpcServer: RpcServer, node: BeaconNode) =
+  rpcServer.rpc("getBeaconHead") do () -> Slot:
+    return node.currentSlot
+
+  template requireOneOf(x, y: distinct Option) =
+    if x.isNone xor y.isNone:
+      raise newException(CatchableError,
+       "Please specify one of " & astToStr(x) & " or " & astToStr(y))
+
+  template jsonResult(x: auto): auto =
+    # TODO, yes this is silly, but teching json-rpc about
+    # all beacon node types will require quite a lot of work.
+    # A minor refactoring in json-rpc can solve this. We need
+    # to allow the handlers to return raw/literal json strings.
+    parseJson(Json.encode(x))
+
+  rpcServer.rpc("getBeaconBlock") do (slot: Option[Slot],
+                                      root: Option[Eth2Digest]) -> JsonNode:
+    requireOneOf(slot, root)
+    var blockHash: Eth2Digest
+    if root.isSome:
+      blockHash = root.get
+    else:
+      let foundRef = node.blockPool.getBlockByPreciseSlot(slot.get)
+      if foundRef.isSome:
+        blockHash = foundRef.get.root
+      else:
+        return newJNull()
+
+    let dbBlock = node.db.getBlock(blockHash)
+    if dbBlock.isSome:
+      return jsonResult(dbBlock.get)
+    else:
+      return newJNull()
+
+  rpcServer.rpc("getBeaconState") do (slot: Option[Slot],
+                                      root: Option[Eth2Digest]) -> JsonNode:
+    requireOneOf(slot, root)
+    if slot.isSome:
+      let blk = node.blockPool.head.blck.atSlot(slot.get)
+      var tmpState: StateData
+      node.blockPool.withState(tmpState, blk):
+        return jsonResult(state)
+    else:
+      let state = node.db.getState(root.get)
+      if state.isSome:
+        return jsonResult(state.get)
+      else:
+        return newJNull()
+
+  rpcServer.rpc("getNetworkPeerId") do () -> string:
+    when networkBackend != libp2p:
+      raise newException(CatchableError, "Unsupported operation")
+    else:
+      return $publicKey(node.network)
+
+  rpcServer.rpc("getNetworkPeers") do () -> seq[string]:
+    when networkBackend != libp2p:
+      if true:
+        raise newException(CatchableError, "Unsupported operation")
+
+    for peerId, peer in node.network.peerPool:
+      result.add $peerId
+
+  rpcServer.rpc("getNetworkEnr") do () -> string:
+    return $node.network.discovery.localNode.record
+
+proc installDebugApiHandlers(rpcServer: RpcServer, node: BeaconNode) =
+  discard
+
+proc installRpcHandlers(rpcServer: RpcServer, node: BeaconNode) =
+  rpcServer.installValidatorApiHandlers(node)
+  rpcServer.installBeaconApiHandlers(node)
+  rpcServer.installDebugApiHandlers(node)
+
 proc run*(node: BeaconNode) =
+  if node.rpcServer != nil:
+    node.rpcServer.installRpcHandlers(node)
+    node.rpcServer.start()
+
   waitFor node.network.subscribe(topicBeaconBlocks) do (signedBlock: SignedBeaconBlock):
     onBeaconBlock(node, signedBlock)
 
@@ -955,11 +1066,6 @@ when hasPrompt:
       else:
         p[].writeLine("Unknown command: " & cmd)
 
-  proc slotOrZero(time: BeaconTime): Slot =
-    let exSlot = time.toSlot
-    if exSlot.afterGenesis: exSlot.slot
-    else: Slot(0)
-
   proc initPrompt(node: BeaconNode) =
     if isatty(stdout) and node.config.statusBarEnabled:
       enableTrueColors()
@@ -982,7 +1088,7 @@ when hasPrompt:
         # arbitrary expression that is resolvable through this API.
         case expr.toLowerAscii
         of "connected_peers":
-          $(libp2p_peers.value.int)
+          $(node.connectedPeersCount)
 
         of "last_finalized_epoch":
           var head = node.blockPool.finalizedHead
@@ -999,7 +1105,7 @@ when hasPrompt:
           $SLOTS_PER_EPOCH
 
         of "slot":
-          $node.beaconClock.now.slotOrZero
+          $node.currentSlot
 
         of "slot_trailing_digits":
           var slotStr = $node.beaconClock.now.slotOrZero
@@ -1115,9 +1221,9 @@ when isMainModule:
       let
         networkKeys = getPersistentNetKeys(config)
         bootstrapAddress = enode.Address(
-          ip: parseIpAddress(config.bootstrapAddress),
-          tcpPort: Port config.bootstrapPort,
-          udpPort: Port config.bootstrapPort)
+          ip: config.bootstrapAddress,
+          tcpPort: config.bootstrapPort,
+          udpPort: config.bootstrapPort)
 
         bootstrapEnr = enr.Record.init(
           1, # sequence number
@@ -1151,11 +1257,11 @@ when isMainModule:
       initPrompt(node)
 
     when useInsecureFeatures:
-      if config.metricsServer:
-        let metricsAddress = config.metricsServerAddress
+      if config.metricsEnabled:
+        let metricsAddress = config.metricsAddress
         info "Starting metrics HTTP server",
-          address = metricsAddress, port = config.metricsServerPort
-        metrics.startHttpServer(metricsAddress, Port(config.metricsServerPort))
+          address = metricsAddress, port = config.metricsPort
+        metrics.startHttpServer($metricsAddress, config.metricsPort)
 
     if node.nickname != "":
       dynamicLogScope(node = node.nickname): node.start()

beacon_chain/block_pool.nim

@@ -514,6 +514,17 @@ proc getBlockRange*(pool: BlockPool, headBlock: Eth2Digest,
     trace "getBlockRange result", position = result, blockSlot = b.slot
     skip skipStep
 
+func getBlockBySlot*(pool: BlockPool, slot: Slot): BlockRef =
+  ## Retrieves the first block in the current canonical chain
+  ## with slot number less or equal to `slot`.
+  pool.head.blck.findAncestorBySlot(slot).blck
+
+func getBlockByPreciseSlot*(pool: BlockPool, slot: Slot): Option[BlockRef] =
+  ## Retrieves a block from the canonical chain with a slot
+  ## number equal to `slot`.
+  let found = pool.getBlockBySlot(slot)
+  if found.slot != slot: some(found) else: none(BlockRef)
+
 proc get*(pool: BlockPool, blck: BlockRef): BlockData =
   ## Retrieve the associated block body of a block reference
   doAssert (not blck.isNil), "Trying to get nil BlockRef"

beacon_chain/conf.nim

@@ -1,14 +1,12 @@
 import
   os, options, strformat, strutils,
   chronicles, confutils, json_serialization,
-  confutils/defs, chronicles/options as chroniclesOptions,
+  confutils/defs, confutils/std/net,
+  chronicles/options as chroniclesOptions,
   spec/[crypto]
 
 export
-  defs, enabledLogLevel
-
-const
-  DEFAULT_NETWORK* {.strdefine.} = "testnet0"
+  defs, enabledLogLevel, parseCmdArg, completeCmdArg
 
 type
   ValidatorKeyPath* = TypedInputFile[ValidatorPrivKey, Txt, "privkey"]
@@ -75,6 +73,21 @@ type
       desc: "Textual template for the contents of the status bar."
       name: "status-bar-contents" }: string
 
+    rpcEnabled* {.
+      defaultValue: false
+      desc: "Enable the JSON-RPC server"
+      name: "rpc" }: bool
+
+    rpcPort* {.
+      defaultValue: defaultEth2RpcPort
+      desc: "HTTP port for the JSON-RPC service."
+      name: "rpc-port" }: Port
+
+    rpcAddress* {.
+      defaultValue: defaultListenAddress(config)
+      desc: "Listening address of the RPC server"
+      name: "rpc-address" }: IpAddress
+
     case cmd* {.
       command
       defaultValue: noCommand }: StartUpCmd
@@ -91,14 +104,14 @@ type
         name: "bootstrap-file" }: InputFile
 
       tcpPort* {.
-        defaultValue: defaultPort(config)
+        defaultValue: defaultEth2TcpPort
         desc: "TCP listening port."
-        name: "tcp-port" }: int
+        name: "tcp-port" }: Port
 
       udpPort* {.
-        defaultValue: defaultPort(config)
+        defaultValue: defaultEth2TcpPort
         desc: "UDP listening port."
-        name: "udp-port" }: int
+        name: "udp-port" }: Port
 
       maxPeers* {.
         defaultValue: 10
@@ -137,20 +150,20 @@ type
         desc: "A positive epoch selects the epoch at which to stop."
         name: "stop-at-epoch" }: uint64
 
-      metricsServer* {.
+      metricsEnabled* {.
         defaultValue: false
         desc: "Enable the metrics server."
-        name: "metrics-server" }: bool
+        name: "metrics" }: bool
 
-      metricsServerAddress* {.
-        defaultValue: "0.0.0.0"
+      metricsAddress* {.
+        defaultValue: defaultListenAddress(config)
         desc: "Listening address of the metrics server."
-        name: "metrics-server-address" }: string # TODO: use a validated type here
+        name: "metrics-address" }: IpAddress
 
-      metricsServerPort* {.
+      metricsPort* {.
         defaultValue: 8008
         desc: "Listening HTTP port of the metrics server."
-        name: "metrics-server-port" }: uint16
+        name: "metrics-port" }: Port
 
       dump* {.
         defaultValue: false
@@ -178,14 +191,14 @@ type
         name: "last-user-validator" }: uint64
 
       bootstrapAddress* {.
-        defaultValue: "127.0.0.1"
+        defaultValue: parseIpAddress("127.0.0.1")
         desc: "The public IP address that will be advertised as a bootstrap node for the testnet."
-        name: "bootstrap-address" }: string
+        name: "bootstrap-address" }: IpAddress
 
       bootstrapPort* {.
-        defaultValue: defaultPort(config)
+        defaultValue: defaultEth2TcpPort
         desc: "The TCP/UDP port that will be used by the bootstrap node."
-        name: "bootstrap-port" }: int
+        name: "bootstrap-port" }: Port
 
       genesisOffset* {.
         defaultValue: 5
@@ -248,9 +261,6 @@ type
           argument
           desc: "REST API path to evaluate" }: string
 
-proc defaultPort*(config: BeaconNodeConf): int =
-  9000
-
 proc defaultDataDir*(conf: BeaconNodeConf): string =
   let dataDir = when defined(windows):
     "AppData" / "Roaming" / "Nimbus"
@@ -274,6 +284,11 @@ func localValidatorsDir*(conf: BeaconNodeConf): string =
 func databaseDir*(conf: BeaconNodeConf): string =
   conf.dataDir / "db"
 
+func defaultListenAddress*(conf: BeaconNodeConf): IpAddress =
+  # TODO: How should we select between IPv4 and IPv6
+  # Maybe there should be a config option for this.
+  parseIpAddress("0.0.0.0")
+
 iterator validatorKeys*(conf: BeaconNodeConf): ValidatorPrivKey =
   for validatorKeyFile in conf.validators:
     try:

beacon_chain/libp2p_backend.nim

@@ -277,6 +277,9 @@ proc init*(T: type Eth2Node, conf: BeaconNodeConf,
       if msg.protocolMounter != nil:
         msg.protocolMounter result
 
+template publicKey*(node: Eth2Node): keys.PublicKey =
+  node.discovery.privKey.getPublicKey
+
 template addKnownPeer*(node: Eth2Node, peer: ENode|enr.Record) =
   node.discovery.addNode peer
 

beacon_chain/peer_pool.nim

@@ -55,6 +55,10 @@ proc fireNotFullEvent[A, B](pool: PeerPool[A, B],
   elif item.peerType == PeerType.Outgoing:
     pool.outNotFullEvent.fire()
 
+iterator pairs*[A, B](pool: PeerPool[A, B]): (B, A) =
+  for peerId, peerIdx in pool.registry:
+    yield (peerId, pool.storage[peerIdx.data].data)
+
 proc waitNotEmptyEvent[A, B](pool: PeerPool[A, B],
                              filter: set[PeerType]) {.async.} =
   if filter == {PeerType.Incoming, PeerType.Outgoing} or filter == {}:

beacon_chain/spec/eth2_apis/beacon_callsigs.nim

@@ -0,0 +1,13 @@
+import
+  options,
+  ../datatypes
+
+# https://github.com/ethereum/eth2.0-APIs/blob/master/apis/beacon/basic.md
+#
+proc getBeaconHead(): Slot
+proc getBeaconBlock(slot = none(Slot), root = none(Eth2Digest)): BeaconBlock
+proc getBeaconState(slot = none(Slot), root = none(Eth2Digest)): BeaconState
+proc getNetworkPeerId()
+proc getNetworkPeers()
+proc getNetworkEnr()
+

beacon_chain/spec/eth2_apis/validator_callsigs.nim

@@ -0,0 +1,36 @@
+import
+  options,
+  ../datatypes
+
+# https://github.com/ethereum/eth2.0-APIs/tree/master/apis/validator
+
+type
+  SyncStatus* = object
+    starting_slot*: Slot
+    current_slot*: Slot
+    highest_slot*: Slot
+
+  SyncingStatusResponse* = object
+    is_syncing*: bool
+    sync_status*: SyncStatus
+
+  ValidatorDuty* = object
+    validator_pubkey: ValidatorPubKey
+    attestation_slot: Slot
+    attestation_shard: uint
+    block_proposal_slot: Slot
+
+proc getNodeVersion(): string
+proc getGenesisTime(): uint64
+proc getSyncingStatus(): SyncingStatusResponse
+proc getValidator(key: ValidatorPubKey): Validator
+proc getValidatorDuties(validators: openarray[ValidatorPubKey], epoch: Epoch): seq[ValidatorDuty]
+proc getBlockForSigning(slot: Slot, randaoReveal: string): BeaconBlock
+proc postBlock(blk: BeaconBlock)
+proc getAttestationForSigning(validatorKey: ValidatorPubKey, pocBit: int, slot: Slot, shard: uint): Attestation
+proc postAttestation(attestation: Attestation)
+
+# Optional RPCs
+
+proc getForkId()
+

beacon_chain/spec/network.nim

@@ -17,6 +17,11 @@ const
   # https://github.com/ethereum/eth2.0-specs/blob/v0.11.0/specs/phase0/p2p-interface.md#configuration
   ATTESTATION_SUBNET_COUNT* = 64
 
+  defaultEth2TcpPort* = 9000
+
+  # This is not part of the spec yet!
+  defaultEth2RpcPort* = 9090
+
 func getAttestationTopic*(committeeIndex: uint64): string =
   # https://github.com/ethereum/eth2.0-specs/blob/v0.11.0/specs/phase0/validator.md#broadcast-attestation
   let topicIndex = committeeIndex mod ATTESTATION_SUBNET_COUNT

tests/simulation/run_node.sh

@@ -64,9 +64,12 @@ cd "$DATA_DIR" && $NODE_BIN \
   --state-snapshot=$SNAPSHOT_FILE \
   $DEPOSIT_WEB3_URL_ARG \
   --deposit-contract=$DEPOSIT_CONTRACT_ADDRESS \
-  --verify-finalization=on \
-  --metrics-server=on \
-  --metrics-server-address="127.0.0.1" \
-  --metrics-server-port="$(( $BASE_METRICS_PORT + $NODE_ID ))" \
+  --verify-finalization \
+  --rpc \
+  --rpc-address="127.0.0.1" \
+  --rpc-port="$(( $BASE_RPC_PORT + $NODE_ID ))" \
+  --metrics \
+  --metrics-address="127.0.0.1" \
+  --metrics-port="$(( $BASE_METRICS_PORT + $NODE_ID ))" \
   "$@"
 

tests/simulation/vars.sh

@@ -41,6 +41,7 @@ DEPLOY_DEPOSIT_CONTRACT_BIN="${SIMULATION_DIR}/deploy_deposit_contract"
 MASTER_NODE_ADDRESS_FILE="${SIMULATION_DIR}/node-${MASTER_NODE}/beacon_node.address"
 
 BASE_P2P_PORT=30000
+BASE_RPC_PORT=7000
 BASE_METRICS_PORT=8008
 # Set DEPOSIT_WEB3_URL_ARG to empty to get genesis state from file, not using web3
 # DEPOSIT_WEB3_URL_ARG=--web3-url=ws://localhost:8545