json cleanups (#2456)

* move json-rpc specific marshalling to rpc
* serialize Epoch/Slot with cast to avoid Defect
* avoid a few eth1 deps
* simplify imports

beacon_chain/beacon_chain_db_immutable.nim

@@ -9,14 +9,14 @@
 
 import
   tables,
-  stew/[assign2, endians2, io2, objects, results],
-  serialization, chronicles,
+  stew/[assign2, io2, objects, results],
+  serialization,
   eth/db/[kvstore, kvstore_sqlite3],
   ./spec/[crypto, datatypes, digest],
   ./ssz/[ssz_serialization, merkleization],
   filepath
 
-type 
+type
   # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#beaconstate
   # Memory-representation-equivalent to a v1.0.1 BeaconState for in-place SSZ reading and writing
   BeaconStateNoImmutableValidators* = object

beacon_chain/beacon_node_common.nim

@@ -11,7 +11,7 @@ import
   std/osproc,
 
   # Nimble packages
-  chronos, json_rpc/rpcserver,
+  chronos, json_rpc/servers/httpserver,
 
   # Local modules
   ./conf, ./beacon_clock, ./beacon_chain_db,
@@ -24,7 +24,7 @@ import
   ./sync/[sync_manager, request_manager]
 
 export
-  osproc, chronos, rpcserver, conf, beacon_clock, beacon_chain_db,
+  osproc, chronos, httpserver, conf, beacon_clock, beacon_chain_db,
   attestation_pool, eth2_network, beacon_node_types, eth1_monitor,
   request_manager, sync_manager, eth2_processor, blockchain_dag, block_quarantine,
   datatypes

beacon_chain/networking/network_metadata.nim

@@ -16,6 +16,7 @@ import
   json_serialization,
   json_serialization/std/[options, sets, net], serialization/errors,
   ../ssz/navigator,
+  eth/common/eth_types_json_serialization,
   ../spec/[presets, datatypes, digest]
 
 # ATTENTION! This file will produce a large C file, because we are inlining

beacon_chain/nimbus_beacon_node.nim

@@ -16,7 +16,6 @@ import
   # Nimble packages
   stew/[objects, byteutils, endians2, io2], stew/shims/macros,
   chronos, confutils, metrics, metrics/chronos_httpserver,
-  json_rpc/[rpcclient, rpcserver, jsonmarshal],
   chronicles, bearssl, blscurve,
   json_serialization/std/[options, sets, net], serialization/errors,
 

beacon_chain/nimbus_validator_client.nim

@@ -9,11 +9,11 @@
 
 import
   # Standard library
-  std/[os, json, random, strutils],
+  std/[os, random, strutils],
 
   # Nimble packages
   stew/shims/[tables, macros],
-  chronos, confutils, metrics, json_rpc/[rpcclient, jsonmarshal],
+  chronos, confutils, metrics,
   chronicles,
   json_serialization/std/[options, net],
 
@@ -23,7 +23,6 @@ import
   ./sync/sync_manager,
   "."/[conf, beacon_clock, version],
   ./networking/[eth2_network, eth2_discovery],
-  ./rpc/eth2_json_rpc_serialization,
   ./beacon_node_types,
   ./nimbus_binary_common,
   ./ssz/merkleization,

beacon_chain/rpc/beacon_api.nim

@@ -10,7 +10,7 @@
 import
   std/[parseutils, sequtils, strutils, deques, sets],
   stew/results,
-  json_rpc/[rpcserver, jsonmarshal],
+  json_rpc/servers/httpserver,
   chronicles,
   nimcrypto/utils as ncrutils,
   ../beacon_node_common,

beacon_chain/rpc/config_api.nim

@@ -9,7 +9,7 @@
 
 import
   stew/endians2,
-  json_rpc/[rpcserver, jsonmarshal],
+  json_rpc/servers/httpserver,
   chronicles,
   nimcrypto/utils as ncrutils,
   ../beacon_node_common,

beacon_chain/rpc/debug_api.nim

@@ -9,7 +9,7 @@
 
 import
   std/sequtils,
-  json_rpc/[rpcserver, jsonmarshal],
+  json_rpc/servers/httpserver,
   chronicles,
   ../version, ../beacon_node_common,
   ../networking/[eth2_network, peer_pool],

beacon_chain/rpc/eth2_json_rpc_serialization.nim

@@ -7,9 +7,22 @@
 
 {.push raises: [Defect].}
 
+# The serializations in this file are approximations of
+# https://ethereum.github.io/eth2.0-APIs/#/ but where written before the standard
+# had materialized - they've now made it out to releases which means the easiest
+# thing to do is to maintain them as-is, even if there are mismatches. In
+# particular, numbers are serialized as strings in the eth2 API - here, they
+# use numbers instead.
+#
+# Using numbers creates problems - uint64 which often appears in eth2 can't
+# portably be represented since many json parsers balk at anything >2^53 and
+# start losing precision. The other issue is the json parser in nim - it can't
+# handle numbers >2^63, either crashing or giving wrong results:
+# https://github.com/status-im/nimbus-eth2/issues/2430
+
 import
   # Standard library
-  std/[tables, json, typetraits],
+  std/[tables, typetraits],
 
   # Nimble packages
   stew/byteutils,
@@ -19,6 +32,8 @@ import
   ../ssz/types,
   ../spec/[datatypes, crypto, digest]
 
+export jsonmarshal, datatypes, crypto, digest
+
 proc toJsonHex(data: openArray[byte]): string =
   # Per the eth2 API spec, hex arrays are printed with leading 0x
   "0x" & toHex(data)
@@ -92,6 +107,17 @@ genFromJsonForIntType(Slot)
 genFromJsonForIntType(CommitteeIndex)
 genFromJsonForIntType(ValidatorIndex)
 
+proc `%`*(value: Epoch): JsonNode =
+  # In nim <= 1.2.6, `uint64` was silently cast to int64 resulting in
+  # FAR_FUTURE_EPOCH showing as -1 - this is a hack to maintain that behaviour
+  # in a world where a Defect or an actual correct value is used - the eth2
+  # REST api instead prints all epochs and similar large numbers as strings!
+  # See also https://github.com/status-im/nimbus-eth2/issues/2430
+  newJInt(cast[int64](value))
+
+proc `%`*(value: Slot): JsonNode =
+  newJInt(cast[int64](value))
+
 proc `%`*(value: GraffitiBytes): JsonNode =
   newJString(toJsonHex(distinctBase(value)))
 

beacon_chain/rpc/event_api.nim

@@ -8,7 +8,7 @@
 {.push raises: [Defect].}
 
 import
-  json_rpc/[rpcserver, jsonmarshal],
+  json_rpc/servers/httpserver,
   chronicles,
   ../beacon_node_common
 

beacon_chain/rpc/nimbus_api.nim

@@ -10,18 +10,16 @@
 import
   std/[deques, sequtils, sets],
   chronos,
-  stew/shims/macros,
   stew/byteutils,
-  json_rpc/[rpcserver, jsonmarshal],
+  json_rpc/servers/httpserver,
+  libp2p/protocols/pubsub/pubsubpeer,
 
-  rpc_utils,
-  ../beacon_node_common, ../nimbus_binary_common,
-  ../networking/eth2_network,
-  ../eth1/eth1_monitor,
-  ../validators/validator_duties,
-  ../spec/[digest, datatypes, presets],
+  "."/[rpc_utils, eth2_json_rpc_serialization],
+  ".."/[
+    beacon_node_common, nimbus_binary_common, networking/eth2_network,
+    eth1/eth1_monitor, validators/validator_duties],
+  ../spec/[digest, datatypes, presets]
 
-  libp2p/protocols/pubsub/pubsubpeer
 
 logScope: topics = "nimbusapi"
 

beacon_chain/rpc/node_api.nim

@@ -9,10 +9,11 @@
 
 import std/options,
   chronicles,
-  json_rpc/[rpcserver, jsonmarshal],
+  json_rpc/servers/httpserver,
   eth/p2p/discoveryv5/enr,
   libp2p/[multiaddress, multicodec],
   nimcrypto/utils as ncrutils,
+  ./eth2_json_rpc_serialization,
   ../beacon_node_common, ../version,
   ../networking/[eth2_network, peer_pool],
   ../sync/sync_manager,
@@ -24,9 +25,6 @@ logScope: topics = "nodeapi"
 type
   RpcServer = RpcHttpServer
 
-template unimplemented() =
-  raise (ref CatchableError)(msg: "Unimplemented")
-
 proc validateState(state: Option[seq[string]]): Option[set[ConnectionState]] =
   var res: set[ConnectionState]
   if state.isSome():

beacon_chain/rpc/rpc_utils.nim

@@ -63,25 +63,25 @@ proc getBlockSlotFromString*(node: BeaconNode, slot: string): BlockSlot {.raises
   if parseBiggestUInt(slot, parsed) != slot.len:
     raise newException(ValueError, "Not a valid slot number")
   let head = node.doChecksAndGetCurrentHead(parsed.Slot)
-  return head.atSlot(parsed.Slot)
+  head.atSlot(parsed.Slot)
 
 proc stateIdToBlockSlot*(node: BeaconNode, stateId: string): BlockSlot {.raises: [Defect, CatchableError].} =
-  result = case stateId:
-    of "head":
-      node.chainDag.head.toBlockSlot()
-    of "genesis":
-      node.chainDag.getGenesisBlockSlot()
-    of "finalized":
-      node.chainDag.finalizedHead
-    of "justified":
-      node.chainDag.head.atEpochStart(
-        node.chainDag.headState.data.data.current_justified_checkpoint.epoch)
+  case stateId:
+  of "head":
+    node.chainDag.head.toBlockSlot()
+  of "genesis":
+    node.chainDag.getGenesisBlockSlot()
+  of "finalized":
+    node.chainDag.finalizedHead
+  of "justified":
+    node.chainDag.head.atEpochStart(
+      node.chainDag.headState.data.data.current_justified_checkpoint.epoch)
+  else:
+    if stateId.startsWith("0x"):
+      let blckRoot = parseRoot(stateId)
+      let blckRef = node.chainDag.getRef(blckRoot)
+      if blckRef.isNil:
+        raise newException(CatchableError, "Block not found")
+      blckRef.toBlockSlot()
     else:
-      if stateId.startsWith("0x"):
-        let blckRoot = parseRoot(stateId)
-        let blckRef = node.chainDag.getRef(blckRoot)
-        if blckRef.isNil:
-          raise newException(CatchableError, "Block not found")
-        blckRef.toBlockSlot()
-      else:
-        node.getBlockSlotFromString(stateId)
+      node.getBlockSlotFromString(stateId)

beacon_chain/rpc/validator_api.nim

@@ -13,7 +13,7 @@ import
 
   # Nimble packages
   stew/[objects],
-  json_rpc/[rpcserver, jsonmarshal],
+  json_rpc/servers/httpserver,
   chronicles,
 
   # Local modules

beacon_chain/spec/datatypes/base.nim

@@ -25,13 +25,13 @@
 {.push raises: [Defect].}
 
 import
-  std/[macros, hashes, intsets, json, strutils, tables, typetraits],
+  std/[macros, hashes, intsets, strutils, tables, typetraits],
   stew/[assign2, byteutils], chronicles,
-  json_serialization/types as jsonTypes,
+  json_serialization,
   ../../version, ../../ssz/types as sszTypes, ../crypto, ../digest, ../presets
 
 export
-  sszTypes, presets
+  sszTypes, presets, json_serialization
 
 # Presently, we're reusing the data types from the serialization (uint64) in the
 # objects we pass around to the beacon chain logic, thus keeping the two
@@ -702,7 +702,6 @@ template ethTimeUnit(typ: type) {.dirty.} =
   # Nim integration
   proc `$`*(x: typ): string {.borrow, noSideEffect.}
   proc hash*(x: typ): Hash {.borrow, noSideEffect.}
-  proc `%`*(x: typ): JsonNode {.borrow, noSideEffect.}
 
   # Serialization
   proc writeValue*(writer: var JsonWriter, value: typ)
@@ -999,10 +998,6 @@ chronicles.formatIt AttestationData: it.shortLog
 chronicles.formatIt Attestation: it.shortLog
 chronicles.formatIt Checkpoint: it.shortLog
 
-import json_serialization
-export json_serialization
-export writeValue, readValue
-
 const
   # http://facweb.cs.depaul.edu/sjost/it212/documents/ascii-pr.htm
   PrintableAsciiChars = {'!'..'~'}

beacon_chain/spec/digest.nim

@@ -24,17 +24,17 @@
 import
   # Standard library
   std/hashes,
-  #Status libraries
+  # Status libraries
   chronicles,
   nimcrypto/[sha2, hash],
   stew/byteutils,
-  eth/common/eth_types_json_serialization,
+  json_serialization,
   blscurve
 
 export
   # Exports from sha2 / hash are explicit to avoid exporting upper-case `$` and
   # constant-time `==`
-  sha2.update, hash.fromHex, readValue, writeValue
+  sha2.update, hash.fromHex, json_serialization
 
 type
   Eth2Digest* = MDigest[32 * 8] ## `hash32` from spec
@@ -110,3 +110,12 @@ func `==`*(a, b: Eth2Digest): bool =
     # nimcrypto uses a constant-time comparison for all MDigest types which for
     # Eth2Digest is unnecessary - the type should never hold a secret!
     equalMem(unsafeAddr a.data[0], unsafeAddr b.data[0], sizeof(a.data))
+
+proc writeValue*(w: var JsonWriter, a: Eth2Digest) {.raises: [Defect, IOError, SerializationError].} =
+  w.writeValue $a
+
+proc readValue*(r: var JsonReader, a: var Eth2Digest) {.raises: [Defect, IOError, SerializationError].} =
+  try:
+    a = fromHex(type(a), r.readValue(string))
+  except ValueError:
+    raiseUnexpectedValue(r, "Hex string expected")

beacon_chain/spec/eth2_apis/beacon_callsigs.nim

@@ -1,7 +1,5 @@
 import
   options,
-  ../[datatypes, digest, crypto],
-  json_rpc/jsonmarshal,
   callsigs_types
 
 proc get_v1_beacon_genesis(): BeaconGenesisTuple

beacon_chain/spec/eth2_apis/beacon_rpc_client.nim

@@ -1,13 +1,11 @@
 import
-  std/[os, json],
-  json_rpc/[rpcclient, jsonmarshal],
+  std/os,
+  json_rpc/rpcclient,
   ../../rpc/eth2_json_rpc_serialization,
-  ../crypto, ../digest, ../datatypes,
-  callsigs_types
+  ./callsigs_types
 
 export
   rpcclient,
-  crypto, digest, datatypes,
   callsigs_types,
   eth2_json_rpc_serialization
 

beacon_chain/spec/eth2_apis/callsigs_types.nim

@@ -1,11 +1,7 @@
 import
-  # Standard library
-  options,
-  # Local modules
-  # TODO for some reason "../[datatypes, digest, crypto]" results in "Error: cannot open file"
-  ../datatypes,
-  ../digest,
-  ../crypto
+  ".."/[datatypes, digest, crypto]
+
+export datatypes, digest, crypto
 
 type
   AttesterDuties* = tuple

beacon_chain/spec/eth2_apis/debug_callsigs.nim

@@ -1,8 +1,7 @@
 import
-  options,
-  ../[datatypes, digest, crypto],
-  json_rpc/jsonmarshal,
   callsigs_types
 
+export callsigs_types
+
 proc get_v1_debug_beacon_states_stateId(stateId: string): BeaconState
 proc get_v1_debug_beacon_heads(): seq[tuple[root: Eth2Digest, slot: Slot]]

beacon_chain/spec/eth2_apis/nimbus_callsigs.nim

@@ -1,9 +1,8 @@
 import
-  options,
-  ../[datatypes, digest, crypto],
-  json_rpc/jsonmarshal,
   callsigs_types
 
+export callsigs_types
+
 proc getBeaconHead(): Slot
 proc getChainHead(): JsonNode
 proc getSyncing(): bool

beacon_chain/spec/eth2_apis/node_callsigs.nim

@@ -1,7 +1,5 @@
 import
   options,
-  ../[datatypes, digest, crypto],
-  json_rpc/jsonmarshal,
   callsigs_types
 
 proc get_v1_node_identity(): NodeIdentityTuple

beacon_chain/spec/eth2_apis/validator_callsigs.nim

@@ -1,12 +1,7 @@
 import
-  # Standard library
   options,
-  # Local modules
-  ../[datatypes, digest, crypto],
-  json_rpc/jsonmarshal,
   callsigs_types
 
-
 # calls that return a bool are actually without a return type in the main REST API
 # spec but nim-json-rpc requires that all RPC calls have a return type.
 

beacon_chain/validators/validator_duties.nim

@@ -12,8 +12,8 @@ import
   std/[os, osproc, sequtils, streams, tables],
 
   # Nimble packages
-  stew/[assign2, objects, shims/macros],
-  chronos, metrics, json_rpc/[rpcserver, jsonmarshal],
+  stew/[assign2, objects],
+  chronos, metrics,
   chronicles,
   json_serialization/std/[options, sets, net], serialization/errors,
   eth/db/kvstore,

vendor/nim-json-rpc

@@ -1 +1 @@
-Subproject commit 4eb39203ebd391c77d16a1c387dc8a6b7d90bc69
+Subproject commit 64d40d6c1a095761a03d1ba55eb45877596e8e7b

vendor/nim-web3

@@ -1 +1 @@
-Subproject commit 21b465fcd58460e6018dcb1048254f2514696778
+Subproject commit 75a1a0e5d8cc8fc7bb9cdb3bfe68a73e11b5c71a