Bump modules to take advantage of the new Json format flavors support

Since quite a lot of additional procs were now compiled as generics, this lead to compiler bugs that had
to be worked-around:

* The `Domain` type was renamed to `Eth2Domain` to avoid compilation errors
  due to conflicts with `nativesockets.Domain`.
  Similarly, `eth2_network.KeyPair` was renamed to `NetKeyPair`.

* A new more robust version of `hexToByteArray` was added to stew

beacon_chain/beacon_node_common.nim

@@ -31,13 +31,12 @@ export
 
 type
   RpcServer* = RpcHttpServer
-  KeyPair* = eth2_network.KeyPair
 
   BeaconNode* = ref object
     nickname*: string
     graffitiBytes*: GraffitiBytes
     network*: Eth2Node
-    netKeys*: KeyPair
+    netKeys*: NetKeyPair
     db*: BeaconChainDB
     config*: BeaconNodeConf
     attachedValidators*: ref ValidatorPool

beacon_chain/networking/eth2_network.nim

@@ -40,7 +40,7 @@ logScope:
   topics = "networking"
 
 type
-  KeyPair* = crypto.KeyPair
+  NetKeyPair* = crypto.KeyPair
   PublicKey* = crypto.PublicKey
   PrivateKey* = crypto.PrivateKey
 
@@ -1368,7 +1368,7 @@ template tcpEndPoint(address, port): auto =
   MultiAddress.init(address, tcpProtocol, port)
 
 proc getPersistentNetKeys*(rng: var BrHmacDrbgContext,
-                           config: BeaconNodeConf): KeyPair =
+                           config: BeaconNodeConf): NetKeyPair =
   case config.cmd
   of noCommand, record:
     if config.netKeyFile == "random":
@@ -1384,7 +1384,7 @@ proc getPersistentNetKeys*(rng: var BrHmacDrbgContext,
         quit QuitFailure
       info "Generating new networking key", network_public_key = pubKey,
                                             network_peer_id = $pres.get()
-      return KeyPair(seckey: privKey, pubkey: privKey.getKey().tryGet())
+      return NetKeyPair(seckey: privKey, pubkey: privKey.getKey().tryGet())
     else:
       let keyPath =
         if isAbsolute(config.netKeyFile):
@@ -1410,7 +1410,7 @@ proc getPersistentNetKeys*(rng: var BrHmacDrbgContext,
         let pubKey = privKey.getKey().tryGet()
         info "Network key storage was successfully unlocked",
              key_path = keyPath, network_public_key = pubKey
-        return KeyPair(seckey: privKey, pubkey: pubKey)
+        return NetKeyPair(seckey: privKey, pubkey: pubKey)
       else:
         info "Network key storage is missing, creating a new one",
              key_path = keyPath
@@ -1436,7 +1436,7 @@ proc getPersistentNetKeys*(rng: var BrHmacDrbgContext,
 
         info "New network key storage was created", key_path = keyPath,
              network_public_key = pubKey
-        return KeyPair(seckey: privKey, pubkey: pubKey)
+        return NetKeyPair(seckey: privKey, pubkey: pubKey)
 
   of createTestnet:
     if config.netKeyFile == "random":
@@ -1472,7 +1472,7 @@ proc getPersistentNetKeys*(rng: var BrHmacDrbgContext,
     info "New network key storage was created", key_path = keyPath,
          network_public_key = pubKey
 
-    return KeyPair(seckey: privKey, pubkey: privkey.getKey().tryGet())
+    return NetKeyPair(seckey: privKey, pubkey: privkey.getKey().tryGet())
   else:
     let res = PrivateKey.random(Secp256k1, rng)
     if res.isErr():
@@ -1480,7 +1480,7 @@ proc getPersistentNetKeys*(rng: var BrHmacDrbgContext,
       quit QuitFailure
 
     let privKey = res.get()
-    return KeyPair(seckey: privKey, pubkey: privkey.getKey().tryGet())
+    return NetKeyPair(seckey: privKey, pubkey: privkey.getKey().tryGet())
 
 func gossipId(data: openArray[byte], valid: bool): seq[byte] =
   # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/p2p-interface.md#topics-and-messages
@@ -1529,7 +1529,7 @@ proc newBeaconSwitch*(config: BeaconNodeConf, seckey: PrivateKey,
 
 proc createEth2Node*(rng: ref BrHmacDrbgContext,
                      config: BeaconNodeConf,
-                     netKeys: KeyPair,
+                     netKeys: NetKeyPair,
                      enrForkId: ENRForkID): Eth2Node =
   var
     (extIp, extTcpPort, extUdpPort) = setupAddress(config.nat,
@@ -1611,7 +1611,7 @@ proc announcedENR*(node: Eth2Node): enr.Record =
   doAssert node.discovery != nil, "The Eth2Node must be initialized"
   node.discovery.localNode.record
 
-proc shortForm*(id: KeyPair): string =
+proc shortForm*(id: NetKeyPair): string =
   $PeerID.init(id.pubkey)
 
 proc subscribe*(node: Eth2Node, topic: string, topicParams: TopicParams, enableTopicMetrics: bool = false) =

beacon_chain/spec/datatypes/base.nim

@@ -126,7 +126,7 @@ type
     DOMAIN_SYNC_COMMITTEE = 7
 
   # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#custom-types
-  Domain* = array[32, byte]
+  Eth2Domain* = array[32, byte]
 
   # https://github.com/nim-lang/Nim/issues/574 and be consistent across
   # 32-bit and 64-bit word platforms.
@@ -324,7 +324,7 @@ type
   # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#signingdata
   SigningData* = object
     object_root*: Eth2Digest
-    domain*: Domain
+    domain*: Eth2Domain
 
   GraffitiBytes* = distinct array[MAX_GRAFFITI_SIZE, byte]
 
@@ -729,6 +729,14 @@ proc readValue*(reader: var JsonReader, value: var CommitteeIndex)
                {.raises: [IOError, SerializationError, Defect].} =
   value = CommitteeIndex reader.readValue(distinctBase CommitteeIndex)
 
+proc writeValue*(writer: var JsonWriter, value: HashList)
+                {.raises: [IOError, SerializationError, Defect].} =
+  writeValue(writer, value.data)
+
+proc readValue*(reader: var JsonReader, value: var HashList)
+               {.raises: [IOError, SerializationError, Defect].} =
+  readValue(reader, value.data)
+
 template writeValue*(writer: var JsonWriter, value: Version | ForkDigest) =
   writeValue(writer, $value)
 

beacon_chain/spec/helpers.nim

@@ -17,11 +17,6 @@ import
   # Internal
   ./datatypes, ./digest, ./crypto, ../ssz/merkleization
 
-type
-  # This solves an ambiguous identifier Error in some contexts
-  # (other candidate is nativesockets.Domain)
-  Domain = datatypes.Domain
-
 # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#integer_squareroot
 func integer_squareroot*(n: SomeInteger): SomeInteger =
   ## Return the largest integer ``x`` such that ``x**2 <= n``.
@@ -129,7 +124,7 @@ func compute_fork_digest*(current_version: Version,
 func compute_domain*(
     domain_type: DomainType,
     fork_version: Version,
-    genesis_validators_root: Eth2Digest = ZERO_HASH): Domain =
+    genesis_validators_root: Eth2Digest = ZERO_HASH): Eth2Domain =
   ## Return the domain for the ``domain_type`` and ``fork_version``.
   let fork_data_root =
     compute_fork_data_root(fork_version, genesis_validators_root)
@@ -138,7 +133,10 @@ func compute_domain*(
 
 # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#get_domain
 func get_domain*(
-    fork: Fork, domain_type: DomainType, epoch: Epoch, genesis_validators_root: Eth2Digest): Domain =
+    fork: Fork,
+    domain_type: DomainType,
+    epoch: Epoch,
+    genesis_validators_root: Eth2Digest): Eth2Domain =
   ## Return the signature domain (fork version concatenated with domain type)
   ## of a message.
   let fork_version =
@@ -149,13 +147,13 @@ func get_domain*(
   compute_domain(domain_type, fork_version, genesis_validators_root)
 
 func get_domain*(
-    state: BeaconState, domain_type: DomainType, epoch: Epoch): Domain =
+    state: BeaconState, domain_type: DomainType, epoch: Epoch): Eth2Domain =
   ## Return the signature domain (fork version concatenated with domain type)
   ## of a message.
   get_domain(state.fork, domain_type, epoch, state.genesis_validators_root)
 
 # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#compute_signing_root
-func compute_signing_root*(ssz_object: auto, domain: Domain): Eth2Digest =
+func compute_signing_root*(ssz_object: auto, domain: Eth2Domain): Eth2Digest =
   ## Return the signing root of an object by calculating the root of the
   ## object-domain tree.
   let domain_wrapped_object = SigningData(

beacon_chain/ssz/dynamic_navigator.nim

@@ -51,7 +51,7 @@ type
 
 proc jsonPrinterImpl[T](m: MemRange, outStream: OutputStream, pretty: bool) {.raisesssz.} =
   var typedNavigator = sszMount(m, T)
-  var jsonWriter = init(JsonWriter, outStream, pretty)
+  var jsonWriter = Json.Writer.init(outStream, pretty)
   # TODO: it should be possible to serialize the navigator object
   #       without dereferencing it (to avoid the intermediate value).
   writeValue(jsonWriter, typedNavigator[])

beacon_chain/ssz/ssz_serialization.nim

@@ -40,10 +40,10 @@ type
 
   FixedSizedWriterCtx = object
 
-serializationFormat SSZ,
-                    Reader = SszReader,
-                    Writer = SszWriter,
-                    PreferedOutput = seq[byte]
+serializationFormat SSZ
+
+SSZ.setReader SszReader
+SSZ.setWriter SszWriter, PreferredOutput = seq[byte]
 
 template sizePrefixed*[TT](x: TT): untyped =
   type T = TT

beacon_chain/ssz/types.nim

@@ -75,6 +75,11 @@ template layer*(vIdx: int64): int =
   ## index 0 for the mixed-in-length
   log2trunc(vIdx.uint64).int
 
+func hashListIndicesLen(maxChunkIdx: int64): int =
+  # TODO: This exists only to work-around a compilation issue when the complex
+  # expression is used directly in the HastList array size definition below
+  int(layer(maxChunkIdx)) + 1
+
 type
   List*[T; maxLen: static Limit] = distinct seq[T]
   BitList*[maxLen: static Limit] = distinct BitSeq
@@ -86,7 +91,7 @@ type
   HashList*[T; maxLen: static Limit] = object
     data*: List[T, maxLen]
     hashes* {.dontSerialize.}: seq[Eth2Digest]
-    indices* {.dontSerialize.}: array[int(layer(maxChunkIdx(T, maxLen))) + 1, int64]
+    indices* {.dontSerialize.}: array[hashListIndicesLen(maxChunkIdx(T, maxLen)), int64]
 
   # Note for readers:
   # We use `array` for `Vector` and

beacon_chain/validators/slashing_protection_common.nim

@@ -237,7 +237,7 @@ proc writeValue*(writer: var JsonWriter, value: PubKey0x)
 proc readValue*(reader: var JsonReader, value: var PubKey0x)
                {.raises: [SerializationError, IOError, Defect].} =
   try:
-    value = PubKey0x reader.readValue(string).hexToByteArray[:RawPubKeySize]()
+    value = PubKey0x reader.readValue(string).hexToByteArray(RawPubKeySize)
   except ValueError:
     raiseUnexpectedValue(reader, "Hex string expected")
 

tests/official/test_fixture_bls.nim

@@ -25,7 +25,7 @@ type
   BLSSignMsgInput = object
     privkey*: ValidatorPrivKey
     message*: seq[byte]
-    domain*: Domain
+    domain*: Eth2Domain
 
   BLSSignMsg* = object
     input*: BLSSignMsgInput
@@ -39,8 +39,8 @@ type
     input*: seq[ValidatorPubKey]
     output*: ValidatorPubKey
 
-proc readValue*(r: var JsonReader, a: var Domain) =
-  ## Custom deserializer for Domain
+proc readValue*(r: var JsonReader, a: var Eth2Domain) =
+  ## Custom deserializer for Eth2Domain
   # Furthermore Nim parseHex doesn't support uint
   # until https://github.com/nim-lang/Nim/pull/11067
   # (0.20)

vendor/nim-chronicles

@@ -1 +1 @@
-Subproject commit b42899070a7daa5cf6f0843faf3d6d41659e9591
+Subproject commit ea0368cc303b6ed59792a7c2556285adf310e455

vendor/nim-confutils

@@ -1 +1 @@
-Subproject commit cfa95661913b0ff8b1609e3954894f8ab31bbf3e
+Subproject commit f091a70a5bf95ec772c8b4d9978e81b8ae89af0c

vendor/nim-json-serialization

@@ -1 +1 @@
-Subproject commit 32f75d93b0762328d1d85ce62cef84ed919ae31e
+Subproject commit fe8a82ca76150b60a950d5aa4e5baa382441ada4

vendor/nim-protobuf-serialization

@@ -1 +1 @@
-Subproject commit e8e84cfc11de78c7bce0cded800060a29220e76f
+Subproject commit d79b5c884965b2ab395315d8e5f3f8ae134ff99a

vendor/nim-serialization

@@ -1 +1 @@
-Subproject commit 261de741b73601821cb6e749fc9b4092f1cc5377
+Subproject commit f9a1121b8733eb75e624ab59f8d79e707f15f76f

vendor/nim-stew

@@ -1 +1 @@
-Subproject commit 6bcb21184aeb96ce6c62e187a64d678b74609f1e
+Subproject commit ee78822e057ac5f39804ecb6ac1096734be13ef8