From 2eacfc4685e4309af18767aca45434d4453fe52b Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Fri, 19 Mar 2021 04:22:45 +0200 Subject: [PATCH] Bump modules to take advantage of the new Json format flavors support Since quite a lot of additional procs were now compiled as generics, this lead to compiler bugs that had to be worked-around: * The `Domain` type was renamed to `Eth2Domain` to avoid compilation errors due to conflicts with `nativesockets.Domain`. Similarly, `eth2_network.KeyPair` was renamed to `NetKeyPair`. * A new more robust version of `hexToByteArray` was added to stew --- beacon_chain/beacon_node_common.nim | 3 +-- beacon_chain/networking/eth2_network.nim | 18 +++++++++--------- beacon_chain/spec/datatypes/base.nim | 12 ++++++++++-- beacon_chain/spec/helpers.nim | 16 +++++++--------- beacon_chain/ssz/dynamic_navigator.nim | 2 +- beacon_chain/ssz/ssz_serialization.nim | 8 ++++---- beacon_chain/ssz/types.nim | 7 ++++++- .../validators/slashing_protection_common.nim | 2 +- tests/official/test_fixture_bls.nim | 6 +++--- vendor/nim-chronicles | 2 +- vendor/nim-confutils | 2 +- vendor/nim-json-serialization | 2 +- vendor/nim-protobuf-serialization | 2 +- vendor/nim-serialization | 2 +- vendor/nim-stew | 2 +- 15 files changed, 48 insertions(+), 38 deletions(-) diff --git a/beacon_chain/beacon_node_common.nim b/beacon_chain/beacon_node_common.nim index b3fb9e657..0b9684e13 100644 --- a/beacon_chain/beacon_node_common.nim +++ b/beacon_chain/beacon_node_common.nim @@ -31,13 +31,12 @@ export type RpcServer* = RpcHttpServer - KeyPair* = eth2_network.KeyPair BeaconNode* = ref object nickname*: string graffitiBytes*: GraffitiBytes network*: Eth2Node - netKeys*: KeyPair + netKeys*: NetKeyPair db*: BeaconChainDB config*: BeaconNodeConf attachedValidators*: ref ValidatorPool diff --git a/beacon_chain/networking/eth2_network.nim b/beacon_chain/networking/eth2_network.nim index e1e3e724c..b1368cc89 100644 --- a/beacon_chain/networking/eth2_network.nim +++ b/beacon_chain/networking/eth2_network.nim @@ -40,7 +40,7 @@ logScope: topics = "networking" type - KeyPair* = crypto.KeyPair + NetKeyPair* = crypto.KeyPair PublicKey* = crypto.PublicKey PrivateKey* = crypto.PrivateKey @@ -1368,7 +1368,7 @@ template tcpEndPoint(address, port): auto = MultiAddress.init(address, tcpProtocol, port) proc getPersistentNetKeys*(rng: var BrHmacDrbgContext, - config: BeaconNodeConf): KeyPair = + config: BeaconNodeConf): NetKeyPair = case config.cmd of noCommand, record: if config.netKeyFile == "random": @@ -1384,7 +1384,7 @@ proc getPersistentNetKeys*(rng: var BrHmacDrbgContext, quit QuitFailure info "Generating new networking key", network_public_key = pubKey, network_peer_id = $pres.get() - return KeyPair(seckey: privKey, pubkey: privKey.getKey().tryGet()) + return NetKeyPair(seckey: privKey, pubkey: privKey.getKey().tryGet()) else: let keyPath = if isAbsolute(config.netKeyFile): @@ -1410,7 +1410,7 @@ proc getPersistentNetKeys*(rng: var BrHmacDrbgContext, let pubKey = privKey.getKey().tryGet() info "Network key storage was successfully unlocked", key_path = keyPath, network_public_key = pubKey - return KeyPair(seckey: privKey, pubkey: pubKey) + return NetKeyPair(seckey: privKey, pubkey: pubKey) else: info "Network key storage is missing, creating a new one", key_path = keyPath @@ -1436,7 +1436,7 @@ proc getPersistentNetKeys*(rng: var BrHmacDrbgContext, info "New network key storage was created", key_path = keyPath, network_public_key = pubKey - return KeyPair(seckey: privKey, pubkey: pubKey) + return NetKeyPair(seckey: privKey, pubkey: pubKey) of createTestnet: if config.netKeyFile == "random": @@ -1472,7 +1472,7 @@ proc getPersistentNetKeys*(rng: var BrHmacDrbgContext, info "New network key storage was created", key_path = keyPath, network_public_key = pubKey - return KeyPair(seckey: privKey, pubkey: privkey.getKey().tryGet()) + return NetKeyPair(seckey: privKey, pubkey: privkey.getKey().tryGet()) else: let res = PrivateKey.random(Secp256k1, rng) if res.isErr(): @@ -1480,7 +1480,7 @@ proc getPersistentNetKeys*(rng: var BrHmacDrbgContext, quit QuitFailure let privKey = res.get() - return KeyPair(seckey: privKey, pubkey: privkey.getKey().tryGet()) + return NetKeyPair(seckey: privKey, pubkey: privkey.getKey().tryGet()) func gossipId(data: openArray[byte], valid: bool): seq[byte] = # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/p2p-interface.md#topics-and-messages @@ -1529,7 +1529,7 @@ proc newBeaconSwitch*(config: BeaconNodeConf, seckey: PrivateKey, proc createEth2Node*(rng: ref BrHmacDrbgContext, config: BeaconNodeConf, - netKeys: KeyPair, + netKeys: NetKeyPair, enrForkId: ENRForkID): Eth2Node = var (extIp, extTcpPort, extUdpPort) = setupAddress(config.nat, @@ -1611,7 +1611,7 @@ proc announcedENR*(node: Eth2Node): enr.Record = doAssert node.discovery != nil, "The Eth2Node must be initialized" node.discovery.localNode.record -proc shortForm*(id: KeyPair): string = +proc shortForm*(id: NetKeyPair): string = $PeerID.init(id.pubkey) proc subscribe*(node: Eth2Node, topic: string, topicParams: TopicParams, enableTopicMetrics: bool = false) = diff --git a/beacon_chain/spec/datatypes/base.nim b/beacon_chain/spec/datatypes/base.nim index a1d1d6065..adb39f0e4 100644 --- a/beacon_chain/spec/datatypes/base.nim +++ b/beacon_chain/spec/datatypes/base.nim @@ -126,7 +126,7 @@ type DOMAIN_SYNC_COMMITTEE = 7 # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#custom-types - Domain* = array[32, byte] + Eth2Domain* = array[32, byte] # https://github.com/nim-lang/Nim/issues/574 and be consistent across # 32-bit and 64-bit word platforms. @@ -324,7 +324,7 @@ type # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#signingdata SigningData* = object object_root*: Eth2Digest - domain*: Domain + domain*: Eth2Domain GraffitiBytes* = distinct array[MAX_GRAFFITI_SIZE, byte] @@ -729,6 +729,14 @@ proc readValue*(reader: var JsonReader, value: var CommitteeIndex) {.raises: [IOError, SerializationError, Defect].} = value = CommitteeIndex reader.readValue(distinctBase CommitteeIndex) +proc writeValue*(writer: var JsonWriter, value: HashList) + {.raises: [IOError, SerializationError, Defect].} = + writeValue(writer, value.data) + +proc readValue*(reader: var JsonReader, value: var HashList) + {.raises: [IOError, SerializationError, Defect].} = + readValue(reader, value.data) + template writeValue*(writer: var JsonWriter, value: Version | ForkDigest) = writeValue(writer, $value) diff --git a/beacon_chain/spec/helpers.nim b/beacon_chain/spec/helpers.nim index 58ed8e2ed..1b46f6da3 100644 --- a/beacon_chain/spec/helpers.nim +++ b/beacon_chain/spec/helpers.nim @@ -17,11 +17,6 @@ import # Internal ./datatypes, ./digest, ./crypto, ../ssz/merkleization -type - # This solves an ambiguous identifier Error in some contexts - # (other candidate is nativesockets.Domain) - Domain = datatypes.Domain - # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#integer_squareroot func integer_squareroot*(n: SomeInteger): SomeInteger = ## Return the largest integer ``x`` such that ``x**2 <= n``. @@ -129,7 +124,7 @@ func compute_fork_digest*(current_version: Version, func compute_domain*( domain_type: DomainType, fork_version: Version, - genesis_validators_root: Eth2Digest = ZERO_HASH): Domain = + genesis_validators_root: Eth2Digest = ZERO_HASH): Eth2Domain = ## Return the domain for the ``domain_type`` and ``fork_version``. let fork_data_root = compute_fork_data_root(fork_version, genesis_validators_root) @@ -138,7 +133,10 @@ func compute_domain*( # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#get_domain func get_domain*( - fork: Fork, domain_type: DomainType, epoch: Epoch, genesis_validators_root: Eth2Digest): Domain = + fork: Fork, + domain_type: DomainType, + epoch: Epoch, + genesis_validators_root: Eth2Digest): Eth2Domain = ## Return the signature domain (fork version concatenated with domain type) ## of a message. let fork_version = @@ -149,13 +147,13 @@ func get_domain*( compute_domain(domain_type, fork_version, genesis_validators_root) func get_domain*( - state: BeaconState, domain_type: DomainType, epoch: Epoch): Domain = + state: BeaconState, domain_type: DomainType, epoch: Epoch): Eth2Domain = ## Return the signature domain (fork version concatenated with domain type) ## of a message. get_domain(state.fork, domain_type, epoch, state.genesis_validators_root) # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#compute_signing_root -func compute_signing_root*(ssz_object: auto, domain: Domain): Eth2Digest = +func compute_signing_root*(ssz_object: auto, domain: Eth2Domain): Eth2Digest = ## Return the signing root of an object by calculating the root of the ## object-domain tree. let domain_wrapped_object = SigningData( diff --git a/beacon_chain/ssz/dynamic_navigator.nim b/beacon_chain/ssz/dynamic_navigator.nim index b80edd870..263a949af 100644 --- a/beacon_chain/ssz/dynamic_navigator.nim +++ b/beacon_chain/ssz/dynamic_navigator.nim @@ -51,7 +51,7 @@ type proc jsonPrinterImpl[T](m: MemRange, outStream: OutputStream, pretty: bool) {.raisesssz.} = var typedNavigator = sszMount(m, T) - var jsonWriter = init(JsonWriter, outStream, pretty) + var jsonWriter = Json.Writer.init(outStream, pretty) # TODO: it should be possible to serialize the navigator object # without dereferencing it (to avoid the intermediate value). writeValue(jsonWriter, typedNavigator[]) diff --git a/beacon_chain/ssz/ssz_serialization.nim b/beacon_chain/ssz/ssz_serialization.nim index 1ab5a3940..e2fc43111 100644 --- a/beacon_chain/ssz/ssz_serialization.nim +++ b/beacon_chain/ssz/ssz_serialization.nim @@ -40,10 +40,10 @@ type FixedSizedWriterCtx = object -serializationFormat SSZ, - Reader = SszReader, - Writer = SszWriter, - PreferedOutput = seq[byte] +serializationFormat SSZ + +SSZ.setReader SszReader +SSZ.setWriter SszWriter, PreferredOutput = seq[byte] template sizePrefixed*[TT](x: TT): untyped = type T = TT diff --git a/beacon_chain/ssz/types.nim b/beacon_chain/ssz/types.nim index 724cc3e9d..30521abad 100644 --- a/beacon_chain/ssz/types.nim +++ b/beacon_chain/ssz/types.nim @@ -75,6 +75,11 @@ template layer*(vIdx: int64): int = ## index 0 for the mixed-in-length log2trunc(vIdx.uint64).int +func hashListIndicesLen(maxChunkIdx: int64): int = + # TODO: This exists only to work-around a compilation issue when the complex + # expression is used directly in the HastList array size definition below + int(layer(maxChunkIdx)) + 1 + type List*[T; maxLen: static Limit] = distinct seq[T] BitList*[maxLen: static Limit] = distinct BitSeq @@ -86,7 +91,7 @@ type HashList*[T; maxLen: static Limit] = object data*: List[T, maxLen] hashes* {.dontSerialize.}: seq[Eth2Digest] - indices* {.dontSerialize.}: array[int(layer(maxChunkIdx(T, maxLen))) + 1, int64] + indices* {.dontSerialize.}: array[hashListIndicesLen(maxChunkIdx(T, maxLen)), int64] # Note for readers: # We use `array` for `Vector` and diff --git a/beacon_chain/validators/slashing_protection_common.nim b/beacon_chain/validators/slashing_protection_common.nim index 3f3d25558..b769a225c 100644 --- a/beacon_chain/validators/slashing_protection_common.nim +++ b/beacon_chain/validators/slashing_protection_common.nim @@ -237,7 +237,7 @@ proc writeValue*(writer: var JsonWriter, value: PubKey0x) proc readValue*(reader: var JsonReader, value: var PubKey0x) {.raises: [SerializationError, IOError, Defect].} = try: - value = PubKey0x reader.readValue(string).hexToByteArray[:RawPubKeySize]() + value = PubKey0x reader.readValue(string).hexToByteArray(RawPubKeySize) except ValueError: raiseUnexpectedValue(reader, "Hex string expected") diff --git a/tests/official/test_fixture_bls.nim b/tests/official/test_fixture_bls.nim index 76074f6c9..b0daa63a9 100644 --- a/tests/official/test_fixture_bls.nim +++ b/tests/official/test_fixture_bls.nim @@ -25,7 +25,7 @@ type BLSSignMsgInput = object privkey*: ValidatorPrivKey message*: seq[byte] - domain*: Domain + domain*: Eth2Domain BLSSignMsg* = object input*: BLSSignMsgInput @@ -39,8 +39,8 @@ type input*: seq[ValidatorPubKey] output*: ValidatorPubKey -proc readValue*(r: var JsonReader, a: var Domain) = - ## Custom deserializer for Domain +proc readValue*(r: var JsonReader, a: var Eth2Domain) = + ## Custom deserializer for Eth2Domain # Furthermore Nim parseHex doesn't support uint # until https://github.com/nim-lang/Nim/pull/11067 # (0.20) diff --git a/vendor/nim-chronicles b/vendor/nim-chronicles index b42899070..ea0368cc3 160000 --- a/vendor/nim-chronicles +++ b/vendor/nim-chronicles @@ -1 +1 @@ -Subproject commit b42899070a7daa5cf6f0843faf3d6d41659e9591 +Subproject commit ea0368cc303b6ed59792a7c2556285adf310e455 diff --git a/vendor/nim-confutils b/vendor/nim-confutils index cfa956619..f091a70a5 160000 --- a/vendor/nim-confutils +++ b/vendor/nim-confutils @@ -1 +1 @@ -Subproject commit cfa95661913b0ff8b1609e3954894f8ab31bbf3e +Subproject commit f091a70a5bf95ec772c8b4d9978e81b8ae89af0c diff --git a/vendor/nim-json-serialization b/vendor/nim-json-serialization index 32f75d93b..fe8a82ca7 160000 --- a/vendor/nim-json-serialization +++ b/vendor/nim-json-serialization @@ -1 +1 @@ -Subproject commit 32f75d93b0762328d1d85ce62cef84ed919ae31e +Subproject commit fe8a82ca76150b60a950d5aa4e5baa382441ada4 diff --git a/vendor/nim-protobuf-serialization b/vendor/nim-protobuf-serialization index e8e84cfc1..d79b5c884 160000 --- a/vendor/nim-protobuf-serialization +++ b/vendor/nim-protobuf-serialization @@ -1 +1 @@ -Subproject commit e8e84cfc11de78c7bce0cded800060a29220e76f +Subproject commit d79b5c884965b2ab395315d8e5f3f8ae134ff99a diff --git a/vendor/nim-serialization b/vendor/nim-serialization index 261de741b..f9a1121b8 160000 --- a/vendor/nim-serialization +++ b/vendor/nim-serialization @@ -1 +1 @@ -Subproject commit 261de741b73601821cb6e749fc9b4092f1cc5377 +Subproject commit f9a1121b8733eb75e624ab59f8d79e707f15f76f diff --git a/vendor/nim-stew b/vendor/nim-stew index 6bcb21184..ee78822e0 160000 --- a/vendor/nim-stew +++ b/vendor/nim-stew @@ -1 +1 @@ -Subproject commit 6bcb21184aeb96ce6c62e187a64d678b74609f1e +Subproject commit ee78822e057ac5f39804ecb6ac1096734be13ef8