partially integrate eth1 merge changes (#2548)

* partially integrate eth1 merge changes

* use hexToSeqByte() and validate execution engine opaque transaction length

* remove incorrect REST serialization code

beacon_chain/beacon_node_common.nim

@@ -1,5 +1,5 @@
 # beacon_chain
-# Copyright (c) 2018-2020 Status Research & Development GmbH
+# Copyright (c) 2018-2021 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).

beacon_chain/eth1/eth1_monitor.nim

@@ -12,11 +12,13 @@ import
        typetraits, uri],
   # Nimble packages:
   chronos, json, metrics, chronicles/timings,
-  web3, web3/ethtypes as web3Types, eth/common/eth_types, eth/async_utils,
+  web3, web3/ethtypes as web3Types, web3/ethhexstrings, eth/common/eth_types,
+  eth/async_utils, stew/byteutils,
   # Local modules:
   ../spec/[datatypes, digest, crypto, helpers],
   ../networking/network_metadata,
   ../ssz,
+  ../rpc/eth_merge_web3,
   ".."/[beacon_chain_db, beacon_node_status],
   ./merkle_minimal
 
@@ -401,6 +403,38 @@ proc getBlockByNumber*(p: Web3DataProviderRef,
   except ValueError as exc: raiseAssert exc.msg # Never fails
   p.web3.provider.eth_getBlockByNumber(hexNumber, false)
 
+proc setHead*(p: Web3DataProviderRef, hash: Eth2Digest):
+    Future[BoolReturnSuccessRPC] =
+  p.web3.provider.consensus_setHead(hash.data.encodeQuantityHex)
+
+proc assembleBlock*(p: Web3DataProviderRef, parentHash: Eth2Digest,
+                    timestamp: uint64): Future[ExecutionPayloadRPC] =
+  p.web3.provider.consensus_assembleBlock(BlockParams(
+    parentHash: parentHash.data.encodeQuantityHex,
+    timestamp: encodeQuantity(timestamp)))
+
+func encodeOpaqueTransaction(ot: OpaqueTransaction): string =
+  var res = "0x"
+  for b in ot:
+    res &= b.toHex
+  res
+
+proc newBlock*(p: Web3DataProviderRef,
+               executableData: ExecutionPayload): Future[BoolReturnValidRPC] =
+  p.web3.provider.consensus_newBlock(ExecutionPayloadRPC(
+    parentHash: executableData.parent_hash.data.encodeQuantityHex,
+    miner: executableData.coinbase.data.encodeQuantityHex,
+    stateRoot: executableData.state_root.data.encodeQuantityHex,
+    number: executableData.number.encodeQuantity,
+    gasLimit: executableData.gas_limit.encodeQuantity,
+    gasUsed: executableData.gas_used.encodeQuantity,
+    timestamp: executableData.timestamp.encodeQuantity,
+    receiptsRoot: executableData.receipt_root.data.encodeQuantityHex,
+    logsBloom: executableData.logs_bloom.data.encodeQuantityHex,
+    blockHash: executableData.block_hash.data.encodeQuantityHex,
+    transactions: List[string, MAX_EXECUTION_TRANSACTIONS].init(
+      mapIt(executableData.transactions, it.encodeOpaqueTransaction))))
+
 template readJsonField(j: JsonNode, fieldName: string, ValueType: type): untyped =
   var res: ValueType
   fromJson(j[fieldName], fieldName, res)

beacon_chain/rpc/eth2_json_rest_serialization.nim

@@ -257,6 +257,19 @@ proc writeValue*(writer: var JsonWriter[RestJson], value: Eth2Digest) {.
      raises: [IOError, Defect].} =
   writeValue(writer, hexOriginal(value.data))
 
+## BloomLogs
+proc readValue*(reader: var JsonReader[RestJson], value: var BloomLogs) {.
+     raises: [IOError, SerializationError, Defect].} =
+  try:
+    hexToByteArray(reader.readValue(string), value.data)
+  except ValueError:
+    raiseUnexpectedValue(reader,
+                         "BloomLogs value should be a valid hex string")
+
+proc writeValue*(writer: var JsonWriter[RestJson], value: BloomLogs) {.
+     raises: [IOError, Defect].} =
+  writeValue(writer, hexOriginal(value.data))
+
 ## HashArray
 proc readValue*(reader: var JsonReader[RestJson], value: var HashArray) {.
      raises: [IOError, SerializationError, Defect].} =

beacon_chain/rpc/eth_merge_web3.nim

@@ -1,9 +1,9 @@
 import
   strutils,
   json_serialization/std/[sets, net], serialization/errors,
-  ./spec/[datatypes, digest, crypto, eth2_apis/beacon_rpc_client],
+  ../spec/[datatypes, digest, crypto, eth2_apis/beacon_rpc_client],
   json_rpc/[client, jsonmarshal]
 
 from os import DirSep, AltSep
 template sourceDir: string = currentSourcePath.rsplit({DirSep, AltSep}, 1)[0]
-createRpcSigs(RpcClient, sourceDir & "/rpc/eth_merge_sigs.nim")
+createRpcSigs(RpcClient, sourceDir & "/eth_merge_sigs.nim")

beacon_chain/rpc/validator_api.nim

@@ -40,7 +40,7 @@ proc installValidatorApiHandlers*(rpcServer: RpcServer, node: BeaconNode) {.
     let proposer = node.chainDag.getProposer(head, slot)
     if proposer.isNone():
       raise newException(CatchableError, "could not retrieve block for slot: " & $slot)
-    let message = makeBeaconBlockForHeadAndSlot(
+    let message = await makeBeaconBlockForHeadAndSlot(
       node, randao_reveal, proposer.get()[0], graffiti, head, slot)
     if message.isNone():
       raise newException(CatchableError, "could not retrieve block for slot: " & $slot)

beacon_chain/rpc/validator_rest_api.nim

@@ -220,7 +220,7 @@ proc installValidatorApiHandlers*(router: var RestRouter, node: BeaconNode) =
         let proposer = node.chainDag.getProposer(qhead, qslot)
         if proposer.isNone():
           return RestApiResponse.jsonError(Http400, ProposerNotFoundError)
-        let res = makeBeaconBlockForHeadAndSlot(
+        let res = await makeBeaconBlockForHeadAndSlot(
           node, qrandao, proposer.get()[0], qgraffiti, qhead, qslot)
         if res.isNone():
           return RestApiResponse.jsonError(Http400, BlockProduceError)

beacon_chain/spec/datatypes/base.nim

@@ -28,10 +28,11 @@ import
   std/[macros, hashes, intsets, strutils, tables, typetraits],
   stew/[assign2, byteutils], chronicles,
   json_serialization,
-  ../../version, ../../ssz/types as sszTypes, ../crypto, ../digest, ../presets
+  ../../version, ../../ssz/types as sszTypes, ../crypto, ../digest, ../presets,
+  ./merge
 
 export
-  sszTypes, presets, json_serialization
+  sszTypes, merge, presets, json_serialization
 
 # Presently, we're reusing the data types from the serialization (uint64) in the
 # objects we pass around to the beacon chain logic, thus keeping the two

beacon_chain/spec/datatypes/merge.nim

@@ -30,13 +30,11 @@ const
   EVM_BLOCK_ROOTS_SIZE* = 8
 
 type
-  # https://github.com/ethereum/eth2.0-specs/blob/eca6bd7d622a0cfb7343bff742da046ed25b3825/specs/merge/beacon-chain.md#custom-types
-  # TODO is this maneuver sizeof()/memcpy()/SSZ-equivalent? Pretty sure, but not 100% certain
-  OpaqueTransaction* = object
-    data*: List[byte, MAX_BYTES_PER_OPAQUE_TRANSACTION]
+  # https://github.com/ethereum/eth2.0-specs/blob/dev/specs/merge/beacon-chain.md#custom-types
+  OpaqueTransaction* = List[byte, Limit MAX_BYTES_PER_OPAQUE_TRANSACTION]
 
   EthAddress* = object
-    data*: array[20, byte]  # TODO there's a network_metadata type, but the import hierarchy's inconvenient without splitting out aspects of this module
+    data*: array[20, byte]  # TODO there's a network_metadata type, but the import hierarchy's inconvenient
 
   BloomLogs* = object
     data*: array[BYTES_PER_LOGS_BLOOM, byte]
@@ -55,6 +53,20 @@ type
     logs_bloom*: BloomLogs
     transactions*: List[OpaqueTransaction, MAX_EXECUTION_TRANSACTIONS]
 
+  # https://github.com/ethereum/eth2.0-specs/blob/dev/specs/merge/beacon-chain.md#executionpayloadheader
+  ExecutionPayloadHeader* = object
+    block_hash*: Eth2Digest  # Hash of execution block
+    parent_hash*: Eth2Digest
+    coinbase*: EthAddress
+    state_root*: Eth2Digest
+    number*: uint64
+    gas_limit*: uint64
+    gas_used*: uint64
+    timestamp*: uint64
+    receipt_root*: Eth2Digest
+    logs_bloom*: BloomLogs
+    transactions_root*: Eth2Digest
+
   # Empirically derived from Catalyst responses; doesn't seem to match merge
   # spec per commit 1fb9a6dd32b581c912d672634882d7e2eb2775cd from 2021-04-22
   # {"jsonrpc":"2.0","id":1,"result":{"blockHash":"0x35139e42d930c640eee446944f7f8b345771b69dfa10120895057f48680ea27d","parentHash":"0x3a3fdfc9ab6e17ff530b57bc21494da3848ebbeaf9343545fded7a18d221ffec","miner":"0x1000000000000000000000000000000000000000","stateRoot":"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421","number":"0x1","gasLimit":"0x2fefd8","gasUsed":"0x0","timestamp":"0x6087e796","receiptsRoot":"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421","logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","transactions":[]}}
@@ -81,6 +93,9 @@ type
   BoolReturnSuccessRPC* = object
     success*: bool
 
+func encodeQuantityHex*(x: auto): string =
+  "0x" & x.toHex
+
 proc fromHex*(T: typedesc[BloomLogs], s: string): T =
   hexToBytes(s, result.data)
 

beacon_chain/spec/state_transition.nim

@@ -265,6 +265,7 @@ proc makeBeaconBlock*(
     proposerSlashings: seq[ProposerSlashing],
     attesterSlashings: seq[AttesterSlashing],
     voluntaryExits: seq[SignedVoluntaryExit],
+    executionPayload: ExecutionPayload,
     rollback: RollbackHashedProc,
     cache: var StateCache): Option[BeaconBlock] =
   ## Create a block for the given state. The last block applied to it must be
@@ -306,4 +307,4 @@ proc makeBeaconBlock*(
   state.root = hash_tree_root(state.data)
   blck.state_root = state.root
 
-  some(blck)
+  return some(blck)

beacon_chain/validators/validator_duties.nim

@@ -12,7 +12,7 @@ import
   std/[os, osproc, sequtils, streams, tables],
 
   # Nimble packages
-  stew/[assign2, objects],
+  stew/[assign2, byteutils, objects],
   chronos, metrics,
   chronicles,
   json_serialization/std/[options, sets, net], serialization/errors,
@@ -266,12 +266,31 @@ proc getBlockProposalEth1Data*(node: BeaconNode,
       stateData.data.data, finalizedEpochRef.eth1_data,
       finalizedEpochRef.eth1_deposit_index)
 
+func getOpaqueTransaction(s: string): OpaqueTransaction =
+  try:
+    # Effectively an internal logic error in the Eth1/Eth2 client system, as
+    # it's not possible to just omit a malformatted transaction: it would be
+    # the wrong ExecutionPayload blockHash overall, and rejected by newBlock
+    # when one attempted to reinsert it into Geth (which, while not all Eth2
+    # clients might connect to, some will). It's also not possible to skip a
+    # whole ExecutionPayload being that it's an integral part of BeaconBlock
+    # construction. So not much better to do than bail if an incoming string
+    # representation of the OpaqueTransaction is invalid. init() could catch
+    # this, but it'd make its interface clumsier in a way it doesn't .add().
+    let opaqueTransactionSeq = hexToSeqByte(s)
+    if opaqueTransactionSeq.len > MAX_BYTES_PER_OPAQUE_TRANSACTION:
+      raiseAssert "Execution engine returned too-long opaque transaction"
+    OpaqueTransaction(List[byte, MAX_BYTES_PER_OPAQUE_TRANSACTION].init(
+      opaqueTransactionSeq))
+  except ValueError:
+    raiseAssert "Execution engine returned invalidly formatted transaction"
+
 proc makeBeaconBlockForHeadAndSlot*(node: BeaconNode,
                                     randao_reveal: ValidatorSig,
                                     validator_index: ValidatorIndex,
                                     graffiti: GraffitiBytes,
                                     head: BlockRef,
-                                    slot: Slot): Option[BeaconBlock] =
+                                    slot: Slot): Future[Option[BeaconBlock]] {.async.} =
   # Advance state to the slot that we're proposing for
 
   let
@@ -294,7 +313,7 @@ proc makeBeaconBlockForHeadAndSlot*(node: BeaconNode,
       doAssert v.addr == addr proposalStateAddr.data
       assign(proposalStateAddr[], poolPtr.headState)
 
-    makeBeaconBlock(
+    return makeBeaconBlock(
       node.runtimePreset,
       hashedState,
       validator_index,
@@ -307,6 +326,7 @@ proc makeBeaconBlockForHeadAndSlot*(node: BeaconNode,
       node.exitPool[].getProposerSlashingsForBlock(),
       node.exitPool[].getAttesterSlashingsForBlock(),
       node.exitPool[].getVoluntaryExitsForBlock(),
+      default(ExecutionPayload),
       restore,
       cache)
 
@@ -363,7 +383,7 @@ proc proposeBlock(node: BeaconNode,
       getStateField(node.chainDag.headState, genesis_validators_root)
     randao = await validator.genRandaoReveal(
       fork, genesis_validators_root, slot)
-    message = makeBeaconBlockForHeadAndSlot(
+    message = await makeBeaconBlockForHeadAndSlot(
       node, randao, validator_index, node.graffitiBytes, head, slot)
 
   if not message.isSome():

research/block_sim.nim

@@ -150,6 +150,7 @@ cli do(slots = SLOTS_PER_EPOCH * 5,
           @[],
           @[],
           @[],
+          ExecutionPayload(),
           noRollback,
           cache)
 

scripts/run-catalyst.sh

@@ -2,8 +2,8 @@
 # set -Eeuo pipefail
 # https://github.com/prysmaticlabs/bazel-go-ethereum/blob/catalyst/run-catalyst.sh
 
-# To increase verbosity: debug.verbosity(5) or debug.verbosity(6)
-# MetaMask seed phrase for account with balance is
+# To increase verbosity: debug.verbosity(4)
+# MetaMask seed phrase for account with balance is:
 # lecture manual soon title cloth uncle gesture cereal common fruit tooth crater
 
 echo \{ \

tests/testblockutil.nim

@@ -117,6 +117,7 @@ proc addTestBlock*(
       @[],
       @[],
       @[],
+      default(ExecutionPayload),
       noRollback,
       cache)