Activate wire protocol eth/66 (#993)

* Activate wire protocol eth/66 and: Disentangle protocol_eth66.nim from import sections why: Importing the protocol_eth66 module is not necessary. There is no need to know too many details of the underlying wire protocol. All that is needed will be exported by blockchain_sync.nim. * fixes, and rebase * Update nimbus/p2p/blockchain_sync.nim Co-authored-by: Kim De Mey <kim.demey@gmail.com> * Fixes and rebase Co-authored-by: Kim De Mey <kim.demey@gmail.com>
2022-03-21 17:12:07 +00:00 · 2022-03-21 17:12:07 +00:00 · 046c97f18b
parent 74127644db
commit 046c97f18b
11 changed files with 69 additions and 27 deletions
--- a/hive_integration/nodocker/graphql/graphql_sim.nim
+++ b/hive_integration/nodocker/graphql/graphql_sim.nim
@ -10,7 +10,7 @@
 import
  std/[os, json],
  eth/[p2p, trie/db], ../../../nimbus/db/db_chain,
-  ../../../nimbus/sync/protocol_eth65,
+  ../../../nimbus/sync/protocol_ethxx,
  ../../../nimbus/[genesis, config, conf_utils, context],
  ../../../nimbus/graphql/ethapi, ../../../tests/test_helpers,
  ../../../nimbus/utils/tx_pool,
--- a/hive_integration/nodocker/rpc/rpc_sim.nim
+++ b/hive_integration/nodocker/rpc/rpc_sim.nim
@ -15,7 +15,7 @@ import
  stew/results,
  chronos, json_rpc/[rpcserver, rpcclient],
  ../../../nimbus/db/db_chain,
-  ../../../nimbus/sync/protocol_eth65,
+  ../../../nimbus/p2p/protocol_ethxx,
  ../../../nimbus/[config, context, genesis],
  ../../../nimbus/rpc/[common, p2p, debug],
  ../../../tests/test_helpers,
--- a/nimbus/nimbus.nim
+++ b/nimbus/nimbus.nim
@ -17,9 +17,9 @@ import
  eth/common as eth_common, eth/p2p as eth_p2p,
  chronos, json_rpc/rpcserver, chronicles,
  eth/p2p/rlpx_protocols/les_protocol,
+  ./sync/protocol_ethxx,
  ./p2p/blockchain_sync, eth/net/nat, eth/p2p/peer_pool,
  ./p2p/clique/[clique_desc, clique_sealer],
-  ./sync/protocol_eth65,
  config, genesis, rpc/[common, p2p, debug, engine_api], p2p/chain,
  eth/trie/db, metrics, metrics/[chronos_httpserver, chronicles_support],
  graphql/ethapi, context, utils/tx_pool,
--- a/nimbus/p2p/blockchain_sync.nim
+++ b/nimbus/p2p/blockchain_sync.nim
@ -8,9 +8,11 @@
 import
  std/[sets, options, random, hashes, sequtils],
  chronos, chronicles,
+  ../sync/protocol_ethxx,
  eth/common/eth_types,
-  eth/[p2p, p2p/private/p2p_types, p2p/rlpx, p2p/peer_pool],
-  ../sync/protocol_eth65
+  eth/[p2p, p2p/private/p2p_types, p2p/rlpx, p2p/peer_pool]
+
+{.push raises:[Defect].}

 const
  minPeersToStartSync* = 2 # Wait for consensus of at least this
@ -22,6 +24,9 @@ type
    syncNotEnoughPeers
    syncTimeOut

+  BlockchainSyncDefect* = object of Defect
+    ## Catch and relay exception
+
  WantedBlocksState = enum
    Initial,
    Requested,
@ -44,6 +49,17 @@ type
    trustedPeers: HashSet[Peer]
    hasOutOfOrderBlocks: bool

+template catchException(info: string; code: untyped) =
+  try:
+    code
+  except CatchableError as e:
+    raise (ref CatchableError)(msg: e.msg)
+  except Defect as e:
+    raise (ref Defect)(msg: e.msg)
+  except Exception as e:
+    raise newException(
+      BlockchainSyncDefect, info & "(): " & $e.name & " -- " & e.msg)
+
 proc hash*(p: Peer): Hash = hash(cast[pointer](p))

 proc endIndex(b: WantedBlocks): BlockNumber =
@ -89,13 +105,15 @@ proc availableWorkItem(ctx: SyncContext): int =

  # Create new work item when queue was increased, reset when selected work item
  # is at Persisted state.
-  var numBlocks = (ctx.endBlockNumber - nextRequestedBlock).toInt
+  var numBlocks = (ctx.endBlockNumber - nextRequestedBlock).truncate(int)
  if numBlocks > maxHeadersFetch:
    numBlocks = maxHeadersFetch
  ctx.workQueue[result] = WantedBlocks(startIndex: nextRequestedBlock, numBlocks: numBlocks.uint, state: Initial)

-proc persistWorkItem(ctx: SyncContext, wi: var WantedBlocks): ValidationResult =
-  result = ctx.chain.persistBlocks(wi.headers, wi.bodies)
+proc persistWorkItem(ctx: SyncContext, wi: var WantedBlocks): ValidationResult
+    {.gcsafe, raises:[Defect,CatchableError].} =
+  catchException("persistBlocks"):
+    result = ctx.chain.persistBlocks(wi.headers, wi.bodies)
  case result
  of ValidationResult.OK:
    ctx.finalizedBlock = wi.endIndex
@ -106,7 +124,8 @@ proc persistWorkItem(ctx: SyncContext, wi: var WantedBlocks): ValidationResult =
  wi.headers = @[]
  wi.bodies = @[]

-proc persistPendingWorkItems(ctx: SyncContext): (int, ValidationResult) =
+proc persistPendingWorkItems(ctx: SyncContext): (int, ValidationResult)
+    {.gcsafe, raises:[Defect,CatchableError].} =
  var nextStartIndex = ctx.finalizedBlock + 1
  var keepRunning = true
  var hasOutOfOrderBlocks = false
@ -134,7 +153,8 @@ proc persistPendingWorkItems(ctx: SyncContext): (int, ValidationResult) =

  ctx.hasOutOfOrderBlocks = hasOutOfOrderBlocks

-proc returnWorkItem(ctx: SyncContext, workItem: int): ValidationResult =
+proc returnWorkItem(ctx: SyncContext, workItem: int): ValidationResult
+    {.gcsafe, raises:[Defect,CatchableError].} =
  let wi = addr ctx.workQueue[workItem]
  let askedBlocks = wi.numBlocks.int
  let receivedBlocks = wi.headers.len
@ -172,7 +192,8 @@ proc returnWorkItem(ctx: SyncContext, workItem: int): ValidationResult =
      receivedBlocks
    return ValidationResult.Error

-proc newSyncContext(chain: AbstractChainDB, peerPool: PeerPool): SyncContext =
+proc newSyncContext(chain: AbstractChainDB, peerPool: PeerPool): SyncContext
+    {.gcsafe, raises:[Defect,CatchableError].} =
  new result
  result.chain = chain
  result.peerPool = peerPool
@ -193,7 +214,7 @@ proc getBestBlockNumber(p: Peer): Future[BlockNumber] {.async.} =
    reverse: true)

  tracePacket ">> Sending eth.GetBlockHeaders (0x03)", peer=p,
-    startBlock=request.startBlock.hash, max=request.maxResults
+    startBlock=request.startBlock.hash.toHex, max=request.maxResults
  let latestBlock = await p.getBlockHeaders(request)

  if latestBlock.isSome:
@ -324,7 +345,7 @@ proc peersAgreeOnChain(a, b: Peer): Future[bool] {.async.} =
    reverse: true)

  tracePacket ">> Sending eth.GetBlockHeaders (0x03)", peer=a,
-    startBlock=request.startBlock.hash, max=request.maxResults
+    startBlock=request.startBlock.hash.toHex, max=request.maxResults
  let latestBlock = await a.getBlockHeaders(request)

  result = latestBlock.isSome and latestBlock.get.headers.len > 0
--- a/nimbus/sync/protocol_eth66.nim
+++ b/nimbus/sync/protocol_eth66.nim
@ -6,9 +6,9 @@
 #  * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)
 # at your option. This file may not be copied, modified, or distributed except according to those terms.

-## This module implements Ethereum Wire Protocol version 65, `eth/65`.
+## This module implements Ethereum Wire Protocol version 66, `eth/66`.
 ## Specification:
-##   https://github.com/ethereum/devp2p/blob/master/caps/eth.md
+##   `eth/66 <https://github.com/ethereum/devp2p/blob/master/caps/eth.md>`_

 import
  chronos, stint, chronicles, stew/byteutils, macros,
@ -26,6 +26,10 @@ type
    forkHash: array[4, byte] # The RLP encoding must be exactly 4 bytes.
    forkNext: BlockNumber    # The RLP encoding must be variable-length

+  NewBlockAndTotalDiff* = object
+    ethBlock: EthBlock
+    totalDifficulty: DifficultyInt
+
  PeerState = ref object
    initialized*: bool
    bestBlockHash*: KeccakHash
@ -40,7 +44,7 @@ const
  maxBodiesFetch* = 128
  maxReceiptsFetch* = 256
  maxHeadersFetch* = 192
-  ethVersion = 65
+  ethVersion* = 66

 func toHex*(x: KeccakHash): string = x.data.toHex
 macro tracePacket*(msg: static[string], args: varargs[untyped]) =
@ -50,7 +54,7 @@ macro tracePacket*(msg: static[string], args: varargs[untyped]) =

 p2pProtocol eth(version = ethVersion,
                peerState = PeerState,
-                useRequestIds = false):
+                useRequestIds = true):

  onPeerConnected do (peer: Peer):
    let
@ -60,8 +64,7 @@ p2pProtocol eth(version = ethVersion,
      chainForkId = chain.getForkId(bestBlock.blockNumber)
      forkId = ForkId(
        forkHash: chainForkId.crc.toBytesBe,
-        forkNext: chainForkId.nextFork.u256,
-      )
+        forkNext: chainForkId.nextFork.u256)

    tracePacket ">> Sending eth.Status (0x00) [eth/" & $ethVersion & "]",
      peer, td=bestBlock.difficulty,
@ -184,7 +187,8 @@ p2pProtocol eth(version = ethVersion,
    # because either `p2pProtocol` or RLPx doesn't work with an alias.)
    tracePacket "<< Discarding eth.NewBlock (0x07)",
      peer, totalDifficulty,
-      blockNumber=bh.header.blockNumber, blockDifficulty=bh.header.difficulty
+      blockNumber = bh.header.blockNumber,
+      blockDifficulty = bh.header.difficulty
    discard

  # User message 0x08: NewPooledTransactionHashes.
--- a/nimbus/sync/protocol_ethxx.nim
+++ b/nimbus/sync/protocol_ethxx.nim
@ -0,0 +1,12 @@
+# Nimbus
+# Copyright (c) 2018-2021 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at
+#     https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at
+#     https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+import ./protocol_eth66
+export protocol_eth66
--- a/tests/graphql/README.md
+++ b/tests/graphql/README.md
@ -0,0 +1,5 @@
+per protocool folders apply, e.g.
+
+- eth65/queries.toml (obsoleted *eth65* test specs)
+- eth66/queries.toml
+- ...
--- a/tests/graphql/eth66/queries.toml
+++ b/tests/graphql/eth66/queries.toml
@ -486,7 +486,7 @@ mutation {
 """
  result = """
 {
-  "protocolVersion":65
+  "protocolVersion":66
 }
 """

--- a/tests/test_graphql.nim
+++ b/tests/test_graphql.nim
@ -12,7 +12,7 @@ import
  stew/byteutils, unittest2,
  eth/[p2p, common, trie/db, rlp],
  graphql, ../nimbus/graphql/ethapi, graphql/test_common,
-  ../nimbus/sync/protocol_eth65,
+  ../nimbus/sync/protocol_ethxx,
  ../nimbus/[genesis, config, chain_config, context],
  ../nimbus/db/[db_chain],
  ../nimbus/p2p/chain, ./test_helpers,
@ -25,7 +25,7 @@ type
    uncles: seq[BlockHeader]

 const
-  caseFolder = "tests" / "graphql"
+  caseFolder = "tests" / "graphql" / "eth" & $ethVersion
  dataFolder  = "tests" / "fixtures" / "eth_tests" / "BlockchainTests" / "ValidBlocks" / "bcUncleTest"

 proc toBlock(n: JsonNode, key: string): EthBlock =
--- a/tests/test_merge.nim
+++ b/tests/test_merge.nim
@ -12,12 +12,12 @@ import
  unittest2, nimcrypto, eth/common as eth_common,
  json_rpc/[rpcserver, rpcclient], web3/[conversions, engine_api_types],
  eth/[trie/db, p2p/private/p2p_types],
-  ../nimbus/sync/protocol_eth65,
+  ../nimbus/sync/protocol_ethxx,
  ../nimbus/rpc/[common, p2p, hexstrings, rpc_types, rpc_utils, engine_api],
  ../nimbus/db/[db_chain],
  ../nimbus/[chain_config, config, context, genesis, sealer],
  ../nimbus/utils/[tx_pool],
-  ../nimbus/p2p/[chain],
+  ../nimbus/p2p/chain,
  ../nimbus/merge/mergetypes,
  ./test_helpers

--- a/tests/test_rpc.nim
+++ b/tests/test_rpc.nim
@ -14,8 +14,8 @@ import
  ../nimbus/[constants, config, genesis, utils, transaction,
             vm_state, vm_types],
  ../nimbus/db/[accounts_cache, db_chain],
+  ../nimbus/sync/protocol_ethxx,
  ../nimbus/p2p/[chain, executor, executor/executor_helpers],
-  ../nimbus/sync/protocol_eth65,
  ../nimbus/utils/[difficulty, tx_pool],
  ../nimbus/[context, chain_config],
   ./test_helpers, ./macro_assembler
@ -217,7 +217,7 @@ proc rpcMain*() =
      # same expression as the client can hide issues when the value is wrong
      # in both places.  When the expected value genuinely changes, it'll be
      # obvious.  Just change this number.
-      check res == "65"
+      check res == $ethVersion

    test "eth_syncing":
      let res = await client.eth_syncing()