Bump stint to v2.0: new array backend (#1747)

* Bump stint to v2.0: new array backend

fluffy/content_db.nim

@@ -232,19 +232,19 @@ proc contentSize*(db: ContentDB): int64 =
 
 proc get*(db: ContentDB, key: ContentId): Opt[seq[byte]] =
   # TODO: Here it is unfortunate that ContentId is a uint256 instead of Digest256.
-  db.get(key.toByteArrayBE())
+  db.get(key.toBytesBE())
 
 proc put*(db: ContentDB, key: ContentId, value: openArray[byte]) =
-  db.put(key.toByteArrayBE(), value)
+  db.put(key.toBytesBE(), value)
 
 proc contains*(db: ContentDB, key: ContentId): bool =
-  db.contains(key.toByteArrayBE())
+  db.contains(key.toBytesBE())
 
 proc del*(db: ContentDB, key: ContentId) =
-  db.del(key.toByteArrayBE())
+  db.del(key.toBytesBE())
 
 proc getSszDecoded*(db: ContentDB, key: ContentId, T: type auto): Opt[T] =
-  db.getSszDecoded(key.toByteArrayBE(), T)
+  db.getSszDecoded(key.toBytesBE(), T)
 
 proc deleteContentFraction(
   db: ContentDB,
@@ -264,7 +264,7 @@ proc deleteContentFraction(
 
   var ri: RowInfo
   var bytesDeleted: int64 = 0
-  let targetBytes = target.toByteArrayBE()
+  let targetBytes = target.toBytesBE()
   for e in db.getAllOrderedByDistanceStmt.exec(targetBytes, ri):
     if bytesDeleted + ri.payloadLength < bytesToDelete:
       db.del(ri.contentId)

fluffy/fluffy.nim

@@ -160,7 +160,7 @@ proc run(config: PortalConf) {.raises: [CatchableError].} =
   # the selected `Radius`.
   let
     db = ContentDB.new(config.dataDir / "db" / "contentdb_" &
-      d.localNode.id.toByteArrayBE().toOpenArray(0, 8).toHex(), maxSize = config.storageSize)
+      d.localNode.id.toBytesBE().toOpenArray(0, 8).toHex(), maxSize = config.storageSize)
 
     portalConfig = PortalProtocolConfig.init(
       config.tableIpLimit,

fluffy/network/beacon_light_client/beacon_light_client_db.nim

@@ -122,10 +122,10 @@ proc put(db: LightClientDb, key, value: openArray[byte]) =
 
 proc get*(db: LightClientDb, key: ContentId):  results.Opt[seq[byte]] =
   # TODO: Here it is unfortunate that ContentId is a uint256 instead of Digest256.
-  db.get(key.toByteArrayBE())
+  db.get(key.toBytesBE())
 
 proc put*(db: LightClientDb, key: ContentId, value: openArray[byte]) =
-  db.put(key.toByteArrayBE(), value)
+  db.put(key.toBytesBE(), value)
 
 proc createGetHandler*(db: LightClientDb): DbGetHandler =
   return (

fluffy/network/beacon_light_client/beacon_light_client_init_loader.nim

@@ -32,7 +32,7 @@ proc loadNetworkData*(
 
     genesisState =
       try:
-        template genesisData(): auto = metadata.genesisData
+        template genesisData(): auto = metadata.genesis.bakedBytes
         newClone(readSszForkedHashedBeaconState(
           metadata.cfg, genesisData.toOpenArray(genesisData.low, genesisData.high)))
       except CatchableError as err:

fluffy/network/state/state_content.nim

@@ -11,7 +11,7 @@
 {.push raises: [].}
 
 import
-  nimcrypto/[hash, sha2, keccak], stew/[objects, results], stint,
+  nimcrypto/[hash, sha2, keccak], stew/[objects, results, endians2], stint,
   ssz_serialization,
   ../../common/common_types
 

fluffy/network/wire/messages.nim

@@ -11,7 +11,7 @@
 {.push raises: [].}
 
 import
-  stint, stew/[results, objects],
+  stint, stew/[results, objects, endians2],
   ssz_serialization,
   ../../common/common_types
 

fluffy/network/wire/portal_protocol.nim

@@ -12,7 +12,7 @@
 
 import
   std/[sequtils, sets, algorithm],
-  stew/[results, byteutils, leb128], chronicles, chronos, nimcrypto/hash,
+  stew/[results, byteutils, leb128, endians2], chronicles, chronos, nimcrypto/hash,
   bearssl, ssz_serialization, metrics, faststreams,
   eth/rlp, eth/p2p/discoveryv5/[protocol, node, enr, routing_table, random2,
     nodes_verification, lru],

fluffy/network/wire/portal_stream.nim

@@ -9,7 +9,7 @@
 
 import
   std/sequtils,
-  chronos, stew/[byteutils, leb128], chronicles,
+  chronos, stew/[byteutils, leb128, endians2], chronicles,
   eth/utp/utp_discv5_protocol,
   # even though utp_discv5_protocol exports this, import is still needed,
   # perhaps protocol.Protocol type of usage?

fluffy/seed_db.nim

@@ -125,7 +125,7 @@ proc put*(db: SeedDb, contentId: array[32, byte], contentKey: seq[byte], content
   db.putStmt.exec((contentId, contentKey, content)).expectDb()
 
 proc put*(db: SeedDb, contentId: UInt256, contentKey: seq[byte], content: seq[byte]): void =
-  db.put(contentId.toByteArrayBE(), contentKey, content)
+  db.put(contentId.toBytesBE(), contentKey, content)
 
 proc get*(db: SeedDb, contentId: array[32, byte]): Option[ContentData] =
   var res = none[ContentData]()
@@ -133,7 +133,7 @@ proc get*(db: SeedDb, contentId: array[32, byte]): Option[ContentData] =
   return res
 
 proc get*(db: SeedDb, contentId: UInt256): Option[ContentData] =
-  db.get(contentId.toByteArrayBE())
+  db.get(contentId.toBytesBE())
 
 proc getContentInRange*(
     db: SeedDb,
@@ -147,7 +147,7 @@ proc getContentInRange*(
 
   var res: seq[ContentDataDist] = @[]
   var cd: ContentDataDist
-  for e in db.getInRangeStmt.exec((nodeId.toByteArrayBE(), nodeRadius.toByteArrayBE(), max, offset), cd):
+  for e in db.getInRangeStmt.exec((nodeId.toBytesBE(), nodeRadius.toBytesBE(), max, offset), cd):
     res.add(cd)
   return res
 

fluffy/tests/beacon_light_client_tests/test_beacon_light_client.nim

@@ -47,7 +47,7 @@ procSuite "Portal Beacon Light Client":
       metadata = loadMainnetData()
       genesisState =
         try:
-          template genesisData(): auto = metadata.genesisData
+          template genesisData(): auto = metadata.genesis.bakedBytes
           newClone(readSszForkedHashedBeaconState(
             metadata.cfg, genesisData.toOpenArray(genesisData.low, genesisData.high)))
         except CatchableError as err:

fluffy/tests/beacon_light_client_tests/test_beacon_light_client_content.nim

@@ -27,7 +27,7 @@ suite "Beacon Light Client Content Encodings - Mainnet":
     metadata = getMetadataForNetwork("mainnet")
     genesisState =
       try:
-        template genesisData(): auto = metadata.genesisData
+        template genesisData(): auto = metadata.genesis.bakedBytes
         newClone(readSszForkedHashedBeaconState(
           metadata.cfg,
           genesisData.toOpenArray(genesisData.low, genesisData.high)))

fluffy/tools/beacon_lc_bridge/beacon_lc_bridge.nim

@@ -402,7 +402,7 @@ proc run(config: BeaconBridgeConf) {.raises: [CatchableError].} =
   let
     genesisState =
       try:
-        template genesisData(): auto = metadata.genesisData
+        template genesisData(): auto = metadata.genesis.bakedBytes
         newClone(readSszForkedHashedBeaconState(
           cfg, genesisData.toOpenArray(genesisData.low, genesisData.high)))
       except CatchableError as err:

fluffy/tools/eth_data_exporter/cl_data_exporter.nim

@@ -24,7 +24,7 @@ proc getBeaconData*(): (RuntimeConfig, ref ForkDigests) {.raises: [IOError].} =
     metadata = getMetadataForNetwork("mainnet")
     genesisState =
       try:
-        template genesisData(): auto = metadata.genesisData
+        template genesisData(): auto = metadata.genesis.bakedBytes
         newClone(readSszForkedHashedBeaconState(
           metadata.cfg,
           genesisData.toOpenArray(genesisData.low, genesisData.high)))

fluffy/tools/utp_testing/utp_rpc_types.nim

@@ -10,7 +10,7 @@
 import
   std/hashes,
   json_rpc/jsonmarshal,
-  stew/byteutils,
+  stew/[byteutils, endians2],
   eth/p2p/discoveryv5/node,
   eth/utp/[utp_discv5_protocol, utp_router]
 

fluffy/tools/utp_testing/utp_test_app.nim

@@ -10,7 +10,7 @@ import
   std/[hashes, tables],
   chronos, chronicles, confutils,
   confutils/std/net as confNet,
-  stew/byteutils,
+  stew/[byteutils, endians2],
   stew/shims/net,
   json_rpc/servers/httpserver,
   eth/p2p/discoveryv5/protocol,

nimbus/beacon/api_handler/api_utils.nim

@@ -12,7 +12,7 @@ import
   eth/[rlp],
   json_rpc/errors,
   nimcrypto/[hash, sha2],
-  stew/[results],
+  stew/[results, endians2],
   ../../constants,
   ../../db/core_db,
   ../../utils/utils,

nimbus/common/chain_config.nim

@@ -89,7 +89,7 @@ const
 
 proc read(rlp: var Rlp, x: var AddressBalance, _: type EthAddress): EthAddress
     {.gcsafe, raises: [RlpError].} =
-  let val = rlp.read(UInt256).toByteArrayBE()
+  let val = rlp.read(UInt256).toBytesBE()
   result[0 .. ^1] = val.toOpenArray(12, val.high)
 
 proc read(rlp: var Rlp, x: var AddressBalance, _: type GenesisAccount): GenesisAccount

nimbus/common/genesis.nim

@@ -71,7 +71,7 @@ proc toGenesisHeader*(
   elif fork >= London:
     result.baseFee = EIP1559_INITIAL_BASE_FEE.u256
 
-  if g.gasLimit.isZero:
+  if g.gasLimit == 0:
     result.gasLimit = GENESIS_GAS_LIMIT
 
   if g.difficulty.isZero and fork <= London:

nimbus/common/hardforks.nim

@@ -10,6 +10,7 @@
 import
   std/[options, times],
   eth/common,
+  stew/endians2,
   json_serialization,
   ../utils/utils,
   ./evmforks

nimbus/core/executor/executor_helpers.nim

@@ -44,7 +44,7 @@ func createBloom*(receipts: openArray[Receipt]): Bloom =
   var bloom: LogsBloom
   for rec in receipts:
     bloom.value = bloom.value or logsBloom(rec.logs).value
-  result = bloom.value.toByteArrayBE
+  result = bloom.value.toBytesBE
 
 proc makeReceipt*(vmState: BaseVMState; txType: TxType): Receipt =
 
@@ -61,7 +61,7 @@ proc makeReceipt*(vmState: BaseVMState; txType: TxType): Receipt =
   rec.receiptType = txType
   rec.cumulativeGasUsed = vmState.cumulativeGasUsed
   rec.logs = vmState.getAndClearLogEntries()
-  rec.bloom = logsBloom(rec.logs).value.toByteArrayBE
+  rec.bloom = logsBloom(rec.logs).value.toBytesBE
   rec
 
 # ------------------------------------------------------------------------------

nimbus/core/pow.nim

@@ -17,6 +17,7 @@ import
   ../utils/utils,
   ./pow/[pow_cache, pow_dataset],
   eth/[common, keys, p2p, rlp],
+  stew/endians2,
   ethash,
   stint
 

nimbus/db/accounts_cache.nim

@@ -242,7 +242,7 @@ template createTrieKeyFromSlot(slot: UInt256): auto =
   # XXX: This is too expensive. Similar to `createRangeFromAddress`
   # Converts a number to hex big-endian representation including
   # prefix and leading zeros:
-  slot.toByteArrayBE
+  slot.toBytesBE
   # Original py-evm code:
   # pad32(int_to_big_endian(slot))
   # morally equivalent to toByteRange_Unnecessary but with different types

nimbus/db/aristo/aristo_desc/desc_identifiers.nim

@@ -175,7 +175,7 @@ func `-`*(lty: LeafTie, n: int): LeafTie =
 func to*(hid: HashID; T: type Hash256): T =
   result.data = hid.UInt256.toBytesBE
 
-func to*(hid: HashID; T: type HashKey): T =
+proc to*(hid: HashID; T: type HashKey): T =
   hid.UInt256.toBytesBE.T
 
 func to*(key: HashKey; T: type HashID): T =

nimbus/db/aristo/aristo_init/rocks_db/rdb_desc.nim

@@ -18,6 +18,7 @@ import
   eth/common,
   rocksdb,
   stint,
+  stew/endians2,
   ../../aristo_desc,
   ../init_common
 

nimbus/db/aristo/aristo_init/rocks_db/rdb_walk.nim

@@ -16,6 +16,7 @@
 import
   std/sequtils,
   eth/common,
+  stew/endians2,
   rocksdb,
   ../init_common,
   ./rdb_desc

nimbus/db/aristo/aristo_transcode.nim

@@ -13,7 +13,7 @@
 import
   std/[bitops, sequtils, sets],
   eth/[common, rlp, trie/nibbles],
-  results,
+  stew/[results, endians2],
   "."/[aristo_constants, aristo_desc]
 
 # ------------------------------------------------------------------------------

nimbus/db/distinct_tries.nim

@@ -57,7 +57,7 @@ template createTrieKeyFromSlot*(slot: UInt256): auto =
   # XXX: This is too expensive. Similar to `createRangeFromAddress`
   # Converts a number to hex big-endian representation including
   # prefix and leading zeros:
-  slot.toByteArrayBE
+  slot.toBytesBE
   # Original py-evm code:
   # pad32(int_to_big_endian(slot))
   # morally equivalent to toByteRange_Unnecessary but with different types

nimbus/db/state_db.nim

@@ -122,7 +122,7 @@ proc subBalance*(db: AccountStateDB, address: EthAddress, delta: UInt256) =
 template createTrieKeyFromSlot(slot: UInt256): auto =
   # Converts a number to hex big-endian representation including
   # prefix and leading zeros:
-  slot.toByteArrayBE
+  slot.toBytesBE
   # Original py-evm code:
   # pad32(int_to_big_endian(slot))
   # morally equivalent to toByteRange_Unnecessary but with different types
@@ -159,7 +159,7 @@ proc setStorage*(db: AccountStateDB,
   var
     triedb = db.kvt
     # slotHash can be obtained from accountTrie.put?
-    slotHash = keccakHash(slot.toByteArrayBE)
+    slotHash = keccakHash(slot.toBytesBE)
   triedb.put(slotHashToSlotKey(slotHash.data).toOpenArray, rlp.encode(slot))
 
   account.storageRoot = accountTrie.rootHash

nimbus/evm/evmc_helpers.nim

@@ -18,7 +18,7 @@ func toEvmc*(n: UInt256): evmc_uint256be {.inline.} =
   when evmc_native:
     cast[evmc_uint256be](n)
   else:
-    cast[evmc_uint256be](n.toByteArrayBE)
+    cast[evmc_uint256be](n.toBytesBE)
 
 func fromEvmc*(T: type, n: evmc_bytes32): T {.inline.} =
   when T is Hash256 | ContractSalt:

nimbus/evm/interpreter/op_handlers/oph_arithmetic.nim

@@ -24,6 +24,12 @@ import
   ./oph_defs,
   eth/common
 
+func slt(x, y: UInt256): bool =
+  type SignedWord = signedWordType(UInt256)
+  let x_neg = cast[SignedWord](x.mostSignificantWord) < 0
+  let y_neg = cast[SignedWord](y.mostSignificantWord) < 0
+  if x_neg xor y_neg: x_neg else: x < y
+
 {.push raises: [CatchableError].} # basically the annotation type of a `Vm2OpFn`
 
 # ------------------------------------------------------------------------------
@@ -184,16 +190,17 @@ const
     ## 0x12, Signed less-than comparison
     let (lhs, rhs) = k.cpt.stack.popInt(2)
     k.cpt.stack.push:
-      (cast[Int256](lhs) < cast[Int256](rhs)).uint.u256
+      slt(lhs, rhs).uint.u256
 
   sgtOp: Vm2OpFn = proc(k: var Vm2Ctx) =
-    ## 0x14, Signed greater-than comparison
+    ## 0x13, Signed greater-than comparison
     let (lhs, rhs) = k.cpt.stack.popInt(2)
     k.cpt.stack.push:
-      (cast[Int256](lhs) > cast[Int256](rhs)).uint.u256
+      # Arguments are swapped and SLT is used.
+      slt(rhs, lhs).uint.u256
 
   eqOp: Vm2OpFn = proc(k: var Vm2Ctx) =
-    ## 0x14, Signed greater-than comparison
+    ## 0x14, Equality comparison
     let (lhs, rhs) = k.cpt.stack.popInt(2)
     k.cpt.stack.push:
       (lhs == rhs).uint.u256
@@ -272,7 +279,7 @@ const
         k.cpt.stack.push:
           cast[UInt256]((-1).i256)
       else:
-       k.cpt.stack. push:
+        k.cpt.stack. push:
           0
     else:
       # int version of `shr` then force the result

nimbus/evm/interpreter/op_handlers/oph_memory.nim

@@ -151,7 +151,7 @@ const
       reason = "MSTORE: GasVeryLow + memory expansion")
 
     k.cpt.memory.extend(memPos, 32)
-    k.cpt.memory.write(memPos, value.toByteArrayBE)
+    k.cpt.memory.write(memPos, value.toBytesBE)
 
 
   mstore8Op: Vm2OpFn = proc (k: var Vm2Ctx) =

nimbus/evm/interpreter/utils/utils_numeric.nim

@@ -74,8 +74,7 @@ proc rangeToPadded*[T: StUint](x: openArray[byte], first, last, size: int): T =
   var temp: array[N, byte]
   temp[0..hi-lo] = x.toOpenArray(lo, hi)
   result = T.fromBytesBE(
-    temp.toOpenArray(0, size-1),
+    temp.toOpenArray(0, size-1)
-    allowPadding = true
   )
 
 proc rangeToPadded*(x: openArray[byte], first, size: int): seq[byte] =
@@ -90,7 +89,7 @@ proc rangeToPadded*(x: openArray[byte], first, size: int): seq[byte] =
 
 # calculates the memory size required for a step
 func calcMemSize*(offset, length: int): int {.inline.} =
-  if length.isZero: return 0
+  if length == 0: return 0
   result = offset + length
 
 func safeInt*(x: UInt256): int {.inline.} =

nimbus/evm/interpreter_dispatch.nim

@@ -16,7 +16,7 @@ const
 import
   std/[macros, strformat],
   pkg/[chronicles, chronos, stew/byteutils],
-  ".."/[constants, utils/utils, db/accounts_cache],
+  ".."/[constants, db/accounts_cache],
   "."/[code_stream, computation],
   "."/[message, precompiles, state, types],
   ./async/operations,

nimbus/evm/memory.nim

@@ -6,7 +6,7 @@
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
 import
-  chronicles, eth/common/eth_types,
+  chronicles,
   ./validation,
   ./interpreter/utils/utils_numeric
 

nimbus/evm/stack.nim

@@ -29,12 +29,12 @@ proc len*(stack: Stack): int {.inline.} =
 proc toStackElement(v: UInt256, elem: var StackElement) {.inline.} = elem = v
 proc toStackElement(v: uint | int | GasInt, elem: var StackElement) {.inline.} = elem = v.u256
 proc toStackElement(v: EthAddress, elem: var StackElement) {.inline.} = elem.initFromBytesBE(v)
-proc toStackElement(v: MDigest, elem: var StackElement) {.inline.} = elem.initFromBytesBE(v.data, allowPadding = false)
+proc toStackElement(v: MDigest, elem: var StackElement) {.inline.} = elem.initFromBytesBE(v.data)
 
 proc fromStackElement(elem: StackElement, v: var UInt256) {.inline.} = v = elem
-proc fromStackElement(elem: StackElement, v: var EthAddress) {.inline.} = v[0 .. ^1] = elem.toByteArrayBE().toOpenArray(12, 31)
+proc fromStackElement(elem: StackElement, v: var EthAddress) {.inline.} = v[0 .. ^1] = elem.toBytesBE().toOpenArray(12, 31)
-proc fromStackElement(elem: StackElement, v: var Hash256) {.inline.} = v.data = elem.toByteArrayBE()
+proc fromStackElement(elem: StackElement, v: var Hash256) {.inline.} = v.data = elem.toBytesBE()
-proc fromStackElement(elem: StackElement, v: var Topic) {.inline.} = v = elem.toByteArrayBE()
+proc fromStackElement(elem: StackElement, v: var Topic) {.inline.} = v = elem.toBytesBE()
 
 proc toStackElement(v: openArray[byte], elem: var StackElement) {.inline.} =
   # TODO: This needs to go

nimbus/evm/state.nim

@@ -274,7 +274,7 @@ method blockNumber*(vmState: BaseVMState): BlockNumber {.base, gcsafe.} =
 method difficulty*(vmState: BaseVMState): UInt256 {.base, gcsafe.} =
   if vmState.com.consensus == ConsensusType.POS:
     # EIP-4399/EIP-3675
-    UInt256.fromBytesBE(vmState.prevRandao.data, allowPadding = false)
+    UInt256.fromBytesBE(vmState.prevRandao.data)
   else:
     vmState.blockDifficulty
 

nimbus/rpc/hexstrings.nim

@@ -28,7 +28,8 @@
 
 import
   std/strutils,
-  stint, stew/byteutils, eth/keys,
+  stint, stew/[byteutils],
+  eth/keys,
   eth/common/eth_types,
   json_serialization
 

nimbus/rpc/merge/mergeutils.nim

@@ -0,0 +1,240 @@
+# Nimbus
+# Copyright (c) 2022-2023 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+import
+  std/[typetraits, times, strutils],
+  eth/[rlp, common],
+  json_rpc/errors,
+  eth/[trie, rlp, common, common/eth_types, trie/db],
+  stew/[results, byteutils, endians2],
+  ../../constants,
+  ../../db/core_db,
+  ../../utils/utils,
+  ../../rpc/execution_types,
+  ./mergetypes
+
+type Hash256 = eth_types.Hash256
+
+proc computePayloadId*(headBlockHash: Hash256, params: SomePayloadAttributes): PayloadID =
+  var dest: Hash256
+  var ctx: sha256
+  ctx.init()
+  ctx.update(headBlockHash.data)
+  ctx.update(toBytesBE distinctBase params.timestamp)
+  ctx.update(distinctBase params.prevRandao)
+  ctx.update(distinctBase params.suggestedFeeRecipient)
+  # FIXME-Adam: Do we need to include the withdrawals in this calculation?
+  # https://github.com/ethereum/go-ethereum/pull/25838#discussion_r1024340383
+  # "The execution api specs define that this ID can be completely random. It
+  # used to be derived from payload attributes in the past, but maybe it's
+  # time to use a randomized ID to not break it with any changes to the
+  # attributes?"
+  ctx.finish dest.data
+  ctx.clear()
+  (distinctBase result)[0..7] = dest.data[0..7]
+
+proc append*(w: var RlpWriter, q: Quantity) =
+  w.append(uint64(q))
+
+proc append*(w: var RlpWriter, a: Address) =
+  w.append(distinctBase(a))
+
+template unsafeQuantityToInt64(q: Quantity): int64 =
+  int64 q
+
+template asEthHash*(hash: engine_api_types.BlockHash): Hash256 =
+  Hash256(data: distinctBase(hash))
+
+proc calcRootHashRlp*(items: openArray[seq[byte]]): Hash256 =
+  var tr = newCoreDbRef(LegacyDbMemory).mptPrune
+  for i, t in items:
+    tr.put(rlp.encode(i), t)
+  return tr.rootHash()
+
+proc toWithdrawal*(w: WithdrawalV1): Withdrawal =
+  Withdrawal(
+    index: uint64(w.index),
+    validatorIndex: uint64(w.validatorIndex),
+    address: distinctBase(w.address),
+    amount: uint64(w.amount) # AARDVARK: is this wei or gwei or what?
+  )
+
+proc toWithdrawalV1*(w: Withdrawal): WithdrawalV1 =
+  WithdrawalV1(
+    index: Quantity(w.index),
+    validatorIndex: Quantity(w.validatorIndex),
+    address: Address(w.address),
+    amount: Quantity(w.amount) # AARDVARK: is this wei or gwei or what?
+  )
+
+proc maybeWithdrawalsRoot(payload: SomeExecutionPayload): Option[Hash256] =
+  when payload is ExecutionPayloadV1:
+    none(Hash256)
+  else:
+    var wds = newSeqOfCap[Withdrawal](payload.withdrawals.len)
+    for wd in payload.withdrawals:
+      wds.add toWithdrawal(wd)
+    some(utils.calcWithdrawalsRoot(wds))
+
+proc toWithdrawals(withdrawals: openArray[WithdrawalV1]): seq[WithDrawal] =
+  result = newSeqOfCap[Withdrawal](withdrawals.len)
+  for wd in withdrawals:
+    result.add toWithdrawal(wd)
+
+proc maybeBlobGasUsed(payload: SomeExecutionPayload): Option[uint64] =
+  when payload is ExecutionPayloadV3:
+    some(payload.blobGasUsed.uint64)
+  else:
+    none(uint64)
+
+proc maybeExcessBlobGas(payload: SomeExecutionPayload): Option[uint64] =
+  when payload is ExecutionPayloadV3:
+    some(payload.excessBlobGas.uint64)
+  else:
+    none(uint64)
+
+proc toBlockHeader*(payload: SomeExecutionPayload): EthBlockHeader =
+  let transactions = seq[seq[byte]](payload.transactions)
+  let txRoot = calcRootHashRlp(transactions)
+
+  EthBlockHeader(
+    parentHash     : payload.parentHash.asEthHash,
+    ommersHash     : EMPTY_UNCLE_HASH,
+    coinbase       : EthAddress payload.feeRecipient,
+    stateRoot      : payload.stateRoot.asEthHash,
+    txRoot         : txRoot,
+    receiptRoot    : payload.receiptsRoot.asEthHash,
+    bloom          : distinctBase(payload.logsBloom),
+    difficulty     : default(DifficultyInt),
+    blockNumber    : payload.blockNumber.distinctBase.u256,
+    gasLimit       : payload.gasLimit.unsafeQuantityToInt64,
+    gasUsed        : payload.gasUsed.unsafeQuantityToInt64,
+    timestamp      : fromUnix payload.timestamp.unsafeQuantityToInt64,
+    extraData      : bytes payload.extraData,
+    mixDigest      : payload.prevRandao.asEthHash, # EIP-4399 redefine `mixDigest` -> `prevRandao`
+    nonce          : default(BlockNonce),
+    fee            : some payload.baseFeePerGas,
+    withdrawalsRoot: payload.maybeWithdrawalsRoot, # EIP-4895
+    blobGasUsed    : payload.maybeBlobGasUsed, # EIP-4844
+    excessBlobGas  : payload.maybeExcessBlobGas, # EIP-4844
+  )
+
+proc toTypedTransaction*(tx: Transaction): TypedTransaction =
+  TypedTransaction(rlp.encode(tx))
+
+proc toBlockBody*(payload: SomeExecutionPayload): BlockBody =
+  result.transactions.setLen(payload.transactions.len)
+  for i, tx in payload.transactions:
+    result.transactions[i] = rlp.decode(distinctBase tx, Transaction)
+  when payload is ExecutionPayloadV2:
+    result.withdrawals = some(payload.withdrawals.toWithdrawals)
+  when payload is ExecutionPayloadV3:
+    result.withdrawals = some(payload.withdrawals.toWithdrawals)
+
+proc `$`*(x: BlockHash): string =
+  toHex(x)
+
+template toValidHash*(x: Hash256): Option[BlockHash] =
+  some(BlockHash(x.data))
+
+proc validateBlockHash*(header: EthBlockHeader, gotHash: Hash256): Result[void, PayloadStatusV1] =
+  let wantHash = header.blockHash
+  if wantHash != gotHash:
+    let status = PayloadStatusV1(
+      # This used to say invalid_block_hash, but see here:
+      # https://github.com/ethereum/execution-apis/blob/main/src/engine/shanghai.md#engine_newpayloadv2
+      # "INVALID_BLOCK_HASH status value is supplanted by INVALID."
+      status: PayloadExecutionStatus.invalid,
+      validationError: some("blockhash mismatch, want $1, got $2" % [$wantHash, $gotHash])
+    )
+    return err(status)
+
+  return ok()
+
+proc simpleFCU*(status: PayloadExecutionStatus): ForkchoiceUpdatedResponse =
+  ForkchoiceUpdatedResponse(payloadStatus: PayloadStatusV1(status: status))
+
+proc simpleFCU*(status: PayloadExecutionStatus, msg: string): ForkchoiceUpdatedResponse =
+  ForkchoiceUpdatedResponse(
+    payloadStatus: PayloadStatusV1(
+      status: status,
+      validationError: some(msg)
+    )
+  )
+
+proc invalidFCU*(hash: Hash256 = Hash256()): ForkchoiceUpdatedResponse =
+  ForkchoiceUpdatedResponse(payloadStatus:
+    PayloadStatusV1(
+      status: PayloadExecutionStatus.invalid,
+      latestValidHash: toValidHash(hash)
+    )
+  )
+
+proc validFCU*(id: Option[PayloadID], validHash: Hash256): ForkchoiceUpdatedResponse =
+  ForkchoiceUpdatedResponse(
+    payloadStatus: PayloadStatusV1(
+      status: PayloadExecutionStatus.valid,
+      latestValidHash: toValidHash(validHash)
+    ),
+    payloadId: id
+  )
+
+proc invalidStatus*(validHash: Hash256, msg: string): PayloadStatusV1 =
+  PayloadStatusV1(
+    status: PayloadExecutionStatus.invalid,
+    latestValidHash: toValidHash(validHash),
+    validationError: some(msg)
+  )
+
+proc invalidStatus*(validHash: Hash256 = Hash256()): PayloadStatusV1 =
+  PayloadStatusV1(
+    status: PayloadExecutionStatus.invalid,
+    latestValidHash: toValidHash(validHash)
+  )
+
+proc acceptedStatus*(validHash: Hash256): PayloadStatusV1 =
+  PayloadStatusV1(
+    status: PayloadExecutionStatus.accepted,
+    latestValidHash: toValidHash(validHash)
+  )
+
+proc acceptedStatus*(): PayloadStatusV1 =
+  PayloadStatusV1(
+    status: PayloadExecutionStatus.accepted
+  )
+
+proc validStatus*(validHash: Hash256): PayloadStatusV1 =
+  PayloadStatusV1(
+    status: PayloadExecutionStatus.valid,
+    latestValidHash: toValidHash(validHash)
+  )
+
+proc invalidParams*(msg: string): ref InvalidRequest =
+  (ref InvalidRequest)(
+    code: engineApiInvalidParams,
+    msg: msg
+  )
+
+proc unknownPayload*(msg: string): ref InvalidRequest =
+  (ref InvalidRequest)(
+    code: engineApiUnknownPayload,
+    msg: msg
+  )
+
+proc invalidAttr*(msg: string): ref InvalidRequest =
+  (ref InvalidRequest)(
+    code: engineApiInvalidPayloadAttributes,
+    msg: msg
+  )
+
+proc unsupportedFork*(msg: string): ref InvalidRequest =
+  (ref InvalidRequest)(
+    code: engineApiUnsupportedFork,
+    msg: msg
+  )

nimbus/sync/handlers/eth.nim

@@ -13,6 +13,7 @@
 import
   std/[tables, times, hashes, sets],
   chronicles, chronos,
+  stew/endians2,
   eth/p2p,
   eth/p2p/peer_pool,
   ".."/[types, protocol],

nimbus/sync/snap/worker/db/hexary_desc.nim

@@ -13,6 +13,7 @@
 import
   std/[hashes, sets, tables],
   eth/[common, trie/nibbles],
+  stew/endians2,
   stint,
   "../.."/[constants, range_desc],
   ./hexary_error

nimbus/transaction.nim

@@ -50,8 +50,8 @@ proc intrinsicGas*(tx: Transaction, fork: EVMFork): GasInt =
 
 proc getSignature*(tx: Transaction, output: var Signature): bool =
   var bytes: array[65, byte]
-  bytes[0..31] = tx.R.toByteArrayBE()
+  bytes[0..31] = tx.R.toBytesBE()
-  bytes[32..63] = tx.S.toByteArrayBE()
+  bytes[32..63] = tx.S.toBytesBE()
 
   if tx.txType == TxLegacy:
     var v = tx.V

nimbus/transaction/evmc_vm_glue.nim

@@ -69,7 +69,7 @@ proc evmcExecute(vm: ptr evmc_vm, hostInterface: ptr evmc_host_interface,
     # Nim defaults are fine for `create_address` and `padding`, zero bytes.
   )
 
-const evmcName  = "Nimbus EVM (vm2)"
+const evmcName  = "Nimbus EVM"
 const evmcVersion = "0.0.1"
 
 proc evmcGetCapabilities(vm: ptr evmc_vm): evmc_capabilities {.cdecl.} =

nimbus/transaction/host_types.nim

@@ -83,7 +83,7 @@ template fromEvmc*(address: evmc_address): EthAddress =
   cast[EthAddress](address)
 
 template flip256*(word256: evmc_uint256be): evmc_uint256be =
-  cast[evmc_uint256be](UInt256.fromBytesBe(word256.bytes).toBytes)
+  cast[evmc_uint256be](UInt256.fromBytesBE(word256.bytes).toBytes(cpuEndian))
 
 template isCreate*(kind: EvmcCallKind): bool =
   kind in {EVMC_CREATE, EVMC_CREATE2}

nimbus/utils/ec_recover.nim

@@ -54,8 +54,8 @@ type
 proc vrsSerialised(tx: Transaction): Result[array[65,byte],UtilsError] =
   ## Parts copied from `transaction.getSignature`.
   var data: array[65,byte]
-  data[0..31] = tx.R.toByteArrayBE
+  data[0..31] = tx.R.toBytesBE
-  data[32..63] = tx.S.toByteArrayBE
+  data[32..63] = tx.S.toBytesBE
 
   if tx.txType != TxLegacy:
     data[64] = tx.V.byte

nimbus_verified_proxy/nimbus_verified_proxy.nim

@@ -36,6 +36,7 @@ func getConfiguredChainId(networkMetadata: Eth2NetworkMetadata): Quantity =
         of rinkeby: 4.Quantity
         of goerli:  5.Quantity
         of sepolia: 11155111.Quantity
+        of holesky: 17000.Quantity
     return chainId
   else:
     return networkMetadata.cfg.DEPOSIT_CHAIN_ID.Quantity
@@ -61,7 +62,7 @@ proc run(config: VerifiedProxyConf) {.raises: [CatchableError].} =
   let
     genesisState =
       try:
-        template genesisData(): auto = metadata.genesisData
+        template genesisData(): auto = metadata.genesis.bakedBytes
         newClone(readSszForkedHashedBeaconState(
           cfg, genesisData.toOpenArray(genesisData.low, genesisData.high)))
       except CatchableError as err:

premix/hunter.nim

@@ -65,7 +65,7 @@ type
     headers: Table[BlockNumber, BlockHeader]
 
 proc hash*(x: UInt256): Hash =
-  result = hash(x.toByteArrayBE)
+  result = hash(x.toBytesBE)
 
 proc new(T: type HunterVMState; parent, header: BlockHeader, com: CommonRef): T =
   new result

tests/test_aristo/test_backend.nim

@@ -15,6 +15,7 @@ import
   eth/common,
   results,
   unittest2,
+  stew/endians2,
   ../../nimbus/sync/protocol,
   ../../nimbus/db/aristo,
   ../../nimbus/db/aristo/[
@@ -123,7 +124,7 @@ proc verify(
   true
 
 # -----------
-  
+
 proc collectFilter(
     db: AristoDbRef;
     filter: FilterRef;
@@ -289,7 +290,7 @@ proc testBackendConsistency*(
       xCheckRc rc.error == (0,0)
       let collectFilterOk = rdb.collectFilter(mdb.roFilter, filTab, noisy)
       xCheck collectFilterOk
-      
+
     # Store onto backend database
     block:
       #noisy.say "***", "db-dump\n    ", mdb.pp

tests/test_aristo/test_misc.nim

@@ -16,6 +16,7 @@ import
   eth/common,
   results,
   stew/byteutils,
+  stew/endians2,
   unittest2,
   ../../nimbus/db/aristo,
   ../../nimbus/db/aristo/[

tests/test_aristo/test_transcode.nim

@@ -0,0 +1,267 @@
+# Nimbus - Types, data structures and shared utilities used in network sync
+#
+# Copyright (c) 2018-2021 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
+#    http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
+#    http://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or
+# distributed except according to those terms.
+
+## Aristo (aka Patricia) DB trancoder test
+
+import
+  std/sequtils,
+  eth/common,
+  stew/[byteutils, endians2],
+  unittest2,
+  ../../nimbus/db/kvstore_rocksdb,
+  ../../nimbus/db/aristo/[
+    aristo_desc, aristo_debug, aristo_init, aristo_transcode, aristo_vid],
+  "."/[test_aristo_cache, test_helpers]
+
+type
+  TesterDesc = object
+    prng: uint32                       ## random state
+
+# ------------------------------------------------------------------------------
+# Private helpers
+# ------------------------------------------------------------------------------
+
+proc posixPrngRand(state: var uint32): byte =
+  ## POSIX.1-2001 example of a rand() implementation, see manual page rand(3).
+  state = state * 1103515245 + 12345;
+  let val = (state shr 16) and 32767    # mod 2^31
+  (val shr 8).byte                      # Extract second byte
+
+proc rand[W: SomeInteger|VertexID](ap: var TesterDesc; T: type W): T =
+  var a: array[sizeof T,byte]
+  for n in 0 ..< sizeof T:
+    a[n] = ap.prng.posixPrngRand().byte
+  when sizeof(T) == 1:
+    let w = uint8.fromBytesBE(a).T
+  when sizeof(T) == 2:
+    let w = uint16.fromBytesBE(a).T
+  when sizeof(T) == 4:
+    let w = uint32.fromBytesBE(a).T
+  else:
+    let w = uint64.fromBytesBE(a).T
+  when T is SomeUnsignedInt:
+    # That way, `fromBytesBE()` can be applied to `uint`
+    result = w
+  else:
+    # That way the result is independent of endianness
+    (addr result).copyMem(unsafeAddr w, sizeof w)
+
+proc vidRand(td: var TesterDesc; bits = 19): VertexID =
+  if bits < 64:
+    let
+      mask = (1u64 shl max(1,bits)) - 1
+      rval = td.rand uint64
+    (rval and mask).VertexID
+  else:
+    td.rand VertexID
+
+proc init(T: type TesterDesc; seed: int): TesterDesc =
+  result.prng = (seed and 0x7fffffff).uint32
+
+# -----
+
+proc getOrEmpty(rc: Result[Blob,AristoError]; noisy = true): Blob =
+  if rc.isOk:
+    return rc.value
+  noisy.say "***", "error=", rc.error
+
+proc `+`(a: VertexID, b: int): VertexID =
+  (a.uint64 + b.uint64).VertexID
+
+# ------------------------------------------------------------------------------
+# Public test function
+# ------------------------------------------------------------------------------
+
+proc test_transcodeAccounts*(
+    noisy = true;
+    rocky: RocksStoreRef;
+    stopAfter = high(int);
+      ) =
+  ## Transcoder tests on accounts database
+  var
+    adb = newAristoDbRef BackendNone
+    count = -1
+  for (n, key, value) in rocky.walkAllDb():
+    if stopAfter < n:
+      break
+    count = n
+
+    # RLP <-> NIM object mapping
+    let node0 = value.decode(NodeRef)
+    block:
+      let blob0 = rlp.encode node0
+      if value != blob0:
+        check value.len == blob0.len
+        check value == blob0
+        noisy.say "***", "count=", count, " value=", value.rlpFromBytes.inspect
+        noisy.say "***", "count=", count, " blob0=", blob0.rlpFromBytes.inspect
+
+    # Provide DbRecord with dummy links and expanded payload. Registering the
+    # node as vertex and re-converting it does the job
+    var node = node0.updated(VertexID(1), adb)
+    if node.error != AristoError(0):
+      check node.error == AristoError(0)
+    else:
+      case node.vType:
+      of aristo_desc.Leaf:
+        let account = node.lData.rawBlob.decode(Account)
+        node.key[0] = account.storageRoot.to(HashKey)
+        node.lData = PayloadRef(
+          pType:   AccountData,
+          account: AristoAccount(
+            nonce:     account.nonce,
+            balance:   account.balance,
+            codeHash:  account.codehash,
+            storageID: adb.vidAttach HashLabel(
+              root:    VertexID(1),
+              key:     account.storageRoot.to(HashKey))))
+      of aristo_desc.Extension:
+        # key <-> vtx correspondence
+        check node.key[0] == node0.key[0]
+        check node.eVid != VertexID(0)
+      of aristo_desc.Branch:
+        for n in 0..15:
+          # key[n] <-> vtx[n] correspondence
+          check node.key[n] == node0.key[n]
+          if node.key[n].isValid != node.bVid[n].isValid:
+            check node.key[n].isValid == node.bVid[n].isValid
+            echo ">>> node=", node.pp
+
+    # This NIM object must match to the same RLP encoded byte stream
+    block:
+      var blob1 = rlp.encode node
+      if value != blob1:
+        check value.len == blob1.len
+        check value == blob1
+        noisy.say "***", "count=", count, " value=", value.rlpFromBytes.inspect
+        noisy.say "***", "count=", count, " blob1=", blob1.rlpFromBytes.inspect
+
+    # NIM object <-> DbRecord mapping
+    let dbr = node.blobify.getOrEmpty(noisy)
+    var node1 = dbr.deblobify(VertexRef).asNode(adb)
+    if node1.error != AristoError(0):
+      check node1.error == AristoError(0)
+
+    block:
+      if node != node1:
+        check node == node1
+        noisy.say "***", "count=", count, " node=", node.pp(adb)
+        noisy.say "***", "count=", count, " node1=", node1.pp(adb)
+
+    # Serialise back with expanded `AccountData` type payload (if any)
+    let dbr1 = node1.blobify.getOrEmpty(noisy)
+    block:
+      if dbr != dbr1:
+        check dbr == dbr1
+        noisy.say "***", "count=", count, " dbr=", dbr.toHex
+        noisy.say "***", "count=", count, " dbr1=", dbr1.toHex
+
+    # Serialise back as is
+    let dbr2 = dbr.deblobify(VertexRef).asNode(adb).blobify.getOrEmpty(noisy)
+    block:
+      if dbr != dbr2:
+        check dbr == dbr2
+        noisy.say "***", "count=", count, " dbr=", dbr.toHex
+        noisy.say "***", "count=", count, " dbr2=", dbr2.toHex
+
+  noisy.say "***", "records visited: ", count + 1
+
+
+proc test_transcodeVidRecycleLists*(noisy = true; seed = 42) =
+  ## Transcode VID lists held in `AristoDb` descriptor
+  var td = TesterDesc.init seed
+  let db = newAristoDbRef BackendNone
+
+  # Add some randum numbers
+  block:
+    let first = td.vidRand()
+    db.vidDispose first
+
+    var
+      expectedVids = 1
+      count = 1
+    # Feed some numbers used and some discaded
+    while expectedVids < 5 or count < 5 + expectedVids:
+      count.inc
+      let vid = td.vidRand()
+      expectedVids += (vid < first).ord
+      db.vidDispose vid
+
+    check db.top.vGen.len == expectedVids
+    noisy.say "***", "vids=", db.top.vGen.len, " discarded=", count-expectedVids
+
+  # Serialise/deserialise
+  block:
+    let dbBlob = db.top.vGen.blobify
+
+    # Deserialise
+    let
+      db1 = newAristoDbRef BackendNone
+      rc = dbBlob.deblobify seq[VertexID]
+    if rc.isErr:
+      check rc.error == AristoError(0)
+    else:
+      db1.top.vGen = rc.value
+
+    check db.top.vGen == db1.top.vGen
+
+  # Make sure that recycled numbers are fetched first
+  let topVid = db.top.vGen[^1]
+  while 1 < db.top.vGen.len:
+    let w = db.vidFetch()
+    check w < topVid
+  check db.top.vGen.len == 1 and db.top.vGen[0] == topVid
+
+  # Get some consecutive vertex IDs
+  for n in 0 .. 5:
+    let w = db.vidFetch()
+    check w == topVid + n
+    check db.top.vGen.len == 1
+
+  # Repeat last test after clearing the cache
+  db.top.vGen.setLen(0)
+  for n in 0 .. 5:
+    let w = db.vidFetch()
+    check w == VertexID(2) + n # VertexID(1) is default root ID
+    check db.top.vGen.len == 1
+
+  # Recycling and re-org tests
+  db.top.vGen = @[8, 7,  3, 4, 5,  9].mapIt(VertexID(it))
+  db.vidReorg()
+  check db.top.vGen == @[3, 4, 5, 7].mapIt(VertexID(it))
+
+  db.top.vGen = @[8, 7, 6,  3, 4, 5,  9].mapIt(VertexID(it))
+  db.vidReorg()
+  check db.top.vGen == @[3].mapIt(VertexID(it))
+
+  db.top.vGen = @[5, 4, 3,  7].mapIt(VertexID(it))
+  db.vidReorg()
+  check db.top.vGen == @[5, 4, 3,  7].mapIt(VertexID(it))
+
+  db.top.vGen = @[5].mapIt(VertexID(it))
+  db.vidReorg()
+  check db.top.vGen == @[5].mapIt(VertexID(it))
+
+  db.top.vGen = @[3, 5].mapIt(VertexID(it))
+  db.vidReorg()
+  check db.top.vGen == @[3, 5].mapIt(VertexID(it))
+
+  db.top.vGen = @[4, 5].mapIt(VertexID(it))
+  db.vidReorg()
+  check db.top.vGen == @[4].mapIt(VertexID(it))
+
+  db.top.vGen.setLen(0)
+  db.vidReorg()
+  check db.top.vGen.len == 0
+
+# ------------------------------------------------------------------------------
+# End
+# ------------------------------------------------------------------------------

tests/test_aristo/test_tx.nim

@@ -16,6 +16,7 @@ import
   eth/common,
   results,
   unittest2,
+  stew/endians2,
   ../../nimbus/db/aristo/[
     aristo_check, aristo_delete, aristo_desc, aristo_get, aristo_merge],
   ../../nimbus/db/[aristo, aristo/aristo_init/persistent],

tests/test_pow.nim

@@ -13,6 +13,7 @@ import
   ./replay/[pp, gunzip],
   ../nimbus/core/[pow, pow/pow_cache, pow/pow_dataset],
   eth/[common],
+  stew/endians2,
   unittest2
 
 const

vendor/nim-blscurve

@@ -1 +1 @@
-Subproject commit 81ed3ff376e6549aead42e506861b3a6ef63b7fa
+Subproject commit 93df4f2cb8f65ddb34ec1c681df6e453b2276ee3

vendor/nim-chronos

@@ -1 +1 @@
-Subproject commit db6410f835c51676f78002a9aa786630e16fbb08
+Subproject commit 00614476c68f0553432b4bb505e24d6ad5586ae4

vendor/nim-libp2p

@@ -1 +1 @@
-Subproject commit 1de7508b64e29c2bcad7a393c36cf8390a8c50fe
+Subproject commit 20b0e40f7d89da3f9f1a0a7e823112e938365337

vendor/nim-stint

@@ -1 +1 @@
-Subproject commit 94fc521ee0f1e113d09ceeaa3568d4d7a6c0b67d
+Subproject commit 54e24cae415b1bed39a987ecd08c19a34f740972

vendor/nimbus-eth2

@@ -1 +1 @@
-Subproject commit 8f184911311d5b891fc2e83420f21206463d422b
+Subproject commit 5c88e74c08bf2c15d230b8b05d959809065a2894