Add new Portal BlockBody type for Shanghai fork (#1589)

fluffy/common/common_types.nim

@@ -8,8 +8,9 @@
 {.push raises: [].}
 
 import
-  ssz_serialization/types,
-  stew/byteutils, nimcrypto/hash
+  ssz_serialization,
+  eth/rlp,
+  stew/[byteutils, results], nimcrypto/hash
 
 export hash
 
@@ -23,3 +24,21 @@ type
 
 func `$`*(x: ByteList): string =
   x.asSeq.toHex()
+
+func decodeRlp*(input: openArray[byte], T: type): Result[T, string] =
+  try:
+    ok(rlp.decode(input, T))
+  except RlpError as e:
+    err(e.msg)
+
+func decodeSsz*(input: openArray[byte], T: type): Result[T, string] =
+  try:
+    ok(SSZ.decode(input, T))
+  except SszError as e:
+    err(e.msg)
+
+func decodeSszOrRaise*(input: openArray[byte], T: type): T =
+  try:
+    SSZ.decode(input, T)
+  except SszError as e:
+    raiseAssert(e.msg)

fluffy/network/beacon_light_client/beacon_light_client_content.nim

@@ -97,12 +97,6 @@ func toContentId*(contentKey: ByteList): ContentId =
 func toContentId*(contentKey: ContentKey): ContentId =
   toContentId(encode(contentKey))
 
-func decodeSsz*(input: openArray[byte], T: type): Result[T, string] =
-  try:
-    ok(SSZ.decode(input, T))
-  except SszError as e:
-    err(e.msg)
-
 # Yes, this API is odd as you pass a SomeForkedLightClientObject yet still have
 # to also pass the ForkDigest. This is because we can't just select the right
 # digest through the LightClientDataFork here as LightClientDataFork and

fluffy/network/history/history_content.nim

@@ -15,6 +15,9 @@ import
   ssz_serialization,
   ../../common/common_types
 
+from beacon_chain/spec/datatypes/capella import Withdrawal
+from beacon_chain/spec/presets/mainnet import MAX_WITHDRAWALS_PER_PAYLOAD
+
 export ssz_serialization, common_types, hash, results
 
 ## Types and calls for history network content keys
@@ -117,6 +120,8 @@ const
   MAX_RECEIPT_LENGTH* = 2^27  # ~= 134 million
   MAX_HEADER_LENGTH = 2^13  # = 8192
   MAX_ENCODED_UNCLES_LENGTH* = MAX_HEADER_LENGTH * 2^4  # = 2**17 ~= 131k
+  MAX_WITHDRAWAL_LENGTH = 64
+  MAX_WITHDRAWALS_COUNT = MAX_WITHDRAWALS_PER_PAYLOAD
 
 type
   ## Types for content
@@ -126,12 +131,23 @@ type
   Transactions* = List[TransactionByteList, MAX_TRANSACTION_COUNT]
   Uncles* = List[byte, MAX_ENCODED_UNCLES_LENGTH] # RLP data
 
-  BlockBodySSZ* = object
+  WithdrawalByteList* = List[byte, MAX_WITHDRAWAL_LENGTH] # RLP data
+  Withdrawals* = List[WithdrawalByteList, MAX_WITHDRAWALS_COUNT]
+
+  # Pre-shanghai block body
+  # Post-merge this block body is required to have an empty list for uncles
+  PortalBlockBodyLegacy* = object
     transactions*: Transactions
     uncles*: Uncles
 
+  # Post-shanghai block body, added withdrawals
+  PortalBlockBodyShanghai* = object
+    transactions*: Transactions
+    uncles*: Uncles
+    withdrawals*: Withdrawals
+
   ReceiptByteList* = List[byte, MAX_RECEIPT_LENGTH] # RLP data
-  ReceiptsSSZ* = List[ReceiptByteList, MAX_TRANSACTION_COUNT]
+  PortalReceipts* = List[ReceiptByteList, MAX_TRANSACTION_COUNT]
 
   AccumulatorProof* = array[15, Digest]
 

fluffy/network/history/history_network.nim

@@ -11,11 +11,15 @@ import
   stew/results, chronos, chronicles,
   eth/[common/eth_types_rlp, rlp, trie, trie/db],
   eth/p2p/discoveryv5/[protocol, enr],
+  ../../common/common_types,
   ../../content_db,
+  ../../network_metadata,
   ../../../nimbus/constants,
   ../wire/[portal_protocol, portal_stream, portal_protocol_config],
   "."/[history_content, accumulator]
 
+from std/times import toUnix
+
 logScope:
   topics = "portal_hist"
 
@@ -61,23 +65,12 @@ type
 func toContentIdHandler(contentKey: ByteList): results.Opt[ContentId] =
   ok(toContentId(contentKey))
 
-func decodeRlp*(input: openArray[byte], T: type): Result[T, string] =
-  try:
-    ok(rlp.decode(input, T))
-  except RlpError as e:
-    err(e.msg)
-
-func decodeSsz*(input: openArray[byte], T: type): Result[T, string] =
-  try:
-    ok(SSZ.decode(input, T))
-  except SszError as e:
-    err(e.msg)
-
-## Calls to go from SSZ decoded types to RLP fully decoded types
+## Calls to go from SSZ decoded Portal types to RLP fully decoded EL types
 
 func fromPortalBlockBody*(
-    T: type BlockBody, body: BlockBodySSZ): Result[T, string] =
-  ## Get the full decoded BlockBody from the SSZ-decoded `PortalBlockBody`.
+    T: type BlockBody, body: PortalBlockBodyLegacy):
+    Result[T, string] =
+  ## Get the EL BlockBody from the SSZ-decoded `PortalBlockBodyLegacy`.
   try:
     var transactions: seq[Transaction]
     for tx in body.transactions:
@@ -89,9 +82,42 @@ func fromPortalBlockBody*(
   except RlpError as e:
     err("RLP decoding failed: " & e.msg)
 
-func fromReceipts*(
-    T: type seq[Receipt], receipts: ReceiptsSSZ): Result[T, string] =
-  ## Get the full decoded seq[Receipt] from the SSZ-decoded `Receipts`.
+func fromPortalBlockBody*(
+    T: type BlockBody, body: PortalBlockBodyShanghai): Result[T, string] =
+  ## Get the EL BlockBody from the SSZ-decoded `PortalBlockBodyShanghai`.
+  try:
+    var transactions: seq[Transaction]
+    for tx in body.transactions:
+      transactions.add(rlp.decode(tx.asSeq(), Transaction))
+
+    var withdrawals: seq[Withdrawal]
+    for w in body.withdrawals:
+      withdrawals.add(rlp.decode(w.asSeq(), Withdrawal))
+
+    ok(BlockBody(
+      transactions: transactions,
+      uncles: @[], # Uncles must be empty: TODO where validation?
+      withdrawals: some(withdrawals)))
+  except RlpError as e:
+    err("RLP decoding failed: " & e.msg)
+
+func fromPortalBlockBodyOrRaise*(
+    T: type BlockBody,
+    body: PortalBlockBodyLegacy | PortalBlockBodyShanghai):
+    T =
+  ## Get the EL BlockBody from one of the SSZ-decoded Portal BlockBody types.
+  ## Will raise Assertion in case of invalid RLP encodings. Only use of data
+  ## has been validated before!
+  # TODO: Using ValueOr here gives compile error
+  let res = BlockBody.fromPortalBlockBody(body)
+  if res.isOk():
+    res.get()
+  else:
+    raiseAssert(res.error)
+
+func fromPortalReceipts*(
+    T: type seq[Receipt], receipts: PortalReceipts): Result[T, string] =
+  ## Get the full decoded EL seq[Receipt] from the SSZ-decoded `PortalReceipts`.
   try:
     var res: seq[Receipt]
     for receipt in receipts:
@@ -101,31 +127,55 @@ func fromReceipts*(
   except RlpError as e:
     err("RLP decoding failed: " & e.msg)
 
-## Calls to encode Block types to the SSZ types.
+## Calls to encode EL block types to the SSZ encoded Portal types.
 
-func fromBlockBody(T: type BlockBodySSZ, body: BlockBody): T =
+# TODO: The fact that we have different Portal BlockBody types for the different
+# forks but not for the EL BlockBody (usage of Option) does not play so well
+# together.
+
+func fromBlockBody(T: type PortalBlockBodyLegacy, body: BlockBody): T =
   var transactions: Transactions
   for tx in body.transactions:
     discard transactions.add(TransactionByteList(rlp.encode(tx)))
 
   let uncles = Uncles(rlp.encode(body.uncles))
 
-  BlockBodySSZ(transactions: transactions, uncles: uncles)
+  PortalBlockBodyLegacy(transactions: transactions, uncles: uncles)
 
-func fromReceipts*(T: type ReceiptsSSZ, receipts: seq[Receipt]): T =
-  var receiptsSSZ: ReceiptsSSZ
+func fromBlockBody(T: type PortalBlockBodyShanghai, body: BlockBody): T =
+  var transactions: Transactions
+  for tx in body.transactions:
+    discard transactions.add(TransactionByteList(rlp.encode(tx)))
+
+  let uncles = Uncles(rlp.encode(body.uncles))
+
+  doAssert(body.withdrawals.isSome())
+
+  var withdrawals: Withdrawals
+  for w in body.withdrawals.get():
+    discard withdrawals.add(WithdrawalByteList(rlp.encode(w)))
+  PortalBlockBodyShanghai(transactions: transactions, uncles: uncles, withdrawals: withdrawals)
+
+func fromReceipts*(T: type PortalReceipts, receipts: seq[Receipt]): T =
+  var portalReceipts: PortalReceipts
   for receipt in receipts:
-    discard receiptsSSZ.add(ReceiptByteList(rlp.encode(receipt)))
+    discard portalReceipts.add(ReceiptByteList(rlp.encode(receipt)))
 
-  receiptsSSZ
+  portalReceipts
 
 func encode*(blockBody: BlockBody): seq[byte] =
-  let portalBlockBody = BlockBodySSZ.fromBlockBody(blockBody)
+  if blockBody.withdrawals.isSome():
+    SSZ.encode(PortalBlockBodyShanghai.fromBlockBody(blockBody))
+  else:
+    SSZ.encode(PortalBlockBodyLegacy.fromBlockBody(blockBody))
+
+func encode*(blockBody: BlockBody, T: type PortalBlockBodyShanghai): seq[byte] =
+  let portalBlockBody = PortalBlockBodyShanghai.fromBlockBody(blockBody)
 
   SSZ.encode(portalBlockBody)
 
 func encode*(receipts: seq[Receipt]): seq[byte] =
-  let portalReceipts = ReceiptsSSZ.fromReceipts(receipts)
+  let portalReceipts = PortalReceipts.fromReceipts(receipts)
 
   SSZ.encode(portalReceipts)
 
@@ -133,7 +183,7 @@ func encode*(receipts: seq[Receipt]): seq[byte] =
 # TODO: Failures on validation and perhaps deserialisation should be punished
 # for if/when peer scoring/banning is added.
 
-proc calcRootHash(items: Transactions | ReceiptsSSZ): Hash256 =
+proc calcRootHash(items: Transactions | PortalReceipts| Withdrawals): Hash256 =
   var tr = initHexaryTrie(newMemoryDB())
   for i, item in items:
     try:
@@ -148,7 +198,10 @@ proc calcRootHash(items: Transactions | ReceiptsSSZ): Hash256 =
 template calcTxsRoot*(transactions: Transactions): Hash256 =
   calcRootHash(transactions)
 
-template calcReceiptsRoot*(receipts: ReceiptsSSZ): Hash256 =
+template calcReceiptsRoot*(receipts: PortalReceipts): Hash256 =
+  calcRootHash(receipts)
+
+template calcWithdrawalsRoot*(receipts: Withdrawals): Hash256 =
   calcRootHash(receipts)
 
 func validateBlockHeaderBytes*(
@@ -168,34 +221,91 @@ func validateBlockHeaderBytes*(
     ok(header)
 
 proc validateBlockBody(
-    body: BlockBodySSZ, txsRoot, ommersHash: KeccakHash):
+    body: PortalBlockBodyLegacy, header: BlockHeader):
     Result[void, string] =
-  ## Validate the block body against the txRoot amd ommersHash from the header.
-  # TODO: should be checked for hash for empty uncles after merge block
+  ## Validate the block body against the txRoot and ommersHash from the header.
   let calculatedOmmersHash = keccakHash(body.uncles.asSeq())
-  if calculatedOmmersHash != ommersHash:
+  if calculatedOmmersHash != header.ommersHash:
     return err("Invalid ommers hash")
 
   let calculatedTxsRoot = calcTxsRoot(body.transactions)
-  if calculatedTxsRoot != txsRoot:
+  if calculatedTxsRoot != header.txRoot:
     return err("Invalid transactions root")
 
-  # TODO: Add root check for withdrawals after Shanghai
-
   ok()
 
+proc validateBlockBody(
+    body: PortalBlockBodyShanghai, header: BlockHeader):
+    Result[void, string] =
+  ## Validate the block body against the txRoot, ommersHash and withdrawalsRoot
+  ## from the header.
+  # Shortcutting the ommersHash calculation as uncles needs to be empty
+  # TODO: This is since post-merge, however, we would need an additional object
+  # type for that period to do this.
+  if body.uncles.len > 0:
+    return err("Invalid ommers hash")
+
+  let calculatedTxsRoot = calcTxsRoot(body.transactions)
+  if calculatedTxsRoot != header.txRoot:
+    return err("Invalid transactions root")
+
+  # TODO: This check is done higher up but perhaps this can become cleaner with
+  # some refactor.
+  doAssert(header.withdrawalsRoot.isSome())
+
+  let calculatedWithdrawalsRoot = calcWithdrawalsRoot(body.withdrawals)
+  if calculatedWithdrawalsRoot != header.txRoot:
+    return err("Invalid transactions root")
+
+  ok()
+
+proc decodeBlockBodyBytes*(bytes: openArray[byte]): Result[BlockBody, string] =
+  if (let body = decodeSsz(bytes, PortalBlockBodyShanghai); body.isOk()):
+    BlockBody.fromPortalBlockBody(body.get())
+  elif (let body = decodeSsz(bytes, PortalBlockBodyLegacy); body.isOk()):
+    BlockBody.fromPortalBlockBody(body.get())
+  else:
+    err("All Portal block body decodings failed")
+
 proc validateBlockBodyBytes*(
-    bytes: openArray[byte], txRoot, ommersHash: KeccakHash):
+    bytes: openArray[byte], header: BlockHeader):
     Result[BlockBody, string] =
-  ## Fully decode the SSZ Block Body and validate it against the header.
-  let body = ? decodeSsz(bytes, BlockBodySSZ)
-
-  ? validateBlockBody(body, txRoot, ommersHash)
-
-  BlockBody.fromPortalBlockBody(body)
+  ## Fully decode the SSZ encoded Portal Block Body and validate it against the
+  ## header.
+  ## TODO: improve this decoding in combination with the block body validation
+  ## calls.
+  let timestamp = Moment.init(header.timestamp.toUnix(), Second)
+  # TODO: The additional header checks are not needed as header is implicitly
+  # verified by means of the accumulator? Except that we don't use this yet
+  # post merge, so the checks are still useful, for now.
+  if isShanghai(chainConfig, timestamp):
+    if header.withdrawalsRoot.isNone():
+      return err("Expected withdrawalsRoot for Shanghai block")
+    elif header.ommersHash != EMPTY_UNCLE_HASH:
+      return err("Expected empty uncles for a Shanghai block")
+    else:
+      let body = ? decodeSsz(bytes, PortalBlockBodyShanghai)
+      ? validateBlockBody(body, header)
+      BlockBody.fromPortalBlockBody(body)
+  elif isPoSBlock(chainConfig, header.blockNumber.truncate(uint64)):
+    if header.withdrawalsRoot.isSome():
+      return err("Expected no withdrawalsRoot for pre Shanghai block")
+    elif header.ommersHash != EMPTY_UNCLE_HASH:
+      return err("Expected empty uncles for a PoS block")
+    else:
+      let body = ? decodeSsz(bytes, PortalBlockBodyLegacy)
+      ? validateBlockBody(body, header)
+      BlockBody.fromPortalBlockBody(body)
+  else:
+    if header.withdrawalsRoot.isSome():
+      return err("Expected no withdrawalsRoot for pre Shanghai block")
+    else:
+      let body = ? decodeSsz(bytes, PortalBlockBodyLegacy)
+      ? validateBlockBody(body, header)
+      BlockBody.fromPortalBlockBody(body)
 
 proc validateReceipts*(
-    receipts: ReceiptsSSZ, receiptsRoot: KeccakHash): Result[void, string] =
+    receipts: PortalReceipts, receiptsRoot: KeccakHash): Result[void, string] =
   let calculatedReceiptsRoot = calcReceiptsRoot(receipts)
 
   if calculatedReceiptsRoot != receiptsRoot:
@@ -207,11 +317,11 @@ proc validateReceiptsBytes*(
     bytes: openArray[byte],
     receiptsRoot: KeccakHash): Result[seq[Receipt], string] =
   ## Fully decode the SSZ Block Body and validate it against the header.
-  let receipts = ? decodeSsz(bytes, ReceiptsSSZ)
+  let receipts = ? decodeSsz(bytes, PortalReceipts)
 
   ? validateReceipts(receipts, receiptsRoot)
 
-  seq[Receipt].fromReceipts(receipts)
+  seq[Receipt].fromPortalReceipts(receipts)
 
 ## ContentDB helper calls for specific history network types
 
@@ -232,21 +342,30 @@ proc get(db: ContentDB, T: type BlockHeader, contentId: ContentId): Opt[T] =
   else:
     Opt.none(T)
 
-proc get(db: ContentDB, T: type BlockBody, contentId: ContentId): Opt[T] =
-  let contentFromDB = db.getSszDecoded(contentId, BlockBodySSZ)
-  if contentFromDB.isSome():
-    let res = T.fromPortalBlockBody(contentFromDB.get())
-    if res.isErr():
-      raiseAssert(res.error)
+proc get(db: ContentDB, T: type BlockBody, contentId: ContentId,
+    header: BlockHeader): Opt[T] =
+  let encoded = db.get(contentId)
+  if encoded.isNone():
+    return Opt.none(T)
+
+  let timestamp = Moment.init(header.timestamp.toUnix(), Second)
+  let body =
+    if isShanghai(chainConfig, timestamp):
+      BlockBody.fromPortalBlockBodyOrRaise(
+        decodeSszOrRaise(encoded.get(), PortalBlockBodyShanghai))
+    elif isPoSBlock(chainConfig, header.blockNumber.truncate(uint64)):
+      BlockBody.fromPortalBlockBodyOrRaise(
+        decodeSszOrRaise(encoded.get(), PortalBlockBodyLegacy))
     else:
-      Opt.some(res.get())
-  else:
-    Opt.none(T)
+      BlockBody.fromPortalBlockBodyOrRaise(
+        decodeSszOrRaise(encoded.get(), PortalBlockBodyLegacy))
+
+  Opt.some(body)
 
 proc get(db: ContentDB, T: type seq[Receipt], contentId: ContentId): Opt[T] =
-  let contentFromDB = db.getSszDecoded(contentId, ReceiptsSSZ)
+  let contentFromDB = db.getSszDecoded(contentId, PortalReceipts)
   if contentFromDB.isSome():
-    let res = T.fromReceipts(contentFromDB.get())
+    let res = T.fromPortalReceipts(contentFromDB.get())
     if res.isErr():
       raiseAssert(res.error)
     else:
@@ -352,7 +471,7 @@ proc getBlockBody*(
     hash
     contentKey
 
-  let bodyFromDb = n.getContentFromDb(BlockBody, contentId)
+  let bodyFromDb = n.contentDB.get(BlockBody, contentId, header)
   if bodyFromDb.isSome():
     info "Fetched block body from database"
     return bodyFromDb
@@ -365,7 +484,7 @@ proc getBlockBody*(
         return Opt.none(BlockBody)
 
       body = validateBlockBodyBytes(
-          bodyContent.content, header.txRoot, header.ommersHash).valueOr:
+          bodyContent.content, header).valueOr:
         warn "Validation of block body failed", error
         continue
 
@@ -536,7 +655,7 @@ proc validateContent(
       warn "Failed getting canonical header for block"
       return false
 
-    let res = validateBlockBodyBytes(content, header.txRoot, header.ommersHash)
+    let res = validateBlockBodyBytes(content, header)
     if res.isErr():
       warn "Failed validating block body", error = res.error
       return false

fluffy/network_metadata.nim

@@ -8,7 +8,10 @@
 {.push raises: [].}
 
 import
-  std/[sequtils, strutils, os, macros]
+  std/[sequtils, strutils, os, macros],
+  stew/results,
+  chronos/timer#,
+  # eth/common/eth_types
 
 proc loadBootstrapNodes(
     path: string): seq[string] {.raises: [IOError].} =
@@ -62,3 +65,40 @@ const
 
   finishedAccumulator* = loadEncodedAccumulator(
     portalTestDir / "mainnet" / "history" / "accumulator" / "finished_accumulator.ssz")
+
+type
+  # TODO: I guess we could use the nimbus ChainConfig but:
+  # - Only need some of the values right now
+  # - `EthTime` uses std/times while chronos Moment is sufficient and more
+  # sensible
+  ChainConfig* = object
+    mergeForkBlock*     : uint64
+    shanghaiTime*       : Opt[Moment]
+    cancunTime*         : Opt[Moment]
+
+const
+  # Allow this to be adjusted at compile time for testing. If more constants
+  # need to be adjusted we can add some more ChainConfig presets either at
+  # compile or runtime.
+  mergeBlockNumber* {.intdefine.}: uint64 = 15537394
+
+  chainConfig* = ChainConfig(
+    mergeForkBlock: mergeBlockNumber,
+    shanghaiTime: Opt.some(Moment.init(1681338455'i64, Second)),
+    cancunTime: Opt.none(Moment)
+  )
+
+func isTimestampForked(forkTime: Opt[Moment], timestamp: Moment): bool =
+  if forkTime.isNone():
+    false
+  else:
+    forkTime.get() <= timestamp
+
+func isPoSBlock*(c: ChainConfig, blockNumber: uint64): bool =
+  c.mergeForkBlock <= blockNumber
+
+func isShanghai*(c: ChainConfig, timestamp: Moment): bool =
+  isTimestampForked(c.shanghaiTime, timestamp)
+
+func isCancun*(c: ChainConfig, timestamp: Moment): bool =
+  isTimestampForked(c.cancunTime, timestamp)

fluffy/tests/all_fluffy_tests.nim

@@ -12,7 +12,6 @@ import
   ./test_state_distance,
   ./test_state_network,
   ./test_accumulator,
-  ./test_history_validation,
   ./test_history_network,
   ./test_content_db,
   ./test_discovery_rpc,

fluffy/tests/portal_spec_tests/mainnet/all_fluffy_portal_spec_tests.nim

@@ -11,6 +11,7 @@ import
   ./test_portal_wire_encoding,
   ./test_history_content_keys,
   ./test_history_content,
+  ./test_history_content_validation,
   ./test_header_content,
   ./test_state_content,
   ./test_accumulator_root

fluffy/tests/portal_spec_tests/mainnet/test_history_content.nim

@@ -134,7 +134,7 @@ suite "History Content Encodings":
         SSZ.encode(blockHeaderWithProof) == contentValueEncoded
         encode(contentKey.get()).asSeq() == contentKeyEncoded
 
-  test "Block Body Encoding/Decoding and Verification":
+  test "PortalBlockBody (Legacy) Encoding/Decoding and Verification":
     const
       dataFile =
         "./vendor/portal-spec-tests/tests/mainnet/history/bodies/14764013.json"
@@ -171,7 +171,7 @@ suite "History Content Encodings":
 
       # Decode (SSZ + RLP decode step) and validate block body
       let contentValue = validateBlockBodyBytes(
-        contentValueEncoded, header.txRoot, header.ommersHash)
+        contentValueEncoded, header)
       check contentValue.isOk()
 
       # Encode content and content key
@@ -179,6 +179,35 @@ suite "History Content Encodings":
         encode(contentValue.get()) == contentValueEncoded
         encode(contentKey.get()).asSeq() == contentKeyEncoded
 
+  test "PortalBlockBody (Shanghai) Encoding/Decoding":
+    # TODO: We don't have the header (without proof) ready here so cannot do
+    # full validation for now. Add this header and then we can do like above.
+    const
+      dataFile =
+        "./vendor/portal-spec-tests/tests/mainnet/history/bodies/17139055.json"
+
+    let res = readJsonType(dataFile, JsonPortalContentTable)
+    check res.isOk()
+    let content = res.get()
+
+    for k, v in content:
+      let
+        contentKeyEncoded = v.content_key.hexToSeqByte()
+        contentValueEncoded = v.content_value.hexToSeqByte()
+
+      # Decode content key
+      let contentKey = decodeSsz(contentKeyEncoded, ContentKey)
+      check contentKey.isOk()
+
+      # Decode (SSZ + RLP decode step) and validate block body
+      let contentValue = decodeBlockBodyBytes(
+        contentValueEncoded)
+      check contentValue.isOk()
+
+      # Encode content and content key
+      check:
+        encode(contentValue.get()) == contentValueEncoded
+        encode(contentKey.get()).asSeq() == contentKeyEncoded
 
   test "Receipts Encoding/Decoding and Verification":
     const

fluffy/tests/portal_spec_tests/mainnet/test_history_content_validation.nim

@@ -13,9 +13,9 @@ import
   unittest2, stint,
   stew/[byteutils, results],
   eth/[common/eth_types, rlp],
-  ../common/common_types,
-  ../eth_data/history_data_json_store,
-  ../network/history/history_network
+  ../../../common/common_types,
+  ../../../eth_data/history_data_json_store,
+  ../../../network/history/history_network
 
 const
   dataFile = "./fluffy/tests/blocks/mainnet_blocks_selected.json"
@@ -43,7 +43,7 @@ suite "History Network Content Validation":
     blockHeader = decodeRlp(blockHeaderBytes, BlockHeader).expect(
       "Valid header should decode")
     blockBody = validateBlockBodyBytes(
-      blockBodyBytes, blockHeader.txRoot, blockHeader.ommersHash).expect(
+      blockBodyBytes, blockHeader).expect(
         "Should be Valid decoded block body")
     receipts = validateReceiptsBytes(
       receiptsBytes, blockHeader.receiptRoot).expect(
@@ -68,13 +68,13 @@ suite "History Network Content Validation":
 
   test "Valid Block Body":
     check validateBlockBodyBytes(
-      blockBodyBytes, blockHeader.txRoot, blockHeader.ommersHash).isOk()
+      blockBodyBytes, blockHeader).isOk()
 
   test "Malformed Block Body":
     let malformedBytes = blockBodyBytes[10..blockBodyBytes.high]
 
     check validateBlockBodyBytes(
-      malformedBytes, blockHeader.txRoot, blockHeader.ommersHash).isErr()
+      malformedBytes, blockHeader).isErr()
 
   test "Invalid Block Body - Modified Transaction List":
     var modifiedBody = blockBody
@@ -88,7 +88,7 @@ suite "History Network Content Validation":
     let modifiedBodyBytes = encode(modifiedBody)
 
     check validateBlockBodyBytes(
-      modifiedBodyBytes, blockHeader.txRoot, blockHeader.ommersHash).isErr()
+      modifiedBodyBytes, blockHeader).isErr()
 
   test "Invalid Block Body - Modified Uncles List":
     var modifiedBody = blockBody
@@ -98,7 +98,7 @@ suite "History Network Content Validation":
     let modifiedBodyBytes = encode(modifiedBody)
 
     check validateBlockBodyBytes(
-      modifiedBodyBytes, blockHeader.txRoot, blockHeader.ommersHash).isErr()
+      modifiedBodyBytes, blockHeader).isErr()
 
   test "Valid Receipts":
     check validateReceiptsBytes(receiptsBytes, blockHeader.receiptRoot).isOk()

fluffy/tools/bridge/beacon_chain_bridge.nim

@@ -185,53 +185,49 @@ func asReceipt(
   else:
     err("No root nor status field in the JSON receipt object")
 
+proc calculateTransactionData(
+    items: openArray[TypedTransaction]):
+    Hash256 {.raises: [].} =
+
+  var tr = initHexaryTrie(newMemoryDB())
+  for i, t in items:
+    try:
+      let tx = distinctBase(t)
+      tr.put(rlp.encode(i), tx)
+    except RlpError as e:
+      # TODO: Investigate this RlpError as it doesn't sound like this is
+      # something that can actually occur.
+      raiseAssert(e.msg)
+
+  return tr.rootHash()
+
+# TODO: Since Capella we can also access ExecutionPayloadHeader and thus
+# could get the Roots through there instead.
+proc calculateWithdrawalsRoot(
+  items: openArray[WithdrawalV1]):
+    Hash256 {.raises: [].} =
+
+  var tr = initHexaryTrie(newMemoryDB())
+  for i, w in items:
+    try:
+      let withdrawal = etypes.Withdrawal(
+        index: distinctBase(w.index),
+        validatorIndex: distinctBase(w.validatorIndex),
+        address: distinctBase(w.address),
+        amount: distinctBase(w.amount)
+      )
+      tr.put(rlp.encode(i), rlp.encode(withdrawal))
+    except RlpError as e:
+      raiseAssert(e.msg)
+
+  return tr.rootHash()
+
 proc asPortalBlockData*(
-    payload: ExecutionPayloadV1 | ExecutionPayloadV2 | ExecutionPayloadV3):
-    (common_types.BlockHash, BlockHeaderWithProof, BlockBodySSZ) =
-  proc calculateTransactionData(
-      items: openArray[TypedTransaction]):
-      Hash256 {.raises: [].} =
-
-    var tr = initHexaryTrie(newMemoryDB())
-    for i, t in items:
-      try:
-        let tx = distinctBase(t)
-        tr.put(rlp.encode(i), tx)
-      except RlpError as e:
-        # TODO: Investigate this RlpError as it doesn't sound like this is
-        # something that can actually occur.
-        raiseAssert(e.msg)
-
-    return tr.rootHash()
-
-  # TODO: Since Capella we can also access ExecutionPayloadHeader and thus
-  # could get the Roots through there instead.
-  proc calculateWithdrawalsRoot(
-    items: openArray[WithdrawalV1]):
-      Hash256 {.raises: [].} =
-
-    var tr = initHexaryTrie(newMemoryDB())
-    for i, w in items:
-      try:
-        let withdrawal = Withdrawal(
-          index: distinctBase(w.index),
-          validatorIndex: distinctBase(w.validatorIndex),
-          address: distinctBase(w.address),
-          amount: distinctBase(w.amount)
-        )
-        tr.put(rlp.encode(i), rlp.encode(withdrawal))
-      except RlpError as e:
-        raiseAssert(e.msg)
-
-    return tr.rootHash()
-
+    payload: ExecutionPayloadV1):
+    (common_types.BlockHash, BlockHeaderWithProof, PortalBlockBodyLegacy) =
   let
     txRoot = calculateTransactionData(payload.transactions)
-    withdrawalsRoot =
-      when type(payload) is ExecutionPayloadV1:
-        options.none(Hash256)
-      else:
-        some(calculateWithdrawalsRoot(payload.withdrawals))
+    withdrawalsRoot = options.none(Hash256)
 
     header = etypes.BlockHeader(
       parentHash: payload.parentHash.asEthHash,
@@ -251,7 +247,7 @@ proc asPortalBlockData*(
       nonce: default(BlockNonce),
       fee: some(payload.baseFeePerGas),
       withdrawalsRoot: withdrawalsRoot,
-      excessDataGas: options.none(UInt256) # TODO: Update later according to fork
+      excessDataGas: options.none(UInt256)
     )
 
     headerWithProof = BlockHeaderWithProof(
@@ -262,9 +258,7 @@ proc asPortalBlockData*(
   for tx in payload.transactions:
     discard transactions.add(TransactionByteList(distinctBase(tx)))
 
-  # TODO: Specifications are not ready for Shanghai/Capella on how to add the
-  # withdrawals here.
-  let body = BlockBodySSZ(
+  let body = PortalBlockBodyLegacy(
     transactions: transactions,
     uncles: Uncles(@[byte 0xc0]))
 
@@ -272,6 +266,64 @@ proc asPortalBlockData*(
 
   (hash, headerWithProof, body)
 
+proc asPortalBlockData*(
+    payload: ExecutionPayloadV2 | ExecutionPayloadV3):
+    (common_types.BlockHash, BlockHeaderWithProof, PortalBlockBodyShanghai) =
+  let
+    txRoot = calculateTransactionData(payload.transactions)
+    withdrawalsRoot = some(calculateWithdrawalsRoot(payload.withdrawals))
+
+    header = etypes.BlockHeader(
+      parentHash: payload.parentHash.asEthHash,
+      ommersHash: EMPTY_UNCLE_HASH,
+      coinbase: EthAddress payload.feeRecipient,
+      stateRoot: payload.stateRoot.asEthHash,
+      txRoot: txRoot,
+      receiptRoot: payload.receiptsRoot.asEthHash,
+      bloom: distinctBase(payload.logsBloom),
+      difficulty: default(DifficultyInt),
+      blockNumber: payload.blockNumber.distinctBase.u256,
+      gasLimit: payload.gasLimit.unsafeQuantityToInt64,
+      gasUsed: payload.gasUsed.unsafeQuantityToInt64,
+      timestamp: fromUnix payload.timestamp.unsafeQuantityToInt64,
+      extraData: bytes payload.extraData,
+      mixDigest: payload.prevRandao.asEthHash,
+      nonce: default(BlockNonce),
+      fee: some(payload.baseFeePerGas),
+      withdrawalsRoot: withdrawalsRoot,
+      excessDataGas: options.none(UInt256) # TODO: adjust later according to deneb fork
+    )
+
+    headerWithProof = BlockHeaderWithProof(
+      header: ByteList(rlp.encode(header)),
+      proof: BlockHeaderProof.init())
+
+  var transactions: Transactions
+  for tx in payload.transactions:
+    discard transactions.add(TransactionByteList(distinctBase(tx)))
+
+  func toWithdrawal(x: WithdrawalV1): Withdrawal =
+    Withdrawal(
+      index: x.index.uint64,
+      validatorIndex: x.validatorIndex.uint64,
+      address: x.address.EthAddress,
+      amount: x.amount.uint64
+    )
+
+  var withdrawals: Withdrawals
+  for w in payload.withdrawals:
+    discard withdrawals.add(WithdrawalByteList(rlp.encode(toWithdrawal(w))))
+
+  let body = PortalBlockBodyShanghai(
+    transactions: transactions,
+    uncles: Uncles(@[byte 0xc0]),
+    withdrawals: withdrawals
+    )
+
+  let hash = common_types.BlockHash(data: distinctBase(payload.blockHash))
+
+  (hash, headerWithProof, body)
+
 func forkDigestAtEpoch(
     forkDigests: ForkDigests, epoch: Epoch, cfg: RuntimeConfig): ForkDigest =
   forkDigests.atEpoch(epoch, cfg)
@@ -453,8 +505,8 @@ proc run(config: BeaconBridgeConf) {.raises: [CatchableError].} =
                   error "Error getting block receipts", error
                   return
 
-              let sszReceipts = ReceiptsSSZ.fromReceipts(receipts)
-              if validateReceipts(sszReceipts, payload.receiptsRoot).isErr():
+              let portalReceipts = PortalReceipts.fromReceipts(receipts)
+              if validateReceipts(portalReceipts, payload.receiptsRoot).isErr():
                 error "Receipts root is invalid"
                 return
 
@@ -466,7 +518,7 @@ proc run(config: BeaconBridgeConf) {.raises: [CatchableError].} =
               try:
                 let peers = await portalRpcClient.portal_historyGossip(
                   encodedContentKeyHex,
-                  SSZ.encode(sszReceipts).toHex())
+                  SSZ.encode(portalReceipts).toHex())
                 info "Block receipts gossiped", peers,
                     contentKey = encodedContentKeyHex
               except CatchableError as e:

vendor/portal-spec-tests

@@ -1 +1 @@
-Subproject commit 6adbbe43830a8b9329f650e341972f9ec23980f4
+Subproject commit df86ab856782f5e24b983b7ec85c2b811d27bc31