Move BlockHeaderWithProof content to content key selector 0 (#1379)

* Move BlockHeaderWithProof content to content key selector 0

- Remove as content type with content key selector 4
- Replace regular block header with BlockHeaderWithProof at
content key selector 0

* Apply blockHeader content key also to bridge

* Add tests for header with proof generation and verification

fluffy/eth_data/history_data_json_store.nim

@@ -5,7 +5,10 @@
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import
   json_serialization, json_serialization/std/tables,
@@ -33,14 +36,16 @@ type
 
   BlockDataTable* = Table[string, BlockData]
 
+proc toString(v: IoErrorCode): string =
+  try: ioErrorMsg(v)
+  except Exception as e: raiseAssert e.msg
+
 proc readJsonType*(dataFile: string, T: type): Result[T, string] =
-  let data = readAllFile(dataFile)
-  if data.isErr(): # TODO: map errors
-    return err("Failed reading data-file")
+  let data = ? readAllFile(dataFile).mapErr(toString)
 
   let decoded =
     try:
-      Json.decode(data.get(), T)
+      Json.decode(data, T)
     except SerializationError as e:
       return err("Failed decoding json data-file: " & e.msg)
 

fluffy/eth_data/history_data_seeding.nim

@@ -181,8 +181,8 @@ proc historyPropagateHeadersWithProof*(
       let
         content = headerWithProof.get()
         contentKey = ContentKey(
-          contentType: blockHeaderWithProof,
-          blockHeaderWithProofKey: BlockKey(blockHash: header.blockHash()))
+          contentType: blockHeader,
+          blockHeaderKey: BlockKey(blockHash: header.blockHash()))
         encKey = history_content.encode(contentKey)
         contentId = history_content.toContentId(encKey)
         encodedContent = SSZ.encode(content)

fluffy/network/history/history_content.nim

@@ -32,7 +32,6 @@ type
     blockBody = 0x01
     receipts = 0x02
     epochAccumulator = 0x03
-    blockHeaderWithProof = 0x04
 
   BlockKey* = object
     blockHash*: BlockHash
@@ -50,8 +49,6 @@ type
       receiptsKey*: BlockKey
     of epochAccumulator:
       epochAccumulatorKey*: EpochAccumulatorKey
-    of blockHeaderWithProof:
-      blockHeaderWithProofKey*: BlockKey
 
 func init*(
     T: type ContentKey, contentType: ContentType,
@@ -70,10 +67,6 @@ func init*(
     ContentKey(
       contentType: contentType,
       epochAccumulatorKey: EpochAccumulatorKey(epochHash: hash))
-  of blockHeaderWithProof:
-    ContentKey(
-      contentType: contentType,
-      blockHeaderWithProofKey: BlockKey(blockHash: hash))
 
 func encode*(contentKey: ContentKey): ByteList =
   ByteList.init(SSZ.encode(contentKey))
@@ -111,8 +104,6 @@ func `$`*(x: ContentKey): string =
   of epochAccumulator:
     let key = x.epochAccumulatorKey
     res.add("epochHash: " & $key.epochHash)
-  of blockHeaderWithProof:
-    res.add($x.blockHeaderWithProofKey)
 
   res.add(")")
 

fluffy/network/history/history_network.nim

@@ -260,7 +260,7 @@ proc getVerifiedBlockHeader*(
     n: HistoryNetwork, hash: BlockHash):
     Future[Opt[BlockHeader]] {.async.} =
   let
-    contentKey = ContentKey.init(blockHeaderWithProof, hash).encode()
+    contentKey = ContentKey.init(blockHeader, hash).encode()
     contentId = contentKey.toContentId()
 
   logScope:
@@ -310,49 +310,6 @@ proc getVerifiedBlockHeader*(
   # Headers were requested `requestRetries` times and all failed on validation
   return Opt.none(BlockHeader)
 
-# TODO: To be deprecated or not? Should there be the case for requesting a
-# block header without proofs?
-proc getBlockHeader*(
-    n: HistoryNetwork, hash: BlockHash):
-    Future[Opt[BlockHeader]] {.async.} =
-  let
-    contentKey = ContentKey.init(blockHeader, hash).encode()
-    contentId = contentKey.toContentId()
-
-  logScope:
-    hash
-    contentKey
-
-  let headerFromDb = n.getContentFromDb(BlockHeader, contentId)
-  if headerFromDb.isSome():
-    info "Fetched block header from database"
-    return headerFromDb
-
-  for i in 0..<requestRetries:
-    let
-      headerContent = (await n.portalProtocol.contentLookup(
-          contentKey, contentId)).valueOr:
-        warn "Failed fetching block header from the network"
-        return Opt.none(BlockHeader)
-
-      header = validateBlockHeaderBytes(headerContent.content, hash).valueOr:
-        warn "Validation of block header failed", error
-        continue
-
-    info "Fetched valid block header from the network"
-    # Content is valid, it can be stored and propagated to interested peers
-    n.portalProtocol.storeContent(contentKey, contentId, headerContent.content)
-    n.portalProtocol.triggerPoke(
-      headerContent.nodesInterestedInContent,
-      contentKey,
-      headerContent.content
-    )
-
-    return Opt.some(header)
-
-  # Headers were requested `requestRetries` times and all failed on validation
-  return Opt.none(BlockHeader)
-
 proc getBlockBody*(
     n: HistoryNetwork, hash: BlockHash, header: BlockHeader):
     Future[Opt[BlockBody]] {.async.} =
@@ -528,19 +485,20 @@ proc validateContent(
 
   case key.contentType:
   of blockHeader:
-    # Note: For now we still accept regular block header type to remain
-    # compatible with the current specs. However, a verification is done by
-    # basically requesting the header with proofs from somewhere else.
-    # This all doesn't make much sense aside from compatibility and should
-    # eventually be removed.
-    let header = validateBlockHeaderBytes(
-        content, key.blockHeaderKey.blockHash).valueOr:
-      warn "Invalid block header offered", error
-      return false
+    let
+      headerWithProof = decodeSsz(content, BlockHeaderWithProof).valueOr:
+        warn "Failed decoding header with proof", error
+        return false
+      header = validateBlockHeaderBytes(
+          headerWithProof.header.asSeq(),
+          key.blockHeaderKey.blockHash).valueOr:
+        warn "Invalid block header offered", error
+        return false
 
-    let res = await n.getVerifiedBlockHeader(key.blockHeaderKey.blockHash)
-    if res.isNone():
-      warn "Block header failed canonical verification"
+    let res = n.verifyHeader(header, headerWithProof.proof)
+    if res.isErr():
+      warn "Failed on check if header is part of canonical chain",
+        error = res.error
       return false
     else:
       return true
@@ -594,25 +552,6 @@ proc validateContent(
     else:
       return true
 
-  of blockHeaderWithProof:
-    let
-      headerWithProof = decodeSsz(content, BlockHeaderWithProof).valueOr:
-        warn "Failed decoding header with proof", error
-        return false
-      header = validateBlockHeaderBytes(
-          headerWithProof.header.asSeq(),
-          key.blockHeaderWithProofKey.blockHash).valueOr:
-        warn "Invalid block header offered", error
-        return false
-
-    let res = n.verifyHeader(header, headerWithProof.proof)
-    if res.isErr():
-      warn "Failed on check if header is part of canonical chain",
-        error = res.error
-      return false
-    else:
-      return true
-
 proc new*(
     T: type HistoryNetwork,
     baseProtocol: protocol.Protocol,

fluffy/scripts/test_portal_testnet.nim

@@ -19,7 +19,8 @@ import
     history_data_json_store,
     history_data_ssz_e2s],
   ../network/history/[history_content, accumulator],
-  ../seed_db
+  ../seed_db,
+  ../tests/test_history_util
 
 type
   FutureCallback[A] = proc (): Future[A] {.gcsafe, raises: [Defect].}
@@ -42,24 +43,6 @@ type
       desc: "Port of the JSON-RPC service of the bootstrap (first) node"
       name: "base-rpc-port" .}: uint16
 
-proc buildHeadersWithProof*(
-    blockHeaders: seq[BlockHeader],
-    epochAccumulator: EpochAccumulatorCached):
-    Result[seq[(seq[byte], seq[byte])], string] =
-  var blockHeadersWithProof: seq[(seq[byte], seq[byte])]
-  for header in blockHeaders:
-    if header.isPreMerge():
-      let
-        content = ? buildHeaderWithProof(header, epochAccumulator)
-        contentKey = ContentKey(
-          contentType: blockHeaderWithProof,
-          blockHeaderWithProofKey: BlockKey(blockHash: header.blockHash()))
-
-      blockHeadersWithProof.add(
-        (encode(contentKey).asSeq(), SSZ.encode(content)))
-
-  ok(blockHeadersWithProof)
-
 proc connectToRpcServers(config: PortalTestnetConf):
     Future[seq[RpcClient]] {.async.} =
   var clients: seq[RpcClient]

fluffy/tests/portal_spec_tests/mainnet/all_fluffy_portal_spec_tests.nim

@@ -10,6 +10,7 @@
 import
   ./test_portal_wire_encoding,
   ./test_history_content,
+  ./test_headers_with_proof,
   ./test_header_content,
   ./test_state_content,
   ./test_accumulator_root

fluffy/tests/portal_spec_tests/mainnet/test_headers_with_proof.nim

@@ -0,0 +1,97 @@
+# Nimbus
+# Copyright (c) 2023 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.used.}
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import
+  unittest2, stew/byteutils,
+  eth/common/eth_types_rlp,
+  ../../../network_metadata,
+  ../../../eth_data/[history_data_json_store, history_data_ssz_e2s],
+  ../../../network/history/[history_content, history_network, accumulator],
+  ../../test_history_util
+
+type
+  JsonHeaderContent* = object
+    content_key*: string
+    content_value*: string
+
+  JsonHeaderContentTable* = Table[string, JsonHeaderContent]
+
+suite "Headers with Proof":
+  test "HeaderWithProof Decoding and Verifying":
+    const dataFile =
+      "./vendor/portal-spec-tests/tests/mainnet/history/headers_with_proof/1000001-1000010.json"
+
+    let accumulator =
+      try:
+        SSZ.decode(finishedAccumulator, FinishedAccumulator)
+      except SszError as err:
+        raiseAssert "Invalid baked-in accumulator: " & err.msg
+
+    let res = readJsonType(dataFile, JsonHeaderContentTable)
+    check res.isOk()
+    let headerContent = res.get()
+
+    for k, v in headerContent:
+      let
+        # TODO: strange assignment failure when using try/except ValueError
+        # for the hexToSeqByte() here.
+        contentKey = decodeSsz(
+          v.content_key.hexToSeqByte(), ContentKey)
+        contentValue = decodeSsz(
+          v.content_value.hexToSeqByte(), BlockHeaderWithProof)
+
+      check:
+        contentKey.isOk()
+        contentValue.isOk()
+
+      let blockHeaderWithProof = contentValue.get()
+
+      let res = decodeRlp(blockHeaderWithProof.header.asSeq(), BlockHeader)
+      check res.isOk()
+      let header = res.get()
+
+      check accumulator.verifyHeader(header, blockHeaderWithProof.proof).isOk()
+
+  test "HeaderWithProof Building and Encoding":
+    const
+      headerFile = "./vendor/portal-spec-tests/tests/mainnet/history/headers/1000001-1000010.e2s"
+      accumulatorFile = "./vendor/portal-spec-tests/tests/mainnet/history/accumulator/epoch-accumulator-00122.ssz"
+      headersWithProofFile = "./vendor/portal-spec-tests/tests/mainnet/history/headers_with_proof/1000001-1000010.json"
+
+    let
+      blockHeaders = readBlockHeaders(headerFile).valueOr:
+        raiseAssert "Invalid header file: " & headerFile
+      epochAccumulator = readEpochAccumulatorCached(accumulatorFile).valueOr:
+        raiseAssert "Invalid epoch accumulator file: " & accumulatorFile
+      blockHeadersWithProof =
+        buildHeadersWithProof(blockHeaders, epochAccumulator).valueOr:
+          raiseAssert "Could not build headers with proof"
+
+    let res = readJsonType(headersWithProofFile, JsonHeaderContentTable)
+    check res.isOk()
+    let headerContent = res.get()
+
+    # Go over all content keys and headers with generated proofs and compare
+    # them with the ones from the test vectors.
+    for i, (headerContentKey, headerWithProof) in blockHeadersWithProof:
+      let
+        blockNumber = blockHeaders[i].blockNumber
+        contentKey =
+          headerContent[blockNumber.toString()].content_key.hexToSeqByte()
+        contentValue =
+          headerContent[blockNumber.toString()].content_value.hexToSeqByte()
+
+      check:
+        contentKey == headerContentKey
+        contentValue == headerWithProof

fluffy/tests/test_history_network.nim

@@ -70,7 +70,7 @@ proc headersToContentInfo(
       headerHash = header.blockHash()
       blockKey = BlockKey(blockHash: headerHash)
       contentKey = encode(ContentKey(
-        contentType: blockHeaderWithProof, blockHeaderWithProofKey: blockKey))
+        contentType: blockHeader, blockHeaderKey: blockKey))
       contentInfo = ContentInfo(
         contentKey: contentKey, content: SSZ.encode(headerWithProof))
     contentInfos.add(contentInfo)
@@ -119,7 +119,7 @@ procSuite "History Content Network":
         headerHash = header.blockHash()
         blockKey = BlockKey(blockHash: headerHash)
         contentKey = ContentKey(
-          contentType: blockHeaderWithProof, blockHeaderWithProofKey: blockKey)
+          contentType: blockHeader, blockHeaderKey: blockKey)
         encKey = encode(contentKey)
         contentId = toContentId(contentKey)
       historyNode2.portalProtocol().storeContent(

fluffy/tests/test_history_util.nim

@@ -0,0 +1,36 @@
+# Nimbus
+# Copyright (c) 2023 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import
+  stew/results,
+  eth/common/eth_types_rlp,
+  ../network/history/[history_content, accumulator]
+
+export results, accumulator, history_content
+
+proc buildHeadersWithProof*(
+    blockHeaders: seq[BlockHeader],
+    epochAccumulator: EpochAccumulatorCached):
+    Result[seq[(seq[byte], seq[byte])], string] =
+  var blockHeadersWithProof: seq[(seq[byte], seq[byte])]
+  for header in blockHeaders:
+    if header.isPreMerge():
+      let
+        content = ? buildHeaderWithProof(header, epochAccumulator)
+        contentKey = ContentKey(
+          contentType: blockHeader,
+          blockHeaderKey: BlockKey(blockHash: header.blockHash()))
+
+      blockHeadersWithProof.add(
+        (encode(contentKey).asSeq(), SSZ.encode(content)))
+
+  ok(blockHeadersWithProof)

fluffy/tools/bridge/beacon_chain_bridge.nim

@@ -231,7 +231,7 @@ proc run() {.raises: [Exception, Defect].} =
               blockhash = history_content.`$`hash
 
             block: # gossip header
-              let contentKey = ContentKey.init(blockHeaderWithProof, hash)
+              let contentKey = ContentKey.init(blockHeader, hash)
               let encodedContentKey = contentKey.encode.asSeq()
 
               try:

vendor/portal-spec-tests

@@ -1 +1 @@
-Subproject commit 5a1d2e553d97c04339b6227624d4ebab4da88701
+Subproject commit ee863fecdc6ec9cc81effe355e74459cd5dda28d