Add headers with proof content type and use it for verification (#1281)

* Add headers with proof content type and use it for verification

- Add BlockHeaderWithProof content type & content
- Use BlockHeaderWithProof content to verify if chain data is
part of the canonical chain
- Adjust parser & seeder code to be able to seed these headers
with proof
- Adjust eth_data_exporter to be able to export custom header
ranges for which to build proofs (mostly for testing)

There is currently quite some ugliness & clean-up needed for which
a big part is due tos upporting both BlockHeader and
BlockHeaderWithProof on the network.

* Change accumulator proof to array / SSZ vector type

- Change accumulator proof to SSZ vector instead of SSZ list.
- Add and use general buildProof and buildHeaderWithProof func.

* Make the BlockHeaderWithProof an SSZ Union with None option

* Update portal-spec-tests to master commit

fluffy/data/history_data_parser.nim

@@ -11,6 +11,7 @@ import
   json_serialization, json_serialization/std/tables,
   stew/[byteutils, io2, results], chronicles,
   eth/[rlp, common/eth_types],
+  ncli/e2store,
   ../../nimbus/[chain_config, genesis],
   ../network/history/[history_content, accumulator]
 
@@ -31,9 +32,6 @@ type
     # Fix in nim-json-serialization or should I overload something here?
     number*: int
 
-  EpochAccumulatorObject = object
-    epochAccumulator: string
-
   BlockDataTable* = Table[string, BlockData]
 
 proc readJsonType*(dataFile: string, T: type): Result[T, string] =
@@ -190,7 +188,6 @@ proc getGenesisHeader*(id: NetworkId = MainNet): BlockHeader =
   except RlpError:
     raise (ref Defect)(msg: "Genesis should be valid")
 
-
 proc toString*(v: IoErrorCode): string =
   try: ioErrorMsg(v)
   except Exception as e: raiseAssert e.msg
@@ -211,3 +208,44 @@ proc readEpochAccumulator*(file: string): Result[EpochAccumulator, string] =
     ok(SSZ.decode(encodedAccumulator, EpochAccumulator))
   except SszError as e:
     err("Decoding epoch accumulator failed: " & e.msg)
+
+proc readEpochAccumulatorCached*(file: string): Result[EpochAccumulatorCached, string] =
+  let encodedAccumulator = ? readAllFile(file).mapErr(toString)
+
+  try:
+    ok(SSZ.decode(encodedAccumulator, EpochAccumulatorCached))
+  except SszError as e:
+    err("Decoding epoch accumulator failed: " & e.msg)
+
+const
+  # Using the e2s format to store data, but without the specific structure
+  # like in an era file, as we currently don't really need that.
+  # See: https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md
+  # Added one type for now, with numbers not formally specified.
+  # Note:
+  # Snappy compression for `ExecutionBlockHeaderRecord` only helps for the
+  # first ~1M (?) block headers, after that there is no gain so we don't do it.
+  ExecutionBlockHeaderRecord* = [byte 0xFF, 0x00]
+
+proc readBlockHeaders*(file: string): Result[seq[BlockHeader], string] =
+  let fh = ? openFile(file, {OpenFlags.Read}).mapErr(toString)
+  defer: discard closeFile(fh)
+
+  var data: seq[byte]
+  var blockHeaders: seq[BlockHeader]
+  while true:
+    let header = readRecord(fh, data).valueOr:
+      break
+
+    if header.typ == ExecutionBlockHeaderRecord:
+      let blockHeader =
+        try:
+          rlp.decode(data, BlockHeader)
+        except RlpError as e:
+          return err("Invalid block header in " & file & ": " & e.msg)
+
+      blockHeaders.add(blockHeader)
+    else:
+      warn "Skipping record, not a block header", typ = toHex(header.typ)
+
+  ok(blockHeaders)

fluffy/data/history_data_seeding.nim

@@ -10,7 +10,7 @@
 import
   std/[strformat, os],
   stew/results, chronos, chronicles,
-  eth/common/eth_types,
+  eth/common/eth_types, eth/rlp,
   ../network/wire/portal_protocol,
   ../network/history/[history_content, accumulator],
   ./history_data_parser
@@ -150,6 +150,62 @@ proc historyPropagateBlock*(
   else:
     return err(blockDataTable.error)
 
+proc historyPropagateHeadersWithProof*(
+    p: PortalProtocol, epochHeadersFile: string, epochAccumulatorFile: string):
+    Future[Result[void, string]] {.async.} =
+  let res = readBlockHeaders(epochHeadersFile)
+  if res.isErr():
+    return err(res.error)
+
+  let blockHeaders = res.get()
+
+  let epochAccumulatorRes = readEpochAccumulatorCached(epochAccumulatorFile)
+  if epochAccumulatorRes.isErr():
+    return err(res.error)
+
+  let epochAccumulator = epochAccumulatorRes.get()
+  for header in blockHeaders:
+    if header.isPreMerge():
+      let headerWithProof = buildHeaderWithProof(header, epochAccumulator)
+      if headerWithProof.isErr:
+        return err(headerWithProof.error)
+
+      let
+        content = headerWithProof.get()
+        contentKey = ContentKey(
+          contentType: blockHeaderWithProof,
+          blockHeaderWithProofKey: BlockKey(blockHash: header.blockHash()))
+        contentId = history_content.toContentId(contentKey)
+        encodedContent = SSZ.encode(content)
+
+      p.storeContent(contentId, encodedContent)
+
+      let keys = ContentKeysList(@[encode(contentKey)])
+      discard await p.neighborhoodGossip(keys, @[encodedContent])
+
+  return ok()
+
+proc historyPropagateHeadersWithProof*(
+    p: PortalProtocol, dataDir: string):
+    Future[Result[void, string]] {.async.} =
+  for i in 0..<preMergeEpochs:
+    let
+      epochHeadersfile =
+        try: dataDir / &"mainnet-headers-epoch-{i.uint64:05}.e2s"
+        except ValueError as e: raiseAssert e.msg
+      epochAccumulatorFile =
+        try: dataDir / &"mainnet-epoch-accumulator-{i.uint64:05}.ssz"
+        except ValueError as e: raiseAssert e.msg
+
+    let res = await p.historyPropagateHeadersWithProof(
+      epochHeadersfile, epochAccumulatorFile)
+    if res.isOk():
+      info "Finished gossiping 1 epoch of headers with proof", i
+    else:
+      return err(res.error)
+
+  return ok()
+
 proc historyPropagateHeaders*(
     p: PortalProtocol, dataFile: string, verify = false):
     Future[Result[void, string]] {.async.} =

fluffy/network/history/accumulator.nim

@@ -8,7 +8,7 @@
 {.push raises: [Defect].}
 
 import
-  eth/common/eth_types_rlp,
+  eth/rlp, eth/common/eth_types_rlp,
   ssz_serialization, ssz_serialization/[proofs, merkleization],
   ../../common/common_types,
   ./history_content
@@ -21,7 +21,7 @@ export ssz_serialization, merkleization, proofs, eth_types_rlp
 
 const
   epochSize* = 8192 # blocks
-  # Allow this to be adjusted at compile time fir testing. If more constants
+  # Allow this to be adjusted at compile time for testing. If more constants
   # need to be adjusted we can add some presets file.
   mergeBlockNumber* {.intdefine.}: uint64 = 15537394
 
@@ -30,8 +30,9 @@ const
   preMergeEpochs* = (mergeBlockNumber + epochSize - 1) div epochSize
 
   # TODO:
-  # Currently disabled, because of testing issues, but could be used as value to
+  # Currently disabled, because issue when testing with other
-  # double check on at merge block.
+  # `mergeBlockNumber`, but it could be used as value to double check on at
+  # merge block.
   # TODO: Could also be used as value to actual finish the accumulator, instead
   # of `mergeBlockNumber`, but:
   # - Still need to store the actual `mergeBlockNumber` and run-time somewhere
@@ -47,6 +48,14 @@ type
 
   EpochAccumulator* = List[HeaderRecord, epochSize]
 
+  # In the core code of Fluffy the `EpochAccumulator` type is solely used, as
+  # `hash_tree_root` is done either once or never on this object after
+  # serialization.
+  # However for the generation of the proofs for all the headers in an epoch, it
+  # needs to be run many times and the cached version of the SSZ list is
+  # obviously much faster, so this second type is added for this usage.
+  EpochAccumulatorCached* = HashList[HeaderRecord, epochSize]
+
   Accumulator* = object
     historicalEpochs*: List[Bytes32, int(preMergeEpochs)]
     currentEpoch*: EpochAccumulator
@@ -127,7 +136,7 @@ func isPreMerge*(blockNumber: uint64): bool =
 func isPreMerge*(header: BlockHeader): bool =
   isPreMerge(header.blockNumber.truncate(uint64))
 
-func verifyProof*(
+func verifyProof(
     a: FinishedAccumulator, header: BlockHeader, proof: openArray[Digest]): bool =
   let
     epochIndex = getEpochIndex(header)
@@ -141,10 +150,12 @@ func verifyProof*(
 
   verify_merkle_multiproof(@[leave], proof, @[gIndex], epochAccumulatorHash)
 
-func verifyHeader*(
+func verifyAccumulatorProof*(
-    a: FinishedAccumulator, header: BlockHeader, proof: openArray[Digest]):
+    a: FinishedAccumulator, header: BlockHeader, proof: AccumulatorProof):
     Result[void, string] =
   if header.isPreMerge():
+    # Note: The proof is typed with correct depth, so no check on this is
+    # required here.
     if a.verifyProof(header, proof):
       ok()
     else:
@@ -152,6 +163,43 @@ func verifyHeader*(
   else:
     err("Cannot verify post merge header with accumulator proof")
 
+func verifyHeader*(
+    a: FinishedAccumulator, header: BlockHeader, proof: BlockHeaderProof):
+    Result[void, string] =
+  case proof.proofType:
+  of BlockHeaderProofType.accumulatorProof:
+    a.verifyAccumulatorProof(header, proof.accumulatorProof)
+  of BlockHeaderProofType.none:
+    err("cannot verify header without proof")
+
+func buildProof*(
+    header: BlockHeader,
+    epochAccumulator: EpochAccumulator | EpochAccumulatorCached):
+    Result[AccumulatorProof, string] =
+  doAssert(header.isPreMerge(), "Must be pre merge header")
+
+  let
+    epochIndex = getEpochIndex(header)
+    headerRecordIndex = getHeaderRecordIndex(header, epochIndex)
+
+    # TODO: Implement more generalized `get_generalized_index`
+    gIndex = GeneralizedIndex(epochSize*2*2 + (headerRecordIndex*2))
+
+  var proof: AccumulatorProof
+  ? epochAccumulator.build_proof(gIndex, proof)
+
+  ok(proof)
+
+func buildHeaderWithProof*(
+    header: BlockHeader,
+    epochAccumulator: EpochAccumulator | EpochAccumulatorCached):
+    Result[BlockHeaderWithProof, string] =
+  let proof = ? buildProof(header, epochAccumulator)
+
+  ok(BlockHeaderWithProof(
+    header: ByteList.init(rlp.encode(header)),
+    proof: BlockHeaderProof.init(proof)))
+
 func getBlockEpochDataForBlockNumber*(
     a: FinishedAccumulator, bn: UInt256): Result[BlockEpochData, string] =
   let blockNumber = bn.truncate(uint64)

fluffy/network/history/history_content.nim

@@ -32,6 +32,7 @@ type
     blockBody = 0x01
     receipts = 0x02
     epochAccumulator = 0x03
+    blockHeaderWithProof = 0x04
 
   BlockKey* = object
     blockHash*: BlockHash
@@ -49,6 +50,8 @@ type
       receiptsKey*: BlockKey
     of epochAccumulator:
       epochAccumulatorKey*: EpochAccumulatorKey
+    of blockHeaderWithProof:
+      blockHeaderWithProofKey*: BlockKey
 
 func encode*(contentKey: ContentKey): ByteList =
   ByteList.init(SSZ.encode(contentKey))
@@ -86,6 +89,8 @@ func `$`*(x: ContentKey): string =
   of epochAccumulator:
     let key = x.epochAccumulatorKey
     res.add("epochHash: " & $key.epochHash)
+  of blockHeaderWithProof:
+    res.add($x.blockHeaderWithProofKey)
 
   res.add(")")
 
@@ -114,3 +119,23 @@ type
 
   ReceiptByteList* = List[byte, MAX_RECEIPT_LENGTH] # RLP data
   ReceiptsSSZ* = List[ReceiptByteList, MAX_TRANSACTION_COUNT]
+
+  AccumulatorProof* = array[15, Digest]
+
+  BlockHeaderProofType* = enum
+    none = 0x00 # An SSZ Union None
+    accumulatorProof = 0x01
+
+  BlockHeaderProof* = object
+    case proofType*: BlockHeaderProofType
+    of none:
+      discard
+    of accumulatorProof:
+      accumulatorProof*: AccumulatorProof
+
+  BlockHeaderWithProof* = object
+    header*: ByteList # RLP data
+    proof*: BlockHeaderProof
+
+func init*(T: type BlockHeaderProof, proof: AccumulatorProof): T =
+  BlockHeaderProof(proofType: accumulatorProof, accumulatorProof: proof)

fluffy/network/history/history_network.nim

@@ -22,16 +22,6 @@ logScope:
 
 export accumulator
 
-# TODO: To currently verify if content is from the canonical chain it is
-# required to download the right epoch accunulator, which is ~0.5 MB. This is
-# too much, at least for the local testnet tests. This needs to be improved
-# by adding the proofs to the block header content. Another independent
-# improvement would be to have a content cache (LRU or so). The latter would
-# probably help mostly for the local testnet tests.
-# For now, we disable this verification default until further improvements are
-# made.
-const canonicalVerify* {.booldefine.} = false
-
 const
   historyProtocolId* = [byte 0x50, 0x0B]
 
@@ -67,15 +57,23 @@ func getEncodedKeyForContent(
       ContentKey(contentType: cType, receiptsKey: contentKeyType)
     of epochAccumulator:
       raiseAssert("Not implemented")
+    of blockHeaderWithProof:
+      ContentKey(contentType: cType, blockHeaderWithProofKey: contentKeyType)
 
   return encodeKey(contentKey)
 
-func decodeRlp*(bytes: openArray[byte], T: type): Result[T, string] =
+func decodeRlp*(input: openArray[byte], T: type): Result[T, string] =
   try:
-    ok(rlp.decode(bytes, T))
+    ok(rlp.decode(input, T))
   except RlpError as e:
     err(e.msg)
 
+func decodeSsz*(input: openArray[byte], T: type): Result[T, string] =
+  try:
+    ok(SSZ.decode(input, T))
+  except SszError as e:
+    err(e.msg)
+
 ## Calls to go from SSZ decoded types to RLP fully decoded types
 
 func fromPortalBlockBody(
@@ -182,11 +180,7 @@ proc validateBlockBodyBytes*(
     bytes: openArray[byte], txRoot, ommersHash: KeccakHash):
     Result[BlockBody, string] =
   ## Fully decode the SSZ Block Body and validate it against the header.
-  let body =
+  let body = ? decodeSsz(bytes, BlockBodySSZ)
-    try:
-      SSZ.decode(bytes, BlockBodySSZ)
-    except SszError as e:
-      return err("Failed to decode block body: " & e.msg)
 
   ? validateBlockBody(body, txRoot, ommersHash)
 
@@ -205,11 +199,7 @@ proc validateReceiptsBytes*(
     bytes: openArray[byte],
     receiptsRoot: KeccakHash): Result[seq[Receipt], string] =
   ## Fully decode the SSZ Block Body and validate it against the header.
-  let receipts =
+  let receipts = ? decodeSsz(bytes, ReceiptsSSZ)
-    try:
-      SSZ.decode(bytes, ReceiptsSSZ)
-    except SszError as e:
-      return err("Failed to decode receipts: " & e.msg)
 
   ? validateReceipts(receipts, receiptsRoot)
 
@@ -220,7 +210,13 @@ proc validateReceiptsBytes*(
 proc get(db: ContentDB, T: type BlockHeader, contentId: ContentId): Option[T] =
   let contentFromDB = db.get(contentId)
   if contentFromDB.isSome():
-    let res = decodeRlp(contentFromDB.get(), T)
+    let headerWithProof =
+      try:
+        SSZ.decode(contentFromDB.get(), BlockHeaderWithProof)
+      except SszError as e:
+        raiseAssert(e.msg)
+
+    let res = decodeRlp(headerWithProof.header.asSeq(), T)
     if res.isErr():
       raiseAssert(res.error)
     else:
@@ -277,6 +273,67 @@ const requestRetries = 4
 # ongoing requests are cancelled after the receival of the first response,
 # however that response is not yet validated at that moment.
 
+func verifyHeader(
+    n: HistoryNetwork, header: BlockHeader, proof: BlockHeaderProof):
+    Result[void, string] =
+  verifyHeader(n.accumulator, header, proof)
+
+proc getVerifiedBlockHeader*(
+    n: HistoryNetwork, hash: BlockHash):
+    Future[Option[BlockHeader]] {.async.} =
+  let (keyEncoded, contentId) =
+    getEncodedKeyForContent(blockHeaderWithProof, hash)
+
+  # Note: This still requests a BlockHeaderWithProof from the database, as that
+  # is what is stored. But the proof doesn't need to be checked as everthing
+  # should get checked before storing.
+  let headerFromDb = n.getContentFromDb(BlockHeader, contentId)
+
+  if headerFromDb.isSome():
+    info "Fetched block header from database", hash, contentKey = keyEncoded
+    return headerFromDb
+
+  for i in 0..<requestRetries:
+    let headerContentLookup =
+      await n.portalProtocol.contentLookup(keyEncoded, contentId)
+    if headerContentLookup.isNone():
+      warn "Failed fetching block header with proof from the network",
+        hash, contentKey = keyEncoded
+      return none(BlockHeader)
+
+    let headerContent = headerContentLookup.unsafeGet()
+
+    let headerWithProofRes = decodeSsz(headerContent.content, BlockHeaderWithProof)
+    if headerWithProofRes.isErr():
+      warn "Failed decoding header with proof", err = headerWithProofRes.error
+      return none(BlockHeader)
+
+    let headerWithProof = headerWithProofRes.get()
+
+    let res = validateBlockHeaderBytes(headerWithProof.header.asSeq(), hash)
+    if res.isOk():
+      let isCanonical = n.verifyHeader(res.get(), headerWithProof.proof)
+
+      if isCanonical.isOk():
+        info "Fetched block header from the network", hash, contentKey = keyEncoded
+        # Content is valid, it can be propagated to interested peers
+        n.portalProtocol.triggerPoke(
+          headerContent.nodesInterestedInContent,
+          keyEncoded,
+          headerContent.content
+        )
+
+        n.portalProtocol.storeContent(contentId, headerContent.content)
+
+        return some(res.get())
+    else:
+      warn "Validation of block header failed", err = res.error, hash, contentKey = keyEncoded
+
+  # Headers were requested `requestRetries` times and all failed on validation
+  return none(BlockHeader)
+
+# TODO: To be deprecated or not? Should there be the case for requesting a
+# block header without proofs?
 proc getBlockHeader*(
     n: HistoryNetwork, hash: BlockHash):
     Future[Option[BlockHeader]] {.async.} =
@@ -367,7 +424,10 @@ proc getBlock*(
     Future[Option[Block]] {.async.} =
   debug "Trying to retrieve block with hash", hash
 
-  let headerOpt = await n.getBlockHeader(hash)
+  # Note: Using `getVerifiedBlockHeader` instead of getBlockHeader even though
+  # proofs are not necessiarly needed, in order to avoid having to inject
+  # also the original type into the network.
+  let headerOpt = await n.getVerifiedBlockHeader(hash)
   if headerOpt.isNone():
     warn "Failed to get header when getting block with hash", hash
     # Cannot validate block without header.
@@ -506,43 +566,6 @@ proc getBlock*(
   else:
     return err(epochDataRes.error)
 
-proc buildProof*(n: HistoryNetwork, header: BlockHeader):
-    Future[Result[seq[Digest], string]] {.async.} =
-  # Note: Temporarily needed proc until proofs are send over with headers.
-  let
-    epochIndex = getEpochIndex(header)
-    epochHash = Digest(data: n.accumulator.historicalEpochs[epochIndex])
-
-    epochAccumulatorOpt = await n.getEpochAccumulator(epochHash)
-
-  if epochAccumulatorOpt.isNone():
-    return err("Epoch accumulator not found")
-
-  let
-    epochAccumulator = epochAccumulatorOpt.get()
-    headerRecordIndex = getHeaderRecordIndex(header, epochIndex)
-    # TODO: Implement more generalized `get_generalized_index`
-    gIndex = GeneralizedIndex(epochSize*2*2 + (headerRecordIndex*2))
-
-  return epochAccumulator.build_proof(gIndex)
-
-proc verifyCanonicalChain(
-    n: HistoryNetwork, header: BlockHeader):
-    Future[Result[void, string]] {.async.} =
-  when not canonicalVerify:
-    return ok()
-
-  # Note: It is a bit silly to build a proof, as we still need to request the
-  # epoch accumulators for it, and could just verify it with those. But the
-  # idea here is that eventually this gets changed so that the proof is send
-  # together with the header.
-  let proof = await n.buildProof(header)
-  if proof.isOk():
-    return verifyHeader(n.accumulator, header, proof.get())
-  else:
-    # Can't verify without master and epoch accumulators
-    return err("Cannot build proof: " & proof.error)
-
 proc validateContent(
     n: HistoryNetwork, content: seq[byte], contentKey: ByteList):
     Future[bool] {.async.} =
@@ -555,6 +578,11 @@ proc validateContent(
 
   case key.contentType:
   of blockHeader:
+    # Note: For now we still accept regular block header type to remain
+    # compatible with the current specs. However, a verification is done by
+    # basically requesting the header with proofs from somewhere else.
+    # This all doesn't make much sense aside from compatibility and should
+    # eventually be removed.
     let validateResult =
       validateBlockHeaderBytes(content, key.blockHeaderKey.blockHash)
     if validateResult.isErr():
@@ -563,57 +591,45 @@ proc validateContent(
 
     let header = validateResult.get()
 
-    let verifyResult = await n.verifyCanonicalChain(header)
+    let res = await n.getVerifiedBlockHeader(key.blockHeaderKey.blockHash)
-    if verifyResult.isErr():
+    if res.isNone():
-      warn "Failed on check if header is part of canonical chain",
+      warn "Block header failed canonical verification"
-        error = verifyResult.error
       return false
     else:
       return true
-  of blockBody:
-    let headerOpt = await n.getBlockHeader(key.blockBodyKey.blockHash)
 
-    if headerOpt.isNone():
+  of blockBody:
-      warn "Cannot find the header, no way to validate the block body"
+    let res = await n.getVerifiedBlockHeader(key.blockBodyKey.blockHash)
+    if res.isNone():
+      warn "Block body Failed canonical verification"
       return false
 
-    let header = headerOpt.get()
+    let header = res.get()
     let validationResult =
       validateBlockBodyBytes(content, header.txRoot, header.ommersHash)
 
     if validationResult.isErr():
       warn "Failed validating block body", error = validationResult.error
       return false
-
-    let verifyResult = await n.verifyCanonicalChain(header)
-    if verifyResult.isErr():
-      warn "Failed on check if header is part of canonical chain",
-        error = verifyResult.error
-      return false
     else:
       return true
-  of receipts:
-    let headerOpt = await n.getBlockHeader(key.receiptsKey.blockHash)
 
-    if headerOpt.isNone():
+  of receipts:
-      warn "Cannot find the header, no way to validate the receipts"
+    let res = await n.getVerifiedBlockHeader(key.receiptsKey.blockHash)
+    if res.isNone():
+      warn "Receipts failed canonical verification"
       return false
 
-    let header = headerOpt.get()
+    let header = res.get()
     let validationResult =
       validateReceiptsBytes(content, header.receiptRoot)
 
     if validationResult.isErr():
       warn "Failed validating receipts", error = validationResult.error
       return false
-
-    let verifyResult = await n.verifyCanonicalChain(header)
-    if verifyResult.isErr():
-      warn "Failed on check if header is part of canonical chain",
-        error = verifyResult.error
-      return false
     else:
       return true
+
   of epochAccumulator:
     # Check first if epochHash is part of master accumulator
     let epochHash = key.epochAccumulatorKey.epochHash
@@ -637,6 +653,30 @@ proc validateContent(
     else:
       return true
 
+  of blockHeaderWithProof:
+    let headerWithProofRes = decodeSsz(content, BlockHeaderWithProof)
+    if headerWithProofRes.isErr():
+      warn "Failed decoding header with proof", err = headerWithProofRes.error
+      return false
+
+    let headerWithProof = headerWithProofRes.get()
+
+    let validateResult = validateBlockHeaderBytes(
+      headerWithProof.header.asSeq(), key.blockHeaderWithProofKey.blockHash)
+    if validateResult.isErr():
+      warn "Invalid block header offered", error = validateResult.error
+      return false
+
+    let header = validateResult.get()
+
+    let isCanonical = n.verifyHeader(header, headerWithProof.proof)
+    if isCanonical.isErr():
+      warn "Failed on check if header is part of canonical chain",
+        error = isCanonical.error
+      return false
+    else:
+      return true
+
 proc new*(
     T: type HistoryNetwork,
     baseProtocol: protocol.Protocol,

fluffy/network/network_seed.nim

@@ -250,7 +250,6 @@ proc offerContentInNodeRange*(
   else:
     return err(offerResult.error)
 
-
 proc storeContentInNodeRange*(
     p: PortalProtocol,
     seedDbPath: string,

fluffy/network_metadata.nim

@@ -61,4 +61,4 @@ const
     portalNetworksDir / "testnet0" / "bootstrap_nodes.txt")
 
   finishedAccumulator* = loadEncodedAccumulator(
-    portalTestDir / "mainnet" / "accumulator" / "finished_accumulator.ssz")
+    portalTestDir / "mainnet" / "history" / "accumulator" / "finished_accumulator.ssz")

fluffy/rpc/rpc_eth_api.nim

@@ -273,7 +273,7 @@ proc installEthApiHandlers*(
     else:
       let hash = filterOptions.blockHash.unsafeGet()
 
-      let headerOpt = await historyNetwork.getBlockHeader(hash)
+      let headerOpt = await historyNetwork.getVerifiedBlockHeader(hash)
       if headerOpt.isNone():
         raise newException(ValueError,
           "Could not find header with requested hash")

fluffy/rpc/rpc_portal_api.nim

@@ -140,7 +140,6 @@ proc installPortalApiHandlers*(
 
   rpcServer.rpc("portal_" & network & "Offer") do(
       contentKey: string, content: string) -> int:
-
     let
       ck = hexToSeqByte(contentKey)
       ct = hexToSeqByte(content)

fluffy/rpc/rpc_portal_debug_api.nim

@@ -49,8 +49,17 @@ proc installPortalDebugApiHandlers*(
       raise newException(ValueError, $res.error)
 
   rpcServer.rpc("portal_" & network & "_propagateHeaders") do(
-      dataFile: string) -> bool:
+      dataDir: string) -> bool:
-    let res = await p.historyPropagateHeaders(dataFile)
+    let res = await p.historyPropagateHeadersWithProof(dataDir)
+    if res.isOk():
+      return true
+    else:
+      raise newException(ValueError, $res.error)
+
+  rpcServer.rpc("portal_" & network & "_propagateHeaders") do(
+      epochHeadersFile: string, epochAccumulatorFile: string) -> bool:
+    let res = await p.historyPropagateHeadersWithProof(
+      epochHeadersFile, epochAccumulatorFile)
     if res.isOk():
       return true
     else:

fluffy/scripts/test_portal_testnet.nim

@@ -15,6 +15,7 @@ import
   ../rpc/portal_rpc_client,
   ../rpc/eth_rpc_client,
   ../data/[history_data_seeding, history_data_parser],
+  ../network/history/[history_content, accumulator],
   ../seed_db
 
 type
@@ -38,6 +39,24 @@ type
       desc: "Port of the JSON-RPC service of the bootstrap (first) node"
       name: "base-rpc-port" .}: uint16
 
+proc buildHeadersWithProof*(
+    blockHeaders: seq[BlockHeader],
+    epochAccumulator: EpochAccumulatorCached):
+    Result[seq[(seq[byte], seq[byte])], string] =
+  var blockHeadersWithProof: seq[(seq[byte], seq[byte])]
+  for header in blockHeaders:
+    if header.isPreMerge():
+      let
+        content = ? buildHeaderWithProof(header, epochAccumulator)
+        contentKey = ContentKey(
+          contentType: blockHeaderWithProof,
+          blockHeaderWithProofKey: BlockKey(blockHash: header.blockHash()))
+
+      blockHeadersWithProof.add(
+        (encode(contentKey).asSeq(), SSZ.encode(content)))
+
+  ok(blockHeadersWithProof)
+
 proc connectToRpcServers(config: PortalTestnetConf):
     Future[seq[RpcClient]] {.async.} =
   var clients: seq[RpcClient]
@@ -91,7 +110,7 @@ proc retryUntil[A](
   checkFailMessage: string,
   nodeIdx: int): Future[A] =
   # some reasonable limits, which will cause waits as: 1, 2, 4, 8, 16, 32 seconds
-  return withRetries(f, c, 6, seconds(1), checkFailMessage, nodeIdx)
+  return withRetries(f, c, 1, seconds(1), checkFailMessage, nodeIdx)
 
 # Note:
 # When doing json-rpc requests following `RpcPostError` can occur:
@@ -225,40 +244,40 @@ procSuite "Portal testnet tests":
       check enr == randomNodeInfo.nodeENR
 
   asyncTest "Portal History - Propagate blocks and do content lookups":
-    let clients = await connectToRpcServers(config)
+    const
+      headerFile = "./vendor/portal-spec-tests/tests/mainnet/history/headers/1000001-1000010.e2s"
+      accumulatorFile = "./vendor/portal-spec-tests/tests/mainnet/history/accumulator/epoch-accumulator-00122.ssz"
+      blockDataFile = "./fluffy/tests/blocks/mainnet_blocks_1000001_1000010.json"
 
-    var nodeInfos: seq[NodeInfo]
+    let
-    for client in clients:
+      blockHeaders = readBlockHeaders(headerFile).valueOr:
-      let nodeInfo = await client.portal_history_nodeInfo()
+        raiseAssert "Invalid header file: " & headerFile
-      await client.close()
+      epochAccumulator = readEpochAccumulatorCached(accumulatorFile).valueOr:
-      nodeInfos.add(nodeInfo)
+        raiseAssert "Invalid epoch accumulator file: " & accumulatorFile
+      blockHeadersWithProof =
+        buildHeadersWithProof(blockHeaders, epochAccumulator).valueOr:
+          raiseAssert "Could not build headers with proof"
+      blockData =
+        readJsonType(blockDataFile, BlockDataTable).valueOr:
+          raiseAssert "Invalid block data file" & blockDataFile
 
-    # const dataFileEpoch = "./fluffy/scripts/eth-epoch-accumulator.json"
+      clients = await connectToRpcServers(config)
-    # check (await clients[0].portal_history_propagateEpochAccumulator(dataFileEpoch))
-    # await clients[0].close()
-    # await sleepAsync(60.seconds)
 
-    const dataFile = "./fluffy/tests/blocks/mainnet_blocks_1000001_1000010.json"
+    # Gossiping all block headers with proof first, as bodies and receipts
-
+    # require them for validation.
-    check (await clients[0].portal_history_propagateHeaders(dataFile))
+    for (content, contentKey) in blockHeadersWithProof:
-    await clients[0].close()
+      discard (await clients[0].portal_history_offer(
-
+        content.toHex(), contentKey.toHex()))
-    # Short sleep between propagation of block headers and propagation of block
-    # bodies and receipts as the latter two require the first for validation.
-    await sleepAsync(5.seconds)
 
     # This will fill the first node its db with blocks from the data file. Next,
     # this node wil offer all these blocks their headers one by one.
-    check (await clients[0].portal_history_propagate(dataFile))
+    check (await clients[0].portal_history_propagate(blockDataFile))
     await clients[0].close()
 
-    let blockData = readJsonType(dataFile, BlockDataTable)
-    check blockData.isOk()
-
     for i, client in clients:
       # Note: Once there is the Canonical Indices Network, we don't need to
       # access this file anymore here for the block hashes.
-      for hash in blockData.get().blockHashes():
+      for hash in blockData.blockHashes():
         # Note: More flexible approach instead of generic retries could be to
         # add a json-rpc debug proc that returns whether the offer queue is empty or
         # not. And then poll every node until all nodes have an empty queue.
@@ -315,99 +334,35 @@ procSuite "Portal testnet tests":
       await client.close()
 
   asyncTest "Portal History - Propagate content from seed db":
-    let clients = await connectToRpcServers(config)
+    # Skipping this as it seems to fail now at offerContentInNodeRange, likely
-
+    # due to not being possibly to validate block bodies. This would mean the
-    var nodeInfos: seq[NodeInfo]
+    # test is flawed and block headers should be offered before bodies and
-    for client in clients:
+    # receipts.
-      let nodeInfo = await client.portal_history_nodeInfo()
-      await client.close()
-      nodeInfos.add(nodeInfo)
-
-    const dataPath = "./fluffy/tests/blocks/mainnet_blocks_1000011_1000030.json"
-
-    # path for temporary db, separate dir is used as sqlite usually also creates
-    # wal files, and we do not want for those to linger in filesystem
-    const tempDbPath = "./fluffy/tests/blocks/tempDir/mainnet_blocks_1000011_1000030.sqlite3"
-
-    let (dbFile, dbName) = getDbBasePathAndName(tempDbPath).unsafeGet()
-
-    let blockData = readJsonType(dataPath, BlockDataTable)
-    check blockData.isOk()
-    let bd = blockData.get()
-
-    createDir(dbFile)
-    let db = SeedDb.new(path = dbFile, name = dbName)
-
-    try:
-      let lastNodeIdx = len(nodeInfos) - 1
-
-      # populate temp database from json file
-      for t in blocksContent(bd, false):
-        db.put(t[0], t[1], t[2])
-
-      # store content in node0 database
-      check (await clients[0].portal_history_storeContentInNodeRange(tempDbPath, 100, 0))
-      await clients[0].close()
-
-      # offer content to node 1..63
-      for i in 1..lastNodeIdx:
-        let receipientId = nodeInfos[i].nodeId
-        let offerResponse = await retryUntil(
-          proc (): Future[int] {.async.} =
-            try:
-              let res = await clients[0].portal_history_offerContentInNodeRange(tempDbPath, receipientId, 64, 0)
-              await clients[0].close()
-              return res
-            except CatchableError as exc:
-              await clients[0].close()
-              raise exc
-          ,
-          proc (os: int): bool = return true,
-          "Offer failed",
-          i
-        )
-        check:
-          offerResponse > 0
-
-      for i, client in clients:
-        # Note: Once there is the Canonical Indices Network, we don't need to
-        # access this file anymore here for the block hashes.
-        for hash in bd.blockHashes():
-          let content = await retryUntil(
-            proc (): Future[Option[BlockObject]] {.async.} =
-              try:
-                let res = await client.eth_getBlockByHash(hash.ethHashStr(), false)
-                await client.close()
-                return res
-              except CatchableError as exc:
-                await client.close()
-                raise exc
-            ,
-            proc (mc: Option[BlockObject]): bool = return mc.isSome(),
-            "Did not receive expected Block with hash " & hash.data.toHex(),
-            i
-          )
-          check content.isSome()
-
-          let blockObj = content.get()
-          check blockObj.hash.get() == hash
-
-          for tx in blockObj.transactions:
-            var txObj: TransactionObject
-            tx.fromJson("tx", txObj)
-            check txObj.blockHash.get() == hash
-
-        await client.close()
-    finally:
-      db.close()
-      removeDir(dbFile)
-
-  asyncTest "Portal History - Propagate content from seed db in depth first fashion":
-    # Skipping this test as it is flawed considering block headers should be
-    # offered before bodies and receipts.
     # TODO: Split this up and activate test
     skip()
-    # let clients = await connectToRpcServers(config)
+
+    # const
+    #   headerFile = "./vendor/portal-spec-tests/tests/mainnet/history/headers/1000011-1000030.e2s"
+    #   accumulatorFile = "./vendor/portal-spec-tests/tests/mainnet/history/accumulator/epoch-accumulator-00122.ssz"
+    #   blockDataFile = "./fluffy/tests/blocks/mainnet_blocks_1000011_1000030.json"
+
+    #   # Path for the temporary db. A separate dir is used as sqlite usually also
+    #   # creates wal files.
+    #   tempDbPath = "./fluffy/tests/blocks/tempDir/mainnet_blocks_1000011_1000030.sqlite3"
+
+    # let
+    #   blockHeaders = readBlockHeaders(headerFile).valueOr:
+    #     raiseAssert "Invalid header file: " & headerFile
+    #   epochAccumulator = readEpochAccumulatorCached(accumulatorFile).valueOr:
+    #     raiseAssert "Invalid epoch accumulator file: " & accumulatorFile
+    #   blockHeadersWithProof =
+    #     buildHeadersWithProof(blockHeaders, epochAccumulator).valueOr:
+    #       raiseAssert "Could not build headers with proof"
+    #   blockData =
+    #     readJsonType(blockDataFile, BlockDataTable).valueOr:
+    #       raiseAssert "Invalid block data file" & blockDataFile
+
+    #   clients = await connectToRpcServers(config)
 
     # var nodeInfos: seq[NodeInfo]
     # for client in clients:
@@ -415,35 +370,54 @@ procSuite "Portal testnet tests":
     #   await client.close()
     #   nodeInfos.add(nodeInfo)
 
-    # # different set of data for each test as tests are statefull so previously propagated
-    # # block are already in the network
-    # const dataPath = "./fluffy/tests/blocks/mainnet_blocks_1000040_1000050.json"
-
-    # # path for temporary db, separate dir is used as sqlite usually also creates
-    # # wal files, and we do not want for those to linger in filesystem
-    # const tempDbPath = "./fluffy/tests/blocks/tempDir/mainnet_blocks_1000040_100050.sqlite3"
-
     # let (dbFile, dbName) = getDbBasePathAndName(tempDbPath).unsafeGet()
-
-    # let blockData = readJsonType(dataPath, BlockDataTable)
-    # check blockData.isOk()
-    # let bd = blockData.get()
-
     # createDir(dbFile)
     # let db = SeedDb.new(path = dbFile, name = dbName)
+    # defer:
+    #   db.close()
+    #   removeDir(dbFile)
 
-    # try:
+    # # Fill seed db with block headers with proof
-    #   # populate temp database from json file
+    # for (content, contentKey) in blockHeadersWithProof:
-    #   for t in blocksContent(bd, false):
+    #   let contentId = history_content.toContentId(ByteList(contentKey))
+    #   db.put(contentId, contentKey, content)
+
+    # # Fill seed db with block bodies and receipts
+    # for t in blocksContent(blockData, false):
     #   db.put(t[0], t[1], t[2])
 
-    #   check (await clients[0].portal_history_depthContentPropagate(tempDbPath, 64))
+    # let lastNodeIdx = len(nodeInfos) - 1
+
+    # # Store content in node 0 database
+    # check (await clients[0].portal_history_storeContentInNodeRange(
+    #   tempDbPath, 100, 0))
     # await clients[0].close()
 
+    # # Offer content to node 1..63
+    # for i in 1..lastNodeIdx:
+    #   let recipientId = nodeInfos[i].nodeId
+    #   let offerResponse = await retryUntil(
+    #     proc (): Future[int] {.async.} =
+    #       try:
+    #         let res = await clients[0].portal_history_offerContentInNodeRange(
+    #           tempDbPath, recipientId, 64, 0)
+    #         await clients[0].close()
+    #         return res
+    #       except CatchableError as exc:
+    #         await clients[0].close()
+    #         raise exc
+    #     ,
+    #     proc (os: int): bool = return true,
+    #     "Offer failed",
+    #     i
+    #   )
+    #   check:
+    #     offerResponse > 0
+
     # for i, client in clients:
     #   # Note: Once there is the Canonical Indices Network, we don't need to
     #   # access this file anymore here for the block hashes.
-    #     for hash in bd.blockHashes():
+    #   for hash in blockData.blockHashes():
     #     let content = await retryUntil(
     #       proc (): Future[Option[BlockObject]] {.async.} =
     #         try:
@@ -469,6 +443,89 @@ procSuite "Portal testnet tests":
     #       check txObj.blockHash.get() == hash
 
     #   await client.close()
-    # finally:
+
+  asyncTest "Portal History - Propagate content from seed db in depth first fashion":
+    # Skipping this test as it is flawed considering block headers should be
+    # offered before bodies and receipts.
+    # TODO: Split this up and activate test
+    skip()
+
+    # const
+    #   headerFile = "./vendor/portal-spec-tests/tests/mainnet/history/headers/1000011-1000030.e2s"
+    #   accumulatorFile = "./vendor/portal-spec-tests/tests/mainnet/history/accumulator/epoch-accumulator-00122.ssz"
+    #   # Different set of data for each test as tests are statefull so previously
+    #   # propagated content is still in the network
+    #   blockDataFile = "./fluffy/tests/blocks/mainnet_blocks_1000040_1000050.json"
+
+    #   # Path for the temporary db. A separate dir is used as sqlite usually also
+    #   # creates wal files.
+    #   tempDbPath = "./fluffy/tests/blocks/tempDir/mainnet_blocks_1000040_100050.sqlite3"
+
+    # let
+    #   blockHeaders = readBlockHeaders(headerFile).valueOr:
+    #     raiseAssert "Invalid header file: " & headerFile
+    #   epochAccumulator = readEpochAccumulatorCached(accumulatorFile).valueOr:
+    #     raiseAssert "Invalid epoch accumulator file: " & accumulatorFile
+    #   blockHeadersWithProof =
+    #     buildHeadersWithProof(blockHeaders, epochAccumulator).valueOr:
+    #       raiseAssert "Could not build headers with proof"
+    #   blockData =
+    #     readJsonType(blockDataFile, BlockDataTable).valueOr:
+    #       raiseAssert "Invalid block data file" & blockDataFile
+
+    #   clients = await connectToRpcServers(config)
+
+    # var nodeInfos: seq[NodeInfo]
+    # for client in clients:
+    #   let nodeInfo = await client.portal_history_nodeInfo()
+    #   await client.close()
+    #   nodeInfos.add(nodeInfo)
+
+    # let (dbFile, dbName) = getDbBasePathAndName(tempDbPath).unsafeGet()
+    # createDir(dbFile)
+    # let db = SeedDb.new(path = dbFile, name = dbName)
+    # defer:
     #   db.close()
     #   removeDir(dbFile)
+
+    # # Fill seed db with block headers with proof
+    # for (content, contentKey) in blockHeadersWithProof:
+    #   let contentId = history_content.toContentId(ByteList(contentKey))
+    #   db.put(contentId, contentKey, content)
+
+    # # Fill seed db with block bodies and receipts
+    # for t in blocksContent(blockData, false):
+    #   db.put(t[0], t[1], t[2])
+
+    # check (await clients[0].portal_history_depthContentPropagate(tempDbPath, 64))
+    # await clients[0].close()
+
+    # for i, client in clients:
+    #   # Note: Once there is the Canonical Indices Network, we don't need to
+    #   # access this file anymore here for the block hashes.
+    #   for hash in blockData.blockHashes():
+    #     let content = await retryUntil(
+    #       proc (): Future[Option[BlockObject]] {.async.} =
+    #         try:
+    #           let res = await client.eth_getBlockByHash(hash.ethHashStr(), false)
+    #           await client.close()
+    #           return res
+    #         except CatchableError as exc:
+    #           await client.close()
+    #           raise exc
+    #       ,
+    #       proc (mc: Option[BlockObject]): bool = return mc.isSome(),
+    #       "Did not receive expected Block with hash " & hash.data.toHex(),
+    #       i
+    #     )
+    #     check content.isSome()
+
+    #     let blockObj = content.get()
+    #     check blockObj.hash.get() == hash
+
+    #     for tx in blockObj.transactions:
+    #       var txObj: TransactionObject
+    #       tx.fromJson("tx", txObj)
+    #       check txObj.blockHash.get() == hash
+
+    #   await client.close()

fluffy/tests/test_accumulator.nim

@@ -16,18 +16,6 @@ import
   ../network/history/[history_content, accumulator],
   ./test_helpers
 
-func buildProof(
-    epochAccumulators: seq[EpochAccumulator], header: BlockHeader):
-    Result[seq[Digest], string] =
-  let
-    epochIndex = getEpochIndex(header)
-    epochAccumulator = epochAccumulators[epochIndex]
-
-    headerRecordIndex = getHeaderRecordIndex(header, epochIndex)
-    gIndex = GeneralizedIndex(epochSize*2*2 + (headerRecordIndex*2))
-
-  return epochAccumulator.build_proof(gIndex)
-
 suite "Header Accumulator":
   test "Header Accumulator Canonical Verification":
     const
@@ -61,35 +49,33 @@ suite "Header Accumulator":
     block: # Test valid headers
       for i in headersToTest:
         let header = headers[i]
-        let proof = buildProof(epochAccumulators, header)
+        let proof = buildProof(header, epochAccumulators)
         check:
           proof.isOk()
-          verifyHeader(accumulator, header, proof.get()).isOk()
+          verifyAccumulatorProof(accumulator, header, proof.get()).isOk()
 
     block: # Test invalid headers
       # Post merge block number must fail (> than latest header in accumulator)
+      var proof: AccumulatorProof
       let header = BlockHeader(blockNumber: mergeBlockNumber.stuint(256))
-      check verifyHeader(accumulator, header, @[]).isErr()
+      check verifyAccumulatorProof(accumulator, header, proof).isErr()
 
       # Test altered block headers by altering the difficulty
       for i in headersToTest:
-        let proof = buildProof( epochAccumulators, headers[i])
+        let proof = buildProof(headers[i], epochAccumulators)
         check:
           proof.isOk()
         # Alter the block header so the proof no longer matches
         let header = BlockHeader(
           blockNumber: i.stuint(256), difficulty: 2.stuint(256))
 
-        check verifyHeader(accumulator, header, proof.get()).isErr()
+        check verifyAccumulatorProof(accumulator, header, proof.get()).isErr()
 
     block: # Test invalid proofs
-      var proof: seq[Digest]
+      var proof: AccumulatorProof
-      for i in 0..14:
-        var digest: Digest
-        proof.add(digest)
 
       for i in headersToTest:
-        check verifyHeader(accumulator, headers[i], proof).isErr()
+        check verifyAccumulatorProof(accumulator, headers[i], proof).isErr()
 
   test "Header Accumulator - Not Finished":
     # Less headers than needed to finish the accumulator

fluffy/tests/test_helpers.nim

@@ -7,10 +7,10 @@
 
 import
   stew/shims/net,
-  eth/keys,
+  eth/[keys, rlp],
   eth/p2p/discoveryv5/[enr, node, routing_table],
   eth/p2p/discoveryv5/protocol as discv5_protocol,
-  ../network/history/accumulator
+  ../network/history/[accumulator, history_content]
 
 proc localAddress*(port: int): Address =
   Address(ip: ValidIpAddress.init("127.0.0.1"), port: Port(port))
@@ -72,3 +72,39 @@ func buildAccumulatorData*(headers: seq[BlockHeader]):
       return ok((finishAccumulator(accumulator), epochAccumulators))
 
   err("Not enough headers provided to finish the accumulator")
+
+func buildProof*(
+    header: BlockHeader, epochAccumulators: seq[EpochAccumulator]):
+    Result[AccumulatorProof, string] =
+  let epochIndex = getEpochIndex(header)
+  doAssert(epochIndex < uint64(epochAccumulators.len()))
+  let epochAccumulator = epochAccumulators[epochIndex]
+
+  buildProof(header, epochAccumulator)
+
+func buildHeaderWithProof*(
+    header: BlockHeader,
+    epochAccumulators: seq[EpochAccumulator]):
+    Result[BlockHeaderWithProof, string] =
+  ## Construct the accumulator proof for a specific header.
+  ## Returns the block header with the proof
+  if header.isPreMerge():
+    let epochIndex = getEpochIndex(header)
+    doAssert(epochIndex < uint64(epochAccumulators.len()))
+    let epochAccumulator = epochAccumulators[epochIndex]
+
+    buildHeaderWithProof(header, epochAccumulator)
+
+  else:
+    err("Cannot build accumulator proof for post merge header")
+
+func buildHeadersWithProof*(
+    headers: seq[BlockHeader],
+    epochAccumulators: seq[EpochAccumulator]):
+    Result[seq[BlockHeaderWithProof], string] =
+  var headersWithProof: seq[BlockHeaderWithProof]
+  for header in headers:
+    headersWithProof.add(
+      ? buildHeaderWithProof(header, epochAccumulators))
+
+  ok(headersWithProof)

fluffy/tests/test_history_network.nim

@@ -60,16 +60,20 @@ proc createEmptyHeaders(fromNum: int, toNum: int): seq[BlockHeader] =
     headers.add(bh)
   return headers
 
-proc headersToContentInfo(headers: seq[BlockHeader]): seq[ContentInfo] =
+proc headersToContentInfo(
+    headersWithProof: seq[BlockHeaderWithProof]): seq[ContentInfo] =
   var contentInfos: seq[ContentInfo]
-  for h in headers:
+  for headerWithProof in headersWithProof:
     let
-      headerHash = h.blockHash()
+      # TODO: Decoding step could be avoided
-      bk = BlockKey(blockHash: headerHash)
+      header = rlp.decode(headerWithProof.header.asSeq(), BlockHeader)
-      ck = encode(ContentKey(contentType: blockHeader, blockHeaderKey: bk))
+      headerHash = header.blockHash()
-      headerEncoded = rlp.encode(h)
+      blockKey = BlockKey(blockHash: headerHash)
-      ci = ContentInfo(contentKey: ck, content: headerEncoded)
+      contentKey = encode(ContentKey(
-    contentInfos.add(ci)
+        contentType: blockHeaderWithProof, blockHeaderWithProofKey: blockKey))
+      contentInfo = ContentInfo(
+        contentKey: contentKey, content: SSZ.encode(headerWithProof))
+    contentInfos.add(contentInfo)
   return contentInfos
 
 procSuite "History Content Network":
@@ -99,16 +103,26 @@ procSuite "History Content Network":
       historyNode1 = newHistoryNode(rng, 20302, masterAccumulator)
       historyNode2 = newHistoryNode(rng, 20303, masterAccumulator)
 
-    # Only node 2 stores all headers and all epoch accumulators.
+    var selectedHeaders: seq[BlockHeader]
-    for h in headers:
+    for i in headersToTest:
+      selectedHeaders.add(headers[i])
+
+    let headersWithProof =
+      buildHeadersWithProof(selectedHeaders, epochAccumulators)
+
+    check headersWithProof.isOk()
+
+    # Only node 2 stores the headers and all epoch accumulators.
+    for headerWithProof in headersWithProof.get():
       let
-        headerHash = h.blockHash()
+        header = rlp.decode(headerWithProof.header.asSeq(), BlockHeader)
+        headerHash = header.blockHash()
         blockKey = BlockKey(blockHash: headerHash)
         contentKey = ContentKey(
-          contentType: blockHeader, blockHeaderKey: blockKey)
+          contentType: blockHeaderWithProof, blockHeaderWithProofKey: blockKey)
         contentId = toContentId(contentKey)
-        headerEncoded = rlp.encode(h)
+      historyNode2.portalProtocol().storeContent(
-      historyNode2.portalProtocol().storeContent(contentId, headerEncoded)
+        contentId, SSZ.encode(headerWithProof))
 
     # Need to store the epoch accumulators to be able to do the block to hash
     # mapping
@@ -172,20 +186,12 @@ procSuite "History Content Network":
     let maxOfferedHistoryContent = getMaxOfferedContentKeys(
         uint32(len(historyProtocolId)), maxContentKeySize)
 
-    # One of the nodes needs to have the epochAccumulator to build proofs from
+    let headersWithProof = buildHeadersWithProof(
-    # for the offered headers.
+      headers[0..maxOfferedHistoryContent], epochAccumulators)
-    for epochAccumulator in epochAccumulators:
+    check headersWithProof.isOk()
-      let
-        rootHash = epochAccumulator.hash_tree_root()
-        contentKey = ContentKey(
-          contentType: ContentType.epochAccumulator,
-          epochAccumulatorKey: EpochAccumulatorKey(epochHash: rootHash))
-        contentId = toContentId(contentKey)
-      historyNode2.portalProtocol().storeContent(
-        contentId, SSZ.encode(epochAccumulator))
 
     # This is one header more than maxOfferedHistoryContent
-    let contentInfos = headersToContentInfo(headers[0..maxOfferedHistoryContent])
+    let contentInfos = headersToContentInfo(headersWithProof.get())
 
     # node 1 will offer the content so it needs to have it in its database
     for contentInfo in contentInfos:
@@ -254,37 +260,31 @@ procSuite "History Content Network":
       (await historyNode1.portalProtocol().ping(historyNode2.localNode())).isOk()
       (await historyNode2.portalProtocol().ping(historyNode1.localNode())).isOk()
 
-    # Need to store the epochAccumulators, because else the headers can't be
-    # verified if being part of the canonical chain currently
-    for epochAccumulator in epochAccumulators:
-      let
-        rootHash = epochAccumulator.hash_tree_root()
-        contentKey = ContentKey(
-          contentType: ContentType.epochAccumulator,
-          epochAccumulatorKey: EpochAccumulatorKey(epochHash: rootHash))
-        contentId = toContentId(contentKey)
-      historyNode1.portalProtocol.storeContent(
-        contentId, SSZ.encode(epochAccumulator))
-
     # Need to run start to get the processContentLoop running
     historyNode1.start()
     historyNode2.start()
 
-    let contentInfos = headersToContentInfo(headers)
+    var selectedHeaders: seq[BlockHeader]
+    for i in headersToTest:
+      selectedHeaders.add(headers[i])
 
-    for header in headersToTest:
+    let headersWithProof = buildHeadersWithProof(
-      let id = toContentId(contentInfos[header].contentKey)
+      selectedHeaders, epochAccumulators)
-      historyNode1.portalProtocol.storeContent(id, contentInfos[header].content)
+    check headersWithProof.isOk()
+
+    let contentInfos = headersToContentInfo(headersWithProof.get())
+
+    for contentInfo in contentInfos:
+      let id = toContentId(contentInfo.contentKey)
+      historyNode1.portalProtocol.storeContent(id, contentInfo.content)
 
       let offerResult = await historyNode1.portalProtocol.offer(
-        historyNode2.localNode(),
+        historyNode2.localNode(), @[contentInfo])
-        contentInfos[header..header]
-      )
 
       check offerResult.isOk()
 
-    for header in headersToTest:
+    for contentInfo in contentInfos:
-      let id = toContentId(contentInfos[header].contentKey)
+      let id = toContentId(contentInfo.contentKey)
       check historyNode2.containsId(id) == true
 
     await historyNode1.stop()

fluffy/tools/eth_data_exporter.nim

@@ -75,13 +75,13 @@ type
     # doesn't work well together with confutils.
     exportBlockData =
       """
-      Export block data (headers, bodies and receipts) to ajson format or a
+      Export block data (headers, bodies and receipts) to a json format or a
       database. Some of this functionality is likely to get deprecated"""
-    exportHeaders =
+    exportEpochHeaders =
       """
       Export block headers from an Ethereum JSON RPC Execution endpoint to
       *.e2s files arranged per epoch (8192 blocks)"""
-    verifyHeaders =
+    verifyEpochHeaders =
       """
       Verify *.e2s files containing block headers. Verify currently only
       means being able to RLP decode the block headers"""
@@ -95,6 +95,10 @@ type
       Print the root hash of the master accumulator and of all historical
       epoch accumulators. Requires data generated by exportAccumulatorData
       command"""
+    exportHeaderRange =
+      """
+      Export block headers from an Ethereum JSON RPC Execution endpoint to
+      *.e2s files (unlimited amount)"""
 
   StorageMode* = enum
     Json, Db
@@ -105,14 +109,6 @@ type
       defaultValueDesc: $LogLevel.INFO
       desc: "Sets the log level"
       name: "log-level" .}: LogLevel
-    initialBlock* {.
-      desc: "Number of the first block which should be downloaded"
-      defaultValue: 0
-      name: "initial-block" .}: uint64
-    endBlock* {.
-      desc: "Number of the last block which should be downloaded"
-      defaultValue: 0
-      name: "end-block" .}: uint64
     dataDir* {.
       desc: "The directory where generated data files will be exported to"
       defaultValue: defaultDataDir()
@@ -122,6 +118,14 @@ type
       command
       defaultValue: exportBlockData .}: ExporterCmd
     of exportBlockData:
+      startBlock* {.
+        desc: "Number of the first block to be exported"
+        defaultValue: 0
+        name: "start-block" .}: uint64
+      endBlock* {.
+        desc: "Number of the last block to be exported"
+        defaultValue: 0
+        name: "end-block" .}: uint64
       fileName* {.
         desc: "File name (minus extension) where block data will be exported to"
         defaultValue: defaultBlockFileName
@@ -135,7 +139,7 @@ type
         desc: "Only export the headers instead of full blocks and receipts"
         defaultValue: false
         name: "headers-only" .}: bool
-    of exportHeaders:
+    of exportEpochHeaders:
       startEpoch* {.
         desc: "Number of the first epoch which should be downloaded"
         defaultValue: 0
@@ -147,7 +151,7 @@ type
     # TODO:
     # Although options are the same as for exportHeaders, we can't drop them
     # under the same case of as confutils does not agree with that.
-    of verifyHeaders:
+    of verifyEpochHeaders:
       startEpochVerify* {.
         desc: "Number of the first epoch which should be downloaded"
         defaultValue: 0
@@ -172,6 +176,13 @@ type
         defaultValue: defaultAccumulatorFileName
         defaultValueDesc: $defaultAccumulatorFileName
         name: "accumulator-file-name" .}: string
+    of exportHeaderRange:
+      startBlockNumber* {.
+        desc: "Number of the first block header to be exported"
+        name: "start-block" .}: uint64
+      endBlockNumber* {.
+        desc: "Number of the last block header to be exported"
+        name: "end-block" .}: uint64
 
   HeaderRecord = object
     header: string
@@ -273,10 +284,10 @@ proc writeHeadersToJson(config: ExporterConf, client: RpcClient) =
   try:
     var writer = JsonWriter[DefaultFlavor].init(fh.s, pretty = true)
     writer.beginRecord()
-    for i in config.initialBlock..config.endBlock:
+    for i in config.startBlock..config.endBlock:
       let blck = client.downloadHeader(i)
       writer.writeHeaderRecord(blck)
-      if ((i - config.initialBlock) mod 8192) == 0 and i != config.initialBlock:
+      if ((i - config.startBlock) mod 8192) == 0 and i != config.startBlock:
         info "Downloaded 8192 new block headers", currentHeader = i
     writer.endRecord()
     info "File successfully written", path = config.dataDir / config.fileName
@@ -296,10 +307,10 @@ proc writeBlocksToJson(config: ExporterConf, client: RpcClient) =
   try:
     var writer = JsonWriter[DefaultFlavor].init(fh.s, pretty = true)
     writer.beginRecord()
-    for i in config.initialBlock..config.endBlock:
+    for i in config.startBlock..config.endBlock:
       let blck = downloadBlock(i, client)
       writer.writeBlockRecord(blck)
-      if ((i - config.initialBlock) mod 8192) == 0 and i != config.initialBlock:
+      if ((i - config.startBlock) mod 8192) == 0 and i != config.startBlock:
         info "Downloaded 8192 new blocks", currentBlock = i
     writer.endRecord()
     info "File successfully written", path = config.dataDir / config.fileName
@@ -319,7 +330,7 @@ proc writeBlocksToDb(config: ExporterConf, client: RpcClient) =
   defer:
     db.close()
 
-  for i in config.initialBlock..config.endBlock:
+  for i in config.startBlock..config.endBlock:
     let
       blck = downloadBlock(i, client)
       blockHash = blck.header.blockHash()
@@ -358,16 +369,6 @@ proc exportBlocks(config: ExporterConf, client: RpcClient) =
     else:
       writeBlocksToDb(config, client)
 
-const
-  # Using the e2s format to store data, but without the specific structure
-  # like in an era file, as we currently don't really need that.
-  # See: https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md
-  # Added one type for now, with numbers not formally specified.
-  # Note:
-  # Snappy compression for `ExecutionBlockHeaderRecord` only helps for the
-  # first ~1M (?) block headers, after that there is no gain so we don't do it.
-  ExecutionBlockHeaderRecord = [byte 0xFF, 0x00]
-
 when isMainModule:
   {.pop.}
   let config = ExporterConf.load()
@@ -375,12 +376,6 @@ when isMainModule:
 
   setLogLevel(config.logLevel)
 
-  if (config.endBlock < config.initialBlock):
-    fatal "Initial block number should be smaller than end block number",
-      initialBlock = config.initialBlock,
-      endBlock = config.endBlock
-    quit 1
-
   let dataDir = config.dataDir.string
   if not isDir(dataDir):
     let res = createPath(dataDir)
@@ -402,12 +397,18 @@ when isMainModule:
 
   case config.cmd
   of ExporterCmd.exportBlockData:
+    if (config.endBlock < config.startBlock):
+      fatal "Initial block number should be smaller than end block number",
+        startBlock = config.startBlock,
+        endBlock = config.endBlock
+      quit 1
+
     try:
       exportBlocks(config, client)
     finally:
       waitFor client.close()
 
-  of ExporterCmd.exportHeaders:
+  of ExporterCmd.exportEpochHeaders:
     proc exportEpochHeaders(file: string, epoch: uint64): Result[void, string] =
       # Downloading headers from JSON RPC endpoint
       info "Requesting epoch headers", epoch
@@ -441,7 +442,7 @@ when isMainModule:
 
     waitFor client.close()
 
-  of ExporterCmd.verifyHeaders:
+  of ExporterCmd.verifyEpochHeaders:
     proc verifyEpochHeaders(file: string, epoch: uint64): Result[void, string] =
       let fh = ? openFile(file, {OpenFlags.Read}).mapErr(toString)
       defer: discard closeFile(fh)
@@ -476,10 +477,6 @@ when isMainModule:
         info "Successfully decoded epoch headers", file
 
   of ExporterCmd.exportAccumulatorData:
-    # TODO:
-    # Also write epoch accumulators to files. These can be re-used for creation
-    # of headers with proofs.
-
     # Lets first check if the accumulator file already exists before starting
     # to build it.
     let accumulatorFile = dataDir / config.accumulatorFileName
@@ -598,3 +595,43 @@ when isMainModule:
     echo "Epoch Root"
     for i, root in accumulator.historicalEpochs:
       echo &"{i.uint64:05} 0x{root.toHex()}"
+
+  of ExporterCmd.exportHeaderRange:
+    let
+      startBlockNumber = config.startBlockNumber
+      endBlockNumber = config.endBlockNumber
+
+    if (endBlockNumber < startBlockNumber):
+      fatal "Start block number should be smaller than end block number",
+        startBlockNumber, endBlockNumber
+      quit 1
+
+    proc exportHeaders(
+        file: string, startBlockNumber, endBlockNumber: uint64):
+        Result[void, string] =
+      # Downloading headers from JSON RPC endpoint
+      info "Requesting headers", startBlockNumber, endBlockNumber
+      var headers: seq[BlockHeader]
+      for j in startBlockNumber..endBlockNumber:
+        debug "Requesting block", number = j
+        let header = client.downloadHeader(j)
+        headers.add(header)
+
+      let fh = ? openFile(
+        file, {OpenFlags.Write, OpenFlags.Create}).mapErr(toString)
+      defer: discard closeFile(fh)
+
+      info "Writing headers to file", file
+      for header in headers:
+        discard ? fh.appendRecord(ExecutionBlockHeaderRecord, rlp.encode(header))
+
+      ok()
+
+    let file =
+      try: dataDir / &"mainnet-headers-{startBlockNumber:05}-{endBlockNumber:05}.e2s"
+      except ValueError as e: raiseAssert e.msg
+
+    let res = exportHeaders(file, startBlockNumber, endBlockNumber)
+    if res.isErr():
+      fatal "Failed exporting headers", error = res.error
+      quit 1

nimbus.nimble

@@ -77,11 +77,11 @@ task fluffy, "Build fluffy":
 task fluffy_test, "Run fluffy tests":
   # Need the nimbus_db_backend in state network tests as we need a Hexary to
   # start from, even though it only uses the MemoryDb.
-  test "fluffy/tests/portal_spec_tests/mainnet", "all_fluffy_portal_spec_tests", "-d:chronicles_log_level=ERROR -d:chronosStrictException -d:nimbus_db_backend=sqlite -d:PREFER_BLST_SHA256=false -d:canonicalVerify=true"
+  test "fluffy/tests/portal_spec_tests/mainnet", "all_fluffy_portal_spec_tests", "-d:chronicles_log_level=ERROR -d:chronosStrictException -d:nimbus_db_backend=sqlite -d:PREFER_BLST_SHA256=false"
   # Running tests with a low `mergeBlockNumber` to make the tests faster.
   # Using the real mainnet merge block number is not realistic for these tests.
-  test "fluffy/tests", "all_fluffy_tests", "-d:chronicles_log_level=ERROR -d:chronosStrictException -d:nimbus_db_backend=sqlite -d:PREFER_BLST_SHA256=false -d:canonicalVerify=true -d:mergeBlockNumber:38130"
+  test "fluffy/tests", "all_fluffy_tests", "-d:chronicles_log_level=ERROR -d:chronosStrictException -d:nimbus_db_backend=sqlite -d:PREFER_BLST_SHA256=false -d:mergeBlockNumber:38130"
-  test "fluffy/tests/beacon_light_client_tests", "all_beacon_light_client_tests", "-d:chronicles_log_level=ERROR -d:chronosStrictException -d:nimbus_db_backend=sqlite -d:PREFER_BLST_SHA256=false -d:canonicalVerify=true"
+  test "fluffy/tests/beacon_light_client_tests", "all_beacon_light_client_tests", "-d:chronicles_log_level=ERROR -d:chronosStrictException -d:nimbus_db_backend=sqlite -d:PREFER_BLST_SHA256=false"
 
 task fluffy_tools, "Build fluffy tools":
   buildBinary "portalcli", "fluffy/tools/", "-d:chronicles_log_level=TRACE -d:chronosStrictException -d:PREFER_BLST_SHA256=false"

vendor/portal-spec-tests

@@ -1 +1 @@
-Subproject commit 98cf0875eaa5d9b74aed87f1a356e0f1dd8c2db5
+Subproject commit 5a1d2e553d97c04339b6227624d4ebab4da88701