Add headers with proof content type and use it for verification (#1281)
* Add headers with proof content type and use it for verification - Add BlockHeaderWithProof content type & content - Use BlockHeaderWithProof content to verify if chain data is part of the canonical chain - Adjust parser & seeder code to be able to seed these headers with proof - Adjust eth_data_exporter to be able to export custom header ranges for which to build proofs (mostly for testing) There is currently quite some ugliness & clean-up needed for which a big part is due tos upporting both BlockHeader and BlockHeaderWithProof on the network. * Change accumulator proof to array / SSZ vector type - Change accumulator proof to SSZ vector instead of SSZ list. - Add and use general buildProof and buildHeaderWithProof func. * Make the BlockHeaderWithProof an SSZ Union with None option * Update portal-spec-tests to master commit
This commit is contained in:
Kim De Mey
GitHub
parent
36a478afa7
commit
36d430aaa2
|
|
@ -11,6 +11,7 @@ import
|
|||
json_serialization, json_serialization/std/tables,
|
||||
stew/[byteutils, io2, results], chronicles,
|
||||
eth/[rlp, common/eth_types],
|
||||
ncli/e2store,
|
||||
../../nimbus/[chain_config, genesis],
|
||||
../network/history/[history_content, accumulator]
|
||||
|
||||
|
|
@ -31,9 +32,6 @@ type
|
|||
# Fix in nim-json-serialization or should I overload something here?
|
||||
number*: int
|
||||
|
||||
EpochAccumulatorObject = object
|
||||
epochAccumulator: string
|
||||
|
||||
BlockDataTable* = Table[string, BlockData]
|
||||
|
||||
proc readJsonType*(dataFile: string, T: type): Result[T, string] =
|
||||
|
|
@ -190,7 +188,6 @@ proc getGenesisHeader*(id: NetworkId = MainNet): BlockHeader =
|
|||
except RlpError:
|
||||
raise (ref Defect)(msg: "Genesis should be valid")
|
||||
|
||||
|
||||
proc toString*(v: IoErrorCode): string =
|
||||
try: ioErrorMsg(v)
|
||||
except Exception as e: raiseAssert e.msg
|
||||
|
|
@ -211,3 +208,44 @@ proc readEpochAccumulator*(file: string): Result[EpochAccumulator, string] =
|
|||
ok(SSZ.decode(encodedAccumulator, EpochAccumulator))
|
||||
except SszError as e:
|
||||
err("Decoding epoch accumulator failed: " & e.msg)
|
||||
|
||||
proc readEpochAccumulatorCached*(file: string): Result[EpochAccumulatorCached, string] =
|
||||
let encodedAccumulator = ? readAllFile(file).mapErr(toString)
|
||||
|
||||
try:
|
||||
ok(SSZ.decode(encodedAccumulator, EpochAccumulatorCached))
|
||||
except SszError as e:
|
||||
err("Decoding epoch accumulator failed: " & e.msg)
|
||||
|
||||
const
|
||||
# Using the e2s format to store data, but without the specific structure
|
||||
# like in an era file, as we currently don't really need that.
|
||||
# See: https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md
|
||||
# Added one type for now, with numbers not formally specified.
|
||||
# Note:
|
||||
# Snappy compression for `ExecutionBlockHeaderRecord` only helps for the
|
||||
# first ~1M (?) block headers, after that there is no gain so we don't do it.
|
||||
ExecutionBlockHeaderRecord* = [byte 0xFF, 0x00]
|
||||
|
||||
proc readBlockHeaders*(file: string): Result[seq[BlockHeader], string] =
|
||||
let fh = ? openFile(file, {OpenFlags.Read}).mapErr(toString)
|
||||
defer: discard closeFile(fh)
|
||||
|
||||
var data: seq[byte]
|
||||
var blockHeaders: seq[BlockHeader]
|
||||
while true:
|
||||
let header = readRecord(fh, data).valueOr:
|
||||
break
|
||||
|
||||
if header.typ == ExecutionBlockHeaderRecord:
|
||||
let blockHeader =
|
||||
try:
|
||||
rlp.decode(data, BlockHeader)
|
||||
except RlpError as e:
|
||||
return err("Invalid block header in " & file & ": " & e.msg)
|
||||
|
||||
blockHeaders.add(blockHeader)
|
||||
else:
|
||||
warn "Skipping record, not a block header", typ = toHex(header.typ)
|
||||
|
||||
ok(blockHeaders)
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@
|
|||
import
|
||||
std/[strformat, os],
|
||||
stew/results, chronos, chronicles,
|
||||
eth/common/eth_types,
|
||||
eth/common/eth_types, eth/rlp,
|
||||
../network/wire/portal_protocol,
|
||||
../network/history/[history_content, accumulator],
|
||||
./history_data_parser
|
||||
|
|
@ -150,6 +150,62 @@ proc historyPropagateBlock*(
|
|||
else:
|
||||
return err(blockDataTable.error)
|
||||
|
||||
proc historyPropagateHeadersWithProof*(
|
||||
p: PortalProtocol, epochHeadersFile: string, epochAccumulatorFile: string):
|
||||
Future[Result[void, string]] {.async.} =
|
||||
let res = readBlockHeaders(epochHeadersFile)
|
||||
if res.isErr():
|
||||
return err(res.error)
|
||||
|
||||
let blockHeaders = res.get()
|
||||
|
||||
let epochAccumulatorRes = readEpochAccumulatorCached(epochAccumulatorFile)
|
||||
if epochAccumulatorRes.isErr():
|
||||
return err(res.error)
|
||||
|
||||
let epochAccumulator = epochAccumulatorRes.get()
|
||||
for header in blockHeaders:
|
||||
if header.isPreMerge():
|
||||
let headerWithProof = buildHeaderWithProof(header, epochAccumulator)
|
||||
if headerWithProof.isErr:
|
||||
return err(headerWithProof.error)
|
||||
|
||||
let
|
||||
content = headerWithProof.get()
|
||||
contentKey = ContentKey(
|
||||
contentType: blockHeaderWithProof,
|
||||
blockHeaderWithProofKey: BlockKey(blockHash: header.blockHash()))
|
||||
contentId = history_content.toContentId(contentKey)
|
||||
encodedContent = SSZ.encode(content)
|
||||
|
||||
p.storeContent(contentId, encodedContent)
|
||||
|
||||
let keys = ContentKeysList(@[encode(contentKey)])
|
||||
discard await p.neighborhoodGossip(keys, @[encodedContent])
|
||||
|
||||
return ok()
|
||||
|
||||
proc historyPropagateHeadersWithProof*(
|
||||
p: PortalProtocol, dataDir: string):
|
||||
Future[Result[void, string]] {.async.} =
|
||||
for i in 0..<preMergeEpochs:
|
||||
let
|
||||
epochHeadersfile =
|
||||
try: dataDir / &"mainnet-headers-epoch-{i.uint64:05}.e2s"
|
||||
except ValueError as e: raiseAssert e.msg
|
||||
epochAccumulatorFile =
|
||||
try: dataDir / &"mainnet-epoch-accumulator-{i.uint64:05}.ssz"
|
||||
except ValueError as e: raiseAssert e.msg
|
||||
|
||||
let res = await p.historyPropagateHeadersWithProof(
|
||||
epochHeadersfile, epochAccumulatorFile)
|
||||
if res.isOk():
|
||||
info "Finished gossiping 1 epoch of headers with proof", i
|
||||
else:
|
||||
return err(res.error)
|
||||
|
||||
return ok()
|
||||
|
||||
proc historyPropagateHeaders*(
|
||||
p: PortalProtocol, dataFile: string, verify = false):
|
||||
Future[Result[void, string]] {.async.} =
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
{.push raises: [Defect].}
|
||||
|
||||
import
|
||||
eth/common/eth_types_rlp,
|
||||
eth/rlp, eth/common/eth_types_rlp,
|
||||
ssz_serialization, ssz_serialization/[proofs, merkleization],
|
||||
../../common/common_types,
|
||||
./history_content
|
||||
|
|
@ -21,7 +21,7 @@ export ssz_serialization, merkleization, proofs, eth_types_rlp
|
|||
|
||||
const
|
||||
epochSize* = 8192 # blocks
|
||||
# Allow this to be adjusted at compile time fir testing. If more constants
|
||||
# Allow this to be adjusted at compile time for testing. If more constants
|
||||
# need to be adjusted we can add some presets file.
|
||||
mergeBlockNumber* {.intdefine.}: uint64 = 15537394
|
||||
|
||||
|
|
@ -30,8 +30,9 @@ const
|
|||
preMergeEpochs* = (mergeBlockNumber + epochSize - 1) div epochSize
|
||||
|
||||
# TODO:
|
||||
# Currently disabled, because of testing issues, but could be used as value to
|
||||
# double check on at merge block.
|
||||
# Currently disabled, because issue when testing with other
|
||||
# `mergeBlockNumber`, but it could be used as value to double check on at
|
||||
# merge block.
|
||||
# TODO: Could also be used as value to actual finish the accumulator, instead
|
||||
# of `mergeBlockNumber`, but:
|
||||
# - Still need to store the actual `mergeBlockNumber` and run-time somewhere
|
||||
|
|
@ -47,6 +48,14 @@ type
|
|||
|
||||
EpochAccumulator* = List[HeaderRecord, epochSize]
|
||||
|
||||
# In the core code of Fluffy the `EpochAccumulator` type is solely used, as
|
||||
# `hash_tree_root` is done either once or never on this object after
|
||||
# serialization.
|
||||
# However for the generation of the proofs for all the headers in an epoch, it
|
||||
# needs to be run many times and the cached version of the SSZ list is
|
||||
# obviously much faster, so this second type is added for this usage.
|
||||
EpochAccumulatorCached* = HashList[HeaderRecord, epochSize]
|
||||
|
||||
Accumulator* = object
|
||||
historicalEpochs*: List[Bytes32, int(preMergeEpochs)]
|
||||
currentEpoch*: EpochAccumulator
|
||||
|
|
@ -127,7 +136,7 @@ func isPreMerge*(blockNumber: uint64): bool =
|
|||
func isPreMerge*(header: BlockHeader): bool =
|
||||
isPreMerge(header.blockNumber.truncate(uint64))
|
||||
|
||||
func verifyProof*(
|
||||
func verifyProof(
|
||||
a: FinishedAccumulator, header: BlockHeader, proof: openArray[Digest]): bool =
|
||||
let
|
||||
epochIndex = getEpochIndex(header)
|
||||
|
|
@ -141,10 +150,12 @@ func verifyProof*(
|
|||
|
||||
verify_merkle_multiproof(@[leave], proof, @[gIndex], epochAccumulatorHash)
|
||||
|
||||
func verifyHeader*(
|
||||
a: FinishedAccumulator, header: BlockHeader, proof: openArray[Digest]):
|
||||
func verifyAccumulatorProof*(
|
||||
a: FinishedAccumulator, header: BlockHeader, proof: AccumulatorProof):
|
||||
Result[void, string] =
|
||||
if header.isPreMerge():
|
||||
# Note: The proof is typed with correct depth, so no check on this is
|
||||
# required here.
|
||||
if a.verifyProof(header, proof):
|
||||
ok()
|
||||
else:
|
||||
|
|
@ -152,6 +163,43 @@ func verifyHeader*(
|
|||
else:
|
||||
err("Cannot verify post merge header with accumulator proof")
|
||||
|
||||
func verifyHeader*(
|
||||
a: FinishedAccumulator, header: BlockHeader, proof: BlockHeaderProof):
|
||||
Result[void, string] =
|
||||
case proof.proofType:
|
||||
of BlockHeaderProofType.accumulatorProof:
|
||||
a.verifyAccumulatorProof(header, proof.accumulatorProof)
|
||||
of BlockHeaderProofType.none:
|
||||
err("cannot verify header without proof")
|
||||
|
||||
func buildProof*(
|
||||
header: BlockHeader,
|
||||
epochAccumulator: EpochAccumulator | EpochAccumulatorCached):
|
||||
Result[AccumulatorProof, string] =
|
||||
doAssert(header.isPreMerge(), "Must be pre merge header")
|
||||
|
||||
let
|
||||
epochIndex = getEpochIndex(header)
|
||||
headerRecordIndex = getHeaderRecordIndex(header, epochIndex)
|
||||
|
||||
# TODO: Implement more generalized `get_generalized_index`
|
||||
gIndex = GeneralizedIndex(epochSize*2*2 + (headerRecordIndex*2))
|
||||
|
||||
var proof: AccumulatorProof
|
||||
? epochAccumulator.build_proof(gIndex, proof)
|
||||
|
||||
ok(proof)
|
||||
|
||||
func buildHeaderWithProof*(
|
||||
header: BlockHeader,
|
||||
epochAccumulator: EpochAccumulator | EpochAccumulatorCached):
|
||||
Result[BlockHeaderWithProof, string] =
|
||||
let proof = ? buildProof(header, epochAccumulator)
|
||||
|
||||
ok(BlockHeaderWithProof(
|
||||
header: ByteList.init(rlp.encode(header)),
|
||||
proof: BlockHeaderProof.init(proof)))
|
||||
|
||||
func getBlockEpochDataForBlockNumber*(
|
||||
a: FinishedAccumulator, bn: UInt256): Result[BlockEpochData, string] =
|
||||
let blockNumber = bn.truncate(uint64)
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ type
|
|||
blockBody = 0x01
|
||||
receipts = 0x02
|
||||
epochAccumulator = 0x03
|
||||
blockHeaderWithProof = 0x04
|
||||
|
||||
BlockKey* = object
|
||||
blockHash*: BlockHash
|
||||
|
|
@ -49,6 +50,8 @@ type
|
|||
receiptsKey*: BlockKey
|
||||
of epochAccumulator:
|
||||
epochAccumulatorKey*: EpochAccumulatorKey
|
||||
of blockHeaderWithProof:
|
||||
blockHeaderWithProofKey*: BlockKey
|
||||
|
||||
func encode*(contentKey: ContentKey): ByteList =
|
||||
ByteList.init(SSZ.encode(contentKey))
|
||||
|
|
@ -86,6 +89,8 @@ func `$`*(x: ContentKey): string =
|
|||
of epochAccumulator:
|
||||
let key = x.epochAccumulatorKey
|
||||
res.add("epochHash: " & $key.epochHash)
|
||||
of blockHeaderWithProof:
|
||||
res.add($x.blockHeaderWithProofKey)
|
||||
|
||||
res.add(")")
|
||||
|
||||
|
|
@ -114,3 +119,23 @@ type
|
|||
|
||||
ReceiptByteList* = List[byte, MAX_RECEIPT_LENGTH] # RLP data
|
||||
ReceiptsSSZ* = List[ReceiptByteList, MAX_TRANSACTION_COUNT]
|
||||
|
||||
AccumulatorProof* = array[15, Digest]
|
||||
|
||||
BlockHeaderProofType* = enum
|
||||
none = 0x00 # An SSZ Union None
|
||||
accumulatorProof = 0x01
|
||||
|
||||
BlockHeaderProof* = object
|
||||
case proofType*: BlockHeaderProofType
|
||||
of none:
|
||||
discard
|
||||
of accumulatorProof:
|
||||
accumulatorProof*: AccumulatorProof
|
||||
|
||||
BlockHeaderWithProof* = object
|
||||
header*: ByteList # RLP data
|
||||
proof*: BlockHeaderProof
|
||||
|
||||
func init*(T: type BlockHeaderProof, proof: AccumulatorProof): T =
|
||||
BlockHeaderProof(proofType: accumulatorProof, accumulatorProof: proof)
|
||||
|
|
|
|||
|
|
@ -22,16 +22,6 @@ logScope:
|
|||
|
||||
export accumulator
|
||||
|
||||
# TODO: To currently verify if content is from the canonical chain it is
|
||||
# required to download the right epoch accunulator, which is ~0.5 MB. This is
|
||||
# too much, at least for the local testnet tests. This needs to be improved
|
||||
# by adding the proofs to the block header content. Another independent
|
||||
# improvement would be to have a content cache (LRU or so). The latter would
|
||||
# probably help mostly for the local testnet tests.
|
||||
# For now, we disable this verification default until further improvements are
|
||||
# made.
|
||||
const canonicalVerify* {.booldefine.} = false
|
||||
|
||||
const
|
||||
historyProtocolId* = [byte 0x50, 0x0B]
|
||||
|
||||
|
|
@ -67,15 +57,23 @@ func getEncodedKeyForContent(
|
|||
ContentKey(contentType: cType, receiptsKey: contentKeyType)
|
||||
of epochAccumulator:
|
||||
raiseAssert("Not implemented")
|
||||
of blockHeaderWithProof:
|
||||
ContentKey(contentType: cType, blockHeaderWithProofKey: contentKeyType)
|
||||
|
||||
return encodeKey(contentKey)
|
||||
|
||||
func decodeRlp*(bytes: openArray[byte], T: type): Result[T, string] =
|
||||
func decodeRlp*(input: openArray[byte], T: type): Result[T, string] =
|
||||
try:
|
||||
ok(rlp.decode(bytes, T))
|
||||
ok(rlp.decode(input, T))
|
||||
except RlpError as e:
|
||||
err(e.msg)
|
||||
|
||||
func decodeSsz*(input: openArray[byte], T: type): Result[T, string] =
|
||||
try:
|
||||
ok(SSZ.decode(input, T))
|
||||
except SszError as e:
|
||||
err(e.msg)
|
||||
|
||||
## Calls to go from SSZ decoded types to RLP fully decoded types
|
||||
|
||||
func fromPortalBlockBody(
|
||||
|
|
@ -182,11 +180,7 @@ proc validateBlockBodyBytes*(
|
|||
bytes: openArray[byte], txRoot, ommersHash: KeccakHash):
|
||||
Result[BlockBody, string] =
|
||||
## Fully decode the SSZ Block Body and validate it against the header.
|
||||
let body =
|
||||
try:
|
||||
SSZ.decode(bytes, BlockBodySSZ)
|
||||
except SszError as e:
|
||||
return err("Failed to decode block body: " & e.msg)
|
||||
let body = ? decodeSsz(bytes, BlockBodySSZ)
|
||||
|
||||
? validateBlockBody(body, txRoot, ommersHash)
|
||||
|
||||
|
|
@ -205,11 +199,7 @@ proc validateReceiptsBytes*(
|
|||
bytes: openArray[byte],
|
||||
receiptsRoot: KeccakHash): Result[seq[Receipt], string] =
|
||||
## Fully decode the SSZ Block Body and validate it against the header.
|
||||
let receipts =
|
||||
try:
|
||||
SSZ.decode(bytes, ReceiptsSSZ)
|
||||
except SszError as e:
|
||||
return err("Failed to decode receipts: " & e.msg)
|
||||
let receipts = ? decodeSsz(bytes, ReceiptsSSZ)
|
||||
|
||||
? validateReceipts(receipts, receiptsRoot)
|
||||
|
||||
|
|
@ -220,7 +210,13 @@ proc validateReceiptsBytes*(
|
|||
proc get(db: ContentDB, T: type BlockHeader, contentId: ContentId): Option[T] =
|
||||
let contentFromDB = db.get(contentId)
|
||||
if contentFromDB.isSome():
|
||||
let res = decodeRlp(contentFromDB.get(), T)
|
||||
let headerWithProof =
|
||||
try:
|
||||
SSZ.decode(contentFromDB.get(), BlockHeaderWithProof)
|
||||
except SszError as e:
|
||||
raiseAssert(e.msg)
|
||||
|
||||
let res = decodeRlp(headerWithProof.header.asSeq(), T)
|
||||
if res.isErr():
|
||||
raiseAssert(res.error)
|
||||
else:
|
||||
|
|
@ -277,6 +273,67 @@ const requestRetries = 4
|
|||
# ongoing requests are cancelled after the receival of the first response,
|
||||
# however that response is not yet validated at that moment.
|
||||
|
||||
func verifyHeader(
|
||||
n: HistoryNetwork, header: BlockHeader, proof: BlockHeaderProof):
|
||||
Result[void, string] =
|
||||
verifyHeader(n.accumulator, header, proof)
|
||||
|
||||
proc getVerifiedBlockHeader*(
|
||||
n: HistoryNetwork, hash: BlockHash):
|
||||
Future[Option[BlockHeader]] {.async.} =
|
||||
let (keyEncoded, contentId) =
|
||||
getEncodedKeyForContent(blockHeaderWithProof, hash)
|
||||
|
||||
# Note: This still requests a BlockHeaderWithProof from the database, as that
|
||||
# is what is stored. But the proof doesn't need to be checked as everthing
|
||||
# should get checked before storing.
|
||||
let headerFromDb = n.getContentFromDb(BlockHeader, contentId)
|
||||
|
||||
if headerFromDb.isSome():
|
||||
info "Fetched block header from database", hash, contentKey = keyEncoded
|
||||
return headerFromDb
|
||||
|
||||
for i in 0..<requestRetries:
|
||||
let headerContentLookup =
|
||||
await n.portalProtocol.contentLookup(keyEncoded, contentId)
|
||||
if headerContentLookup.isNone():
|
||||
warn "Failed fetching block header with proof from the network",
|
||||
hash, contentKey = keyEncoded
|
||||
return none(BlockHeader)
|
||||
|
||||
let headerContent = headerContentLookup.unsafeGet()
|
||||
|
||||
let headerWithProofRes = decodeSsz(headerContent.content, BlockHeaderWithProof)
|
||||
if headerWithProofRes.isErr():
|
||||
warn "Failed decoding header with proof", err = headerWithProofRes.error
|
||||
return none(BlockHeader)
|
||||
|
||||
let headerWithProof = headerWithProofRes.get()
|
||||
|
||||
let res = validateBlockHeaderBytes(headerWithProof.header.asSeq(), hash)
|
||||
if res.isOk():
|
||||
let isCanonical = n.verifyHeader(res.get(), headerWithProof.proof)
|
||||
|
||||
if isCanonical.isOk():
|
||||
info "Fetched block header from the network", hash, contentKey = keyEncoded
|
||||
# Content is valid, it can be propagated to interested peers
|
||||
n.portalProtocol.triggerPoke(
|
||||
headerContent.nodesInterestedInContent,
|
||||
keyEncoded,
|
||||
headerContent.content
|
||||
)
|
||||
|
||||
n.portalProtocol.storeContent(contentId, headerContent.content)
|
||||
|
||||
return some(res.get())
|
||||
else:
|
||||
warn "Validation of block header failed", err = res.error, hash, contentKey = keyEncoded
|
||||
|
||||
# Headers were requested `requestRetries` times and all failed on validation
|
||||
return none(BlockHeader)
|
||||
|
||||
# TODO: To be deprecated or not? Should there be the case for requesting a
|
||||
# block header without proofs?
|
||||
proc getBlockHeader*(
|
||||
n: HistoryNetwork, hash: BlockHash):
|
||||
Future[Option[BlockHeader]] {.async.} =
|
||||
|
|
@ -367,7 +424,10 @@ proc getBlock*(
|
|||
Future[Option[Block]] {.async.} =
|
||||
debug "Trying to retrieve block with hash", hash
|
||||
|
||||
let headerOpt = await n.getBlockHeader(hash)
|
||||
# Note: Using `getVerifiedBlockHeader` instead of getBlockHeader even though
|
||||
# proofs are not necessiarly needed, in order to avoid having to inject
|
||||
# also the original type into the network.
|
||||
let headerOpt = await n.getVerifiedBlockHeader(hash)
|
||||
if headerOpt.isNone():
|
||||
warn "Failed to get header when getting block with hash", hash
|
||||
# Cannot validate block without header.
|
||||
|
|
@ -506,43 +566,6 @@ proc getBlock*(
|
|||
else:
|
||||
return err(epochDataRes.error)
|
||||
|
||||
proc buildProof*(n: HistoryNetwork, header: BlockHeader):
|
||||
Future[Result[seq[Digest], string]] {.async.} =
|
||||
# Note: Temporarily needed proc until proofs are send over with headers.
|
||||
let
|
||||
epochIndex = getEpochIndex(header)
|
||||
epochHash = Digest(data: n.accumulator.historicalEpochs[epochIndex])
|
||||
|
||||
epochAccumulatorOpt = await n.getEpochAccumulator(epochHash)
|
||||
|
||||
if epochAccumulatorOpt.isNone():
|
||||
return err("Epoch accumulator not found")
|
||||
|
||||
let
|
||||
epochAccumulator = epochAccumulatorOpt.get()
|
||||
headerRecordIndex = getHeaderRecordIndex(header, epochIndex)
|
||||
# TODO: Implement more generalized `get_generalized_index`
|
||||
gIndex = GeneralizedIndex(epochSize*2*2 + (headerRecordIndex*2))
|
||||
|
||||
return epochAccumulator.build_proof(gIndex)
|
||||
|
||||
proc verifyCanonicalChain(
|
||||
n: HistoryNetwork, header: BlockHeader):
|
||||
Future[Result[void, string]] {.async.} =
|
||||
when not canonicalVerify:
|
||||
return ok()
|
||||
|
||||
# Note: It is a bit silly to build a proof, as we still need to request the
|
||||
# epoch accumulators for it, and could just verify it with those. But the
|
||||
# idea here is that eventually this gets changed so that the proof is send
|
||||
# together with the header.
|
||||
let proof = await n.buildProof(header)
|
||||
if proof.isOk():
|
||||
return verifyHeader(n.accumulator, header, proof.get())
|
||||
else:
|
||||
# Can't verify without master and epoch accumulators
|
||||
return err("Cannot build proof: " & proof.error)
|
||||
|
||||
proc validateContent(
|
||||
n: HistoryNetwork, content: seq[byte], contentKey: ByteList):
|
||||
Future[bool] {.async.} =
|
||||
|
|
@ -555,6 +578,11 @@ proc validateContent(
|
|||
|
||||
case key.contentType:
|
||||
of blockHeader:
|
||||
# Note: For now we still accept regular block header type to remain
|
||||
# compatible with the current specs. However, a verification is done by
|
||||
# basically requesting the header with proofs from somewhere else.
|
||||
# This all doesn't make much sense aside from compatibility and should
|
||||
# eventually be removed.
|
||||
let validateResult =
|
||||
validateBlockHeaderBytes(content, key.blockHeaderKey.blockHash)
|
||||
if validateResult.isErr():
|
||||
|
|
@ -563,57 +591,45 @@ proc validateContent(
|
|||
|
||||
let header = validateResult.get()
|
||||
|
||||
let verifyResult = await n.verifyCanonicalChain(header)
|
||||
if verifyResult.isErr():
|
||||
warn "Failed on check if header is part of canonical chain",
|
||||
error = verifyResult.error
|
||||
let res = await n.getVerifiedBlockHeader(key.blockHeaderKey.blockHash)
|
||||
if res.isNone():
|
||||
warn "Block header failed canonical verification"
|
||||
return false
|
||||
else:
|
||||
return true
|
||||
of blockBody:
|
||||
let headerOpt = await n.getBlockHeader(key.blockBodyKey.blockHash)
|
||||
|
||||
if headerOpt.isNone():
|
||||
warn "Cannot find the header, no way to validate the block body"
|
||||
of blockBody:
|
||||
let res = await n.getVerifiedBlockHeader(key.blockBodyKey.blockHash)
|
||||
if res.isNone():
|
||||
warn "Block body Failed canonical verification"
|
||||
return false
|
||||
|
||||
let header = headerOpt.get()
|
||||
let header = res.get()
|
||||
let validationResult =
|
||||
validateBlockBodyBytes(content, header.txRoot, header.ommersHash)
|
||||
|
||||
if validationResult.isErr():
|
||||
warn "Failed validating block body", error = validationResult.error
|
||||
return false
|
||||
|
||||
let verifyResult = await n.verifyCanonicalChain(header)
|
||||
if verifyResult.isErr():
|
||||
warn "Failed on check if header is part of canonical chain",
|
||||
error = verifyResult.error
|
||||
return false
|
||||
else:
|
||||
return true
|
||||
of receipts:
|
||||
let headerOpt = await n.getBlockHeader(key.receiptsKey.blockHash)
|
||||
|
||||
if headerOpt.isNone():
|
||||
warn "Cannot find the header, no way to validate the receipts"
|
||||
of receipts:
|
||||
let res = await n.getVerifiedBlockHeader(key.receiptsKey.blockHash)
|
||||
if res.isNone():
|
||||
warn "Receipts failed canonical verification"
|
||||
return false
|
||||
|
||||
let header = headerOpt.get()
|
||||
let header = res.get()
|
||||
let validationResult =
|
||||
validateReceiptsBytes(content, header.receiptRoot)
|
||||
|
||||
if validationResult.isErr():
|
||||
warn "Failed validating receipts", error = validationResult.error
|
||||
return false
|
||||
|
||||
let verifyResult = await n.verifyCanonicalChain(header)
|
||||
if verifyResult.isErr():
|
||||
warn "Failed on check if header is part of canonical chain",
|
||||
error = verifyResult.error
|
||||
return false
|
||||
else:
|
||||
return true
|
||||
|
||||
of epochAccumulator:
|
||||
# Check first if epochHash is part of master accumulator
|
||||
let epochHash = key.epochAccumulatorKey.epochHash
|
||||
|
|
@ -637,6 +653,30 @@ proc validateContent(
|
|||
else:
|
||||
return true
|
||||
|
||||
of blockHeaderWithProof:
|
||||
let headerWithProofRes = decodeSsz(content, BlockHeaderWithProof)
|
||||
if headerWithProofRes.isErr():
|
||||
warn "Failed decoding header with proof", err = headerWithProofRes.error
|
||||
return false
|
||||
|
||||
let headerWithProof = headerWithProofRes.get()
|
||||
|
||||
let validateResult = validateBlockHeaderBytes(
|
||||
headerWithProof.header.asSeq(), key.blockHeaderWithProofKey.blockHash)
|
||||
if validateResult.isErr():
|
||||
warn "Invalid block header offered", error = validateResult.error
|
||||
return false
|
||||
|
||||
let header = validateResult.get()
|
||||
|
||||
let isCanonical = n.verifyHeader(header, headerWithProof.proof)
|
||||
if isCanonical.isErr():
|
||||
warn "Failed on check if header is part of canonical chain",
|
||||
error = isCanonical.error
|
||||
return false
|
||||
else:
|
||||
return true
|
||||
|
||||
proc new*(
|
||||
T: type HistoryNetwork,
|
||||
baseProtocol: protocol.Protocol,
|
||||
|
|
|
|||
|
|
@ -250,7 +250,6 @@ proc offerContentInNodeRange*(
|
|||
else:
|
||||
return err(offerResult.error)
|
||||
|
||||
|
||||
proc storeContentInNodeRange*(
|
||||
p: PortalProtocol,
|
||||
seedDbPath: string,
|
||||
|
|
|
|||
|
|
@ -61,4 +61,4 @@ const
|
|||
portalNetworksDir / "testnet0" / "bootstrap_nodes.txt")
|
||||
|
||||
finishedAccumulator* = loadEncodedAccumulator(
|
||||
portalTestDir / "mainnet" / "accumulator" / "finished_accumulator.ssz")
|
||||
portalTestDir / "mainnet" / "history" / "accumulator" / "finished_accumulator.ssz")
|
||||
|
|
|
|||
|
|
@ -273,7 +273,7 @@ proc installEthApiHandlers*(
|
|||
else:
|
||||
let hash = filterOptions.blockHash.unsafeGet()
|
||||
|
||||
let headerOpt = await historyNetwork.getBlockHeader(hash)
|
||||
let headerOpt = await historyNetwork.getVerifiedBlockHeader(hash)
|
||||
if headerOpt.isNone():
|
||||
raise newException(ValueError,
|
||||
"Could not find header with requested hash")
|
||||
|
|
|
|||
|
|
@ -139,8 +139,7 @@ proc installPortalApiHandlers*(
|
|||
some(foundContent.nodes.map(proc(n: Node): Record = n.record)))
|
||||
|
||||
rpcServer.rpc("portal_" & network & "Offer") do(
|
||||
contentKey: string, content: string) -> int:
|
||||
|
||||
contentKey: string, content: string) -> int:
|
||||
let
|
||||
ck = hexToSeqByte(contentKey)
|
||||
ct = hexToSeqByte(content)
|
||||
|
|
|
|||
|
|
@ -49,8 +49,17 @@ proc installPortalDebugApiHandlers*(
|
|||
raise newException(ValueError, $res.error)
|
||||
|
||||
rpcServer.rpc("portal_" & network & "_propagateHeaders") do(
|
||||
dataFile: string) -> bool:
|
||||
let res = await p.historyPropagateHeaders(dataFile)
|
||||
dataDir: string) -> bool:
|
||||
let res = await p.historyPropagateHeadersWithProof(dataDir)
|
||||
if res.isOk():
|
||||
return true
|
||||
else:
|
||||
raise newException(ValueError, $res.error)
|
||||
|
||||
rpcServer.rpc("portal_" & network & "_propagateHeaders") do(
|
||||
epochHeadersFile: string, epochAccumulatorFile: string) -> bool:
|
||||
let res = await p.historyPropagateHeadersWithProof(
|
||||
epochHeadersFile, epochAccumulatorFile)
|
||||
if res.isOk():
|
||||
return true
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import
|
|||
../rpc/portal_rpc_client,
|
||||
../rpc/eth_rpc_client,
|
||||
../data/[history_data_seeding, history_data_parser],
|
||||
../network/history/[history_content, accumulator],
|
||||
../seed_db
|
||||
|
||||
type
|
||||
|
|
@ -38,6 +39,24 @@ type
|
|||
desc: "Port of the JSON-RPC service of the bootstrap (first) node"
|
||||
name: "base-rpc-port" .}: uint16
|
||||
|
||||
proc buildHeadersWithProof*(
|
||||
blockHeaders: seq[BlockHeader],
|
||||
epochAccumulator: EpochAccumulatorCached):
|
||||
Result[seq[(seq[byte], seq[byte])], string] =
|
||||
var blockHeadersWithProof: seq[(seq[byte], seq[byte])]
|
||||
for header in blockHeaders:
|
||||
if header.isPreMerge():
|
||||
let
|
||||
content = ? buildHeaderWithProof(header, epochAccumulator)
|
||||
contentKey = ContentKey(
|
||||
contentType: blockHeaderWithProof,
|
||||
blockHeaderWithProofKey: BlockKey(blockHash: header.blockHash()))
|
||||
|
||||
blockHeadersWithProof.add(
|
||||
(encode(contentKey).asSeq(), SSZ.encode(content)))
|
||||
|
||||
ok(blockHeadersWithProof)
|
||||
|
||||
proc connectToRpcServers(config: PortalTestnetConf):
|
||||
Future[seq[RpcClient]] {.async.} =
|
||||
var clients: seq[RpcClient]
|
||||
|
|
@ -91,7 +110,7 @@ proc retryUntil[A](
|
|||
checkFailMessage: string,
|
||||
nodeIdx: int): Future[A] =
|
||||
# some reasonable limits, which will cause waits as: 1, 2, 4, 8, 16, 32 seconds
|
||||
return withRetries(f, c, 6, seconds(1), checkFailMessage, nodeIdx)
|
||||
return withRetries(f, c, 1, seconds(1), checkFailMessage, nodeIdx)
|
||||
|
||||
# Note:
|
||||
# When doing json-rpc requests following `RpcPostError` can occur:
|
||||
|
|
@ -225,40 +244,40 @@ procSuite "Portal testnet tests":
|
|||
check enr == randomNodeInfo.nodeENR
|
||||
|
||||
asyncTest "Portal History - Propagate blocks and do content lookups":
|
||||
let clients = await connectToRpcServers(config)
|
||||
const
|
||||
headerFile = "./vendor/portal-spec-tests/tests/mainnet/history/headers/1000001-1000010.e2s"
|
||||
accumulatorFile = "./vendor/portal-spec-tests/tests/mainnet/history/accumulator/epoch-accumulator-00122.ssz"
|
||||
blockDataFile = "./fluffy/tests/blocks/mainnet_blocks_1000001_1000010.json"
|
||||
|
||||
var nodeInfos: seq[NodeInfo]
|
||||
for client in clients:
|
||||
let nodeInfo = await client.portal_history_nodeInfo()
|
||||
await client.close()
|
||||
nodeInfos.add(nodeInfo)
|
||||
let
|
||||
blockHeaders = readBlockHeaders(headerFile).valueOr:
|
||||
raiseAssert "Invalid header file: " & headerFile
|
||||
epochAccumulator = readEpochAccumulatorCached(accumulatorFile).valueOr:
|
||||
raiseAssert "Invalid epoch accumulator file: " & accumulatorFile
|
||||
blockHeadersWithProof =
|
||||
buildHeadersWithProof(blockHeaders, epochAccumulator).valueOr:
|
||||
raiseAssert "Could not build headers with proof"
|
||||
blockData =
|
||||
readJsonType(blockDataFile, BlockDataTable).valueOr:
|
||||
raiseAssert "Invalid block data file" & blockDataFile
|
||||
|
||||
# const dataFileEpoch = "./fluffy/scripts/eth-epoch-accumulator.json"
|
||||
# check (await clients[0].portal_history_propagateEpochAccumulator(dataFileEpoch))
|
||||
# await clients[0].close()
|
||||
# await sleepAsync(60.seconds)
|
||||
clients = await connectToRpcServers(config)
|
||||
|
||||
const dataFile = "./fluffy/tests/blocks/mainnet_blocks_1000001_1000010.json"
|
||||
|
||||
check (await clients[0].portal_history_propagateHeaders(dataFile))
|
||||
await clients[0].close()
|
||||
|
||||
# Short sleep between propagation of block headers and propagation of block
|
||||
# bodies and receipts as the latter two require the first for validation.
|
||||
await sleepAsync(5.seconds)
|
||||
# Gossiping all block headers with proof first, as bodies and receipts
|
||||
# require them for validation.
|
||||
for (content, contentKey) in blockHeadersWithProof:
|
||||
discard (await clients[0].portal_history_offer(
|
||||
content.toHex(), contentKey.toHex()))
|
||||
|
||||
# This will fill the first node its db with blocks from the data file. Next,
|
||||
# this node wil offer all these blocks their headers one by one.
|
||||
check (await clients[0].portal_history_propagate(dataFile))
|
||||
check (await clients[0].portal_history_propagate(blockDataFile))
|
||||
await clients[0].close()
|
||||
|
||||
let blockData = readJsonType(dataFile, BlockDataTable)
|
||||
check blockData.isOk()
|
||||
|
||||
for i, client in clients:
|
||||
# Note: Once there is the Canonical Indices Network, we don't need to
|
||||
# access this file anymore here for the block hashes.
|
||||
for hash in blockData.get().blockHashes():
|
||||
for hash in blockData.blockHashes():
|
||||
# Note: More flexible approach instead of generic retries could be to
|
||||
# add a json-rpc debug proc that returns whether the offer queue is empty or
|
||||
# not. And then poll every node until all nodes have an empty queue.
|
||||
|
|
@ -315,99 +334,35 @@ procSuite "Portal testnet tests":
|
|||
await client.close()
|
||||
|
||||
asyncTest "Portal History - Propagate content from seed db":
|
||||
let clients = await connectToRpcServers(config)
|
||||
|
||||
var nodeInfos: seq[NodeInfo]
|
||||
for client in clients:
|
||||
let nodeInfo = await client.portal_history_nodeInfo()
|
||||
await client.close()
|
||||
nodeInfos.add(nodeInfo)
|
||||
|
||||
const dataPath = "./fluffy/tests/blocks/mainnet_blocks_1000011_1000030.json"
|
||||
|
||||
# path for temporary db, separate dir is used as sqlite usually also creates
|
||||
# wal files, and we do not want for those to linger in filesystem
|
||||
const tempDbPath = "./fluffy/tests/blocks/tempDir/mainnet_blocks_1000011_1000030.sqlite3"
|
||||
|
||||
let (dbFile, dbName) = getDbBasePathAndName(tempDbPath).unsafeGet()
|
||||
|
||||
let blockData = readJsonType(dataPath, BlockDataTable)
|
||||
check blockData.isOk()
|
||||
let bd = blockData.get()
|
||||
|
||||
createDir(dbFile)
|
||||
let db = SeedDb.new(path = dbFile, name = dbName)
|
||||
|
||||
try:
|
||||
let lastNodeIdx = len(nodeInfos) - 1
|
||||
|
||||
# populate temp database from json file
|
||||
for t in blocksContent(bd, false):
|
||||
db.put(t[0], t[1], t[2])
|
||||
|
||||
# store content in node0 database
|
||||
check (await clients[0].portal_history_storeContentInNodeRange(tempDbPath, 100, 0))
|
||||
await clients[0].close()
|
||||
|
||||
# offer content to node 1..63
|
||||
for i in 1..lastNodeIdx:
|
||||
let receipientId = nodeInfos[i].nodeId
|
||||
let offerResponse = await retryUntil(
|
||||
proc (): Future[int] {.async.} =
|
||||
try:
|
||||
let res = await clients[0].portal_history_offerContentInNodeRange(tempDbPath, receipientId, 64, 0)
|
||||
await clients[0].close()
|
||||
return res
|
||||
except CatchableError as exc:
|
||||
await clients[0].close()
|
||||
raise exc
|
||||
,
|
||||
proc (os: int): bool = return true,
|
||||
"Offer failed",
|
||||
i
|
||||
)
|
||||
check:
|
||||
offerResponse > 0
|
||||
|
||||
for i, client in clients:
|
||||
# Note: Once there is the Canonical Indices Network, we don't need to
|
||||
# access this file anymore here for the block hashes.
|
||||
for hash in bd.blockHashes():
|
||||
let content = await retryUntil(
|
||||
proc (): Future[Option[BlockObject]] {.async.} =
|
||||
try:
|
||||
let res = await client.eth_getBlockByHash(hash.ethHashStr(), false)
|
||||
await client.close()
|
||||
return res
|
||||
except CatchableError as exc:
|
||||
await client.close()
|
||||
raise exc
|
||||
,
|
||||
proc (mc: Option[BlockObject]): bool = return mc.isSome(),
|
||||
"Did not receive expected Block with hash " & hash.data.toHex(),
|
||||
i
|
||||
)
|
||||
check content.isSome()
|
||||
|
||||
let blockObj = content.get()
|
||||
check blockObj.hash.get() == hash
|
||||
|
||||
for tx in blockObj.transactions:
|
||||
var txObj: TransactionObject
|
||||
tx.fromJson("tx", txObj)
|
||||
check txObj.blockHash.get() == hash
|
||||
|
||||
await client.close()
|
||||
finally:
|
||||
db.close()
|
||||
removeDir(dbFile)
|
||||
|
||||
asyncTest "Portal History - Propagate content from seed db in depth first fashion":
|
||||
# Skipping this test as it is flawed considering block headers should be
|
||||
# offered before bodies and receipts.
|
||||
# Skipping this as it seems to fail now at offerContentInNodeRange, likely
|
||||
# due to not being possibly to validate block bodies. This would mean the
|
||||
# test is flawed and block headers should be offered before bodies and
|
||||
# receipts.
|
||||
# TODO: Split this up and activate test
|
||||
skip()
|
||||
# let clients = await connectToRpcServers(config)
|
||||
|
||||
# const
|
||||
# headerFile = "./vendor/portal-spec-tests/tests/mainnet/history/headers/1000011-1000030.e2s"
|
||||
# accumulatorFile = "./vendor/portal-spec-tests/tests/mainnet/history/accumulator/epoch-accumulator-00122.ssz"
|
||||
# blockDataFile = "./fluffy/tests/blocks/mainnet_blocks_1000011_1000030.json"
|
||||
|
||||
# # Path for the temporary db. A separate dir is used as sqlite usually also
|
||||
# # creates wal files.
|
||||
# tempDbPath = "./fluffy/tests/blocks/tempDir/mainnet_blocks_1000011_1000030.sqlite3"
|
||||
|
||||
# let
|
||||
# blockHeaders = readBlockHeaders(headerFile).valueOr:
|
||||
# raiseAssert "Invalid header file: " & headerFile
|
||||
# epochAccumulator = readEpochAccumulatorCached(accumulatorFile).valueOr:
|
||||
# raiseAssert "Invalid epoch accumulator file: " & accumulatorFile
|
||||
# blockHeadersWithProof =
|
||||
# buildHeadersWithProof(blockHeaders, epochAccumulator).valueOr:
|
||||
# raiseAssert "Could not build headers with proof"
|
||||
# blockData =
|
||||
# readJsonType(blockDataFile, BlockDataTable).valueOr:
|
||||
# raiseAssert "Invalid block data file" & blockDataFile
|
||||
|
||||
# clients = await connectToRpcServers(config)
|
||||
|
||||
# var nodeInfos: seq[NodeInfo]
|
||||
# for client in clients:
|
||||
|
|
@ -415,60 +370,162 @@ procSuite "Portal testnet tests":
|
|||
# await client.close()
|
||||
# nodeInfos.add(nodeInfo)
|
||||
|
||||
# # different set of data for each test as tests are statefull so previously propagated
|
||||
# # block are already in the network
|
||||
# const dataPath = "./fluffy/tests/blocks/mainnet_blocks_1000040_1000050.json"
|
||||
|
||||
# # path for temporary db, separate dir is used as sqlite usually also creates
|
||||
# # wal files, and we do not want for those to linger in filesystem
|
||||
# const tempDbPath = "./fluffy/tests/blocks/tempDir/mainnet_blocks_1000040_100050.sqlite3"
|
||||
|
||||
# let (dbFile, dbName) = getDbBasePathAndName(tempDbPath).unsafeGet()
|
||||
|
||||
# let blockData = readJsonType(dataPath, BlockDataTable)
|
||||
# check blockData.isOk()
|
||||
# let bd = blockData.get()
|
||||
|
||||
# createDir(dbFile)
|
||||
# let db = SeedDb.new(path = dbFile, name = dbName)
|
||||
|
||||
# try:
|
||||
# # populate temp database from json file
|
||||
# for t in blocksContent(bd, false):
|
||||
# db.put(t[0], t[1], t[2])
|
||||
|
||||
# check (await clients[0].portal_history_depthContentPropagate(tempDbPath, 64))
|
||||
# await clients[0].close()
|
||||
|
||||
# for i, client in clients:
|
||||
# # Note: Once there is the Canonical Indices Network, we don't need to
|
||||
# # access this file anymore here for the block hashes.
|
||||
# for hash in bd.blockHashes():
|
||||
# let content = await retryUntil(
|
||||
# proc (): Future[Option[BlockObject]] {.async.} =
|
||||
# try:
|
||||
# let res = await client.eth_getBlockByHash(hash.ethHashStr(), false)
|
||||
# await client.close()
|
||||
# return res
|
||||
# except CatchableError as exc:
|
||||
# await client.close()
|
||||
# raise exc
|
||||
# ,
|
||||
# proc (mc: Option[BlockObject]): bool = return mc.isSome(),
|
||||
# "Did not receive expected Block with hash " & hash.data.toHex(),
|
||||
# i
|
||||
# )
|
||||
# check content.isSome()
|
||||
|
||||
# let blockObj = content.get()
|
||||
# check blockObj.hash.get() == hash
|
||||
|
||||
# for tx in blockObj.transactions:
|
||||
# var txObj: TransactionObject
|
||||
# tx.fromJson("tx", txObj)
|
||||
# check txObj.blockHash.get() == hash
|
||||
|
||||
# await client.close()
|
||||
# finally:
|
||||
# defer:
|
||||
# db.close()
|
||||
# removeDir(dbFile)
|
||||
|
||||
# # Fill seed db with block headers with proof
|
||||
# for (content, contentKey) in blockHeadersWithProof:
|
||||
# let contentId = history_content.toContentId(ByteList(contentKey))
|
||||
# db.put(contentId, contentKey, content)
|
||||
|
||||
# # Fill seed db with block bodies and receipts
|
||||
# for t in blocksContent(blockData, false):
|
||||
# db.put(t[0], t[1], t[2])
|
||||
|
||||
# let lastNodeIdx = len(nodeInfos) - 1
|
||||
|
||||
# # Store content in node 0 database
|
||||
# check (await clients[0].portal_history_storeContentInNodeRange(
|
||||
# tempDbPath, 100, 0))
|
||||
# await clients[0].close()
|
||||
|
||||
# # Offer content to node 1..63
|
||||
# for i in 1..lastNodeIdx:
|
||||
# let recipientId = nodeInfos[i].nodeId
|
||||
# let offerResponse = await retryUntil(
|
||||
# proc (): Future[int] {.async.} =
|
||||
# try:
|
||||
# let res = await clients[0].portal_history_offerContentInNodeRange(
|
||||
# tempDbPath, recipientId, 64, 0)
|
||||
# await clients[0].close()
|
||||
# return res
|
||||
# except CatchableError as exc:
|
||||
# await clients[0].close()
|
||||
# raise exc
|
||||
# ,
|
||||
# proc (os: int): bool = return true,
|
||||
# "Offer failed",
|
||||
# i
|
||||
# )
|
||||
# check:
|
||||
# offerResponse > 0
|
||||
|
||||
# for i, client in clients:
|
||||
# # Note: Once there is the Canonical Indices Network, we don't need to
|
||||
# # access this file anymore here for the block hashes.
|
||||
# for hash in blockData.blockHashes():
|
||||
# let content = await retryUntil(
|
||||
# proc (): Future[Option[BlockObject]] {.async.} =
|
||||
# try:
|
||||
# let res = await client.eth_getBlockByHash(hash.ethHashStr(), false)
|
||||
# await client.close()
|
||||
# return res
|
||||
# except CatchableError as exc:
|
||||
# await client.close()
|
||||
# raise exc
|
||||
# ,
|
||||
# proc (mc: Option[BlockObject]): bool = return mc.isSome(),
|
||||
# "Did not receive expected Block with hash " & hash.data.toHex(),
|
||||
# i
|
||||
# )
|
||||
# check content.isSome()
|
||||
|
||||
# let blockObj = content.get()
|
||||
# check blockObj.hash.get() == hash
|
||||
|
||||
# for tx in blockObj.transactions:
|
||||
# var txObj: TransactionObject
|
||||
# tx.fromJson("tx", txObj)
|
||||
# check txObj.blockHash.get() == hash
|
||||
|
||||
# await client.close()
|
||||
|
||||
asyncTest "Portal History - Propagate content from seed db in depth first fashion":
|
||||
# Skipping this test as it is flawed considering block headers should be
|
||||
# offered before bodies and receipts.
|
||||
# TODO: Split this up and activate test
|
||||
skip()
|
||||
|
||||
# const
|
||||
# headerFile = "./vendor/portal-spec-tests/tests/mainnet/history/headers/1000011-1000030.e2s"
|
||||
# accumulatorFile = "./vendor/portal-spec-tests/tests/mainnet/history/accumulator/epoch-accumulator-00122.ssz"
|
||||
# # Different set of data for each test as tests are statefull so previously
|
||||
# # propagated content is still in the network
|
||||
# blockDataFile = "./fluffy/tests/blocks/mainnet_blocks_1000040_1000050.json"
|
||||
|
||||
# # Path for the temporary db. A separate dir is used as sqlite usually also
|
||||
# # creates wal files.
|
||||
# tempDbPath = "./fluffy/tests/blocks/tempDir/mainnet_blocks_1000040_100050.sqlite3"
|
||||
|
||||
# let
|
||||
# blockHeaders = readBlockHeaders(headerFile).valueOr:
|
||||
# raiseAssert "Invalid header file: " & headerFile
|
||||
# epochAccumulator = readEpochAccumulatorCached(accumulatorFile).valueOr:
|
||||
# raiseAssert "Invalid epoch accumulator file: " & accumulatorFile
|
||||
# blockHeadersWithProof =
|
||||
# buildHeadersWithProof(blockHeaders, epochAccumulator).valueOr:
|
||||
# raiseAssert "Could not build headers with proof"
|
||||
# blockData =
|
||||
# readJsonType(blockDataFile, BlockDataTable).valueOr:
|
||||
# raiseAssert "Invalid block data file" & blockDataFile
|
||||
|
||||
# clients = await connectToRpcServers(config)
|
||||
|
||||
# var nodeInfos: seq[NodeInfo]
|
||||
# for client in clients:
|
||||
# let nodeInfo = await client.portal_history_nodeInfo()
|
||||
# await client.close()
|
||||
# nodeInfos.add(nodeInfo)
|
||||
|
||||
# let (dbFile, dbName) = getDbBasePathAndName(tempDbPath).unsafeGet()
|
||||
# createDir(dbFile)
|
||||
# let db = SeedDb.new(path = dbFile, name = dbName)
|
||||
# defer:
|
||||
# db.close()
|
||||
# removeDir(dbFile)
|
||||
|
||||
# # Fill seed db with block headers with proof
|
||||
# for (content, contentKey) in blockHeadersWithProof:
|
||||
# let contentId = history_content.toContentId(ByteList(contentKey))
|
||||
# db.put(contentId, contentKey, content)
|
||||
|
||||
# # Fill seed db with block bodies and receipts
|
||||
# for t in blocksContent(blockData, false):
|
||||
# db.put(t[0], t[1], t[2])
|
||||
|
||||
# check (await clients[0].portal_history_depthContentPropagate(tempDbPath, 64))
|
||||
# await clients[0].close()
|
||||
|
||||
# for i, client in clients:
|
||||
# # Note: Once there is the Canonical Indices Network, we don't need to
|
||||
# # access this file anymore here for the block hashes.
|
||||
# for hash in blockData.blockHashes():
|
||||
# let content = await retryUntil(
|
||||
# proc (): Future[Option[BlockObject]] {.async.} =
|
||||
# try:
|
||||
# let res = await client.eth_getBlockByHash(hash.ethHashStr(), false)
|
||||
# await client.close()
|
||||
# return res
|
||||
# except CatchableError as exc:
|
||||
# await client.close()
|
||||
# raise exc
|
||||
# ,
|
||||
# proc (mc: Option[BlockObject]): bool = return mc.isSome(),
|
||||
# "Did not receive expected Block with hash " & hash.data.toHex(),
|
||||
# i
|
||||
# )
|
||||
# check content.isSome()
|
||||
|
||||
# let blockObj = content.get()
|
||||
# check blockObj.hash.get() == hash
|
||||
|
||||
# for tx in blockObj.transactions:
|
||||
# var txObj: TransactionObject
|
||||
# tx.fromJson("tx", txObj)
|
||||
# check txObj.blockHash.get() == hash
|
||||
|
||||
# await client.close()
|
||||
|
|
|
|||
|
|
@ -16,18 +16,6 @@ import
|
|||
../network/history/[history_content, accumulator],
|
||||
./test_helpers
|
||||
|
||||
func buildProof(
|
||||
epochAccumulators: seq[EpochAccumulator], header: BlockHeader):
|
||||
Result[seq[Digest], string] =
|
||||
let
|
||||
epochIndex = getEpochIndex(header)
|
||||
epochAccumulator = epochAccumulators[epochIndex]
|
||||
|
||||
headerRecordIndex = getHeaderRecordIndex(header, epochIndex)
|
||||
gIndex = GeneralizedIndex(epochSize*2*2 + (headerRecordIndex*2))
|
||||
|
||||
return epochAccumulator.build_proof(gIndex)
|
||||
|
||||
suite "Header Accumulator":
|
||||
test "Header Accumulator Canonical Verification":
|
||||
const
|
||||
|
|
@ -61,35 +49,33 @@ suite "Header Accumulator":
|
|||
block: # Test valid headers
|
||||
for i in headersToTest:
|
||||
let header = headers[i]
|
||||
let proof = buildProof(epochAccumulators, header)
|
||||
let proof = buildProof(header, epochAccumulators)
|
||||
check:
|
||||
proof.isOk()
|
||||
verifyHeader(accumulator, header, proof.get()).isOk()
|
||||
verifyAccumulatorProof(accumulator, header, proof.get()).isOk()
|
||||
|
||||
block: # Test invalid headers
|
||||
# Post merge block number must fail (> than latest header in accumulator)
|
||||
var proof: AccumulatorProof
|
||||
let header = BlockHeader(blockNumber: mergeBlockNumber.stuint(256))
|
||||
check verifyHeader(accumulator, header, @[]).isErr()
|
||||
check verifyAccumulatorProof(accumulator, header, proof).isErr()
|
||||
|
||||
# Test altered block headers by altering the difficulty
|
||||
for i in headersToTest:
|
||||
let proof = buildProof( epochAccumulators, headers[i])
|
||||
let proof = buildProof(headers[i], epochAccumulators)
|
||||
check:
|
||||
proof.isOk()
|
||||
# Alter the block header so the proof no longer matches
|
||||
let header = BlockHeader(
|
||||
blockNumber: i.stuint(256), difficulty: 2.stuint(256))
|
||||
|
||||
check verifyHeader(accumulator, header, proof.get()).isErr()
|
||||
check verifyAccumulatorProof(accumulator, header, proof.get()).isErr()
|
||||
|
||||
block: # Test invalid proofs
|
||||
var proof: seq[Digest]
|
||||
for i in 0..14:
|
||||
var digest: Digest
|
||||
proof.add(digest)
|
||||
var proof: AccumulatorProof
|
||||
|
||||
for i in headersToTest:
|
||||
check verifyHeader(accumulator, headers[i], proof).isErr()
|
||||
check verifyAccumulatorProof(accumulator, headers[i], proof).isErr()
|
||||
|
||||
test "Header Accumulator - Not Finished":
|
||||
# Less headers than needed to finish the accumulator
|
||||
|
|
|
|||
|
|
@ -7,10 +7,10 @@
|
|||
|
||||
import
|
||||
stew/shims/net,
|
||||
eth/keys,
|
||||
eth/[keys, rlp],
|
||||
eth/p2p/discoveryv5/[enr, node, routing_table],
|
||||
eth/p2p/discoveryv5/protocol as discv5_protocol,
|
||||
../network/history/accumulator
|
||||
../network/history/[accumulator, history_content]
|
||||
|
||||
proc localAddress*(port: int): Address =
|
||||
Address(ip: ValidIpAddress.init("127.0.0.1"), port: Port(port))
|
||||
|
|
@ -72,3 +72,39 @@ func buildAccumulatorData*(headers: seq[BlockHeader]):
|
|||
return ok((finishAccumulator(accumulator), epochAccumulators))
|
||||
|
||||
err("Not enough headers provided to finish the accumulator")
|
||||
|
||||
func buildProof*(
|
||||
header: BlockHeader, epochAccumulators: seq[EpochAccumulator]):
|
||||
Result[AccumulatorProof, string] =
|
||||
let epochIndex = getEpochIndex(header)
|
||||
doAssert(epochIndex < uint64(epochAccumulators.len()))
|
||||
let epochAccumulator = epochAccumulators[epochIndex]
|
||||
|
||||
buildProof(header, epochAccumulator)
|
||||
|
||||
func buildHeaderWithProof*(
|
||||
header: BlockHeader,
|
||||
epochAccumulators: seq[EpochAccumulator]):
|
||||
Result[BlockHeaderWithProof, string] =
|
||||
## Construct the accumulator proof for a specific header.
|
||||
## Returns the block header with the proof
|
||||
if header.isPreMerge():
|
||||
let epochIndex = getEpochIndex(header)
|
||||
doAssert(epochIndex < uint64(epochAccumulators.len()))
|
||||
let epochAccumulator = epochAccumulators[epochIndex]
|
||||
|
||||
buildHeaderWithProof(header, epochAccumulator)
|
||||
|
||||
else:
|
||||
err("Cannot build accumulator proof for post merge header")
|
||||
|
||||
func buildHeadersWithProof*(
|
||||
headers: seq[BlockHeader],
|
||||
epochAccumulators: seq[EpochAccumulator]):
|
||||
Result[seq[BlockHeaderWithProof], string] =
|
||||
var headersWithProof: seq[BlockHeaderWithProof]
|
||||
for header in headers:
|
||||
headersWithProof.add(
|
||||
? buildHeaderWithProof(header, epochAccumulators))
|
||||
|
||||
ok(headersWithProof)
|
||||
|
|
|
|||
|
|
@ -60,16 +60,20 @@ proc createEmptyHeaders(fromNum: int, toNum: int): seq[BlockHeader] =
|
|||
headers.add(bh)
|
||||
return headers
|
||||
|
||||
proc headersToContentInfo(headers: seq[BlockHeader]): seq[ContentInfo] =
|
||||
proc headersToContentInfo(
|
||||
headersWithProof: seq[BlockHeaderWithProof]): seq[ContentInfo] =
|
||||
var contentInfos: seq[ContentInfo]
|
||||
for h in headers:
|
||||
for headerWithProof in headersWithProof:
|
||||
let
|
||||
headerHash = h.blockHash()
|
||||
bk = BlockKey(blockHash: headerHash)
|
||||
ck = encode(ContentKey(contentType: blockHeader, blockHeaderKey: bk))
|
||||
headerEncoded = rlp.encode(h)
|
||||
ci = ContentInfo(contentKey: ck, content: headerEncoded)
|
||||
contentInfos.add(ci)
|
||||
# TODO: Decoding step could be avoided
|
||||
header = rlp.decode(headerWithProof.header.asSeq(), BlockHeader)
|
||||
headerHash = header.blockHash()
|
||||
blockKey = BlockKey(blockHash: headerHash)
|
||||
contentKey = encode(ContentKey(
|
||||
contentType: blockHeaderWithProof, blockHeaderWithProofKey: blockKey))
|
||||
contentInfo = ContentInfo(
|
||||
contentKey: contentKey, content: SSZ.encode(headerWithProof))
|
||||
contentInfos.add(contentInfo)
|
||||
return contentInfos
|
||||
|
||||
procSuite "History Content Network":
|
||||
|
|
@ -99,16 +103,26 @@ procSuite "History Content Network":
|
|||
historyNode1 = newHistoryNode(rng, 20302, masterAccumulator)
|
||||
historyNode2 = newHistoryNode(rng, 20303, masterAccumulator)
|
||||
|
||||
# Only node 2 stores all headers and all epoch accumulators.
|
||||
for h in headers:
|
||||
var selectedHeaders: seq[BlockHeader]
|
||||
for i in headersToTest:
|
||||
selectedHeaders.add(headers[i])
|
||||
|
||||
let headersWithProof =
|
||||
buildHeadersWithProof(selectedHeaders, epochAccumulators)
|
||||
|
||||
check headersWithProof.isOk()
|
||||
|
||||
# Only node 2 stores the headers and all epoch accumulators.
|
||||
for headerWithProof in headersWithProof.get():
|
||||
let
|
||||
headerHash = h.blockHash()
|
||||
header = rlp.decode(headerWithProof.header.asSeq(), BlockHeader)
|
||||
headerHash = header.blockHash()
|
||||
blockKey = BlockKey(blockHash: headerHash)
|
||||
contentKey = ContentKey(
|
||||
contentType: blockHeader, blockHeaderKey: blockKey)
|
||||
contentType: blockHeaderWithProof, blockHeaderWithProofKey: blockKey)
|
||||
contentId = toContentId(contentKey)
|
||||
headerEncoded = rlp.encode(h)
|
||||
historyNode2.portalProtocol().storeContent(contentId, headerEncoded)
|
||||
historyNode2.portalProtocol().storeContent(
|
||||
contentId, SSZ.encode(headerWithProof))
|
||||
|
||||
# Need to store the epoch accumulators to be able to do the block to hash
|
||||
# mapping
|
||||
|
|
@ -172,20 +186,12 @@ procSuite "History Content Network":
|
|||
let maxOfferedHistoryContent = getMaxOfferedContentKeys(
|
||||
uint32(len(historyProtocolId)), maxContentKeySize)
|
||||
|
||||
# One of the nodes needs to have the epochAccumulator to build proofs from
|
||||
# for the offered headers.
|
||||
for epochAccumulator in epochAccumulators:
|
||||
let
|
||||
rootHash = epochAccumulator.hash_tree_root()
|
||||
contentKey = ContentKey(
|
||||
contentType: ContentType.epochAccumulator,
|
||||
epochAccumulatorKey: EpochAccumulatorKey(epochHash: rootHash))
|
||||
contentId = toContentId(contentKey)
|
||||
historyNode2.portalProtocol().storeContent(
|
||||
contentId, SSZ.encode(epochAccumulator))
|
||||
let headersWithProof = buildHeadersWithProof(
|
||||
headers[0..maxOfferedHistoryContent], epochAccumulators)
|
||||
check headersWithProof.isOk()
|
||||
|
||||
# This is one header more than maxOfferedHistoryContent
|
||||
let contentInfos = headersToContentInfo(headers[0..maxOfferedHistoryContent])
|
||||
let contentInfos = headersToContentInfo(headersWithProof.get())
|
||||
|
||||
# node 1 will offer the content so it needs to have it in its database
|
||||
for contentInfo in contentInfos:
|
||||
|
|
@ -254,37 +260,31 @@ procSuite "History Content Network":
|
|||
(await historyNode1.portalProtocol().ping(historyNode2.localNode())).isOk()
|
||||
(await historyNode2.portalProtocol().ping(historyNode1.localNode())).isOk()
|
||||
|
||||
# Need to store the epochAccumulators, because else the headers can't be
|
||||
# verified if being part of the canonical chain currently
|
||||
for epochAccumulator in epochAccumulators:
|
||||
let
|
||||
rootHash = epochAccumulator.hash_tree_root()
|
||||
contentKey = ContentKey(
|
||||
contentType: ContentType.epochAccumulator,
|
||||
epochAccumulatorKey: EpochAccumulatorKey(epochHash: rootHash))
|
||||
contentId = toContentId(contentKey)
|
||||
historyNode1.portalProtocol.storeContent(
|
||||
contentId, SSZ.encode(epochAccumulator))
|
||||
|
||||
# Need to run start to get the processContentLoop running
|
||||
historyNode1.start()
|
||||
historyNode2.start()
|
||||
|
||||
let contentInfos = headersToContentInfo(headers)
|
||||
var selectedHeaders: seq[BlockHeader]
|
||||
for i in headersToTest:
|
||||
selectedHeaders.add(headers[i])
|
||||
|
||||
for header in headersToTest:
|
||||
let id = toContentId(contentInfos[header].contentKey)
|
||||
historyNode1.portalProtocol.storeContent(id, contentInfos[header].content)
|
||||
let headersWithProof = buildHeadersWithProof(
|
||||
selectedHeaders, epochAccumulators)
|
||||
check headersWithProof.isOk()
|
||||
|
||||
let contentInfos = headersToContentInfo(headersWithProof.get())
|
||||
|
||||
for contentInfo in contentInfos:
|
||||
let id = toContentId(contentInfo.contentKey)
|
||||
historyNode1.portalProtocol.storeContent(id, contentInfo.content)
|
||||
|
||||
let offerResult = await historyNode1.portalProtocol.offer(
|
||||
historyNode2.localNode(),
|
||||
contentInfos[header..header]
|
||||
)
|
||||
historyNode2.localNode(), @[contentInfo])
|
||||
|
||||
check offerResult.isOk()
|
||||
|
||||
for header in headersToTest:
|
||||
let id = toContentId(contentInfos[header].contentKey)
|
||||
for contentInfo in contentInfos:
|
||||
let id = toContentId(contentInfo.contentKey)
|
||||
check historyNode2.containsId(id) == true
|
||||
|
||||
await historyNode1.stop()
|
||||
|
|
|
|||
|
|
@ -75,13 +75,13 @@ type
|
|||
# doesn't work well together with confutils.
|
||||
exportBlockData =
|
||||
"""
|
||||
Export block data (headers, bodies and receipts) to ajson format or a
|
||||
Export block data (headers, bodies and receipts) to a json format or a
|
||||
database. Some of this functionality is likely to get deprecated"""
|
||||
exportHeaders =
|
||||
exportEpochHeaders =
|
||||
"""
|
||||
Export block headers from an Ethereum JSON RPC Execution endpoint to
|
||||
*.e2s files arranged per epoch (8192 blocks)"""
|
||||
verifyHeaders =
|
||||
verifyEpochHeaders =
|
||||
"""
|
||||
Verify *.e2s files containing block headers. Verify currently only
|
||||
means being able to RLP decode the block headers"""
|
||||
|
|
@ -95,6 +95,10 @@ type
|
|||
Print the root hash of the master accumulator and of all historical
|
||||
epoch accumulators. Requires data generated by exportAccumulatorData
|
||||
command"""
|
||||
exportHeaderRange =
|
||||
"""
|
||||
Export block headers from an Ethereum JSON RPC Execution endpoint to
|
||||
*.e2s files (unlimited amount)"""
|
||||
|
||||
StorageMode* = enum
|
||||
Json, Db
|
||||
|
|
@ -105,14 +109,6 @@ type
|
|||
defaultValueDesc: $LogLevel.INFO
|
||||
desc: "Sets the log level"
|
||||
name: "log-level" .}: LogLevel
|
||||
initialBlock* {.
|
||||
desc: "Number of the first block which should be downloaded"
|
||||
defaultValue: 0
|
||||
name: "initial-block" .}: uint64
|
||||
endBlock* {.
|
||||
desc: "Number of the last block which should be downloaded"
|
||||
defaultValue: 0
|
||||
name: "end-block" .}: uint64
|
||||
dataDir* {.
|
||||
desc: "The directory where generated data files will be exported to"
|
||||
defaultValue: defaultDataDir()
|
||||
|
|
@ -122,6 +118,14 @@ type
|
|||
command
|
||||
defaultValue: exportBlockData .}: ExporterCmd
|
||||
of exportBlockData:
|
||||
startBlock* {.
|
||||
desc: "Number of the first block to be exported"
|
||||
defaultValue: 0
|
||||
name: "start-block" .}: uint64
|
||||
endBlock* {.
|
||||
desc: "Number of the last block to be exported"
|
||||
defaultValue: 0
|
||||
name: "end-block" .}: uint64
|
||||
fileName* {.
|
||||
desc: "File name (minus extension) where block data will be exported to"
|
||||
defaultValue: defaultBlockFileName
|
||||
|
|
@ -135,7 +139,7 @@ type
|
|||
desc: "Only export the headers instead of full blocks and receipts"
|
||||
defaultValue: false
|
||||
name: "headers-only" .}: bool
|
||||
of exportHeaders:
|
||||
of exportEpochHeaders:
|
||||
startEpoch* {.
|
||||
desc: "Number of the first epoch which should be downloaded"
|
||||
defaultValue: 0
|
||||
|
|
@ -147,7 +151,7 @@ type
|
|||
# TODO:
|
||||
# Although options are the same as for exportHeaders, we can't drop them
|
||||
# under the same case of as confutils does not agree with that.
|
||||
of verifyHeaders:
|
||||
of verifyEpochHeaders:
|
||||
startEpochVerify* {.
|
||||
desc: "Number of the first epoch which should be downloaded"
|
||||
defaultValue: 0
|
||||
|
|
@ -172,6 +176,13 @@ type
|
|||
defaultValue: defaultAccumulatorFileName
|
||||
defaultValueDesc: $defaultAccumulatorFileName
|
||||
name: "accumulator-file-name" .}: string
|
||||
of exportHeaderRange:
|
||||
startBlockNumber* {.
|
||||
desc: "Number of the first block header to be exported"
|
||||
name: "start-block" .}: uint64
|
||||
endBlockNumber* {.
|
||||
desc: "Number of the last block header to be exported"
|
||||
name: "end-block" .}: uint64
|
||||
|
||||
HeaderRecord = object
|
||||
header: string
|
||||
|
|
@ -273,10 +284,10 @@ proc writeHeadersToJson(config: ExporterConf, client: RpcClient) =
|
|||
try:
|
||||
var writer = JsonWriter[DefaultFlavor].init(fh.s, pretty = true)
|
||||
writer.beginRecord()
|
||||
for i in config.initialBlock..config.endBlock:
|
||||
for i in config.startBlock..config.endBlock:
|
||||
let blck = client.downloadHeader(i)
|
||||
writer.writeHeaderRecord(blck)
|
||||
if ((i - config.initialBlock) mod 8192) == 0 and i != config.initialBlock:
|
||||
if ((i - config.startBlock) mod 8192) == 0 and i != config.startBlock:
|
||||
info "Downloaded 8192 new block headers", currentHeader = i
|
||||
writer.endRecord()
|
||||
info "File successfully written", path = config.dataDir / config.fileName
|
||||
|
|
@ -296,10 +307,10 @@ proc writeBlocksToJson(config: ExporterConf, client: RpcClient) =
|
|||
try:
|
||||
var writer = JsonWriter[DefaultFlavor].init(fh.s, pretty = true)
|
||||
writer.beginRecord()
|
||||
for i in config.initialBlock..config.endBlock:
|
||||
for i in config.startBlock..config.endBlock:
|
||||
let blck = downloadBlock(i, client)
|
||||
writer.writeBlockRecord(blck)
|
||||
if ((i - config.initialBlock) mod 8192) == 0 and i != config.initialBlock:
|
||||
if ((i - config.startBlock) mod 8192) == 0 and i != config.startBlock:
|
||||
info "Downloaded 8192 new blocks", currentBlock = i
|
||||
writer.endRecord()
|
||||
info "File successfully written", path = config.dataDir / config.fileName
|
||||
|
|
@ -319,7 +330,7 @@ proc writeBlocksToDb(config: ExporterConf, client: RpcClient) =
|
|||
defer:
|
||||
db.close()
|
||||
|
||||
for i in config.initialBlock..config.endBlock:
|
||||
for i in config.startBlock..config.endBlock:
|
||||
let
|
||||
blck = downloadBlock(i, client)
|
||||
blockHash = blck.header.blockHash()
|
||||
|
|
@ -358,16 +369,6 @@ proc exportBlocks(config: ExporterConf, client: RpcClient) =
|
|||
else:
|
||||
writeBlocksToDb(config, client)
|
||||
|
||||
const
|
||||
# Using the e2s format to store data, but without the specific structure
|
||||
# like in an era file, as we currently don't really need that.
|
||||
# See: https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md
|
||||
# Added one type for now, with numbers not formally specified.
|
||||
# Note:
|
||||
# Snappy compression for `ExecutionBlockHeaderRecord` only helps for the
|
||||
# first ~1M (?) block headers, after that there is no gain so we don't do it.
|
||||
ExecutionBlockHeaderRecord = [byte 0xFF, 0x00]
|
||||
|
||||
when isMainModule:
|
||||
{.pop.}
|
||||
let config = ExporterConf.load()
|
||||
|
|
@ -375,12 +376,6 @@ when isMainModule:
|
|||
|
||||
setLogLevel(config.logLevel)
|
||||
|
||||
if (config.endBlock < config.initialBlock):
|
||||
fatal "Initial block number should be smaller than end block number",
|
||||
initialBlock = config.initialBlock,
|
||||
endBlock = config.endBlock
|
||||
quit 1
|
||||
|
||||
let dataDir = config.dataDir.string
|
||||
if not isDir(dataDir):
|
||||
let res = createPath(dataDir)
|
||||
|
|
@ -402,12 +397,18 @@ when isMainModule:
|
|||
|
||||
case config.cmd
|
||||
of ExporterCmd.exportBlockData:
|
||||
if (config.endBlock < config.startBlock):
|
||||
fatal "Initial block number should be smaller than end block number",
|
||||
startBlock = config.startBlock,
|
||||
endBlock = config.endBlock
|
||||
quit 1
|
||||
|
||||
try:
|
||||
exportBlocks(config, client)
|
||||
finally:
|
||||
waitFor client.close()
|
||||
|
||||
of ExporterCmd.exportHeaders:
|
||||
of ExporterCmd.exportEpochHeaders:
|
||||
proc exportEpochHeaders(file: string, epoch: uint64): Result[void, string] =
|
||||
# Downloading headers from JSON RPC endpoint
|
||||
info "Requesting epoch headers", epoch
|
||||
|
|
@ -441,7 +442,7 @@ when isMainModule:
|
|||
|
||||
waitFor client.close()
|
||||
|
||||
of ExporterCmd.verifyHeaders:
|
||||
of ExporterCmd.verifyEpochHeaders:
|
||||
proc verifyEpochHeaders(file: string, epoch: uint64): Result[void, string] =
|
||||
let fh = ? openFile(file, {OpenFlags.Read}).mapErr(toString)
|
||||
defer: discard closeFile(fh)
|
||||
|
|
@ -476,10 +477,6 @@ when isMainModule:
|
|||
info "Successfully decoded epoch headers", file
|
||||
|
||||
of ExporterCmd.exportAccumulatorData:
|
||||
# TODO:
|
||||
# Also write epoch accumulators to files. These can be re-used for creation
|
||||
# of headers with proofs.
|
||||
|
||||
# Lets first check if the accumulator file already exists before starting
|
||||
# to build it.
|
||||
let accumulatorFile = dataDir / config.accumulatorFileName
|
||||
|
|
@ -598,3 +595,43 @@ when isMainModule:
|
|||
echo "Epoch Root"
|
||||
for i, root in accumulator.historicalEpochs:
|
||||
echo &"{i.uint64:05} 0x{root.toHex()}"
|
||||
|
||||
of ExporterCmd.exportHeaderRange:
|
||||
let
|
||||
startBlockNumber = config.startBlockNumber
|
||||
endBlockNumber = config.endBlockNumber
|
||||
|
||||
if (endBlockNumber < startBlockNumber):
|
||||
fatal "Start block number should be smaller than end block number",
|
||||
startBlockNumber, endBlockNumber
|
||||
quit 1
|
||||
|
||||
proc exportHeaders(
|
||||
file: string, startBlockNumber, endBlockNumber: uint64):
|
||||
Result[void, string] =
|
||||
# Downloading headers from JSON RPC endpoint
|
||||
info "Requesting headers", startBlockNumber, endBlockNumber
|
||||
var headers: seq[BlockHeader]
|
||||
for j in startBlockNumber..endBlockNumber:
|
||||
debug "Requesting block", number = j
|
||||
let header = client.downloadHeader(j)
|
||||
headers.add(header)
|
||||
|
||||
let fh = ? openFile(
|
||||
file, {OpenFlags.Write, OpenFlags.Create}).mapErr(toString)
|
||||
defer: discard closeFile(fh)
|
||||
|
||||
info "Writing headers to file", file
|
||||
for header in headers:
|
||||
discard ? fh.appendRecord(ExecutionBlockHeaderRecord, rlp.encode(header))
|
||||
|
||||
ok()
|
||||
|
||||
let file =
|
||||
try: dataDir / &"mainnet-headers-{startBlockNumber:05}-{endBlockNumber:05}.e2s"
|
||||
except ValueError as e: raiseAssert e.msg
|
||||
|
||||
let res = exportHeaders(file, startBlockNumber, endBlockNumber)
|
||||
if res.isErr():
|
||||
fatal "Failed exporting headers", error = res.error
|
||||
quit 1
|
||||
|
|
|
|||
|
|
@ -77,11 +77,11 @@ task fluffy, "Build fluffy":
|
|||
task fluffy_test, "Run fluffy tests":
|
||||
# Need the nimbus_db_backend in state network tests as we need a Hexary to
|
||||
# start from, even though it only uses the MemoryDb.
|
||||
test "fluffy/tests/portal_spec_tests/mainnet", "all_fluffy_portal_spec_tests", "-d:chronicles_log_level=ERROR -d:chronosStrictException -d:nimbus_db_backend=sqlite -d:PREFER_BLST_SHA256=false -d:canonicalVerify=true"
|
||||
test "fluffy/tests/portal_spec_tests/mainnet", "all_fluffy_portal_spec_tests", "-d:chronicles_log_level=ERROR -d:chronosStrictException -d:nimbus_db_backend=sqlite -d:PREFER_BLST_SHA256=false"
|
||||
# Running tests with a low `mergeBlockNumber` to make the tests faster.
|
||||
# Using the real mainnet merge block number is not realistic for these tests.
|
||||
test "fluffy/tests", "all_fluffy_tests", "-d:chronicles_log_level=ERROR -d:chronosStrictException -d:nimbus_db_backend=sqlite -d:PREFER_BLST_SHA256=false -d:canonicalVerify=true -d:mergeBlockNumber:38130"
|
||||
test "fluffy/tests/beacon_light_client_tests", "all_beacon_light_client_tests", "-d:chronicles_log_level=ERROR -d:chronosStrictException -d:nimbus_db_backend=sqlite -d:PREFER_BLST_SHA256=false -d:canonicalVerify=true"
|
||||
test "fluffy/tests", "all_fluffy_tests", "-d:chronicles_log_level=ERROR -d:chronosStrictException -d:nimbus_db_backend=sqlite -d:PREFER_BLST_SHA256=false -d:mergeBlockNumber:38130"
|
||||
test "fluffy/tests/beacon_light_client_tests", "all_beacon_light_client_tests", "-d:chronicles_log_level=ERROR -d:chronosStrictException -d:nimbus_db_backend=sqlite -d:PREFER_BLST_SHA256=false"
|
||||
|
||||
task fluffy_tools, "Build fluffy tools":
|
||||
buildBinary "portalcli", "fluffy/tools/", "-d:chronicles_log_level=TRACE -d:chronosStrictException -d:PREFER_BLST_SHA256=false"
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
Subproject commit 98cf0875eaa5d9b74aed87f1a356e0f1dd8c2db5
|
||||
Subproject commit 5a1d2e553d97c04339b6227624d4ebab4da88701
|
||||
Loading…
Reference in New Issue