From 6374bfb39ce4df18bcbb58640b469d34e0ea113d Mon Sep 17 00:00:00 2001 From: Kim De Mey Date: Wed, 6 Nov 2024 08:49:55 +0100 Subject: [PATCH] Refactor/clean-up/prepping history network code (#2836) - Move any validation related code to new file - Move any type conversion related code to new file - Prepare validation code for adding different type of canonical proofs - Prepare for validation code for json-rpc api - several other clean-ups and renames Sort of a first pass as the validation code specifically can use some further changes. --- fluffy/common/common_types.nim | 6 + fluffy/network/history/history_network.nim | 462 +++--------------- .../history/history_type_conversions.nim | 145 ++++++ fluffy/network/history/history_validation.nim | 177 +++++++ .../historical_hashes_accumulator.nim | 18 - fluffy/rpc/portal_rpc_client.nim | 6 +- .../test_history_content.nim | 11 +- .../test_history_content_validation.nim | 8 +- .../rpc_tests/test_portal_rpc_client.nim | 2 +- .../state_test_helpers.nim | 2 +- .../portal_bridge/portal_bridge_history.nim | 4 +- 11 files changed, 416 insertions(+), 425 deletions(-) create mode 100644 fluffy/network/history/history_type_conversions.nim create mode 100644 fluffy/network/history/history_validation.nim diff --git a/fluffy/common/common_types.nim b/fluffy/common/common_types.nim index 3d67b6472..2f0043154 100644 --- a/fluffy/common/common_types.nim +++ b/fluffy/common/common_types.nim @@ -36,6 +36,12 @@ func decodeRlp*(input: openArray[byte], T: type): Result[T, string] = except RlpError as e: err(e.msg) +func decodeRlpOrRaise*(input: openArray[byte], T: type): T = + try: + rlp.decode(input, T) + except RlpError as e: + raiseAssert(e.msg) + func decodeSsz*(input: openArray[byte], T: type): Result[T, string] = try: ok(SSZ.decode(input, T)) diff --git a/fluffy/network/history/history_network.nim b/fluffy/network/history/history_network.nim index 6707f9470..88dbb6c8f 100644 --- a/fluffy/network/history/history_network.nim +++ b/fluffy/network/history/history_network.nim @@ -18,7 +18,7 @@ import ../../database/content_db, ../../network_metadata, ../wire/[portal_protocol, portal_stream, portal_protocol_config], - "."/[history_content, validation/historical_hashes_accumulator], + "."/[history_content, history_validation, history_type_conversions], ../beacon/beacon_chain_historical_roots, ./content/content_deprecated @@ -28,7 +28,7 @@ from eth/common/accounts import EMPTY_ROOT_HASH logScope: topics = "portal_hist" -export historical_hashes_accumulator, blocks_rlp +export blocks_rlp type HistoryNetwork* = ref object @@ -46,303 +46,25 @@ type func toContentIdHandler(contentKey: ContentKeyByteList): results.Opt[ContentId] = ok(toContentId(contentKey)) -## Calls to go from SSZ decoded Portal types to RLP fully decoded EL types +## Get local content calls -func fromPortalBlockBody*( - T: type BlockBody, body: PortalBlockBodyLegacy -): Result[T, string] = - ## Get the EL BlockBody from the SSZ-decoded `PortalBlockBodyLegacy`. - try: - var transactions: seq[Transaction] - for tx in body.transactions: - transactions.add(rlp.decode(tx.asSeq(), Transaction)) - - let uncles = rlp.decode(body.uncles.asSeq(), seq[Header]) - - ok(BlockBody(transactions: transactions, uncles: uncles)) - except RlpError as e: - err("RLP decoding failed: " & e.msg) - -func fromPortalBlockBody*( - T: type BlockBody, body: PortalBlockBodyShanghai -): Result[T, string] = - ## Get the EL BlockBody from the SSZ-decoded `PortalBlockBodyShanghai`. - try: - var transactions: seq[Transaction] - for tx in body.transactions: - transactions.add(rlp.decode(tx.asSeq(), Transaction)) - - var withdrawals: seq[Withdrawal] - for w in body.withdrawals: - withdrawals.add(rlp.decode(w.asSeq(), Withdrawal)) - - ok( - BlockBody( - transactions: transactions, - uncles: @[], # Uncles must be empty, this is verified in `validateBlockBody` - withdrawals: Opt.some(withdrawals), - ) - ) - except RlpError as e: - err("RLP decoding failed: " & e.msg) - -func fromPortalBlockBodyOrRaise*( - T: type BlockBody, body: PortalBlockBodyLegacy | PortalBlockBodyShanghai -): T = - ## Get the EL BlockBody from one of the SSZ-decoded Portal BlockBody types. - ## Will raise Assertion in case of invalid RLP encodings. Only use of data - ## has been validated before! - let res = BlockBody.fromPortalBlockBody(body) - if res.isOk(): - res.get() - else: - raiseAssert(res.error) - -func fromPortalReceipts*( - T: type seq[Receipt], receipts: PortalReceipts -): Result[T, string] = - ## Get the full decoded EL seq[Receipt] from the SSZ-decoded `PortalReceipts`. - try: - var res: seq[Receipt] - for receipt in receipts: - res.add(rlp.decode(receipt.asSeq(), Receipt)) - - ok(res) - except RlpError as e: - err("RLP decoding failed: " & e.msg) - -## Calls to encode EL block types to the SSZ encoded Portal types. - -# TODO: The fact that we have different Portal BlockBody types for the different -# forks but not for the EL BlockBody (usage of Option) does not play so well -# together. - -func fromBlockBody*(T: type PortalBlockBodyLegacy, body: BlockBody): T = - var transactions: Transactions - for tx in body.transactions: - discard transactions.add(TransactionByteList(rlp.encode(tx))) - - let uncles = Uncles(rlp.encode(body.uncles)) - - PortalBlockBodyLegacy(transactions: transactions, uncles: uncles) - -func fromBlockBody*(T: type PortalBlockBodyShanghai, body: BlockBody): T = - var transactions: Transactions - for tx in body.transactions: - discard transactions.add(TransactionByteList(rlp.encode(tx))) - - let uncles = Uncles(rlp.encode(body.uncles)) - - doAssert(body.withdrawals.isSome()) - - var withdrawals: Withdrawals - for w in body.withdrawals.get(): - discard withdrawals.add(WithdrawalByteList(rlp.encode(w))) - PortalBlockBodyShanghai( - transactions: transactions, uncles: uncles, withdrawals: withdrawals - ) - -func fromReceipts*(T: type PortalReceipts, receipts: seq[Receipt]): T = - var portalReceipts: PortalReceipts - for receipt in receipts: - discard portalReceipts.add(ReceiptByteList(rlp.encode(receipt))) - - portalReceipts - -func encode*(blockBody: BlockBody): seq[byte] = - if blockBody.withdrawals.isSome(): - SSZ.encode(PortalBlockBodyShanghai.fromBlockBody(blockBody)) - else: - SSZ.encode(PortalBlockBodyLegacy.fromBlockBody(blockBody)) - -func encode*(receipts: seq[Receipt]): seq[byte] = - let portalReceipts = PortalReceipts.fromReceipts(receipts) - - SSZ.encode(portalReceipts) - -## Calls and helper calls to do validation of block header, body and receipts -# TODO: Failures on validation and perhaps deserialisation should be punished -# for if/when peer scoring/banning is added. - -func validateBlockHeader*(header: Header, blockHash: Hash32): Result[void, string] = - if not (header.rlpHash() == blockHash): - err("Block header hash does not match") - else: - ok() - -func validateBlockHeader*(header: Header, number: uint64): Result[void, string] = - if not (header.number == number): - err("Block header number does not match") - else: - ok() - -func validateBlockHeaderBytes*( - bytes: openArray[byte], id: uint64 | Hash32 -): Result[Header, string] = - let header = ?decodeRlp(bytes, Header) - - # Note: - # One could do additional quick-checks here such as timestamp vs the optional - # (later forks) added fields. E.g. Shanghai field, Cancun fields, - # zero ommersHash, etc. - # However, the block hash comparison will obviously catch these and it is - # pretty trivial to provide a non-canonical valid header. - # It might be somewhat more useful if just done (temporarily) for the headers - # post-merge which are currently provided without proof. - # For comparison by number this is obviously not sufficient as any other field - # could be manipulated and because of this a block header proof will always - # be needed. - - ?header.validateBlockHeader(id) - - ok(header) - -template append*(w: var RlpWriter, v: TransactionByteList) = - w.appendRawBytes(v.asSeq) - -template append*(w: var RlpWriter, v: WithdrawalByteList) = - w.appendRawBytes(v.asSeq) - -template append*(w: var RlpWriter, v: ReceiptByteList) = - w.appendRawBytes(v.asSeq) - -proc validateBlockBody*( - body: PortalBlockBodyLegacy, header: Header -): Result[void, string] = - ## Validate the block body against the txRoot and ommersHash from the header. - let calculatedOmmersHash = keccak256(body.uncles.asSeq()) - if calculatedOmmersHash != header.ommersHash: - return err("Invalid ommers hash") - - let calculatedTxsRoot = orderedTrieRoot(body.transactions.asSeq) - if calculatedTxsRoot != header.txRoot: - return err( - "Invalid transactions root: expected " & $header.txRoot & " - got " & - $calculatedTxsRoot - ) - - ok() - -proc validateBlockBody*( - body: PortalBlockBodyShanghai, header: Header -): Result[void, string] = - ## Validate the block body against the txRoot, ommersHash and withdrawalsRoot - ## from the header. - # Shortcut the ommersHash calculation as uncles must be an RLP encoded - # empty list - if body.uncles.asSeq() != @[byte 0xc0]: - return err("Invalid ommers hash, uncles list is not empty") - - let calculatedTxsRoot = orderedTrieRoot(body.transactions.asSeq) - if calculatedTxsRoot != header.txRoot: - return err( - "Invalid transactions root: expected " & $header.txRoot & " - got " & - $calculatedTxsRoot - ) - - # TODO: This check is done higher up but perhaps this can become cleaner with - # some refactor. - doAssert(header.withdrawalsRoot.isSome()) - - let - calculatedWithdrawalsRoot = orderedTrieRoot(body.withdrawals.asSeq) - headerWithdrawalsRoot = header.withdrawalsRoot.get() - if calculatedWithdrawalsRoot != headerWithdrawalsRoot: - return err( - "Invalid withdrawals root: expected " & $headerWithdrawalsRoot & " - got " & - $calculatedWithdrawalsRoot - ) - - ok() - -proc decodeBlockBodyBytes*(bytes: openArray[byte]): Result[BlockBody, string] = - if (let body = decodeSsz(bytes, PortalBlockBodyShanghai); body.isOk()): - BlockBody.fromPortalBlockBody(body.get()) - elif (let body = decodeSsz(bytes, PortalBlockBodyLegacy); body.isOk()): - BlockBody.fromPortalBlockBody(body.get()) - else: - err("All Portal block body decodings failed") - -proc validateBlockBodyBytes*( - bytes: openArray[byte], header: Header -): Result[BlockBody, string] = - ## Fully decode the SSZ encoded Portal Block Body and validate it against the - ## header. - ## TODO: improve this decoding in combination with the block body validation - ## calls. - let timestamp = Moment.init(header.timestamp.int64, Second) - # TODO: The additional header checks are not needed as header is implicitly - # verified by means of the accumulator? Except that we don't use this yet - # post merge, so the checks are still useful, for now. - if isShanghai(chainConfig, timestamp): - if header.withdrawalsRoot.isNone(): - err("Expected withdrawalsRoot for Shanghai block") - elif header.ommersHash != EMPTY_UNCLE_HASH: - err("Expected empty uncles for a Shanghai block") - else: - let body = ?decodeSsz(bytes, PortalBlockBodyShanghai) - ?validateBlockBody(body, header) - BlockBody.fromPortalBlockBody(body) - elif isPoSBlock(chainConfig, header.number): - if header.withdrawalsRoot.isSome(): - err("Expected no withdrawalsRoot for pre Shanghai block") - elif header.ommersHash != EMPTY_UNCLE_HASH: - err("Expected empty uncles for a PoS block") - else: - let body = ?decodeSsz(bytes, PortalBlockBodyLegacy) - ?validateBlockBody(body, header) - BlockBody.fromPortalBlockBody(body) - else: - if header.withdrawalsRoot.isSome(): - err("Expected no withdrawalsRoot for pre Shanghai block") - else: - let body = ?decodeSsz(bytes, PortalBlockBodyLegacy) - ?validateBlockBody(body, header) - BlockBody.fromPortalBlockBody(body) - -proc validateReceipts*( - receipts: PortalReceipts, receiptsRoot: Hash32 -): Result[void, string] = - if orderedTrieRoot(receipts.asSeq) != receiptsRoot: - err("Unexpected receipt root") - else: - ok() - -proc validateReceiptsBytes*( - bytes: openArray[byte], receiptsRoot: Hash32 -): Result[seq[Receipt], string] = - ## Fully decode the SSZ encoded receipts and validate it against the header's - ## receipts root. - let receipts = ?decodeSsz(bytes, PortalReceipts) - - ?validateReceipts(receipts, receiptsRoot) - - seq[Receipt].fromPortalReceipts(receipts) - -## Content helper calls for specific history network types - -proc getContent( +proc getLocalContent( n: HistoryNetwork, T: type Header, contentKey: ContentKeyByteList, contentId: ContentId, ): Opt[T] = - let localContent = n.portalProtocol.getLocalContent(contentKey, contentId).valueOr: - return Opt.none(T) + let + localContent = n.portalProtocol.getLocalContent(contentKey, contentId).valueOr: + return Opt.none(T) - let headerWithProof = - try: - SSZ.decode(localContent, BlockHeaderWithProof) - except SerializationError as e: - raiseAssert(e.msg) + # Stored data should always be serialized correctly + headerWithProof = decodeSszOrRaise(localContent, BlockHeaderWithProof) + header = decodeRlpOrRaise(headerWithProof.header.asSeq(), T) - let res = decodeRlp(headerWithProof.header.asSeq(), T) - if res.isErr(): - raiseAssert(res.error) - else: - Opt.some(res.get()) + Opt.some(header) -proc getContent( +proc getLocalContent( n: HistoryNetwork, T: type BlockBody, contentKey: ContentKeyByteList, @@ -370,26 +92,22 @@ proc getContent( Opt.some(body) -proc getContent( +proc getLocalContent( n: HistoryNetwork, T: type seq[Receipt], contentKey: ContentKeyByteList, contentId: ContentId, ): Opt[T] = - let localContent = n.portalProtocol.getLocalContent(contentKey, contentId).valueOr: - return Opt.none(T) + let + localContent = n.portalProtocol.getLocalContent(contentKey, contentId).valueOr: + return Opt.none(T) - let portalReceipts = - try: - SSZ.decode(localContent, PortalReceipts) - except SerializationError: - raiseAssert("Stored data should always be serialized correctly") + # Stored data should always be serialized correctly + portalReceipts = decodeSszOrRaise(localContent, PortalReceipts) + receipts = T.fromPortalReceipts(portalReceipts).valueOr: + raiseAssert(error) - let res = T.fromPortalReceipts(portalReceipts) - if res.isErr(): - raiseAssert(res.error) - else: - Opt.some(res.get()) + Opt.some(receipts) ## Public API to get the history network specific types, either from database ## or through a lookup on the Portal Network @@ -402,11 +120,6 @@ proc getContent( # ongoing requests are cancelled after the receival of the first response, # however that response is not yet validated at that moment. -func verifyHeader( - n: HistoryNetwork, header: Header, proof: BlockHeaderProof -): Result[void, string] = - verifyHeader(n.accumulator, header, proof) - proc getVerifiedBlockHeader*( n: HistoryNetwork, id: Hash32 | uint64 ): Future[Opt[Header]] {.async: (raises: [CancelledError]).} = @@ -421,10 +134,10 @@ proc getVerifiedBlockHeader*( # Note: This still requests a BlockHeaderWithProof from the database, as that # is what is stored. But the proof doesn't need to be verified as it gets # gets verified before storing. - let headerFromDb = n.getContent(Header, contentKey, contentId) - if headerFromDb.isSome(): - info "Fetched block header from database" - return headerFromDb + let localContent = n.getLocalContent(Header, contentKey, contentId) + if localContent.isSome(): + info "Fetched block header locally" + return localContent for i in 0 ..< (1 + n.contentRequestRetries): let @@ -432,18 +145,10 @@ proc getVerifiedBlockHeader*( warn "Failed fetching block header with proof from the network" return Opt.none(Header) - headerWithProof = decodeSsz(headerContent.content, BlockHeaderWithProof).valueOr: - warn "Failed decoding header with proof", error = error - continue - - header = validateBlockHeaderBytes(headerWithProof.header.asSeq(), id).valueOr: + header = validateCanonicalHeaderBytes(headerContent.content, id, n.accumulator).valueOr: warn "Validation of block header failed", error = error continue - if (let r = n.verifyHeader(header, headerWithProof.proof); r.isErr): - warn "Verification of block header failed", error = r.error - continue - info "Fetched valid block header from the network" # Content is valid, it can be stored and propagated to interested peers n.portalProtocol.storeContent( @@ -456,7 +161,7 @@ proc getVerifiedBlockHeader*( return Opt.some(header) # Headers were requested `1 + requestRetries` times and all failed on validation - return Opt.none(Header) + Opt.none(Header) proc getBlockBody*( n: HistoryNetwork, blockHash: Hash32, header: Header @@ -473,10 +178,10 @@ proc getBlockBody*( blockHash contentKey - let bodyFromDb = n.getContent(BlockBody, contentKey, contentId, header) - if bodyFromDb.isSome(): - info "Fetched block body from database" - return bodyFromDb + let localContent = n.getLocalContent(BlockBody, contentKey, contentId, header) + if localContent.isSome(): + info "Fetched block body locally" + return localContent for i in 0 ..< (1 + n.contentRequestRetries): let @@ -500,7 +205,7 @@ proc getBlockBody*( return Opt.some(body) # Bodies were requested `1 + requestRetries` times and all failed on validation - return Opt.none(BlockBody) + Opt.none(BlockBody) proc getBlock*( n: HistoryNetwork, id: Hash32 | uint64 @@ -523,7 +228,7 @@ proc getBlock*( warn "Failed to get body when getting block", hash return Opt.none(Block) - return Opt.some((header, body)) + Opt.some((header, body)) proc getBlockHashByNumber*( n: HistoryNetwork, blockNumber: uint64 @@ -548,10 +253,10 @@ proc getReceipts*( blockHash contentKey - let receiptsFromDb = n.getContent(seq[Receipt], contentKey, contentId) - if receiptsFromDb.isSome(): - info "Fetched receipts from database" - return receiptsFromDb + let localContent = n.getLocalContent(seq[Receipt], contentKey, contentId) + if localContent.isSome(): + info "Fetched receipts locally" + return localContent for i in 0 ..< (1 + n.contentRequestRetries): let @@ -574,71 +279,45 @@ proc getReceipts*( return Opt.some(receipts) # Receipts were requested `1 + requestRetries` times and all failed on validation - return Opt.none(seq[Receipt]) + Opt.none(seq[Receipt]) proc validateContent( - n: HistoryNetwork, content: seq[byte], contentKey: ContentKeyByteList -): Future[bool] {.async: (raises: [CancelledError]).} = - let key = contentKey.decode().valueOr: - return false + n: HistoryNetwork, content: seq[byte], contentKeyBytes: ContentKeyByteList +): Future[Result[void, string]] {.async: (raises: [CancelledError]).} = + let contentKey = contentKeyBytes.decode().valueOr: + return err("Error decoding content key") - case key.contentType + case contentKey.contentType of blockHeader: - let - headerWithProof = decodeSsz(content, BlockHeaderWithProof).valueOr: - warn "Failed decoding header with proof", error - return false - header = validateBlockHeaderBytes( - headerWithProof.header.asSeq(), key.blockHeaderKey.blockHash - ).valueOr: - warn "Invalid block header offered", error - return false + let _ = validateCanonicalHeaderBytes( + content, contentKey.blockHeaderKey.blockHash, n.accumulator + ).valueOr: + return err("Failed validating block header: " & error) - let res = n.verifyHeader(header, headerWithProof.proof) - if res.isErr(): - warn "Failed on check if header is part of canonical chain", error = res.error - return false - else: - return true + ok() of blockBody: - let header = (await n.getVerifiedBlockHeader(key.blockBodyKey.blockHash)).valueOr: - warn "Failed getting canonical header for block" - return false - - let res = validateBlockBodyBytes(content, header) - if res.isErr(): - warn "Failed validating block body", error = res.error - return false - else: - return true - of receipts: - let header = (await n.getVerifiedBlockHeader(key.receiptsKey.blockHash)).valueOr: - warn "Failed getting canonical header for receipts" - return false - - let res = validateReceiptsBytes(content, header.receiptsRoot) - if res.isErr(): - warn "Failed validating receipts", error = res.error - return false - else: - return true - of blockNumber: let - headerWithProof = decodeSsz(content, BlockHeaderWithProof).valueOr: - warn "Failed decoding header with proof", error - return false - header = validateBlockHeaderBytes( - headerWithProof.header.asSeq(), key.blockNumberKey.blockNumber - ).valueOr: - warn "Invalid block header offered", error - return false + header = (await n.getVerifiedBlockHeader(contentKey.blockBodyKey.blockHash)).valueOr: + return err("Failed getting canonical header for block") + _ = validateBlockBodyBytes(content, header).valueOr: + return err("Failed validating block body: " & error) - let res = n.verifyHeader(header, headerWithProof.proof) - if res.isErr(): - warn "Failed on check if header is part of canonical chain", error = res.error - return false - else: - return true + ok() + of receipts: + let + header = (await n.getVerifiedBlockHeader(contentKey.receiptsKey.blockHash)).valueOr: + return err("Failed getting canonical header for receipts") + _ = validateReceiptsBytes(content, header.receiptsRoot).valueOr: + return err("Failed validating receipts: " & error) + + ok() + of blockNumber: + let _ = validateCanonicalHeaderBytes( + content, contentKey.blockNumberKey.blockNumber, n.accumulator + ).valueOr: + return err("Failed validating block header: " & error) + + ok() proc new*( T: type HistoryNetwork, @@ -685,7 +364,8 @@ proc validateContent( # content passed here can have less items then contentKeys, but not more. for i, contentItem in contentItems: let contentKey = contentKeys[i] - if await n.validateContent(contentItem, contentKey): + let res = await n.validateContent(contentItem, contentKey) + if res.isOk(): let contentId = n.portalProtocol.toContentId(contentKey).valueOr: error "Received offered content with invalid content key", contentKey return false @@ -694,7 +374,7 @@ proc validateContent( debug "Received offered content validated successfully", contentKey else: - error "Received offered content failed validation", contentKey + error "Received offered content failed validation", contentKey, error = res.error return false return true diff --git a/fluffy/network/history/history_type_conversions.nim b/fluffy/network/history/history_type_conversions.nim new file mode 100644 index 000000000..0cd3c3276 --- /dev/null +++ b/fluffy/network/history/history_type_conversions.nim @@ -0,0 +1,145 @@ +# Fluffy +# Copyright (c) 2021-2024 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [].} + +import + eth/common/[headers_rlp, blocks_rlp, receipts_rlp, transactions_rlp], + ./history_content + +export history_content, headers_rlp, blocks_rlp, receipts_rlp + +## Calls to go from SSZ decoded Portal types to RLP fully decoded EL types + +func fromPortalBlockBody*( + T: type BlockBody, body: PortalBlockBodyLegacy +): Result[T, string] = + ## Get the EL BlockBody from the SSZ-decoded `PortalBlockBodyLegacy`. + try: + var transactions: seq[Transaction] + for tx in body.transactions: + transactions.add(rlp.decode(tx.asSeq(), Transaction)) + + let uncles = rlp.decode(body.uncles.asSeq(), seq[Header]) + + ok(BlockBody(transactions: transactions, uncles: uncles)) + except RlpError as e: + err("RLP decoding failed: " & e.msg) + +func fromPortalBlockBody*( + T: type BlockBody, body: PortalBlockBodyShanghai +): Result[T, string] = + ## Get the EL BlockBody from the SSZ-decoded `PortalBlockBodyShanghai`. + try: + var transactions: seq[Transaction] + for tx in body.transactions: + transactions.add(rlp.decode(tx.asSeq(), Transaction)) + + var withdrawals: seq[Withdrawal] + for w in body.withdrawals: + withdrawals.add(rlp.decode(w.asSeq(), Withdrawal)) + + ok( + BlockBody( + transactions: transactions, + uncles: @[], # Uncles must be empty, this is verified in `validateBlockBody` + withdrawals: Opt.some(withdrawals), + ) + ) + except RlpError as e: + err("RLP decoding failed: " & e.msg) + +func fromPortalBlockBodyBytes*(bytes: openArray[byte]): Result[BlockBody, string] = + if (let res = decodeSsz(bytes, PortalBlockBodyLegacy); res.isOk()): + BlockBody.fromPortalBlockBody(res.value()) + elif (let res = decodeSsz(bytes, PortalBlockBodyShanghai); res.isOk()): + BlockBody.fromPortalBlockBody(res.value()) + else: + err("Invalid Portal BlockBody encoding") + +func fromPortalBlockBodyOrRaise*( + T: type BlockBody, body: PortalBlockBodyLegacy | PortalBlockBodyShanghai +): T = + ## Get the EL BlockBody from one of the SSZ-decoded Portal BlockBody types. + ## Will raise Assertion in case of invalid RLP encodings. Only use for data + ## has been validated before! + let res = BlockBody.fromPortalBlockBody(body) + if res.isOk(): + res.get() + else: + raiseAssert(res.error) + +func fromPortalReceipts*( + T: type seq[Receipt], receipts: PortalReceipts +): Result[T, string] = + ## Get the full decoded EL seq[Receipt] from the SSZ-decoded `PortalReceipts`. + try: + var res: seq[Receipt] + for receipt in receipts: + res.add(rlp.decode(receipt.asSeq(), Receipt)) + + ok(res) + except RlpError as e: + err("RLP decoding failed: " & e.msg) + +## Calls to convert EL block types to the Portal types. + +func fromBlockBody*(T: type PortalBlockBodyLegacy, body: BlockBody): T = + var transactions: Transactions + for tx in body.transactions: + discard transactions.add(TransactionByteList(rlp.encode(tx))) + + let uncles = Uncles(rlp.encode(body.uncles)) + + PortalBlockBodyLegacy(transactions: transactions, uncles: uncles) + +func fromBlockBody*(T: type PortalBlockBodyShanghai, body: BlockBody): T = + var transactions: Transactions + for tx in body.transactions: + discard transactions.add(TransactionByteList(rlp.encode(tx))) + + let uncles = Uncles(rlp.encode(body.uncles)) + + doAssert(body.withdrawals.isSome()) + + var withdrawals: Withdrawals + for w in body.withdrawals.get(): + discard withdrawals.add(WithdrawalByteList(rlp.encode(w))) + PortalBlockBodyShanghai( + transactions: transactions, uncles: uncles, withdrawals: withdrawals + ) + +func fromReceipts*(T: type PortalReceipts, receipts: seq[Receipt]): T = + var portalReceipts: PortalReceipts + for receipt in receipts: + discard portalReceipts.add(ReceiptByteList(rlp.encode(receipt))) + + portalReceipts + +## Calls to encode EL types to the SSZ encoded Portal types. + +func encode*(blockBody: BlockBody): seq[byte] = + if blockBody.withdrawals.isSome(): + SSZ.encode(PortalBlockBodyShanghai.fromBlockBody(blockBody)) + else: + SSZ.encode(PortalBlockBodyLegacy.fromBlockBody(blockBody)) + +func encode*(receipts: seq[Receipt]): seq[byte] = + let portalReceipts = PortalReceipts.fromReceipts(receipts) + + SSZ.encode(portalReceipts) + +## RLP writer append calls for the Portal/SSZ types + +template append*(w: var RlpWriter, v: TransactionByteList) = + w.appendRawBytes(v.asSeq) + +template append*(w: var RlpWriter, v: WithdrawalByteList) = + w.appendRawBytes(v.asSeq) + +template append*(w: var RlpWriter, v: ReceiptByteList) = + w.appendRawBytes(v.asSeq) diff --git a/fluffy/network/history/history_validation.nim b/fluffy/network/history/history_validation.nim new file mode 100644 index 000000000..90f940673 --- /dev/null +++ b/fluffy/network/history/history_validation.nim @@ -0,0 +1,177 @@ +# Fluffy +# Copyright (c) 2021-2024 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [].} + +import + chronos/timer, + eth/trie/ordered_trie, + ../../network_metadata, + ./history_type_conversions, + ./validation/historical_hashes_accumulator + +from eth/common/eth_types_rlp import rlpHash + +export historical_hashes_accumulator + +func validateHeader(header: Header, blockHash: Hash32): Result[void, string] = + if not (header.rlpHash() == blockHash): + err("Header hash does not match") + else: + ok() + +func validateHeader(header: Header, number: uint64): Result[void, string] = + if not (header.number == number): + err("Header number does not match") + else: + ok() + +func validateHeaderBytes*( + bytes: openArray[byte], id: uint64 | Hash32 +): Result[Header, string] = + # Note: + # No additional quick-checks are addedhere such as timestamp vs the optional + # (later forks) added fields. E.g. Shanghai field, Cancun fields, + # zero ommersHash, etc. + # This is because the block hash comparison + canonical verification will + # catch these. For comparison by number this is will also be caught by the + # canonical verification. + let header = ?decodeRlp(bytes, Header) + + ?header.validateHeader(id) + + ok(header) + +func verifyBlockHeaderProof*( + a: FinishedHistoricalHashesAccumulator, header: Header, proof: BlockHeaderProof +): Result[void, string] = + case proof.proofType + of BlockHeaderProofType.historicalHashesAccumulatorProof: + a.verifyAccumulatorProof(header, proof.historicalHashesAccumulatorProof) + of BlockHeaderProofType.none: + if header.isPreMerge(): + err("Pre merge header requires HistoricalHashesAccumulatorProof") + else: + # TODO: + # Add verification post merge based on historical_roots & historical_summaries + ok() + +func validateCanonicalHeaderBytes*( + bytes: openArray[byte], id: uint64 | Hash32, a: FinishedHistoricalHashesAccumulator +): Result[Header, string] = + let headerWithProof = decodeSsz(bytes, BlockHeaderWithProof).valueOr: + return err("Failed decoding header with proof: " & error) + let header = ?validateHeaderBytes(headerWithProof.header.asSeq(), id) + + ?a.verifyBlockHeaderProof(header, headerWithProof.proof) + + ok(header) + +func validateBlockBody*( + body: PortalBlockBodyLegacy, header: Header +): Result[void, string] = + ## Validate the block body against the txRoot and ommersHash from the header. + let calculatedOmmersHash = keccak256(body.uncles.asSeq()) + if calculatedOmmersHash != header.ommersHash: + return err("Invalid ommers hash") + + let calculatedTxsRoot = orderedTrieRoot(body.transactions.asSeq) + if calculatedTxsRoot != header.txRoot: + return err( + "Invalid transactions root: expected " & $header.txRoot & " - got " & + $calculatedTxsRoot + ) + + ok() + +func validateBlockBody*( + body: PortalBlockBodyShanghai, header: Header +): Result[void, string] = + ## Validate the block body against the txRoot, ommersHash and withdrawalsRoot + ## from the header. + # Shortcut the ommersHash calculation as uncles must be an RLP encoded + # empty list + if body.uncles.asSeq() != @[byte 0xc0]: + return err("Invalid ommers hash, uncles list is not empty") + + let calculatedTxsRoot = orderedTrieRoot(body.transactions.asSeq) + if calculatedTxsRoot != header.txRoot: + return err( + "Invalid transactions root: expected " & $header.txRoot & " - got " & + $calculatedTxsRoot + ) + + # TODO: This check is done higher up but perhaps this can become cleaner with + # some refactor. + doAssert(header.withdrawalsRoot.isSome()) + + let + calculatedWithdrawalsRoot = orderedTrieRoot(body.withdrawals.asSeq) + headerWithdrawalsRoot = header.withdrawalsRoot.get() + if calculatedWithdrawalsRoot != headerWithdrawalsRoot: + return err( + "Invalid withdrawals root: expected " & $headerWithdrawalsRoot & " - got " & + $calculatedWithdrawalsRoot + ) + + ok() + +func validateBlockBodyBytes*( + bytes: openArray[byte], header: Header +): Result[BlockBody, string] = + ## Fully decode the SSZ encoded Portal Block Body and validate it against the + ## header. + ## TODO: improve this decoding in combination with the block body validation + ## calls. + let timestamp = Moment.init(header.timestamp.int64, Second) + # TODO: The additional header checks are not needed as header is implicitly + # verified by means of the accumulator? Except that we don't use this yet + # post merge, so the checks are still useful, for now. + if isShanghai(chainConfig, timestamp): + if header.withdrawalsRoot.isNone(): + err("Expected withdrawalsRoot for Shanghai block") + elif header.ommersHash != EMPTY_UNCLE_HASH: + err("Expected empty uncles for a Shanghai block") + else: + let body = ?decodeSsz(bytes, PortalBlockBodyShanghai) + ?validateBlockBody(body, header) + BlockBody.fromPortalBlockBody(body) + elif isPoSBlock(chainConfig, header.number): + if header.withdrawalsRoot.isSome(): + err("Expected no withdrawalsRoot for pre Shanghai block") + elif header.ommersHash != EMPTY_UNCLE_HASH: + err("Expected empty uncles for a PoS block") + else: + let body = ?decodeSsz(bytes, PortalBlockBodyLegacy) + ?validateBlockBody(body, header) + BlockBody.fromPortalBlockBody(body) + else: + if header.withdrawalsRoot.isSome(): + err("Expected no withdrawalsRoot for pre Shanghai block") + else: + let body = ?decodeSsz(bytes, PortalBlockBodyLegacy) + ?validateBlockBody(body, header) + BlockBody.fromPortalBlockBody(body) + +func validateReceipts*( + receipts: PortalReceipts, receiptsRoot: Hash32 +): Result[void, string] = + if orderedTrieRoot(receipts.asSeq) != receiptsRoot: + err("Unexpected receipt root") + else: + ok() + +func validateReceiptsBytes*( + bytes: openArray[byte], receiptsRoot: Hash32 +): Result[seq[Receipt], string] = + ## Fully decode the SSZ encoded receipts and validate it against the header's + ## receipts root. + let receipts = ?decodeSsz(bytes, PortalReceipts) + + ?validateReceipts(receipts, receiptsRoot) + + seq[Receipt].fromPortalReceipts(receipts) diff --git a/fluffy/network/history/validation/historical_hashes_accumulator.nim b/fluffy/network/history/validation/historical_hashes_accumulator.nim index f46d64d3d..a6979aece 100644 --- a/fluffy/network/history/validation/historical_hashes_accumulator.nim +++ b/fluffy/network/history/validation/historical_hashes_accumulator.nim @@ -175,24 +175,6 @@ func verifyAccumulatorProof*( else: err("Cannot verify post merge header with accumulator proof") -func verifyHeader*( - a: FinishedHistoricalHashesAccumulator, header: Header, proof: BlockHeaderProof -): Result[void, string] = - case proof.proofType - of BlockHeaderProofType.historicalHashesAccumulatorProof: - a.verifyAccumulatorProof(header, proof.historicalHashesAccumulatorProof) - of BlockHeaderProofType.none: - if header.isPreMerge(): - err("Pre merge header requires HistoricalHashesAccumulatorProof") - else: - # TODO: - # Currently there is no proof solution for verifying headers post-merge. - # Skipping canonical verification will allow for nodes to push block data - # that is not part of the canonical chain. - # For now we accept this flaw as the focus lies on testing data - # availability up to the head of the chain. - ok() - func buildProof*( header: Header, epochRecord: EpochRecord | EpochRecordCached ): Result[HistoricalHashesAccumulatorProof, string] = diff --git a/fluffy/rpc/portal_rpc_client.nim b/fluffy/rpc/portal_rpc_client.nim index a33d0836d..7eb493bc3 100644 --- a/fluffy/rpc/portal_rpc_client.nim +++ b/fluffy/rpc/portal_rpc_client.nim @@ -13,7 +13,7 @@ import eth/common/[headers_rlp, blocks_rlp, receipts_rlp], json_rpc/rpcclient, ../common/common_types, - ../network/history/[history_content, history_network], + ../network/history/[history_content, history_type_conversions, history_validation], ./rpc_calls/[rpc_discovery_calls, rpc_portal_calls, rpc_portal_debug_calls] export rpcclient, rpc_discovery_calls, rpc_portal_calls, rpc_portal_debug_calls, results @@ -110,7 +110,7 @@ proc historyGetBlockHeader*( headerBytes = headerWithProof.header.asSeq() if validateContent: - validateBlockHeaderBytes(headerBytes, blockHash).valueOrErr(ContentValidationFailed) + validateHeaderBytes(headerBytes, blockHash).valueOrErr(ContentValidationFailed) else: decodeRlp(headerBytes, Header).valueOrErr(InvalidContentValue) @@ -132,7 +132,7 @@ proc historyGetBlockBody*( ContentValidationFailed ) else: - decodeBlockBodyBytes(content.toBytes()).valueOrErr(InvalidContentValue) + fromPortalBlockBodyBytes(content.toBytes()).valueOrErr(InvalidContentValue) proc historyGetReceipts*( client: PortalRpcClient, blockHash: Hash32, validateContent = true diff --git a/fluffy/tests/history_network_tests/test_history_content.nim b/fluffy/tests/history_network_tests/test_history_content.nim index d9af9e576..bdd99c87a 100644 --- a/fluffy/tests/history_network_tests/test_history_content.nim +++ b/fluffy/tests/history_network_tests/test_history_content.nim @@ -15,8 +15,7 @@ import eth/common/headers_rlp, ../../network_metadata, ../../eth_data/[history_data_json_store, history_data_ssz_e2s], - ../../network/history/ - [history_content, history_network, validation/historical_hashes_accumulator], + ../../network/history/[history_content, history_type_conversions, history_validation], ../../eth_data/yaml_utils, ./test_history_util @@ -73,7 +72,7 @@ suite "History Content Values": check res.isOk() let header = res.get() - check accumulator.verifyHeader(header, blockHeaderWithProof.proof).isOk() + check accumulator.verifyBlockHeaderProof(header, blockHeaderWithProof.proof).isOk() # Encode content check: @@ -108,7 +107,9 @@ suite "History Content Values": check res.isOk() let header = res.get() - check accumulator.verifyHeader(header, blockHeaderWithProof.proof).isOk() + check accumulator + .verifyBlockHeaderProof(header, blockHeaderWithProof.proof) + .isOk() # Encode content check: @@ -172,7 +173,7 @@ suite "History Content Values": check contentKey.isOk() # Decode (SSZ + RLP decode step) and validate block body - let contentValue = decodeBlockBodyBytes(contentValueEncoded) + let contentValue = fromPortalBlockBodyBytes(contentValueEncoded) check contentValue.isOk() # Encode content and content key diff --git a/fluffy/tests/history_network_tests/test_history_content_validation.nim b/fluffy/tests/history_network_tests/test_history_content_validation.nim index 04de1c6b6..bd9f89653 100644 --- a/fluffy/tests/history_network_tests/test_history_content_validation.nim +++ b/fluffy/tests/history_network_tests/test_history_content_validation.nim @@ -17,7 +17,7 @@ import eth/common/headers_rlp, ../../common/common_types, ../../eth_data/history_data_json_store, - ../../network/history/history_network + ../../network/history/[history_type_conversions, history_validation] const dataFile = "./fluffy/tests/blocks/mainnet_blocks_selected.json" @@ -51,12 +51,12 @@ suite "History Content Values Validation": ) test "Valid Header": - check validateBlockHeaderBytes(blockHeaderBytes, blockHash).isOk() + check validateHeaderBytes(blockHeaderBytes, blockHash).isOk() test "Malformed Header": let malformedBytes = blockHeaderBytes[10 .. blockHeaderBytes.high] - check validateBlockHeaderBytes(malformedBytes, blockHash).isErr() + check validateHeaderBytes(malformedBytes, blockHash).isErr() test "Invalid Header - Different gasUsed": var modifiedHeader = blockHeader @@ -65,7 +65,7 @@ suite "History Content Values Validation": let modifiedHeaderBytes = rlp.encode(modifiedHeader) - check validateBlockHeaderBytes(modifiedHeaderBytes, blockHash).isErr() + check validateHeaderBytes(modifiedHeaderBytes, blockHash).isErr() test "Valid Block Body": check validateBlockBodyBytes(blockBodyBytes, blockHeader).isOk() diff --git a/fluffy/tests/rpc_tests/test_portal_rpc_client.nim b/fluffy/tests/rpc_tests/test_portal_rpc_client.nim index 7be6c2257..bb8b9a6b9 100644 --- a/fluffy/tests/rpc_tests/test_portal_rpc_client.nim +++ b/fluffy/tests/rpc_tests/test_portal_rpc_client.nim @@ -19,7 +19,7 @@ import eth/p2p/discoveryv5/protocol as discv5_protocol, ../../network/wire/[portal_protocol, portal_stream, portal_protocol_config], ../../network/history/ - [history_network, history_content, validation/historical_hashes_accumulator], + [history_network, history_content, history_type_conversions, history_validation], ../../database/content_db, ../../rpc/[portal_rpc_client, rpc_portal_history_api], ../test_helpers diff --git a/fluffy/tests/state_network_tests/state_test_helpers.nim b/fluffy/tests/state_network_tests/state_test_helpers.nim index 90260e293..01d76faef 100644 --- a/fluffy/tests/state_network_tests/state_test_helpers.nim +++ b/fluffy/tests/state_network_tests/state_test_helpers.nim @@ -16,7 +16,7 @@ import eth/p2p/discoveryv5/routing_table, ../../network/wire/[portal_protocol, portal_stream, portal_protocol_config], ../../../nimbus/common/chain_config, - ../../network/history/[history_content, history_network], + ../../network/history/[history_content, history_network, history_validation], ../../network/state/[state_content, state_utils, state_network], ../../eth_data/yaml_utils, ../../database/content_db, diff --git a/fluffy/tools/portal_bridge/portal_bridge_history.nim b/fluffy/tools/portal_bridge/portal_bridge_history.nim index db553570c..df7777be7 100644 --- a/fluffy/tools/portal_bridge/portal_bridge_history.nim +++ b/fluffy/tools/portal_bridge/portal_bridge_history.nim @@ -20,7 +20,7 @@ import ../../../nimbus/beacon/web3_eth_conv, ../../../hive_integration/nodocker/engine/engine_client, ../../rpc/portal_rpc_client, - ../../network/history/[history_content, history_network], + ../../network/history/[history_content, history_type_conversions, history_validation], ../../network_metadata, ../../eth_data/[era1, history_data_ssz_e2s, history_data_seeding], ../../database/era1_db, @@ -239,7 +239,7 @@ proc runLatestLoop( let hash = blockObject.hash if validate: - if validateBlockHeaderBytes(headerWithProof.header.asSeq(), hash).isErr(): + if validateHeaderBytes(headerWithProof.header.asSeq(), hash).isErr(): error "Block header is invalid" continue if validateBlockBody(body, ethBlock.header).isErr():