# Nimbus # Copyright (c) 2021-2022 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). # at your option. This file may not be copied, modified, or distributed except according to those terms. {.push raises: [Defect].} import std/[options, tables], stew/results, chronos, chronicles, eth/[common/eth_types_rlp, rlp, trie, trie/db], eth/p2p/discoveryv5/[protocol, enr], ../../content_db, ../../../nimbus/constants, ../wire/[portal_protocol, portal_stream, portal_protocol_config], "."/[history_content, accumulator] logScope: topics = "portal_hist" export accumulator const historyProtocolId* = [byte 0x50, 0x0B] type HistoryNetwork* = ref object portalProtocol*: PortalProtocol contentDB*: ContentDB contentQueue*: AsyncQueue[(ContentKeysList, seq[seq[byte]])] accumulator*: FinishedAccumulator processContentLoop: Future[void] Block* = (BlockHeader, BlockBody) func toContentIdHandler(contentKey: ByteList): results.Opt[ContentId] = ok(toContentId(contentKey)) func encodeKey(k: ContentKey): (ByteList, ContentId) = let keyEncoded = encode(k) return (keyEncoded, toContentId(keyEncoded)) func getEncodedKeyForContent( cType: ContentType, hash: BlockHash): (ByteList, ContentId) = let contentKeyType = BlockKey(blockHash: hash) let contentKey = case cType of blockHeader: ContentKey(contentType: cType, blockHeaderKey: contentKeyType) of blockBody: ContentKey(contentType: cType, blockBodyKey: contentKeyType) of receipts: ContentKey(contentType: cType, receiptsKey: contentKeyType) of epochAccumulator: raiseAssert("Not implemented") of blockHeaderWithProof: ContentKey(contentType: cType, blockHeaderWithProofKey: contentKeyType) return encodeKey(contentKey) func decodeRlp*(input: openArray[byte], T: type): Result[T, string] = try: ok(rlp.decode(input, T)) except RlpError as e: err(e.msg) func decodeSsz*(input: openArray[byte], T: type): Result[T, string] = try: ok(SSZ.decode(input, T)) except SszError as e: err(e.msg) ## Calls to go from SSZ decoded types to RLP fully decoded types func fromPortalBlockBody( T: type BlockBody, body: BlockBodySSZ): Result[T, string] = ## Get the full decoded BlockBody from the SSZ-decoded `PortalBlockBody`. try: var transactions: seq[Transaction] for tx in body.transactions: transactions.add(rlp.decode(tx.asSeq(), Transaction)) let uncles = rlp.decode(body.uncles.asSeq(), seq[BlockHeader]) ok(BlockBody(transactions: transactions, uncles: uncles)) except RlpError as e: err("RLP decoding failed: " & e.msg) func fromReceipts( T: type seq[Receipt], receipts: ReceiptsSSZ): Result[T, string] = ## Get the full decoded seq[Receipt] from the SSZ-decoded `Receipts`. try: var res: seq[Receipt] for receipt in receipts: res.add(rlp.decode(receipt.asSeq(), Receipt)) ok(res) except RlpError as e: err("RLP decoding failed: " & e.msg) ## Calls to encode Block types to the SSZ types. func fromBlockBody(T: type BlockBodySSZ, body: BlockBody): T = var transactions: Transactions for tx in body.transactions: discard transactions.add(TransactionByteList(rlp.encode(tx))) let uncles = Uncles(rlp.encode(body.uncles)) BlockBodySSZ(transactions: transactions, uncles: uncles) func fromReceipts(T: type ReceiptsSSZ, receipts: seq[Receipt]): T = var receiptsSSZ: ReceiptsSSZ for receipt in receipts: discard receiptsSSZ.add(ReceiptByteList(rlp.encode(receipt))) receiptsSSZ func encode*(blockBody: BlockBody): seq[byte] = let portalBlockBody = BlockBodySSZ.fromBlockBody(blockBody) SSZ.encode(portalBlockBody) func encode*(receipts: seq[Receipt]): seq[byte] = let portalReceipts = ReceiptsSSZ.fromReceipts(receipts) SSZ.encode(portalReceipts) ## Calls and helper calls to do validation of block header, body and receipts # TODO: Failures on validation and perhaps deserialisation should be punished # for if/when peer scoring/banning is added. proc calcRootHash(items: Transactions | ReceiptsSSZ): Hash256 = var tr = initHexaryTrie(newMemoryDB()) for i, t in items: try: tr.put(rlp.encode(i), t.asSeq()) except RlpError as e: # TODO: Investigate this RlpError as it doesn't sound like this is # something that can actually occur. raiseAssert(e.msg) return tr.rootHash template calcTxsRoot*(transactions: Transactions): Hash256 = calcRootHash(transactions) template calcReceiptsRoot*(receipts: ReceiptsSSZ): Hash256 = calcRootHash(receipts) func validateBlockHeaderBytes*( bytes: openArray[byte], hash: BlockHash): Result[BlockHeader, string] = let header = ? decodeRlp(bytes, BlockHeader) if header.withdrawalsRoot.isSome: return err("Withdrawals not yet implemented") if not (header.blockHash() == hash): err("Block header hash does not match") else: ok(header) proc validateBlockBody( body: BlockBodySSZ, txsRoot, ommersHash: KeccakHash): Result[void, string] = ## Validate the block body against the txRoot amd ommersHash from the header. let calculatedOmmersHash = keccakHash(body.uncles.asSeq()) if calculatedOmmersHash != ommersHash: return err("Invalid ommers hash") let calculatedTxsRoot = calcTxsRoot(body.transactions) if calculatedTxsRoot != txsRoot: return err("Invalid transactions root") ok() proc validateBlockBodyBytes*( bytes: openArray[byte], txRoot, ommersHash: KeccakHash): Result[BlockBody, string] = ## Fully decode the SSZ Block Body and validate it against the header. let body = ? decodeSsz(bytes, BlockBodySSZ) ? validateBlockBody(body, txRoot, ommersHash) BlockBody.fromPortalBlockBody(body) proc validateReceipts( receipts: ReceiptsSSZ, receiptsRoot: KeccakHash): Result[void, string] = let calculatedReceiptsRoot = calcReceiptsRoot(receipts) if calculatedReceiptsRoot != receiptsRoot: return err("Unexpected receipt root") else: return ok() proc validateReceiptsBytes*( bytes: openArray[byte], receiptsRoot: KeccakHash): Result[seq[Receipt], string] = ## Fully decode the SSZ Block Body and validate it against the header. let receipts = ? decodeSsz(bytes, ReceiptsSSZ) ? validateReceipts(receipts, receiptsRoot) seq[Receipt].fromReceipts(receipts) ## ContentDB helper calls for specific history network types proc get(db: ContentDB, T: type BlockHeader, contentId: ContentId): Option[T] = let contentFromDB = db.get(contentId) if contentFromDB.isSome(): let headerWithProof = try: SSZ.decode(contentFromDB.get(), BlockHeaderWithProof) except SszError as e: raiseAssert(e.msg) let res = decodeRlp(headerWithProof.header.asSeq(), T) if res.isErr(): raiseAssert(res.error) else: some(res.get()) else: none(T) proc get(db: ContentDB, T: type BlockBody, contentId: ContentId): Option[T] = let contentFromDB = db.getSszDecoded(contentId, BlockBodySSZ) if contentFromDB.isSome(): let res = T.fromPortalBlockBody(contentFromDB.get()) if res.isErr(): raiseAssert(res.error) else: some(res.get()) else: none(T) proc get(db: ContentDB, T: type seq[Receipt], contentId: ContentId): Option[T] = let contentFromDB = db.getSszDecoded(contentId, ReceiptsSSZ) if contentFromDB.isSome(): let res = T.fromReceipts(contentFromDB.get()) if res.isErr(): raiseAssert(res.error) else: some(res.get()) else: none(T) proc get( db: ContentDB, T: type EpochAccumulator, contentId: ContentId): Option[T] = db.getSszDecoded(contentId, T) proc getContentFromDb( n: HistoryNetwork, T: type, contentId: ContentId): Option[T] = if n.portalProtocol.inRange(contentId): n.contentDB.get(T, contentId) else: none(T) ## Public API to get the history network specific types, either from database ## or through a lookup on the Portal Network const requestRetries = 4 # TODO: Currently doing 4 retries on lookups but only when the validation fails. # This is to avoid nodes that provide garbage from blocking us with getting the # requested data. Might want to also do that on a failed lookup, as perhaps this # could occur when being really unlucky with nodes timing out on requests. # Additionally, more improvements could be done with the lookup, as currently # ongoing requests are cancelled after the receival of the first response, # however that response is not yet validated at that moment. func verifyHeader( n: HistoryNetwork, header: BlockHeader, proof: BlockHeaderProof): Result[void, string] = verifyHeader(n.accumulator, header, proof) proc getVerifiedBlockHeader*( n: HistoryNetwork, hash: BlockHash): Future[Option[BlockHeader]] {.async.} = let (keyEncoded, contentId) = getEncodedKeyForContent(blockHeaderWithProof, hash) # Note: This still requests a BlockHeaderWithProof from the database, as that # is what is stored. But the proof doesn't need to be checked as everthing # should get checked before storing. let headerFromDb = n.getContentFromDb(BlockHeader, contentId) if headerFromDb.isSome(): info "Fetched block header from database", hash, contentKey = keyEncoded return headerFromDb for i in 0..