From 6325712aa605a78175a80fe0fd9c987e5dd241f3 Mon Sep 17 00:00:00 2001 From: Kim De Mey Date: Tue, 14 Jun 2022 23:38:34 +0200 Subject: [PATCH] Add accumulator content keys for history network (#1123) --- fluffy/network/history/history_content.nim | 66 +++++++++--- fluffy/network/history/history_network.nim | 32 +++--- fluffy/populate_db.nim | 2 +- fluffy/tests/test_history_content.nim | 113 +++++++++++++++++++-- 4 files changed, 178 insertions(+), 35 deletions(-) diff --git a/fluffy/network/history/history_content.nim b/fluffy/network/history/history_content.nim index e58f99534..3eb7e0268 100644 --- a/fluffy/network/history/history_content.nim +++ b/fluffy/network/history/history_content.nim @@ -22,19 +22,39 @@ type blockHeader = 0x00 blockBody = 0x01 receipts = 0x02 + epochAccumulator = 0x03 + masterAccumulator = 0x04 - ContentKeyType* = object + BlockKey* = object chainId*: uint16 blockHash*: BlockHash + EpochAccumulatorKey* = object + epochHash*: Digest + + MasterAccumulatorKeyType* = enum + latest = 0x00 # An SSZ Union None + masterHash = 0x01 + + MasterAccumulatorKey* = object + case accumulaterKeyType*: MasterAccumulatorKeyType + of latest: + discard + of masterHash: + masterHashKey*: Digest + ContentKey* = object case contentType*: ContentType of blockHeader: - blockHeaderKey*: ContentKeyType + blockHeaderKey*: BlockKey of blockBody: - blockBodyKey*: ContentKeyType + blockBodyKey*: BlockKey of receipts: - receiptsKey*: ContentKeyType + receiptsKey*: BlockKey + of epochAccumulator: + epochAccumulatorKey*: EpochAccumulatorKey + of masterAccumulator: + masterAccumulatorKey*: MasterAccumulatorKey func encode*(contentKey: ContentKey): ByteList = ByteList.init(SSZ.encode(contentKey)) @@ -56,16 +76,30 @@ func toContentId*(contentKey: ContentKey): ContentId = func `$`*(x: BlockHash): string = "0x" & x.data.toHex() -func `$`*(x: ContentKey): string = - let key = - case x.contentType: - of blockHeader: - x.blockHeaderKey - of blockBody: - x.blockBodyKey - of receipts: - x.receiptsKey +func `$`*(x: BlockKey): string = + "blockHash: " & $x.blockHash & ", chainId: " & $x.chainId - "(contentType: " & $x.contentType & - ", blockHash: " & $key.blockHash & - ", chainId: " & $key.chainId & ")" +func `$`*(x: ContentKey): string = + var res = "(type: " & $x.contentType & ", " + + case x.contentType: + of blockHeader: + res.add($x.blockHeaderKey) + of blockBody: + res.add($x.blockBodyKey) + of receipts: + res.add($x.receiptsKey) + of epochAccumulator: + let key = x.epochAccumulatorKey + res.add("epochHash: " & $key.epochHash) + of masterAccumulator: + let key = x.masterAccumulatorKey + case key.accumulaterKeyType: + of latest: + res.add($key.accumulaterKeyType) + of masterHash: + res.add($key.accumulaterKeyType & ": " & $key.masterHashKey) + + res.add(")") + + res diff --git a/fluffy/network/history/history_network.nim b/fluffy/network/history/history_network.nim index 0fc5484c2..c0ca7e0da 100644 --- a/fluffy/network/history/history_network.nim +++ b/fluffy/network/history/history_network.nim @@ -44,7 +44,7 @@ func encodeKey(k: ContentKey): (ByteList, ContentId) = func getEncodedKeyForContent( cType: ContentType, chainId: uint16, hash: BlockHash): (ByteList, ContentId) = - let contentKeyType = ContentKeyType(chainId: chainId, blockHash: hash) + let contentKeyType = BlockKey(chainId: chainId, blockHash: hash) let contentKey = case cType @@ -54,6 +54,10 @@ func getEncodedKeyForContent( ContentKey(contentType: cType, blockBodyKey: contentKeyType) of receipts: ContentKey(contentType: cType, receiptsKey: contentKeyType) + of epochAccumulator: + raiseAssert("Not implemented") + of masterAccumulator: + raiseAssert("Not implemented") return encodeKey(contentKey) @@ -67,20 +71,20 @@ proc getContentFromBytes(bytes: openArray[byte], T: type): Result[T, string] = proc validateHeaderBytes*( bytes: openArray[byte], hash: BlockHash): Option[BlockHeader] = - + let headerResult = getContentFromBytes(bytes, BlockHeader) - + if headerResult.isErr(): error "Failed to decode header ", msg = headerResult.error() return none(BlockHeader) - + let header = headerResult.unsafeGet() if not (header.blockHash() == hash): # TODO: Header with different hash than expected, maybe we should punish # peer which sent us this ? return none(BlockHeader) - + return some(header) proc validateExpectedBody( @@ -99,7 +103,7 @@ proc validateExpectedBody( return ok() except RlpError as e: return err(e.msg) - + proc validateBodyBytes*( bytes: openArray[byte], txRoot: KeccakHash, @@ -110,20 +114,20 @@ proc validateBodyBytes*( if bodyResult.isErr(): error "Failed to decode block body", msg = bodyResult.error() return none(BlockBody) - + let blockBody = bodyResult.unsafeGet() let expectedResult = validateExpectedBody(blockBody, txRoot, ommersHash) if expectedResult.isErr(): - error "Failed to validate if block body matches header", + error "Failed to validate if block body matches header", msg = expectedResult.error() # we got block body (bundle of transactions and uncles) which do not match # header. For now just ignore it, but maybe we should penalize peer # sending us such data? return none(BlockBody) - + return some(blockBody) proc getContentFromDb( @@ -238,7 +242,7 @@ proc validateExpectedReceipts( return ok() except RlpError as e: return err(e.msg) - + proc validateReceiptsBytes*( bytes: openArray[byte], receiptRoot: KeccakHash): Option[seq[Receipt]] = @@ -248,13 +252,13 @@ proc validateReceiptsBytes*( if receiptResult.isErr(): error "Failed to decode receipts", msg = receiptResult.error() return none(seq[Receipt]) - + let receipts = receiptResult.unsafeGet() let expectedReceiptsResult = validateExpectedReceipts(receipts, receiptRoot) if expectedReceiptsResult.isErr(): - error "Failed to validate if receipts matches header", + error "Failed to validate if receipts matches header", msg = expectedReceiptsResult.error() # we got receipts which do not match @@ -327,6 +331,10 @@ proc validateContent(content: openArray[byte], contentKey: ByteList): bool = # to deal with this? of receipts: true + of epochAccumulator: + true + of masterAccumulator: + true proc new*( T: type HistoryNetwork, diff --git a/fluffy/populate_db.nim b/fluffy/populate_db.nim index dee8445b5..a1c267be5 100644 --- a/fluffy/populate_db.nim +++ b/fluffy/populate_db.nim @@ -82,7 +82,7 @@ func readBlockData( $blockData.number & ": " & e.msg) let contentKeyType = - ContentKeyType(chainId: 1'u16, blockHash: blockHash) + BlockKey(chainId: 1'u16, blockHash: blockHash) try: # If wanted the hash for the corresponding header can be verified diff --git a/fluffy/tests/test_history_content.nim b/fluffy/tests/test_history_content.nim index 55e37dbae..0c8fb66a5 100644 --- a/fluffy/tests/test_history_content.nim +++ b/fluffy/tests/test_history_content.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2021 Status Research & Development GmbH +# Copyright (c) 2021-2022 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). @@ -12,7 +12,8 @@ import ../network/history/history_content # According to test vectors: -# TODO: Add link once test vectors are merged +# https://github.com/ethereum/portal-network-specs/blob/master/content-keys-test-vectors.md#history-network-keys + suite "History ContentKey Encodings": test "BlockHeader": # Input @@ -32,7 +33,7 @@ suite "History ContentKey Encodings": let contentKey = ContentKey( contentType: blockHeader, - blockHeaderKey: ContentKeyType(chainId: 15'u16, blockHash: blockHash)) + blockHeaderKey: BlockKey(chainId: 15'u16, blockHash: blockHash)) let encoded = encode(contentKey) check encoded.asSeq.toHex == contentKeyHex @@ -66,7 +67,7 @@ suite "History ContentKey Encodings": let contentKey = ContentKey( contentType: blockBody, - blockBodyKey: ContentKeyType(chainId: 20'u16, blockHash: blockHash)) + blockBodyKey: BlockKey(chainId: 20'u16, blockHash: blockHash)) let encoded = encode(contentKey) check encoded.asSeq.toHex == contentKeyHex @@ -99,8 +100,7 @@ suite "History ContentKey Encodings": let contentKey = ContentKey( contentType: receipts, - receiptsKey: ContentKeyType(chainId: 4'u16, blockHash: blockHash)) - + receiptsKey: BlockKey(chainId: 4'u16, blockHash: blockHash)) let encoded = encode(contentKey) check encoded.asSeq.toHex == contentKeyHex @@ -115,3 +115,104 @@ suite "History ContentKey Encodings": toContentId(contentKey) == parse(contentId, Stuint[256], 10) # In stint this does BE hex string toContentId(contentKey).toHex() == contentIdHexBE + + test "Epoch Accumulator": + var epochHash: Digest + epochHash.data = hexToByteArray[sizeof(Digest)]( + "0xe242814b90ed3950e13aac7e56ce116540c71b41d1516605aada26c6c07cc491") + + const + contentKeyHex = + "03e242814b90ed3950e13aac7e56ce116540c71b41d1516605aada26c6c07cc491" + contentId = + "72232402989179419196382321898161638871438419016077939952896528930608027961710" + # or + contentIdHexBE = + "9fb2175e76c6989e0fdac3ee10c40d2a81eb176af32e1c16193e3904fe56896e" + + let contentKey = ContentKey( + contentType: epochAccumulator, + epochAccumulatorKey: EpochAccumulatorKey(epochHash: epochHash)) + + let encoded = encode(contentKey) + check encoded.asSeq.toHex == contentKeyHex + let decoded = decode(encoded) + check decoded.isSome() + + let contentKeyDecoded = decoded.get() + check: + contentKeyDecoded.contentType == contentKey.contentType + contentKeyDecoded.epochAccumulatorKey == contentKey.epochAccumulatorKey + + toContentId(contentKey) == parse(contentId, Stuint[256], 10) + # In stint this does BE hex string + toContentId(contentKey).toHex() == contentIdHexBE + + test "Master Accumulator - Latest": + var accumulatorHash: Digest + accumulatorHash.data = hexToByteArray[sizeof(Digest)]( + "0x88cce8439ebc0c1d007177ffb6831c15c07b4361984cc52235b6fd728434f0c7") + + const + contentKeyHex = + "0400" + contentId = + "87173654316145541646904042090629917349369185510102051783618763191692466404071" + # or + contentIdHexBE = + "c0ba8a33ac67f44abff5984dfbb6f56c46b880ac2b86e1f23e7fa9c402c53ae7" + + let contentKey = ContentKey( + contentType: masterAccumulator, + masterAccumulatorKey: MasterAccumulatorKey(accumulaterKeyType: latest)) + + let encoded = encode(contentKey) + check encoded.asSeq.toHex == contentKeyHex + let decoded = decode(encoded) + check decoded.isSome() + + let contentKeyDecoded = decoded.get() + check: + contentKeyDecoded.contentType == contentKey.contentType + contentKeyDecoded.masterAccumulatorKey.accumulaterKeyType == + contentKey.masterAccumulatorKey.accumulaterKeyType + + toContentId(contentKey) == parse(contentId, Stuint[256], 10) + # In stint this does BE hex string + toContentId(contentKey).toHex() == contentIdHexBE + + test "Master Accumulator - Hash": + var accumulatorHash: Digest + accumulatorHash.data = hexToByteArray[sizeof(Digest)]( + "0x88cce8439ebc0c1d007177ffb6831c15c07b4361984cc52235b6fd728434f0c7") + + const + contentKeyHex = + "040188cce8439ebc0c1d007177ffb6831c15c07b4361984cc52235b6fd728434f0c7" + contentId = + "79362820890138237094338894474079140563693945795365426184460738681339857347750" + # or + contentIdHexBE = + "af75c3c9d0e89a5083361a3334a9c5583955f0dbe9a413eb79ba26400d1824a6" + + let contentKey = ContentKey( + contentType: masterAccumulator, + masterAccumulatorKey: MasterAccumulatorKey( + accumulaterKeyType: masterHash, masterHashKey: accumulatorHash)) + + let encoded = encode(contentKey) + check encoded.asSeq.toHex == contentKeyHex + let decoded = decode(encoded) + check decoded.isSome() + + let contentKeyDecoded = decoded.get() + check: + contentKeyDecoded.contentType == contentKey.contentType + contentKeyDecoded.masterAccumulatorKey.accumulaterKeyType == + contentKey.masterAccumulatorKey.accumulaterKeyType + contentKeyDecoded.masterAccumulatorKey.masterHashKey == + contentKey.masterAccumulatorKey.masterHashKey + + toContentId(contentKey) == parse(contentId, Stuint[256], 10) + # In stint this does BE hex string + toContentId(contentKey).toHex() == contentIdHexBE