From 6e05c7588e2214d48a0ba9d30c50d8a93e73967c Mon Sep 17 00:00:00 2001 From: Kim De Mey Date: Wed, 18 May 2022 22:49:35 +0200 Subject: [PATCH] Add Header Accumulator for the Header Gossip Network (#1094) --- fluffy/common/common_types.nim | 5 +- fluffy/network/header/header_content.nim | 78 +++++++++++++++++++++ fluffy/network/history/history_content.nim | 2 - fluffy/populate_db.nim | 20 +++++- fluffy/tests/all_fluffy_tests.nim | 1 + fluffy/tests/blocks/mainnet_blocks_1-2.json | 10 +++ fluffy/tests/test_header_content.nim | 46 ++++++++++++ nimbus.nimble | 2 +- 8 files changed, 158 insertions(+), 6 deletions(-) create mode 100644 fluffy/network/header/header_content.nim create mode 100644 fluffy/tests/blocks/mainnet_blocks_1-2.json create mode 100644 fluffy/tests/test_header_content.nim diff --git a/fluffy/common/common_types.nim b/fluffy/common/common_types.nim index 725b85f7a..da4d49acb 100644 --- a/fluffy/common/common_types.nim +++ b/fluffy/common/common_types.nim @@ -9,14 +9,15 @@ import ssz_serialization/types, - stew/byteutils + stew/byteutils, nimcrypto/hash type ByteList* = List[byte, 2048] Bytes2* = array[2, byte] Bytes32* = array[32, byte] - ContentId* = Uint256 + ContentId* = UInt256 + BlockHash* = MDigest[32 * 8] # Bytes32 func `$`*(x: ByteList): string = x.asSeq.toHex() diff --git a/fluffy/network/header/header_content.nim b/fluffy/network/header/header_content.nim new file mode 100644 index 000000000..06f4152f9 --- /dev/null +++ b/fluffy/network/header/header_content.nim @@ -0,0 +1,78 @@ +# Nimbus +# Copyright (c) 2022 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +# https://github.com/ethereum/portal-network-specs/blob/master/header-gossip-network.md + +{.push raises: [Defect].} + +import + nimcrypto/[sha2, hash], + ssz_serialization, ssz_serialization/merkleization, + eth/common/eth_types, + ../../common/common_types + +export merkleization + +const + epochSize = 8192 # blocks + maxHistoricalEpochs = 100_000 # Doesn't really need a limit, does it? + +type + # Header Gossip Content Keys + # https://github.com/ethereum/portal-network-specs/blob/master/header-gossip-network.md#content-keys + + ContentType* = enum + accumulatorSnapshot = 0x00 + newBlockHeader = 0x01 + + AccumulatorSnapshotKey* = object + accumulatorRootHash*: Bytes32 + + NewBlockHeaderKey* = object + blockHash*: BlockHash + blockNumber*: UInt256 + + ContentKey* = object + case contentType*: ContentType + of accumulatorSnapshot: + accumulatorSnapshotKey*: AccumulatorSnapshotKey + of newBlockHeader: + newBlockHeaderKey*: NewBlockHeaderKey + + # Header Accumulator + # https://github.com/ethereum/portal-network-specs/blob/master/header-gossip-network.md#accumulator-snapshot + + HeaderRecord = object + blockHash: BlockHash + totalDifficulty: UInt256 + + EpochAccumulator = List[HeaderRecord, epochSize] + + Accumulator* = object + historicalEpochs*: List[Bytes32, maxHistoricalEpochs] + currentEpoch*: EpochAccumulator + +func updateAccumulator*(a: var Accumulator, header: BlockHeader) = + let lastTotalDifficulty = + if a.currentEpoch.len() == 0: + 0.stuint(256) + else: + a.currentEpoch[^1].totalDifficulty + + if a.currentEpoch.len() == epochSize: + let epochHash = hash_tree_root(a.currentEpoch) + + doAssert(a.historicalEpochs.add(epochHash.data)) + a.currentEpoch = EpochAccumulator.init(@[]) + + let headerRecord = + HeaderRecord( + blockHash: header.blockHash(), + totalDifficulty: lastTotalDifficulty + header.difficulty) + + let res = a.currentEpoch.add(headerRecord) + doAssert(res, "Can't fail because of currentEpoch length check") diff --git a/fluffy/network/history/history_content.nim b/fluffy/network/history/history_content.nim index a99e6a14b..dd459f770 100644 --- a/fluffy/network/history/history_content.nim +++ b/fluffy/network/history/history_content.nim @@ -23,8 +23,6 @@ type blockBody = 0x01 receipts = 0x02 - BlockHash* = MDigest[32 * 8] # Bytes32 - ContentKeyType* = object chainId*: uint16 blockHash*: BlockHash diff --git a/fluffy/populate_db.nim b/fluffy/populate_db.nim index aacb1f080..e17b31ee0 100644 --- a/fluffy/populate_db.nim +++ b/fluffy/populate_db.nim @@ -15,6 +15,8 @@ import ./network/wire/portal_protocol, ./network/history/history_content +export results + # Helper calls to, offline, populate the database with the current existing json # files with block data. Might move to some other storage format later on. # Perhaps https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md @@ -137,6 +139,22 @@ iterator blocks*( else: error "Failed reading block from block data", error = res.error +proc readBlockHeader*(blockData: BlockData): Result[BlockHeader, string] = + var rlp = + try: + rlpFromHex(blockData.rlp) + except ValueError as e: + return err("Invalid hex for rlp block data, number " & + $blockData.number & ": " & e.msg) + + if rlp.enterList(): + try: + return ok(rlp.read(BlockHeader)) + except RlpError as e: + return err("Invalid header, number " & $blockData.number & ": " & e.msg) + else: + return err("Item is not a valid rlp list, number " & $blockData.number) + # TODO pass nodeid as uint256 so it will be possible to use put method which # preserves size proc populateHistoryDb*( @@ -146,7 +164,7 @@ proc populateHistoryDb*( for b in blocks(blockData, verify): for value in b: # Note: This is the slowest part due to the hashing that takes place. - # TODO use put method which preserves size + # TODO use put method which preserves size db.put(history_content.toContentId(value[0]), value[1]) ok() diff --git a/fluffy/tests/all_fluffy_tests.nim b/fluffy/tests/all_fluffy_tests.nim index 6a052dc57..f75799792 100644 --- a/fluffy/tests/all_fluffy_tests.nim +++ b/fluffy/tests/all_fluffy_tests.nim @@ -15,6 +15,7 @@ import ./test_state_network, ./test_history_content, ./test_history_validation, + ./test_header_content, ./test_content_db, ./test_discovery_rpc, ./test_bridge_parser diff --git a/fluffy/tests/blocks/mainnet_blocks_1-2.json b/fluffy/tests/blocks/mainnet_blocks_1-2.json new file mode 100644 index 000000000..3a25abc1e --- /dev/null +++ b/fluffy/tests/blocks/mainnet_blocks_1-2.json @@ -0,0 +1,10 @@ +{ + "0x88e96d4537bea4d9c05d12549907b32561d3bf31f45aae734cdc119f13406cb6": { + "rlp": "0xf90216f90211a0d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d493479405a56e2d52c817161883f50c441c3228cfe54d9fa0d67e4d450343046425ae4271474353857ab860dbc0a1dde64b41b5cd3a532bf3a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008503ff80000001821388808455ba422499476574682f76312e302e302f6c696e75782f676f312e342e32a0969b900de27b6ac6a67742365dd65f55a0526c41fd18e1b16f1a1215c2e66f5988539bd4979fef1ec4c0c0", + "number": 1 + }, + "0xb495a1d7e6663152ae92708da4843337b958146015a2802f4193a410044698c9": { + "rlp": "0xf9021df90218a088e96d4537bea4d9c05d12549907b32561d3bf31f45aae734cdc119f13406cb6a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d4934794dd2f1e6e498202e86d8f5442af596580a4f03c2ca04943d941637411107494da9ec8bc04359d731bfd08b72b4d0edcbd4cd2ecb341a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008503ff00100002821388808455ba4241a0476574682f76312e302e302d30636463373634372f6c696e75782f676f312e34a02f0790c5aa31ab94195e1f6443d645af5b75c46c04fbf9911711198a0ce8fdda88b853fa261a86aa9ec0c0", + "number": 2 + } +} diff --git a/fluffy/tests/test_header_content.nim b/fluffy/tests/test_header_content.nim new file mode 100644 index 000000000..fbb8dc12b --- /dev/null +++ b/fluffy/tests/test_header_content.nim @@ -0,0 +1,46 @@ +# Nimbus +# Copyright (c) 2022 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.used.} + +{.push raises: [Defect].} + +import + std/tables, + unittest2, stew/byteutils, + eth/common/eth_types, + ../network/header/header_content, + ../populate_db + +suite "Header Gossip Content": + test "Header Accumulator Update": + const dataFile = "./fluffy/tests/blocks/mainnet_blocks_1-2.json" + let blockDataRes = readBlockDataTable(dataFile) + + check blockDataRes.isOk() + let blockData = blockDataRes.get() + + var headers: seq[BlockHeader] + headers.setLen(blockData.len()) + + for k, v in blockData.pairs: + let res = v.readBlockHeader() + check res.isOk() + let header = res.get() + headers[header.blockNumber.truncate(int) - 1] = header + + var accumulator: Accumulator + + updateAccumulator(accumulator, headers[0]) + + check accumulator.hash_tree_root().data.toHex() == + "411548579b5f6c651e6e1e56c3dc3fae6f389c663c0c910e462a4b806831fef6" + + updateAccumulator(accumulator, headers[1]) + + check accumulator.hash_tree_root().data.toHex() == + "e8dbd17538189d9a5b77001ff80c4ff6d841ceb0a3d374d17ddc4098550f5f93" diff --git a/nimbus.nimble b/nimbus.nimble index 147c5706e..69e6f0c74 100644 --- a/nimbus.nimble +++ b/nimbus.nimble @@ -80,4 +80,4 @@ task test_portal_testnet, "Build test_portal_testnet": task testfluffy, "Run fluffy tests": # Need the nimbus_db_backend in state network tests as we need a Hexary to # start from, even though it only uses the MemoryDb. - test "fluffy/tests", "all_fluffy_tests", "-d:chronicles_log_level=ERROR -d:chronosStrictException -d:nimbus_db_backend=sqlite" + test "fluffy/tests", "all_fluffy_tests", "-d:chronicles_log_level=ERROR -d:chronosStrictException -d:nimbus_db_backend=sqlite -d:PREFER_BLST_SHA256=false"