From a39b51e3e14f84156a0d95e9f0db93f2a5e5f0de Mon Sep 17 00:00:00 2001 From: Kim De Mey Date: Fri, 26 Jan 2024 23:38:12 +0100 Subject: [PATCH] Add historical summaries to Portal beacon network (#1990) --- .../beacon_chain_historical_summaries.nim | 15 ++- fluffy/network/beacon/beacon_content.nim | 13 +- fluffy/network/beacon/beacon_db.nim | 23 ++++ fluffy/network/beacon/beacon_network.nim | 50 +++++++- .../test_beacon_network.nim | 114 +++++++++++++++++- ...test_beacon_chain_historical_summaries.nim | 6 +- 6 files changed, 207 insertions(+), 14 deletions(-) rename fluffy/network/{history/experimental => beacon}/beacon_chain_historical_summaries.nim (79%) diff --git a/fluffy/network/history/experimental/beacon_chain_historical_summaries.nim b/fluffy/network/beacon/beacon_chain_historical_summaries.nim similarity index 79% rename from fluffy/network/history/experimental/beacon_chain_historical_summaries.nim rename to fluffy/network/beacon/beacon_chain_historical_summaries.nim index 4460ccaa7..bd5b305dc 100644 --- a/fluffy/network/history/experimental/beacon_chain_historical_summaries.nim +++ b/fluffy/network/beacon/beacon_chain_historical_summaries.nim @@ -1,5 +1,5 @@ -# Nimbus -# Copyright (c) 2023 Status Research & Development GmbH +# fluffy +# Copyright (c) 2023-2024 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). @@ -25,8 +25,9 @@ type HistoricalSummaries* = HashList[HistoricalSummary, Limit HISTORICAL_ROOTS_LIMIT] HistoricalSummariesProof* = array[5, Digest] HistoricalSummariesWithProof* = object - historical_summaries: HistoricalSummaries - proof: HistoricalSummariesProof + finalized_slot*: Slot + historical_summaries*: HistoricalSummaries + proof*: HistoricalSummariesProof func buildProof*( state: ForkedHashedBeaconState): Result[HistoricalSummariesProof, string] = @@ -47,3 +48,9 @@ func verifyProof*( leave = hash_tree_root(historical_summaries) verify_merkle_multiproof(@[leave], proof, @[gIndex], stateRoot) + +func verifyProof*( + summariesWithProof: HistoricalSummariesWithProof, + stateRoot: Digest): bool = + verifyProof( + summariesWithProof.historical_summaries, summariesWithProof.proof, stateRoot) diff --git a/fluffy/network/beacon/beacon_content.nim b/fluffy/network/beacon/beacon_content.nim index 3a52090b7..5402fcdf2 100644 --- a/fluffy/network/beacon/beacon_content.nim +++ b/fluffy/network/beacon/beacon_content.nim @@ -1,5 +1,5 @@ # Fluffy - Portal Network -# Copyright (c) 2022-2023 Status Research & Development GmbH +# Copyright (c) 2022-2024 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). @@ -36,6 +36,7 @@ type lightClientUpdate = 0x11 lightClientFinalityUpdate = 0x12 lightClientOptimisticUpdate = 0x13 + historicalSummaries = 0x14 # TODO: Consider how we will gossip bootstraps? # In consensus light client operation a node trusts only one bootstrap hash, @@ -59,6 +60,8 @@ type LightClientOptimisticUpdateKey* = object optimisticSlot*: uint64 ## signature_slot of the update + HistoricalSummariesKey* = uint8 + ContentKey* = object case contentType*: ContentType of unused: @@ -71,6 +74,8 @@ type lightClientFinalityUpdateKey*: LightClientFinalityUpdateKey of lightClientOptimisticUpdate: lightClientOptimisticUpdateKey*: LightClientOptimisticUpdateKey + of historicalSummaries: + historicalSummariesKey*: HistoricalSummariesKey # TODO: # ForkedLightClientUpdateBytesList can get pretty big and is send in one go. @@ -267,3 +272,9 @@ func optimisticUpdateContentKey*(optimisticSlot: uint64): ContentKey = optimisticSlot: optimisticSlot ) ) + +func historicalSummariesContentKey*(): ContentKey = + ContentKey( + contentType: historicalSummaries, + historicalSummariesKey: 0 + ) diff --git a/fluffy/network/beacon/beacon_db.nim b/fluffy/network/beacon/beacon_db.nim index c4b75f132..71526d0cc 100644 --- a/fluffy/network/beacon/beacon_db.nim +++ b/fluffy/network/beacon/beacon_db.nim @@ -20,6 +20,7 @@ import beacon_chain/spec/forks, beacon_chain/spec/forks_light_client, ./beacon_content, + ./beacon_chain_historical_summaries, ./beacon_init_loader, ../wire/portal_protocol @@ -245,6 +246,12 @@ proc putUpdateIfBetter*( db.putUpdateIfBetter(period, newUpdate) +proc getLastFinalityUpdate*(db: BeaconDb): Opt[ForkedLightClientFinalityUpdate] = + db.finalityUpdateCache.map( + proc(x: LightClientFinalityUpdateCache): ForkedLightClientFinalityUpdate = + decodeLightClientFinalityUpdateForked(db.forkDigests, x.lastFinalityUpdate).valueOr: + raiseAssert "Stored finality update must be valid") + proc createGetHandler*(db: BeaconDb): DbGetHandler = return ( proc(contentKey: ByteList, contentId: ContentId): results.Opt[seq[byte]] = @@ -299,6 +306,8 @@ proc createGetHandler*(db: BeaconDb): DbGetHandler = Opt.none(seq[byte]) else: Opt.none(seq[byte]) + of beacon_content.ContentType.historicalSummaries: + db.get(contentId) ) proc createStoreHandler*(db: BeaconDb): DbStoreHandler = @@ -343,4 +352,18 @@ proc createStoreHandler*(db: BeaconDb): DbStoreHandler = contentKey.lightClientOptimisticUpdateKey.optimisticSlot, lastOptimisticUpdate: content )) + of beacon_content.ContentType.historicalSummaries: + # TODO: Its probably better to use the kvstore here and instead use a sql + # table with slot as index and move the slot logic to the db store handler. + let current = db.get(contentId) + if current.isSome(): + let summariesWithProof = + decodeSszOrRaise(current.get(), HistoricalSummariesWithProof) + let newSummariesWithProof = + decodeSsz(content, HistoricalSummariesWithProof).valueOr: + return + if newSummariesWithProof.finalized_slot > summariesWithProof.finalized_slot: + db.put(contentId, content) + else: + db.put(contentId, content) ) diff --git a/fluffy/network/beacon/beacon_network.nim b/fluffy/network/beacon/beacon_network.nim index adc8da23b..5f207fe25 100644 --- a/fluffy/network/beacon/beacon_network.nim +++ b/fluffy/network/beacon/beacon_network.nim @@ -1,5 +1,5 @@ -# Nimbus - Portal Network -# Copyright (c) 2022-2023 Status Research & Development GmbH +# fluffy +# Copyright (c) 2022-2024 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). @@ -15,7 +15,7 @@ import beacon_chain/gossip_processing/light_client_processor, ../../../nimbus/constants, ../wire/[portal_protocol, portal_stream, portal_protocol_config], - "."/[beacon_content, beacon_db] + "."/[beacon_content, beacon_db, beacon_chain_historical_summaries] export beacon_content, beacon_db @@ -37,6 +37,29 @@ type func toContentIdHandler(contentKey: ByteList): results.Opt[ContentId] = ok(toContentId(contentKey)) +proc validateHistoricalSummaries( + n: BeaconNetwork, + summariesWithProof: HistoricalSummariesWithProof + ): Result[void, string] = + let + finalityUpdate = getLastFinalityUpdate(n.beaconDb).valueOr: + return err("Require finality update for verification") + + # TODO: compare slots first + stateRoot = + withForkyFinalityUpdate(finalityUpdate): + when lcDataFork > LightClientDataFork.None: + forkyFinalityUpdate.finalized_header.beacon.state_root + else: + # Note: this should always be the case as historical_summaries was + # introduced in Capella. + return err("Require Altair or > for verification") + + if summariesWithProof.verifyProof(stateRoot): + ok() + else: + err("Failed verifying historical_summaries proof") + proc getContent( n: BeaconNetwork, contentKey: ContentKey): Future[results.Opt[seq[byte]]] {.async.} = @@ -148,6 +171,23 @@ proc getLightClientOptimisticUpdate*( else: return Opt.some(decodingResult.value()) +proc getHistoricalSummaries*( + n: BeaconNetwork + ): Future[results.Opt[HistoricalSummaries]] {.async.} = + # Note: when taken from the db, it does not need to verify the proof. + let + contentKey = historicalSummariesContentKey() + content = ? await n.getContent(contentKey) + + summariesWithProof = decodeSsz(content, HistoricalSummariesWithProof).valueOr: + return Opt.none(HistoricalSummaries) + + if n.validateHistoricalSummaries(summariesWithProof).isOk(): + return Opt.some(summariesWithProof.historical_summaries) + else: + return Opt.none(HistoricalSummaries) + + proc new*( T: type BeaconNetwork, baseProtocol: protocol.Protocol, @@ -248,6 +288,10 @@ proc validateContent( err("Error processing update: " & $res.error[1]) else: ok() + of beacon_content.ContentType.historicalSummaries: + let summariesWithProof = ? decodeSsz(content, HistoricalSummariesWithProof) + + n.validateHistoricalSummaries(summariesWithProof) proc validateContent( n: BeaconNetwork, diff --git a/fluffy/tests/beacon_network_tests/test_beacon_network.nim b/fluffy/tests/beacon_network_tests/test_beacon_network.nim index bc38d3e06..d05a1318b 100644 --- a/fluffy/tests/beacon_network_tests/test_beacon_network.nim +++ b/fluffy/tests/beacon_network_tests/test_beacon_network.nim @@ -1,5 +1,5 @@ -# Nimbus - Portal Network -# Copyright (c) 2022-2023 Status Research & Development GmbH +# fluffy +# Copyright (c) 2022-2024 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). @@ -10,8 +10,14 @@ import eth/p2p/discoveryv5/protocol as discv5_protocol, beacon_chain/spec/forks, beacon_chain/spec/datatypes/altair, + # Test helpers + beacon_chain/../tests/testblockutil, + beacon_chain/../tests/mocking/mock_genesis, + beacon_chain/../tests/consensus_spec/fixtures_utils, + ../../network/wire/portal_protocol, - ../../network/beacon/[beacon_network, beacon_init_loader], + ../../network/beacon/[beacon_network, beacon_init_loader, + beacon_chain_historical_summaries], "."/[light_client_test_data, beacon_test_helpers] procSuite "Beacon Content Network": @@ -197,3 +203,105 @@ procSuite "Beacon Content Network": await lcNode1.stop() await lcNode2.stop() + + asyncTest "Get HistoricalSummaries": + let + cfg = genesisTestRuntimeConfig(ConsensusFork.Capella) + state = newClone(initGenesisState(cfg = cfg)) + var cache = StateCache() + + var blocks: seq[capella.SignedBeaconBlock] + # Note: + # Adding 8192 blocks. First block is genesis block and not one of these. + # Then one extra block is needed to get the historical summaries, block + # roots and state roots processed. + # index i = 0 is second block. + # index i = 8190 is 8192th block and last one that is part of the first + # historical root + for i in 0..= ConsensusFork.Capella: + let historical_summaries = forkyState.data.historical_summaries + let res = buildProof(state[]) + check res.isOk() + let + proof = res.get() + + historicalSummariesWithProof = HistoricalSummariesWithProof( + finalized_slot: forkyState.data.slot, + historical_summaries: historical_summaries, + proof: proof + ) + + content = SSZ.encode(historicalSummariesWithProof) + + (content, forkyState.data.slot, forkyState.root) + else: + raiseAssert("Not implemented pre-Capella") + let + networkData = loadNetworkData("mainnet") + lcNode1 = newLCNode(rng, 20302, networkData) + lcNode2 = newLCNode(rng, 20303, networkData) + forkDigests = (newClone networkData.forks)[] + + check: + lcNode1.portalProtocol().addNode(lcNode2.localNode()) == Added + lcNode2.portalProtocol().addNode(lcNode1.localNode()) == Added + + (await lcNode1.portalProtocol().ping(lcNode2.localNode())).isOk() + (await lcNode2.portalProtocol().ping(lcNode1.localNode())).isOk() + + let + contentKeyEncoded = historicalSummariesContentKey().encode() + contentId = toContentId(contentKeyEncoded) + + lcNode2.portalProtocol().storeContent( + contentKeyEncoded, + contentId, + content + ) + + block: + let res = await lcNode1.beaconNetwork.getHistoricalSummaries() + # Should fail as it cannot validate + check res.isErr() + + block: + # Add a (fake) finality update but with correct slot and state root + # so that node 1 can do the validation of the historical summaries. + let + dummyFinalityUpdate = capella.LightClientFinalityUpdate( + finalized_header: capella.LightClientHeader( + beacon: BeaconBlockHeader(slot: slot, state_root: root) + )) + finalityUpdateForked = ForkedLightClientFinalityUpdate( + kind: LightClientDataFork.Capella, capellaData: dummyFinalityUpdate) + forkDigest = forkDigestAtEpoch( + forkDigests, epoch(slot), cfg) + content = encodeFinalityUpdateForked( + forkDigest,finalityUpdateForked) + contentKey = finalityUpdateContentKey(slot.distinctBase()) + contentKeyEncoded = encode(contentKey) + contentId = toContentId(contentKeyEncoded) + + lcNode1.portalProtocol().storeContent( + contentKeyEncoded, + contentId, + content + ) + + block: + let res = await lcNode1.beaconNetwork.getHistoricalSummaries() + check: + res.isOk() + withState(state[]): + when consensusFork >= ConsensusFork.Capella: + res.get() == forkyState.data.historical_summaries + else: + false + + await lcNode1.stop() + await lcNode2.stop() diff --git a/fluffy/tests/test_beacon_chain_historical_summaries.nim b/fluffy/tests/test_beacon_chain_historical_summaries.nim index 13436a3a6..10a5807e8 100644 --- a/fluffy/tests/test_beacon_chain_historical_summaries.nim +++ b/fluffy/tests/test_beacon_chain_historical_summaries.nim @@ -1,5 +1,5 @@ -# Nimbus -# Copyright (c) 2023 Status Research & Development GmbH +# fluffy +# Copyright (c) 2023-2024 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). @@ -18,7 +18,7 @@ import beacon_chain/../tests/mocking/mock_genesis, beacon_chain/../tests/consensus_spec/fixtures_utils, - ../network/history/experimental/beacon_chain_historical_summaries + ../network/beacon/beacon_chain_historical_summaries suite "Beacon Chain Historical Summaries": let