Add historical summaries to Portal beacon network (#1990)

fluffy/network/beacon/beacon_chain_historical_summaries.nim

@@ -1,5 +1,5 @@
-# Nimbus
-# Copyright (c) 2023 Status Research & Development GmbH
+# fluffy
+# Copyright (c) 2023-2024 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -25,8 +25,9 @@ type
   HistoricalSummaries* = HashList[HistoricalSummary, Limit HISTORICAL_ROOTS_LIMIT]
   HistoricalSummariesProof* = array[5, Digest]
   HistoricalSummariesWithProof* = object
-    historical_summaries: HistoricalSummaries
-    proof: HistoricalSummariesProof
+    finalized_slot*: Slot
+    historical_summaries*: HistoricalSummaries
+    proof*: HistoricalSummariesProof
 
 func buildProof*(
     state: ForkedHashedBeaconState): Result[HistoricalSummariesProof, string] =
@@ -47,3 +48,9 @@ func verifyProof*(
     leave = hash_tree_root(historical_summaries)
 
   verify_merkle_multiproof(@[leave], proof, @[gIndex], stateRoot)
+
+func verifyProof*(
+    summariesWithProof: HistoricalSummariesWithProof,
+    stateRoot: Digest): bool =
+  verifyProof(
+    summariesWithProof.historical_summaries, summariesWithProof.proof, stateRoot)

fluffy/network/beacon/beacon_content.nim

@@ -1,5 +1,5 @@
 # Fluffy - Portal Network
-# Copyright (c) 2022-2023 Status Research & Development GmbH
+# Copyright (c) 2022-2024 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -36,6 +36,7 @@ type
     lightClientUpdate = 0x11
     lightClientFinalityUpdate = 0x12
     lightClientOptimisticUpdate = 0x13
+    historicalSummaries = 0x14
 
   # TODO: Consider how we will gossip bootstraps?
   # In consensus light client operation a node trusts only one bootstrap hash,
@@ -59,6 +60,8 @@ type
   LightClientOptimisticUpdateKey* = object
     optimisticSlot*: uint64 ## signature_slot of the update
 
+  HistoricalSummariesKey* = uint8
+
   ContentKey* = object
     case contentType*: ContentType
     of unused:
@@ -71,6 +74,8 @@ type
       lightClientFinalityUpdateKey*: LightClientFinalityUpdateKey
     of lightClientOptimisticUpdate:
       lightClientOptimisticUpdateKey*: LightClientOptimisticUpdateKey
+    of historicalSummaries:
+      historicalSummariesKey*: HistoricalSummariesKey
 
   # TODO:
   # ForkedLightClientUpdateBytesList can get pretty big and is send in one go.
@@ -267,3 +272,9 @@ func optimisticUpdateContentKey*(optimisticSlot: uint64): ContentKey =
       optimisticSlot: optimisticSlot
     )
   )
+
+func historicalSummariesContentKey*(): ContentKey =
+  ContentKey(
+    contentType: historicalSummaries,
+    historicalSummariesKey: 0
+  )

fluffy/network/beacon/beacon_db.nim

@@ -20,6 +20,7 @@ import
   beacon_chain/spec/forks,
   beacon_chain/spec/forks_light_client,
   ./beacon_content,
+  ./beacon_chain_historical_summaries,
   ./beacon_init_loader,
   ../wire/portal_protocol
 
@@ -245,6 +246,12 @@ proc putUpdateIfBetter*(
 
   db.putUpdateIfBetter(period, newUpdate)
 
+proc getLastFinalityUpdate*(db: BeaconDb): Opt[ForkedLightClientFinalityUpdate] =
+  db.finalityUpdateCache.map(
+    proc(x: LightClientFinalityUpdateCache): ForkedLightClientFinalityUpdate =
+      decodeLightClientFinalityUpdateForked(db.forkDigests, x.lastFinalityUpdate).valueOr:
+        raiseAssert "Stored finality update must be valid")
+
 proc createGetHandler*(db: BeaconDb): DbGetHandler =
   return (
     proc(contentKey: ByteList, contentId: ContentId): results.Opt[seq[byte]] =
@@ -299,6 +306,8 @@ proc createGetHandler*(db: BeaconDb): DbGetHandler =
             Opt.none(seq[byte])
         else:
           Opt.none(seq[byte])
+      of beacon_content.ContentType.historicalSummaries:
+        db.get(contentId)
   )
 
 proc createStoreHandler*(db: BeaconDb): DbStoreHandler =
@@ -343,4 +352,18 @@ proc createStoreHandler*(db: BeaconDb): DbStoreHandler =
             contentKey.lightClientOptimisticUpdateKey.optimisticSlot,
           lastOptimisticUpdate: content
         ))
+    of beacon_content.ContentType.historicalSummaries:
+      # TODO: Its probably better to use the kvstore here and instead use a sql
+      # table with slot as index and move the slot logic to the db store handler.
+      let current = db.get(contentId)
+      if current.isSome():
+        let summariesWithProof =
+          decodeSszOrRaise(current.get(), HistoricalSummariesWithProof)
+        let newSummariesWithProof =
+          decodeSsz(content, HistoricalSummariesWithProof).valueOr:
+            return
+        if newSummariesWithProof.finalized_slot > summariesWithProof.finalized_slot:
+          db.put(contentId, content)
+      else:
+        db.put(contentId, content)
   )

fluffy/network/beacon/beacon_network.nim

@@ -1,5 +1,5 @@
-# Nimbus - Portal Network
-# Copyright (c) 2022-2023 Status Research & Development GmbH
+# fluffy
+# Copyright (c) 2022-2024 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -15,7 +15,7 @@ import
   beacon_chain/gossip_processing/light_client_processor,
   ../../../nimbus/constants,
   ../wire/[portal_protocol, portal_stream, portal_protocol_config],
-  "."/[beacon_content, beacon_db]
+  "."/[beacon_content, beacon_db, beacon_chain_historical_summaries]
 
 export beacon_content, beacon_db
 
@@ -37,6 +37,29 @@ type
 func toContentIdHandler(contentKey: ByteList): results.Opt[ContentId] =
   ok(toContentId(contentKey))
 
+proc validateHistoricalSummaries(
+    n: BeaconNetwork,
+    summariesWithProof: HistoricalSummariesWithProof
+    ): Result[void, string] =
+  let
+    finalityUpdate = getLastFinalityUpdate(n.beaconDb).valueOr:
+      return err("Require finality update for verification")
+
+    # TODO: compare slots first
+    stateRoot =
+      withForkyFinalityUpdate(finalityUpdate):
+        when lcDataFork > LightClientDataFork.None:
+          forkyFinalityUpdate.finalized_header.beacon.state_root
+        else:
+          # Note: this should always be the case as historical_summaries was
+          # introduced in Capella.
+          return err("Require Altair or > for verification")
+
+  if summariesWithProof.verifyProof(stateRoot):
+    ok()
+  else:
+    err("Failed verifying historical_summaries proof")
+
 proc getContent(
     n: BeaconNetwork, contentKey: ContentKey):
     Future[results.Opt[seq[byte]]] {.async.} =
@@ -148,6 +171,23 @@ proc getLightClientOptimisticUpdate*(
   else:
     return Opt.some(decodingResult.value())
 
+proc getHistoricalSummaries*(
+    n: BeaconNetwork
+  ): Future[results.Opt[HistoricalSummaries]] {.async.} =
+  # Note: when taken from the db, it does not need to verify the proof.
+  let
+    contentKey = historicalSummariesContentKey()
+    content = ? await n.getContent(contentKey)
+
+    summariesWithProof = decodeSsz(content, HistoricalSummariesWithProof).valueOr:
+      return Opt.none(HistoricalSummaries)
+
+  if n.validateHistoricalSummaries(summariesWithProof).isOk():
+    return Opt.some(summariesWithProof.historical_summaries)
+  else:
+    return Opt.none(HistoricalSummaries)
+
+
 proc new*(
     T: type BeaconNetwork,
     baseProtocol: protocol.Protocol,
@@ -248,6 +288,10 @@ proc validateContent(
       err("Error processing update: " & $res.error[1])
     else:
       ok()
+  of beacon_content.ContentType.historicalSummaries:
+    let summariesWithProof = ? decodeSsz(content, HistoricalSummariesWithProof)
+
+    n.validateHistoricalSummaries(summariesWithProof)
 
 proc validateContent(
     n: BeaconNetwork,

fluffy/tests/beacon_network_tests/test_beacon_network.nim

@@ -1,5 +1,5 @@
-# Nimbus - Portal Network
-# Copyright (c) 2022-2023 Status Research & Development GmbH
+# fluffy
+# Copyright (c) 2022-2024 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -10,8 +10,14 @@ import
   eth/p2p/discoveryv5/protocol as discv5_protocol,
   beacon_chain/spec/forks,
   beacon_chain/spec/datatypes/altair,
+  # Test helpers
+  beacon_chain/../tests/testblockutil,
+  beacon_chain/../tests/mocking/mock_genesis,
+  beacon_chain/../tests/consensus_spec/fixtures_utils,
+
   ../../network/wire/portal_protocol,
-  ../../network/beacon/[beacon_network, beacon_init_loader],
+  ../../network/beacon/[beacon_network, beacon_init_loader,
+    beacon_chain_historical_summaries],
   "."/[light_client_test_data, beacon_test_helpers]
 
 procSuite "Beacon Content Network":
@@ -197,3 +203,105 @@ procSuite "Beacon Content Network":
 
     await lcNode1.stop()
     await lcNode2.stop()
+
+  asyncTest "Get HistoricalSummaries":
+    let
+      cfg = genesisTestRuntimeConfig(ConsensusFork.Capella)
+      state = newClone(initGenesisState(cfg = cfg))
+    var cache = StateCache()
+
+    var blocks: seq[capella.SignedBeaconBlock]
+    # Note:
+    # Adding 8192 blocks. First block is genesis block and not one of these.
+    # Then one extra block is needed to get the historical summaries, block
+    # roots and state roots processed.
+    # index i = 0 is second block.
+    # index i = 8190 is 8192th block and last one that is part of the first
+    # historical root
+    for i in 0..<SLOTS_PER_HISTORICAL_ROOT:
+      blocks.add(addTestBlock(state[], cache, cfg = cfg).capellaData)
+
+    let (content, slot, root) =
+      withState(state[]):
+        when consensusFork >= ConsensusFork.Capella:
+          let historical_summaries = forkyState.data.historical_summaries
+          let res = buildProof(state[])
+          check res.isOk()
+          let
+            proof = res.get()
+
+            historicalSummariesWithProof = HistoricalSummariesWithProof(
+              finalized_slot: forkyState.data.slot,
+              historical_summaries: historical_summaries,
+              proof: proof
+            )
+
+            content = SSZ.encode(historicalSummariesWithProof)
+
+          (content, forkyState.data.slot, forkyState.root)
+        else:
+          raiseAssert("Not implemented pre-Capella")
+    let
+      networkData = loadNetworkData("mainnet")
+      lcNode1 = newLCNode(rng, 20302, networkData)
+      lcNode2 = newLCNode(rng, 20303, networkData)
+      forkDigests = (newClone networkData.forks)[]
+
+    check:
+      lcNode1.portalProtocol().addNode(lcNode2.localNode()) == Added
+      lcNode2.portalProtocol().addNode(lcNode1.localNode()) == Added
+
+      (await lcNode1.portalProtocol().ping(lcNode2.localNode())).isOk()
+      (await lcNode2.portalProtocol().ping(lcNode1.localNode())).isOk()
+
+    let
+      contentKeyEncoded = historicalSummariesContentKey().encode()
+      contentId = toContentId(contentKeyEncoded)
+
+    lcNode2.portalProtocol().storeContent(
+      contentKeyEncoded,
+      contentId,
+      content
+    )
+
+    block:
+      let res = await lcNode1.beaconNetwork.getHistoricalSummaries()
+      # Should fail as it cannot validate
+      check res.isErr()
+
+    block:
+      # Add a (fake) finality update but with correct slot and state root
+      # so that node 1 can do the validation of the historical summaries.
+      let
+        dummyFinalityUpdate = capella.LightClientFinalityUpdate(
+          finalized_header: capella.LightClientHeader(
+            beacon: BeaconBlockHeader(slot: slot, state_root: root)
+        ))
+        finalityUpdateForked = ForkedLightClientFinalityUpdate(
+          kind: LightClientDataFork.Capella, capellaData: dummyFinalityUpdate)
+        forkDigest = forkDigestAtEpoch(
+            forkDigests, epoch(slot), cfg)
+        content = encodeFinalityUpdateForked(
+          forkDigest,finalityUpdateForked)
+        contentKey = finalityUpdateContentKey(slot.distinctBase())
+        contentKeyEncoded = encode(contentKey)
+        contentId = toContentId(contentKeyEncoded)
+
+      lcNode1.portalProtocol().storeContent(
+        contentKeyEncoded,
+        contentId,
+        content
+      )
+
+    block:
+      let res = await lcNode1.beaconNetwork.getHistoricalSummaries()
+      check:
+        res.isOk()
+        withState(state[]):
+          when consensusFork >= ConsensusFork.Capella:
+            res.get() == forkyState.data.historical_summaries
+          else:
+            false
+
+    await lcNode1.stop()
+    await lcNode2.stop()

fluffy/tests/test_beacon_chain_historical_summaries.nim

@@ -1,5 +1,5 @@
-# Nimbus
-# Copyright (c) 2023 Status Research & Development GmbH
+# fluffy
+# Copyright (c) 2023-2024 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -18,7 +18,7 @@ import
   beacon_chain/../tests/mocking/mock_genesis,
   beacon_chain/../tests/consensus_spec/fixtures_utils,
 
-  ../network/history/experimental/beacon_chain_historical_summaries
+  ../network/beacon/beacon_chain_historical_summaries
 
 suite "Beacon Chain Historical Summaries":
   let