add function to build merkle proofs (#2874)

Adds a function that constructs a Merkle proof for a generalized index.
This will be used during light client sync to update light clients with
a new state (see NEXT_SYNC_COMMITTEE_INDEX / FINALIZED_ROOT_INDEX).

AllTests-mainnet.md

@@ -217,9 +217,10 @@ OK: 12/12 Fail: 0/12 Skip: 0/12
 OK: 1/1 Fail: 0/1 Skip: 0/1
 ## Spec helpers
 ```diff
++ build_proof - BeaconState                                                                  OK
 + integer_squareroot                                                                         OK
 ```
-OK: 1/1 Fail: 0/1 Skip: 0/1
+OK: 2/2 Fail: 0/2 Skip: 0/2
 ## Specific field types
 ```diff
 + root update                                                                                OK
@@ -359,4 +360,4 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
 OK: 36/48 Fail: 0/48 Skip: 12/48
 
 ---TOTAL---
-OK: 193/205 Fail: 0/205 Skip: 12/205
+OK: 194/206 Fail: 0/206 Skip: 12/206

FixtureAll-mainnet.md

@@ -381,14 +381,16 @@ FixtureAll-mainnet
 + altair_fork_random_low_balances                                                            OK
 + altair_fork_random_misc_balances                                                           OK
 + altair_fork_random_mismatched_attestations                                                 OK
++ finality_root_merkle_proof                                                                 OK
 + fork_base_state                                                                            OK
 + fork_many_next_epoch                                                                       OK
 + fork_next_epoch                                                                            OK
 + fork_next_epoch_with_block                                                                 OK
 + fork_random_low_balances                                                                   OK
 + fork_random_misc_balances                                                                  OK
++ next_sync_committee_merkle_proof                                                           OK
 ```
-OK: 385/385 Fail: 0/385 Skip: 0/385
+OK: 387/387 Fail: 0/387 Skip: 0/387
 ## Ethereum Foundation - Altair - Epoch Processing - Effective balance updates [Preset: mainnet]
 ```diff
 + Effective balance updates - effective_balance_hysteresis [Preset: mainnet]                 OK
@@ -773,4 +775,4 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
 OK: 27/27 Fail: 0/27 Skip: 0/27
 
 ---TOTAL---
-OK: 639/639 Fail: 0/639 Skip: 0/639
+OK: 641/641 Fail: 0/641 Skip: 0/641

FixtureAll-minimal.md

@@ -383,8 +383,10 @@ FixtureAll-minimal
 + [Valid]   sync_committee_with_participating_exited_member                                  OK
 + [Valid]   sync_committee_with_participating_withdrawable_member                            OK
 + [Valid]   valid_signature_future_committee                                                 OK
++ finality_root_merkle_proof                                                                 OK
++ next_sync_committee_merkle_proof                                                           OK
 ```
-OK: 381/381 Fail: 0/381 Skip: 0/381
+OK: 383/383 Fail: 0/383 Skip: 0/383
 ## Ethereum Foundation - Altair - Epoch Processing - Effective balance updates [Preset: minimal]
 ```diff
 + Effective balance updates - effective_balance_hysteresis [Preset: minimal]                 OK
@@ -803,4 +805,4 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
 OK: 27/27 Fail: 0/27 Skip: 0/27
 
 ---TOTAL---
-OK: 661/661 Fail: 0/661 Skip: 0/661
+OK: 663/663 Fail: 0/663 Skip: 0/663

FixtureSSZConsensus-mainnet.md

@@ -381,14 +381,16 @@ FixtureSSZConsensus-mainnet
 + altair_fork_random_low_balances                                                            OK
 + altair_fork_random_misc_balances                                                           OK
 + altair_fork_random_mismatched_attestations                                                 OK
++ finality_root_merkle_proof                                                                 OK
 + fork_base_state                                                                            OK
 + fork_many_next_epoch                                                                       OK
 + fork_next_epoch                                                                            OK
 + fork_next_epoch_with_block                                                                 OK
 + fork_random_low_balances                                                                   OK
 + fork_random_misc_balances                                                                  OK
++ next_sync_committee_merkle_proof                                                           OK
 ```
-OK: 385/385 Fail: 0/385 Skip: 0/385
+OK: 387/387 Fail: 0/387 Skip: 0/387
 ## Ethereum Foundation - Altair - Epoch Processing - Effective balance updates [Preset: mainnet]
 ```diff
 + Effective balance updates - effective_balance_hysteresis [Preset: mainnet]                 OK
@@ -673,4 +675,4 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
 OK: 27/27 Fail: 0/27 Skip: 0/27
 
 ---TOTAL---
-OK: 579/579 Fail: 0/579 Skip: 0/579
+OK: 581/581 Fail: 0/581 Skip: 0/581

FixtureSSZConsensus-minimal.md

@@ -383,8 +383,10 @@ FixtureSSZConsensus-minimal
 + [Valid]   sync_committee_with_participating_exited_member                                  OK
 + [Valid]   sync_committee_with_participating_withdrawable_member                            OK
 + [Valid]   valid_signature_future_committee                                                 OK
++ finality_root_merkle_proof                                                                 OK
++ next_sync_committee_merkle_proof                                                           OK
 ```
-OK: 381/381 Fail: 0/381 Skip: 0/381
+OK: 383/383 Fail: 0/383 Skip: 0/383
 ## Ethereum Foundation - Altair - Epoch Processing - Effective balance updates [Preset: minimal]
 ```diff
 + Effective balance updates - effective_balance_hysteresis [Preset: minimal]                 OK
@@ -688,4 +690,4 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
 OK: 27/27 Fail: 0/27 Skip: 0/27
 
 ---TOTAL---
-OK: 590/590 Fail: 0/590 Skip: 0/590
+OK: 592/592 Fail: 0/592 Skip: 0/592

beacon_chain/spec/helpers.nim

@@ -67,6 +67,59 @@ func is_valid_merkle_branch*(leaf: Eth2Digest, branch: openArray[Eth2Digest],
     value = eth2digest(buf)
   value == root
 
+# https://github.com/ethereum/consensus-specs/blob/v1.1.0-beta.4/tests/core/pyspec/eth2spec/test/helpers/merkle.py#L4-L21
+func build_proof_impl(anchor: object, leaf_index: uint64, 
+                      proof: var openArray[Eth2Digest]) =
+  let
+    bottom_length = nextPow2(typeof(anchor).totalSerializedFields.uint64)
+    tree_depth = log2trunc(bottom_length)
+    parent_index = 
+      if leaf_index < bottom_length shl 1:
+        0'u64
+      else:
+        var i = leaf_index
+        while i >= bottom_length shl 1:
+          i = i shr 1
+        i
+
+  var 
+    prefix_len = 0
+    proof_len = log2trunc(leaf_index)
+    cache = newSeq[Eth2Digest](bottom_length shl 1)
+  block:
+    var i = bottom_length
+    anchor.enumInstanceSerializedFields(fieldNameVar, fieldVar):
+      if i == parent_index:
+        when fieldVar is object:
+          prefix_len = log2trunc(leaf_index) - tree_depth
+          proof_len -= prefix_len
+          let 
+            bottom_bits = leaf_index and not (uint64.high shl prefix_len)
+            prefix_leaf_index = (1'u64 shl prefix_len) + bottom_bits
+          build_proof_impl(fieldVar, prefix_leaf_index, proof)
+        else: raiseAssert "Invalid leaf_index"
+      cache[i] = hash_tree_root(fieldVar)
+      i += 1
+    for i in countdown(bottom_length - 1, 1):
+      cache[i] = withEth2Hash:
+        h.update cache[i shl 1].data
+        h.update cache[i shl 1 + 1].data
+
+  var i = if parent_index != 0: parent_index
+          else: leaf_index
+  doAssert i > 0 and i < bottom_length shl 1
+  for proof_index in prefix_len ..< prefix_len + proof_len:
+    let b = (i and 1) != 0
+    i = i shr 1
+    proof[proof_index] = if b: cache[i shl 1]
+                         else: cache[i shl 1 + 1]
+
+func build_proof*(anchor: object, leaf_index: uint64, 
+                  proof: var openArray[Eth2Digest]) =
+  doAssert leaf_index > 0
+  doAssert proof.len == log2trunc(leaf_index)
+  build_proof_impl(anchor, leaf_index, proof)
+  
 const SLOTS_PER_SYNC_COMMITTEE_PERIOD* =
   EPOCHS_PER_SYNC_COMMITTEE_PERIOD * SLOTS_PER_EPOCH
 

tests/official/altair/all_altair_fixtures_require_ssz.nim

@@ -12,6 +12,7 @@
 {.used.}
 
 import
+  ./test_fixture_merkle_single_proof,
   ./test_fixture_ssz_consensus_objects,
   ./test_fixture_sanity_slots,
   ./test_fixture_sanity_blocks,

tests/official/altair/test_fixture_merkle_single_proof.nim

@@ -0,0 +1,68 @@
+# beacon_chain
+# Copyright (c) 2021 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.used.}
+
+import 
+  # Standard library
+  std/[os, sequtils, streams],
+  # Status libraries
+  stew/bitops2,
+  # Third-party
+  yaml,
+  # Beacon chain internals
+  ../../../beacon_chain/spec/datatypes/altair,
+  ../../../beacon_chain/spec/helpers,
+  # Test utilities
+  ../../testutil,
+  ../fixtures_utils
+
+const TestsDir = 
+  SszTestsDir/const_preset/"altair"/"merkle"/"single_proof"/"pyspec_tests"
+
+proc runTest(identifier: string) =
+  # We wrap the tests in a proc to avoid running out of globals
+  # in the future: Nim supports up to 3500 globals
+  # but unittest with the macro/templates put everything as globals
+  # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
+
+  let testDir = TestsDir / identifier
+
+  proc `testImpl _ merkle_single_proof _ identifier`() =
+    test identifier:
+      type
+        TestProof = object
+          leaf: string
+          leaf_index: GeneralizedIndex
+          branch: seq[string]
+
+      let 
+        proof = block:
+          var s = openFileStream(testDir/"proof.yaml")
+          defer: close(s)
+          var res: TestProof
+          yaml.load(s, res)
+          res
+
+        state = newClone(parseTest(testDir/"state.ssz_snappy", SSZ,
+                                   altair.BeaconState))
+
+      var computedProof = newSeq[Eth2Digest](log2trunc(proof.leaf_index))
+      build_proof(state[], proof.leaf_index, computedProof)
+
+      check:
+        computedProof == proof.branch.mapIt(Eth2Digest.fromHex(it))
+        is_valid_merkle_branch(Eth2Digest.fromHex(proof.leaf), computedProof, 
+                               log2trunc(proof.leaf_index),
+                               get_subtree_index(proof.leaf_index),
+                               hash_tree_root(state[]))
+
+  `testImpl _ merkle_single_proof _ identifier`()
+
+suite "Ethereum Foundation - Altair - Merkle - Single proof" & preset():    
+  for kind, path in walkDir(TestsDir, relative = true, checkDir = true):
+    runTest(path)

tests/test_helpers.nim

@@ -8,8 +8,12 @@
 {.used.}
 
 import
-  ./unittest2,
+  # Status libraries
-  ../beacon_chain/spec/[helpers]
+  stew/bitops2,
+  # Beacon chain internals
+  ../beacon_chain/spec/[helpers, state_transition],
+  # Test utilities
+  ./unittest2, mocking/mock_genesis
 
 suite "Spec helpers":
   test "integer_squareroot":
@@ -20,3 +24,34 @@ suite "Spec helpers":
       integer_squareroot(3'u64) == 1'u64
       integer_squareroot(4'u64) == 2'u64
       integer_squareroot(5'u64) == 2'u64
+
+  test "build_proof - BeaconState":
+    var
+      forked = newClone(initGenesisState())
+      cache = StateCache()
+      rewards = RewardInfo()
+    doAssert process_slots(defaultRuntimeConfig, forked[], 
+                           Slot(100), cache, rewards, flags = {})
+
+    let
+      state = forked[].hbsPhase0.data
+      root = state.hash_tree_root()
+
+    func numLeaves(obj: object): GeneralizedIndex =
+      nextPow2(typeof(obj).totalSerializedFields.uint64).GeneralizedIndex
+
+    proc process(anchor: object, index: GeneralizedIndex) =
+      var i = index
+      anchor.enumInstanceSerializedFields(fieldNameVar, fieldVar):
+        let depth = log2trunc(i)
+        var proof = newSeq[Eth2Digest](depth)
+        build_proof(state, i, proof)
+        check: is_valid_merkle_branch(hash_tree_root(fieldVar), proof, 
+                                      depth, get_subtree_index(i), root)
+        when fieldVar is object and not (fieldVar is Eth2Digest):
+          let 
+            numChildLeaves = fieldVar.numLeaves
+            childDepth = log2trunc(numChildLeaves)
+          process(fieldVar, i shl childDepth)
+        i += 1
+    process(state, state.numLeaves)