support merkle multiproof verification (#2980)

This introduces spec-compatible support to verify merkle multiproofs for
use, e.g., in light client syncing.

AllTests-mainnet.md

@@ -287,9 +287,13 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
 ## Spec helpers
 ```diff
 + build_proof - BeaconState                                                                  OK
++ get_branch_indices                                                                         OK
++ get_helper_indices                                                                         OK
++ get_path_indices                                                                           OK
 + integer_squareroot                                                                         OK
++ verify_merkle_multiproof                                                                   OK
 ```
-OK: 2/2 Fail: 0/2 Skip: 0/2
+OK: 6/6 Fail: 0/6 Skip: 0/6
 ## Specific field types
 ```diff
 + root update                                                                                OK
@@ -443,4 +447,4 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
 OK: 42/56 Fail: 0/56 Skip: 14/56
 
 ---TOTAL---
-OK: 253/269 Fail: 0/269 Skip: 16/269
+OK: 257/273 Fail: 0/273 Skip: 16/273

FixtureSSZGeneric-mainnet.md

@@ -231,9 +231,13 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
 ## Spec helpers
 ```diff
 + build_proof - BeaconState                                                                  OK
++ get_branch_indices                                                                         OK
++ get_helper_indices                                                                         OK
++ get_path_indices                                                                           OK
 + integer_squareroot                                                                         OK
++ verify_merkle_multiproof                                                                   OK
 ```
-OK: 2/2 Fail: 0/2 Skip: 0/2
+OK: 6/6 Fail: 0/6 Skip: 0/6
 ## Specific field types
 ```diff
 + root update                                                                                OK
@@ -360,4 +364,4 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
 OK: 42/56 Fail: 0/56 Skip: 14/56
 
 ---TOTAL---
-OK: 198/214 Fail: 0/214 Skip: 16/214
+OK: 202/218 Fail: 0/218 Skip: 16/218

beacon_chain/spec/helpers.nim

@@ -11,9 +11,10 @@
 
 import
   # Standard lib
-  std/[math, tables],
+  std/[algorithm, intsets, math, sequtils, tables],
-  # Third-party
+  # Status libraries
   stew/[byteutils, endians2, bitops2],
+  chronicles,
   # Internal
   ./datatypes/[phase0, altair, merge],
   ./eth2_merkleization, ./ssz_codec
@@ -47,6 +48,237 @@ template epoch*(slot: Slot): Epoch =
 template isEpoch*(slot: Slot): bool =
   (slot mod SLOTS_PER_EPOCH) == 0
 
+# https://github.com/ethereum/consensus-specs/blob/v1.1.2/ssz/merkle-proofs.md#generalized_index_sibling
+template generalized_index_sibling*(
+    index: GeneralizedIndex): GeneralizedIndex =
+  index xor 1.GeneralizedIndex
+
+template generalized_index_sibling_left(
+    index: GeneralizedIndex): GeneralizedIndex =
+  index and not 1.GeneralizedIndex
+
+template generalized_index_sibling_right(
+    index: GeneralizedIndex): GeneralizedIndex =
+  index or 1.GeneralizedIndex
+
+# https://github.com/ethereum/consensus-specs/blob/v1.1.2/ssz/merkle-proofs.md#generalized_index_parent
+template generalized_index_parent*(
+    index: GeneralizedIndex): GeneralizedIndex =
+  index shr 1
+
+# https://github.com/ethereum/consensus-specs/blob/v1.1.2/ssz/merkle-proofs.md#merkle-multiproofs
+iterator get_branch_indices*(
+    tree_index: GeneralizedIndex): GeneralizedIndex =
+  ## Get the generalized indices of the sister chunks along the path
+  ## from the chunk with the given tree index to the root.
+  var index = tree_index
+  while index > 1.GeneralizedIndex:
+    yield generalized_index_sibling(index)
+    index = generalized_index_parent(index)
+
+# https://github.com/ethereum/consensus-specs/blob/v1.1.2/ssz/merkle-proofs.md#merkle-multiproofs
+iterator get_path_indices*(
+    tree_index: GeneralizedIndex): GeneralizedIndex =
+  ## Get the generalized indices of the chunks along the path
+  ## from the chunk with the given tree index to the root.
+  var index = tree_index
+  while index > 1.GeneralizedIndex:
+    yield index
+    index = generalized_index_parent(index)
+
+# https://github.com/ethereum/consensus-specs/blob/v1.1.2/ssz/merkle-proofs.md#merkle-multiproofs
+func get_helper_indices*(
+    indices: openArray[GeneralizedIndex]): seq[GeneralizedIndex] =
+  ## Get the generalized indices of all "extra" chunks in the tree needed
+  ## to prove the chunks with the given generalized indices. Note that the
+  ## decreasing order is chosen deliberately to ensure equivalence to the order
+  ## of hashes in a regular single-item Merkle proof in the single-item case.
+  var all_helper_indices = initIntSet()
+  var all_path_indices = initIntSet()
+  for index in indices:
+    for idx in get_branch_indices(index):
+      all_helper_indices.incl idx.int
+    for idx in get_path_indices(index):
+      all_path_indices.incl idx.int
+  all_helper_indices.excl all_path_indices
+
+  result = newSeqOfCap[GeneralizedIndex](all_helper_indices.len)
+  for idx in all_helper_indices:
+    result.add idx.GeneralizedIndex
+  result.sort(SortOrder.Descending)
+
+# https://github.com/ethereum/consensus-specs/blob/v1.1.2/ssz/merkle-proofs.md#merkle-multiproofs
+func check_multiproof_acceptable*(
+    indices: openArray[GeneralizedIndex]): Result[void, string] =
+  # Check that proof verification won't allocate excessive amounts of memory.
+  const max_multiproof_complexity = nextPowerOfTwo(256)
+  if indices.len > max_multiproof_complexity:
+    trace "Max multiproof complexity exceeded",
+      num_indices=indices.len, max_multiproof_complexity
+    return err("Unsupported multiproof complexity (" & $indices.len & ")")
+
+  if indices.len == 0:
+    return err("No indices specified")
+  if indices.anyIt(it == 0.GeneralizedIndex):
+    return err("Invalid index specified")
+  ok()
+
+func calculate_multi_merkle_root_impl(
+    leaves: openArray[Eth2Digest],
+    proof: openArray[Eth2Digest],
+    indices: openArray[GeneralizedIndex],
+    helper_indices: openArray[GeneralizedIndex]): Result[Eth2Digest, string] =
+  # All callers have already verified the checks in check_multiproof_acceptable,
+  # as well as whether lengths of leaves/indices and proof/helper_indices match.
+
+  # Helper to retrieve a value from a table that is statically known to exist.
+  template getExisting[A, B](t: var Table[A, B], key: A): var B =
+    try: t[key]
+    except KeyError: raiseAssert "Unreachable"
+
+  # Populate data structure with all leaves.
+  # This data structure only scales with the number of `leaves`,
+  # in contrast to the spec one that also scales with the number of `proof`
+  # items and the number of all intermediate roots, potentially the entire tree.
+  let capacity = nextPowerOfTwo(leaves.len)
+  var objects = initTable[GeneralizedIndex, Eth2Digest](capacity)
+  for i, index in indices:
+    if objects.mgetOrPut(index, leaves[i]) != leaves[i]:
+      return err("Conflicting roots for same index")
+
+  # Create list with keys of all active nodes that need to be visited.
+  # This list is sorted in descending order, same as `helper_indices`.
+  # Pulling from `objects` instead of from `indices` deduplicates the list.
+  var keys = newSeqOfCap[GeneralizedIndex](objects.len)
+  for index in objects.keys:
+    if index > 1.GeneralizedIndex: # For the root, no work needs to be done.
+      keys.add index
+  keys.sort(SortOrder.Descending)
+
+  # The merkle tree is processed from bottom to top, pulling in helper
+  # indices from `proof` as needed. During processing, the `keys` list
+  # may temporarily end up being split into two parts, sorted individually.
+  # An additional index tracks the current maximum element of the list.
+  var
+    completed = 0         # All key indices before this are fully processed.
+    maxIndex = completed  # Index of the list's largest key.
+    helper = 0            # Helper index from `proof` to be pulled next.
+
+  # Processing is done when there are no more keys to process.
+  while completed < keys.len:
+    let
+      k = keys[maxIndex]
+      sibling = generalized_index_sibling(k)
+      left = generalized_index_sibling_left(k)
+      right = generalized_index_sibling_right(k)
+      parent = generalized_index_parent(k)
+      parentRight = generalized_index_sibling_right(parent)
+
+    # Keys need to be processed in descending order to ensure that intermediate
+    # roots remain available until they are no longer needed. This ensures that
+    # conflicting roots are detected in all cases.
+    keys[maxIndex] =
+      if not objects.hasKey(k):
+        # A previous computation did already merge this key with its sibling.
+        0.GeneralizedIndex
+      else:
+        # Compute expected root for parent. This deletes child roots.
+        # Because the list is sorted in descending order, they are not needed.
+        let root = withEth2Hash:
+          if helper < helper_indices.len and helper_indices[helper] == sibling:
+            # The next proof item is required to form the parent hash.
+            if sibling == left:
+              h.update proof[helper].data
+              h.update objects.getExisting(right).data; objects.del right
+            else:
+              h.update objects.getExisting(left).data;  objects.del left
+              h.update proof[helper].data
+            inc helper
+          else:
+            # Both siblings are already known.
+            h.update objects.getExisting(left).data;  objects.del left
+            h.update objects.getExisting(right).data; objects.del right
+
+        # Store parent root, and replace the current list entry with its parent.
+        if objects.hasKeyOrPut(parent, root):
+          if objects.getExisting(parent) != root:
+            return err("Conflicting roots for same index")
+          0.GeneralizedIndex
+        elif parent > 1.GeneralizedIndex:
+          # Note that the list may contain further nodes that are on a layer
+          # beneath the parent, so this may break the strictly descending order
+          # of the list. For example, given [12, 9], this will lead to [6, 9].
+          # This will resolve itself after the additional nodes are processed,
+          # i.e., [6, 9] -> [6, 4] -> [3, 4] -> [3, 2] -> [1].
+          parent
+        else:
+          0.GeneralizedIndex
+    if keys[maxIndex] != 0.GeneralizedIndex:
+      # The list may have been temporarily split up into two parts that are
+      # individually sorted in descending order. Have to first process further
+      # nodes until the list is sorted once more.
+      inc maxIndex
+
+    # Determine whether descending sort order has been restored.
+    let isSorted =
+      if maxIndex == completed: true
+      else:
+        while maxIndex < keys.len and keys[maxIndex] == 0.GeneralizedIndex:
+          inc maxIndex
+        maxIndex >= keys.len or keys[maxIndex] <= parentRight
+    if isSorted:
+      # List is sorted once more. Reset `maxIndex` to its start.
+      while completed < keys.len and keys[completed] == 0.GeneralizedIndex:
+        inc completed
+      maxIndex = completed
+
+  # Proof is guaranteed to provide all info needed to reach the root.
+  doAssert helper == helper_indices.len
+  doAssert objects.len == 1
+  ok(objects.getExisting(1.GeneralizedIndex))
+
+func calculate_multi_merkle_root*(
+    leaves: openArray[Eth2Digest],
+    proof: openArray[Eth2Digest],
+    indices: openArray[GeneralizedIndex],
+    helper_indices: openArray[GeneralizedIndex]): Result[Eth2Digest, string] =
+  doAssert proof.len == helper_indices.len
+  if leaves.len != indices.len:
+    return err("Length mismatch for leaves and indices")
+  ? check_multiproof_acceptable(indices)
+  calculate_multi_merkle_root_impl(
+    leaves, proof, indices, helper_indices)
+
+func calculate_multi_merkle_root*(
+    leaves: openArray[Eth2Digest],
+    proof: openArray[Eth2Digest],
+    indices: openArray[GeneralizedIndex]): Result[Eth2Digest, string] =
+  if leaves.len != indices.len:
+    return err("Length mismatch for leaves and indices")
+  ? check_multiproof_acceptable(indices)
+  calculate_multi_merkle_root_impl(
+    leaves, proof, indices, get_helper_indices(indices))
+
+# https://github.com/ethereum/consensus-specs/blob/v1.1.2/ssz/merkle-proofs.md#merkle-multiproofs
+func verify_merkle_multiproof*(
+    leaves: openArray[Eth2Digest],
+    proof: openArray[Eth2Digest],
+    indices: openArray[GeneralizedIndex],
+    helper_indices: openArray[GeneralizedIndex],
+    root: Eth2Digest): bool =
+  let calc = calculate_multi_merkle_root(leaves, proof, indices, helper_indices)
+  if calc.isErr: return false
+  calc.get == root
+
+func verify_merkle_multiproof*(
+    leaves: openArray[Eth2Digest],
+    proof: openArray[Eth2Digest],
+    indices: openArray[GeneralizedIndex],
+    root: Eth2Digest): bool =
+  let calc = calculate_multi_merkle_root(leaves, proof, indices)
+  if calc.isErr: return false
+  calc.get == root
+
 # https://github.com/ethereum/consensus-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#is_valid_merkle_branch
 func is_valid_merkle_branch*(leaf: Eth2Digest, branch: openArray[Eth2Digest],
                              depth: int, index: uint64,

tests/test_helpers.nim

@@ -1,5 +1,5 @@
 # beacon_chain
-# Copyright (c) 2018 Status Research & Development GmbH
+# Copyright (c) 2018, 2021 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -8,6 +8,8 @@
 {.used.}
 
 import
+  # Standard library
+  sequtils,
   # Status libraries
   stew/bitops2,
   # Beacon chain internals
@@ -55,3 +57,73 @@ suite "Spec helpers":
           process(fieldVar, i shl childDepth)
         i += 1
     process(state, state.numLeaves)
+
+  test "get_branch_indices":
+    check:
+      toSeq(get_branch_indices(1.GeneralizedIndex)) == []
+      toSeq(get_branch_indices(0b101010.GeneralizedIndex)) ==
+        [
+          0b101011.GeneralizedIndex,
+          0b10100.GeneralizedIndex,
+          0b1011.GeneralizedIndex,
+          0b100.GeneralizedIndex,
+          0b11.GeneralizedIndex
+        ]
+
+  test "get_path_indices":
+    check:
+      toSeq(get_path_indices(1.GeneralizedIndex)) == []
+      toSeq(get_path_indices(0b101010.GeneralizedIndex)) ==
+        [
+          0b101010.GeneralizedIndex,
+          0b10101.GeneralizedIndex,
+          0b1010.GeneralizedIndex,
+          0b101.GeneralizedIndex,
+          0b10.GeneralizedIndex
+        ]
+
+  test "get_helper_indices":
+    check:
+      get_helper_indices(
+        [
+          8.GeneralizedIndex,
+          9.GeneralizedIndex,
+          14.GeneralizedIndex]) ==
+        [
+          15.GeneralizedIndex,
+          6.GeneralizedIndex,
+          5.GeneralizedIndex
+        ]
+
+  test "verify_merkle_multiproof":
+    var nodes: array[16, Eth2Digest]
+    for i in countdown(15, 8):
+      nodes[i] = eth2digest([i.byte])
+    for i in countdown(7, 1):
+      nodes[i] = withEth2Hash:
+        h.update nodes[2 * i + 0].data
+        h.update nodes[2 * i + 1].data
+
+    proc verify(indices_int: openArray[int]) =
+      let
+        indices = indices_int.mapIt(it.GeneralizedIndex)
+        helper_indices = get_helper_indices(indices)
+        leaves = indices.mapIt(nodes[it])
+        proof = helper_indices.mapIt(nodes[it])
+        root = nodes[1]
+      checkpoint "Verifying " & $indices & "---" & $helper_indices
+      check:
+        verify_merkle_multiproof(leaves, proof, indices, root)
+
+    verify([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15])
+
+    for a in 1 .. 15:
+      verify([a])
+      for b in 1 .. 15:
+        verify([a, b])
+        for c in 1 .. 15:
+          verify([a, b, c])
+          for d in 8 .. 15:
+            verify([a, b, c, d])
+            for e in 1 .. 7:
+              verify([a, b, c, d, e])