From 05908859b71ab738eecbd2dfc73353fd469c7486 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Thu, 14 Oct 2021 10:28:22 +0200 Subject: [PATCH] support merkle multiproof verification (#2980) This introduces spec-compatible support to verify merkle multiproofs for use, e.g., in light client syncing. --- AllTests-mainnet.md | 8 +- FixtureSSZGeneric-mainnet.md | 8 +- beacon_chain/spec/helpers.nim | 248 ++++++++++++++++++++++++++++++++-- tests/test_helpers.nim | 74 +++++++++- 4 files changed, 325 insertions(+), 13 deletions(-) diff --git a/AllTests-mainnet.md b/AllTests-mainnet.md index e5728bd7d..3c62d97e7 100644 --- a/AllTests-mainnet.md +++ b/AllTests-mainnet.md @@ -287,9 +287,13 @@ OK: 1/1 Fail: 0/1 Skip: 0/1 ## Spec helpers ```diff + build_proof - BeaconState OK ++ get_branch_indices OK ++ get_helper_indices OK ++ get_path_indices OK + integer_squareroot OK ++ verify_merkle_multiproof OK ``` -OK: 2/2 Fail: 0/2 Skip: 0/2 +OK: 6/6 Fail: 0/6 Skip: 0/6 ## Specific field types ```diff + root update OK @@ -443,4 +447,4 @@ OK: 1/1 Fail: 0/1 Skip: 0/1 OK: 42/56 Fail: 0/56 Skip: 14/56 ---TOTAL--- -OK: 253/269 Fail: 0/269 Skip: 16/269 +OK: 257/273 Fail: 0/273 Skip: 16/273 diff --git a/FixtureSSZGeneric-mainnet.md b/FixtureSSZGeneric-mainnet.md index d00c4d834..30060e35f 100644 --- a/FixtureSSZGeneric-mainnet.md +++ b/FixtureSSZGeneric-mainnet.md @@ -231,9 +231,13 @@ OK: 1/1 Fail: 0/1 Skip: 0/1 ## Spec helpers ```diff + build_proof - BeaconState OK ++ get_branch_indices OK ++ get_helper_indices OK ++ get_path_indices OK + integer_squareroot OK ++ verify_merkle_multiproof OK ``` -OK: 2/2 Fail: 0/2 Skip: 0/2 +OK: 6/6 Fail: 0/6 Skip: 0/6 ## Specific field types ```diff + root update OK @@ -360,4 +364,4 @@ OK: 1/1 Fail: 0/1 Skip: 0/1 OK: 42/56 Fail: 0/56 Skip: 14/56 ---TOTAL--- -OK: 198/214 Fail: 0/214 Skip: 16/214 +OK: 202/218 Fail: 0/218 Skip: 16/218 diff --git a/beacon_chain/spec/helpers.nim b/beacon_chain/spec/helpers.nim index 980a145ac..d6912850c 100644 --- a/beacon_chain/spec/helpers.nim +++ b/beacon_chain/spec/helpers.nim @@ -11,9 +11,10 @@ import # Standard lib - std/[math, tables], - # Third-party + std/[algorithm, intsets, math, sequtils, tables], + # Status libraries stew/[byteutils, endians2, bitops2], + chronicles, # Internal ./datatypes/[phase0, altair, merge], ./eth2_merkleization, ./ssz_codec @@ -47,6 +48,237 @@ template epoch*(slot: Slot): Epoch = template isEpoch*(slot: Slot): bool = (slot mod SLOTS_PER_EPOCH) == 0 +# https://github.com/ethereum/consensus-specs/blob/v1.1.2/ssz/merkle-proofs.md#generalized_index_sibling +template generalized_index_sibling*( + index: GeneralizedIndex): GeneralizedIndex = + index xor 1.GeneralizedIndex + +template generalized_index_sibling_left( + index: GeneralizedIndex): GeneralizedIndex = + index and not 1.GeneralizedIndex + +template generalized_index_sibling_right( + index: GeneralizedIndex): GeneralizedIndex = + index or 1.GeneralizedIndex + +# https://github.com/ethereum/consensus-specs/blob/v1.1.2/ssz/merkle-proofs.md#generalized_index_parent +template generalized_index_parent*( + index: GeneralizedIndex): GeneralizedIndex = + index shr 1 + +# https://github.com/ethereum/consensus-specs/blob/v1.1.2/ssz/merkle-proofs.md#merkle-multiproofs +iterator get_branch_indices*( + tree_index: GeneralizedIndex): GeneralizedIndex = + ## Get the generalized indices of the sister chunks along the path + ## from the chunk with the given tree index to the root. + var index = tree_index + while index > 1.GeneralizedIndex: + yield generalized_index_sibling(index) + index = generalized_index_parent(index) + +# https://github.com/ethereum/consensus-specs/blob/v1.1.2/ssz/merkle-proofs.md#merkle-multiproofs +iterator get_path_indices*( + tree_index: GeneralizedIndex): GeneralizedIndex = + ## Get the generalized indices of the chunks along the path + ## from the chunk with the given tree index to the root. + var index = tree_index + while index > 1.GeneralizedIndex: + yield index + index = generalized_index_parent(index) + +# https://github.com/ethereum/consensus-specs/blob/v1.1.2/ssz/merkle-proofs.md#merkle-multiproofs +func get_helper_indices*( + indices: openArray[GeneralizedIndex]): seq[GeneralizedIndex] = + ## Get the generalized indices of all "extra" chunks in the tree needed + ## to prove the chunks with the given generalized indices. Note that the + ## decreasing order is chosen deliberately to ensure equivalence to the order + ## of hashes in a regular single-item Merkle proof in the single-item case. + var all_helper_indices = initIntSet() + var all_path_indices = initIntSet() + for index in indices: + for idx in get_branch_indices(index): + all_helper_indices.incl idx.int + for idx in get_path_indices(index): + all_path_indices.incl idx.int + all_helper_indices.excl all_path_indices + + result = newSeqOfCap[GeneralizedIndex](all_helper_indices.len) + for idx in all_helper_indices: + result.add idx.GeneralizedIndex + result.sort(SortOrder.Descending) + +# https://github.com/ethereum/consensus-specs/blob/v1.1.2/ssz/merkle-proofs.md#merkle-multiproofs +func check_multiproof_acceptable*( + indices: openArray[GeneralizedIndex]): Result[void, string] = + # Check that proof verification won't allocate excessive amounts of memory. + const max_multiproof_complexity = nextPowerOfTwo(256) + if indices.len > max_multiproof_complexity: + trace "Max multiproof complexity exceeded", + num_indices=indices.len, max_multiproof_complexity + return err("Unsupported multiproof complexity (" & $indices.len & ")") + + if indices.len == 0: + return err("No indices specified") + if indices.anyIt(it == 0.GeneralizedIndex): + return err("Invalid index specified") + ok() + +func calculate_multi_merkle_root_impl( + leaves: openArray[Eth2Digest], + proof: openArray[Eth2Digest], + indices: openArray[GeneralizedIndex], + helper_indices: openArray[GeneralizedIndex]): Result[Eth2Digest, string] = + # All callers have already verified the checks in check_multiproof_acceptable, + # as well as whether lengths of leaves/indices and proof/helper_indices match. + + # Helper to retrieve a value from a table that is statically known to exist. + template getExisting[A, B](t: var Table[A, B], key: A): var B = + try: t[key] + except KeyError: raiseAssert "Unreachable" + + # Populate data structure with all leaves. + # This data structure only scales with the number of `leaves`, + # in contrast to the spec one that also scales with the number of `proof` + # items and the number of all intermediate roots, potentially the entire tree. + let capacity = nextPowerOfTwo(leaves.len) + var objects = initTable[GeneralizedIndex, Eth2Digest](capacity) + for i, index in indices: + if objects.mgetOrPut(index, leaves[i]) != leaves[i]: + return err("Conflicting roots for same index") + + # Create list with keys of all active nodes that need to be visited. + # This list is sorted in descending order, same as `helper_indices`. + # Pulling from `objects` instead of from `indices` deduplicates the list. + var keys = newSeqOfCap[GeneralizedIndex](objects.len) + for index in objects.keys: + if index > 1.GeneralizedIndex: # For the root, no work needs to be done. + keys.add index + keys.sort(SortOrder.Descending) + + # The merkle tree is processed from bottom to top, pulling in helper + # indices from `proof` as needed. During processing, the `keys` list + # may temporarily end up being split into two parts, sorted individually. + # An additional index tracks the current maximum element of the list. + var + completed = 0 # All key indices before this are fully processed. + maxIndex = completed # Index of the list's largest key. + helper = 0 # Helper index from `proof` to be pulled next. + + # Processing is done when there are no more keys to process. + while completed < keys.len: + let + k = keys[maxIndex] + sibling = generalized_index_sibling(k) + left = generalized_index_sibling_left(k) + right = generalized_index_sibling_right(k) + parent = generalized_index_parent(k) + parentRight = generalized_index_sibling_right(parent) + + # Keys need to be processed in descending order to ensure that intermediate + # roots remain available until they are no longer needed. This ensures that + # conflicting roots are detected in all cases. + keys[maxIndex] = + if not objects.hasKey(k): + # A previous computation did already merge this key with its sibling. + 0.GeneralizedIndex + else: + # Compute expected root for parent. This deletes child roots. + # Because the list is sorted in descending order, they are not needed. + let root = withEth2Hash: + if helper < helper_indices.len and helper_indices[helper] == sibling: + # The next proof item is required to form the parent hash. + if sibling == left: + h.update proof[helper].data + h.update objects.getExisting(right).data; objects.del right + else: + h.update objects.getExisting(left).data; objects.del left + h.update proof[helper].data + inc helper + else: + # Both siblings are already known. + h.update objects.getExisting(left).data; objects.del left + h.update objects.getExisting(right).data; objects.del right + + # Store parent root, and replace the current list entry with its parent. + if objects.hasKeyOrPut(parent, root): + if objects.getExisting(parent) != root: + return err("Conflicting roots for same index") + 0.GeneralizedIndex + elif parent > 1.GeneralizedIndex: + # Note that the list may contain further nodes that are on a layer + # beneath the parent, so this may break the strictly descending order + # of the list. For example, given [12, 9], this will lead to [6, 9]. + # This will resolve itself after the additional nodes are processed, + # i.e., [6, 9] -> [6, 4] -> [3, 4] -> [3, 2] -> [1]. + parent + else: + 0.GeneralizedIndex + if keys[maxIndex] != 0.GeneralizedIndex: + # The list may have been temporarily split up into two parts that are + # individually sorted in descending order. Have to first process further + # nodes until the list is sorted once more. + inc maxIndex + + # Determine whether descending sort order has been restored. + let isSorted = + if maxIndex == completed: true + else: + while maxIndex < keys.len and keys[maxIndex] == 0.GeneralizedIndex: + inc maxIndex + maxIndex >= keys.len or keys[maxIndex] <= parentRight + if isSorted: + # List is sorted once more. Reset `maxIndex` to its start. + while completed < keys.len and keys[completed] == 0.GeneralizedIndex: + inc completed + maxIndex = completed + + # Proof is guaranteed to provide all info needed to reach the root. + doAssert helper == helper_indices.len + doAssert objects.len == 1 + ok(objects.getExisting(1.GeneralizedIndex)) + +func calculate_multi_merkle_root*( + leaves: openArray[Eth2Digest], + proof: openArray[Eth2Digest], + indices: openArray[GeneralizedIndex], + helper_indices: openArray[GeneralizedIndex]): Result[Eth2Digest, string] = + doAssert proof.len == helper_indices.len + if leaves.len != indices.len: + return err("Length mismatch for leaves and indices") + ? check_multiproof_acceptable(indices) + calculate_multi_merkle_root_impl( + leaves, proof, indices, helper_indices) + +func calculate_multi_merkle_root*( + leaves: openArray[Eth2Digest], + proof: openArray[Eth2Digest], + indices: openArray[GeneralizedIndex]): Result[Eth2Digest, string] = + if leaves.len != indices.len: + return err("Length mismatch for leaves and indices") + ? check_multiproof_acceptable(indices) + calculate_multi_merkle_root_impl( + leaves, proof, indices, get_helper_indices(indices)) + +# https://github.com/ethereum/consensus-specs/blob/v1.1.2/ssz/merkle-proofs.md#merkle-multiproofs +func verify_merkle_multiproof*( + leaves: openArray[Eth2Digest], + proof: openArray[Eth2Digest], + indices: openArray[GeneralizedIndex], + helper_indices: openArray[GeneralizedIndex], + root: Eth2Digest): bool = + let calc = calculate_multi_merkle_root(leaves, proof, indices, helper_indices) + if calc.isErr: return false + calc.get == root + +func verify_merkle_multiproof*( + leaves: openArray[Eth2Digest], + proof: openArray[Eth2Digest], + indices: openArray[GeneralizedIndex], + root: Eth2Digest): bool = + let calc = calculate_multi_merkle_root(leaves, proof, indices) + if calc.isErr: return false + calc.get == root + # https://github.com/ethereum/consensus-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#is_valid_merkle_branch func is_valid_merkle_branch*(leaf: Eth2Digest, branch: openArray[Eth2Digest], depth: int, index: uint64, @@ -68,12 +300,12 @@ func is_valid_merkle_branch*(leaf: Eth2Digest, branch: openArray[Eth2Digest], value == root # https://github.com/ethereum/consensus-specs/blob/v1.1.0-beta.4/tests/core/pyspec/eth2spec/test/helpers/merkle.py#L4-L21 -func build_proof_impl(anchor: object, leaf_index: uint64, +func build_proof_impl(anchor: object, leaf_index: uint64, proof: var openArray[Eth2Digest]) = let bottom_length = nextPow2(typeof(anchor).totalSerializedFields.uint64) tree_depth = log2trunc(bottom_length) - parent_index = + parent_index = if leaf_index < bottom_length shl 1: 0'u64 else: @@ -82,7 +314,7 @@ func build_proof_impl(anchor: object, leaf_index: uint64, i = i shr 1 i - var + var prefix_len = 0 proof_len = log2trunc(leaf_index) cache = newSeq[Eth2Digest](bottom_length shl 1) @@ -93,7 +325,7 @@ func build_proof_impl(anchor: object, leaf_index: uint64, when fieldVar is object: prefix_len = log2trunc(leaf_index) - tree_depth proof_len -= prefix_len - let + let bottom_bits = leaf_index and not (uint64.high shl prefix_len) prefix_leaf_index = (1'u64 shl prefix_len) + bottom_bits build_proof_impl(fieldVar, prefix_leaf_index, proof) @@ -114,12 +346,12 @@ func build_proof_impl(anchor: object, leaf_index: uint64, proof[proof_index] = if b: cache[i shl 1] else: cache[i shl 1 + 1] -func build_proof*(anchor: object, leaf_index: uint64, +func build_proof*(anchor: object, leaf_index: uint64, proof: var openArray[Eth2Digest]) = doAssert leaf_index > 0 doAssert proof.len == log2trunc(leaf_index) build_proof_impl(anchor, leaf_index, proof) - + const SLOTS_PER_SYNC_COMMITTEE_PERIOD* = EPOCHS_PER_SYNC_COMMITTEE_PERIOD * SLOTS_PER_EPOCH diff --git a/tests/test_helpers.nim b/tests/test_helpers.nim index 3f032697f..db2c3febf 100644 --- a/tests/test_helpers.nim +++ b/tests/test_helpers.nim @@ -1,5 +1,5 @@ # beacon_chain -# Copyright (c) 2018 Status Research & Development GmbH +# Copyright (c) 2018, 2021 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). @@ -8,6 +8,8 @@ {.used.} import + # Standard library + sequtils, # Status libraries stew/bitops2, # Beacon chain internals @@ -55,3 +57,73 @@ suite "Spec helpers": process(fieldVar, i shl childDepth) i += 1 process(state, state.numLeaves) + + test "get_branch_indices": + check: + toSeq(get_branch_indices(1.GeneralizedIndex)) == [] + toSeq(get_branch_indices(0b101010.GeneralizedIndex)) == + [ + 0b101011.GeneralizedIndex, + 0b10100.GeneralizedIndex, + 0b1011.GeneralizedIndex, + 0b100.GeneralizedIndex, + 0b11.GeneralizedIndex + ] + + test "get_path_indices": + check: + toSeq(get_path_indices(1.GeneralizedIndex)) == [] + toSeq(get_path_indices(0b101010.GeneralizedIndex)) == + [ + 0b101010.GeneralizedIndex, + 0b10101.GeneralizedIndex, + 0b1010.GeneralizedIndex, + 0b101.GeneralizedIndex, + 0b10.GeneralizedIndex + ] + + test "get_helper_indices": + check: + get_helper_indices( + [ + 8.GeneralizedIndex, + 9.GeneralizedIndex, + 14.GeneralizedIndex]) == + [ + 15.GeneralizedIndex, + 6.GeneralizedIndex, + 5.GeneralizedIndex + ] + + test "verify_merkle_multiproof": + var nodes: array[16, Eth2Digest] + for i in countdown(15, 8): + nodes[i] = eth2digest([i.byte]) + for i in countdown(7, 1): + nodes[i] = withEth2Hash: + h.update nodes[2 * i + 0].data + h.update nodes[2 * i + 1].data + + proc verify(indices_int: openArray[int]) = + let + indices = indices_int.mapIt(it.GeneralizedIndex) + helper_indices = get_helper_indices(indices) + leaves = indices.mapIt(nodes[it]) + proof = helper_indices.mapIt(nodes[it]) + root = nodes[1] + checkpoint "Verifying " & $indices & "---" & $helper_indices + check: + verify_merkle_multiproof(leaves, proof, indices, root) + + verify([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) + + for a in 1 .. 15: + verify([a]) + for b in 1 .. 15: + verify([a, b]) + for c in 1 .. 15: + verify([a, b, c]) + for d in 8 .. 15: + verify([a, b, c, d]) + for e in 1 .. 7: + verify([a, b, c, d, e])