status-im/nimbus-eth2
2
0
Fork 0
mirror of https://github.com/status-im/nimbus-eth2.git synced 2025-02-22 11:18:25 +00:00
Code Issues Projects Releases Wiki Activity

support merkle multiproof verification (#2980)

Browse Source 
This introduces spec-compatible support to verify merkle multiproofs for
use, e.g., in light client syncing.
This commit is contained in:
Etan Kissling 2021-10-14 10:28:22 +02:00 committed by GitHub
parent 6cc8757930
commit 05908859b7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 325 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
AllTests-mainnet.md
View File

@ -287,9 +287,13 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
## Spec helpers
```diff
+ build_proof - BeaconState OK
+ get_branch_indices OK
+ get_helper_indices OK
+ get_path_indices OK
+ integer_squareroot OK
+ verify_merkle_multiproof OK
```
OK: 2/2 Fail: 0/2 Skip: 0/2
OK: 6/6 Fail: 0/6 Skip: 0/6
## Specific field types
```diff
+ root update OK
@ -443,4 +447,4 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
OK: 42/56 Fail: 0/56 Skip: 14/56
---TOTAL---
OK: 253/269 Fail: 0/269 Skip: 16/269
OK: 257/273 Fail: 0/273 Skip: 16/273

8
FixtureSSZGeneric-mainnet.md
View File

@ -231,9 +231,13 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
## Spec helpers
```diff
+ build_proof - BeaconState OK
+ get_branch_indices OK
+ get_helper_indices OK
+ get_path_indices OK
+ integer_squareroot OK
+ verify_merkle_multiproof OK
```
OK: 2/2 Fail: 0/2 Skip: 0/2
OK: 6/6 Fail: 0/6 Skip: 0/6
## Specific field types
```diff
+ root update OK
@ -360,4 +364,4 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
OK: 42/56 Fail: 0/56 Skip: 14/56
---TOTAL---
OK: 198/214 Fail: 0/214 Skip: 16/214
OK: 202/218 Fail: 0/218 Skip: 16/218

248
beacon_chain/spec/helpers.nim
View File

@ -11,9 +11,10 @@
import
# Standard lib
std/[math, tables],
# Third-party
std/[algorithm, intsets, math, sequtils, tables],
# Status libraries
stew/[byteutils, endians2, bitops2],
chronicles,
# Internal
./datatypes/[phase0, altair, merge],
./eth2_merkleization, ./ssz_codec
@ -47,6 +48,237 @@ template epoch*(slot: Slot): Epoch =
template isEpoch*(slot: Slot): bool =
(slot mod SLOTS_PER_EPOCH) == 0
# https://github.com/ethereum/consensus-specs/blob/v1.1.2/ssz/merkle-proofs.md#generalized_index_sibling
template generalized_index_sibling*(
index: GeneralizedIndex): GeneralizedIndex =
index xor 1.GeneralizedIndex
template generalized_index_sibling_left(
index: GeneralizedIndex): GeneralizedIndex =
index and not 1.GeneralizedIndex
template generalized_index_sibling_right(
index: GeneralizedIndex): GeneralizedIndex =
index or 1.GeneralizedIndex
# https://github.com/ethereum/consensus-specs/blob/v1.1.2/ssz/merkle-proofs.md#generalized_index_parent
template generalized_index_parent*(
index: GeneralizedIndex): GeneralizedIndex =
index shr 1
# https://github.com/ethereum/consensus-specs/blob/v1.1.2/ssz/merkle-proofs.md#merkle-multiproofs
iterator get_branch_indices*(
tree_index: GeneralizedIndex): GeneralizedIndex =
## Get the generalized indices of the sister chunks along the path
## from the chunk with the given tree index to the root.
var index = tree_index
while index > 1.GeneralizedIndex:
yield generalized_index_sibling(index)
index = generalized_index_parent(index)
# https://github.com/ethereum/consensus-specs/blob/v1.1.2/ssz/merkle-proofs.md#merkle-multiproofs
iterator get_path_indices*(
tree_index: GeneralizedIndex): GeneralizedIndex =
## Get the generalized indices of the chunks along the path
## from the chunk with the given tree index to the root.
var index = tree_index
while index > 1.GeneralizedIndex:
yield index
index = generalized_index_parent(index)
# https://github.com/ethereum/consensus-specs/blob/v1.1.2/ssz/merkle-proofs.md#merkle-multiproofs
func get_helper_indices*(
indices: openArray[GeneralizedIndex]): seq[GeneralizedIndex] =
## Get the generalized indices of all "extra" chunks in the tree needed
## to prove the chunks with the given generalized indices. Note that the
## decreasing order is chosen deliberately to ensure equivalence to the order
## of hashes in a regular single-item Merkle proof in the single-item case.
var all_helper_indices = initIntSet()
var all_path_indices = initIntSet()
for index in indices:
for idx in get_branch_indices(index):
all_helper_indices.incl idx.int
for idx in get_path_indices(index):
all_path_indices.incl idx.int
all_helper_indices.excl all_path_indices
result = newSeqOfCap[GeneralizedIndex](all_helper_indices.len)
for idx in all_helper_indices:
result.add idx.GeneralizedIndex
result.sort(SortOrder.Descending)
# https://github.com/ethereum/consensus-specs/blob/v1.1.2/ssz/merkle-proofs.md#merkle-multiproofs
func check_multiproof_acceptable*(
indices: openArray[GeneralizedIndex]): Result[void, string] =
# Check that proof verification won't allocate excessive amounts of memory.
const max_multiproof_complexity = nextPowerOfTwo(256)
if indices.len > max_multiproof_complexity:
trace "Max multiproof complexity exceeded",
num_indices=indices.len, max_multiproof_complexity
return err("Unsupported multiproof complexity (" & $indices.len & ")")
if indices.len == 0:
return err("No indices specified")
if indices.anyIt(it == 0.GeneralizedIndex):
return err("Invalid index specified")
ok()
func calculate_multi_merkle_root_impl(
leaves: openArray[Eth2Digest],
proof: openArray[Eth2Digest],
indices: openArray[GeneralizedIndex],
helper_indices: openArray[GeneralizedIndex]): Result[Eth2Digest, string] =
# All callers have already verified the checks in check_multiproof_acceptable,
# as well as whether lengths of leaves/indices and proof/helper_indices match.
# Helper to retrieve a value from a table that is statically known to exist.
template getExisting[A, B](t: var Table[A, B], key: A): var B =
try: t[key]
except KeyError: raiseAssert "Unreachable"
# Populate data structure with all leaves.
# This data structure only scales with the number of `leaves`,
# in contrast to the spec one that also scales with the number of `proof`
# items and the number of all intermediate roots, potentially the entire tree.
let capacity = nextPowerOfTwo(leaves.len)
var objects = initTable[GeneralizedIndex, Eth2Digest](capacity)
for i, index in indices:
if objects.mgetOrPut(index, leaves[i]) != leaves[i]:
return err("Conflicting roots for same index")
# Create list with keys of all active nodes that need to be visited.
# This list is sorted in descending order, same as `helper_indices`.
# Pulling from `objects` instead of from `indices` deduplicates the list.
var keys = newSeqOfCap[GeneralizedIndex](objects.len)
for index in objects.keys:
if index > 1.GeneralizedIndex: # For the root, no work needs to be done.
keys.add index
keys.sort(SortOrder.Descending)
# The merkle tree is processed from bottom to top, pulling in helper
# indices from `proof` as needed. During processing, the `keys` list
# may temporarily end up being split into two parts, sorted individually.
# An additional index tracks the current maximum element of the list.
var
completed = 0 # All key indices before this are fully processed.
maxIndex = completed # Index of the list's largest key.
helper = 0 # Helper index from `proof` to be pulled next.
# Processing is done when there are no more keys to process.
while completed < keys.len:
let
k = keys[maxIndex]
sibling = generalized_index_sibling(k)
left = generalized_index_sibling_left(k)
right = generalized_index_sibling_right(k)
parent = generalized_index_parent(k)
parentRight = generalized_index_sibling_right(parent)
# Keys need to be processed in descending order to ensure that intermediate
# roots remain available until they are no longer needed. This ensures that
# conflicting roots are detected in all cases.
keys[maxIndex] =
if not objects.hasKey(k):
# A previous computation did already merge this key with its sibling.
0.GeneralizedIndex
else:
# Compute expected root for parent. This deletes child roots.
# Because the list is sorted in descending order, they are not needed.
let root = withEth2Hash:
if helper < helper_indices.len and helper_indices[helper] == sibling:
# The next proof item is required to form the parent hash.
if sibling == left:
h.update proof[helper].data
h.update objects.getExisting(right).data; objects.del right
else:
h.update objects.getExisting(left).data; objects.del left
h.update proof[helper].data
inc helper
else:
# Both siblings are already known.
h.update objects.getExisting(left).data; objects.del left
h.update objects.getExisting(right).data; objects.del right
# Store parent root, and replace the current list entry with its parent.
if objects.hasKeyOrPut(parent, root):
if objects.getExisting(parent) != root:
return err("Conflicting roots for same index")
0.GeneralizedIndex
elif parent > 1.GeneralizedIndex:
# Note that the list may contain further nodes that are on a layer
# beneath the parent, so this may break the strictly descending order
# of the list. For example, given [12, 9], this will lead to [6, 9].
# This will resolve itself after the additional nodes are processed,
# i.e., [6, 9] -> [6, 4] -> [3, 4] -> [3, 2] -> [1].
parent
else:
0.GeneralizedIndex
if keys[maxIndex] != 0.GeneralizedIndex:
# The list may have been temporarily split up into two parts that are
# individually sorted in descending order. Have to first process further
# nodes until the list is sorted once more.
inc maxIndex
# Determine whether descending sort order has been restored.
let isSorted =
if maxIndex == completed: true
else:
while maxIndex < keys.len and keys[maxIndex] == 0.GeneralizedIndex:
inc maxIndex
maxIndex >= keys.len or keys[maxIndex] <= parentRight
if isSorted:
# List is sorted once more. Reset `maxIndex` to its start.
while completed < keys.len and keys[completed] == 0.GeneralizedIndex:
inc completed
maxIndex = completed
# Proof is guaranteed to provide all info needed to reach the root.
doAssert helper == helper_indices.len
doAssert objects.len == 1
ok(objects.getExisting(1.GeneralizedIndex))
func calculate_multi_merkle_root*(
leaves: openArray[Eth2Digest],
proof: openArray[Eth2Digest],
indices: openArray[GeneralizedIndex],
helper_indices: openArray[GeneralizedIndex]): Result[Eth2Digest, string] =
doAssert proof.len == helper_indices.len
if leaves.len != indices.len:
return err("Length mismatch for leaves and indices")
? check_multiproof_acceptable(indices)
calculate_multi_merkle_root_impl(
leaves, proof, indices, helper_indices)
func calculate_multi_merkle_root*(
leaves: openArray[Eth2Digest],
proof: openArray[Eth2Digest],
indices: openArray[GeneralizedIndex]): Result[Eth2Digest, string] =
if leaves.len != indices.len:
return err("Length mismatch for leaves and indices")
? check_multiproof_acceptable(indices)
calculate_multi_merkle_root_impl(
leaves, proof, indices, get_helper_indices(indices))
# https://github.com/ethereum/consensus-specs/blob/v1.1.2/ssz/merkle-proofs.md#merkle-multiproofs
func verify_merkle_multiproof*(
leaves: openArray[Eth2Digest],
proof: openArray[Eth2Digest],
indices: openArray[GeneralizedIndex],
helper_indices: openArray[GeneralizedIndex],
root: Eth2Digest): bool =
let calc = calculate_multi_merkle_root(leaves, proof, indices, helper_indices)
if calc.isErr: return false
calc.get == root
func verify_merkle_multiproof*(
leaves: openArray[Eth2Digest],
proof: openArray[Eth2Digest],
indices: openArray[GeneralizedIndex],
root: Eth2Digest): bool =
let calc = calculate_multi_merkle_root(leaves, proof, indices)
if calc.isErr: return false
calc.get == root
# https://github.com/ethereum/consensus-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#is_valid_merkle_branch
func is_valid_merkle_branch*(leaf: Eth2Digest, branch: openArray[Eth2Digest],
depth: int, index: uint64,
@ -68,12 +300,12 @@ func is_valid_merkle_branch*(leaf: Eth2Digest, branch: openArray[Eth2Digest],
value == root
# https://github.com/ethereum/consensus-specs/blob/v1.1.0-beta.4/tests/core/pyspec/eth2spec/test/helpers/merkle.py#L4-L21
func build_proof_impl(anchor: object, leaf_index: uint64,
func build_proof_impl(anchor: object, leaf_index: uint64,
proof: var openArray[Eth2Digest]) =
let
bottom_length = nextPow2(typeof(anchor).totalSerializedFields.uint64)
tree_depth = log2trunc(bottom_length)
parent_index =
parent_index =
if leaf_index < bottom_length shl 1:
0'u64
else:
@ -82,7 +314,7 @@ func build_proof_impl(anchor: object, leaf_index: uint64,
i = i shr 1
i
var
var
prefix_len = 0
proof_len = log2trunc(leaf_index)
cache = newSeq[Eth2Digest](bottom_length shl 1)
@ -93,7 +325,7 @@ func build_proof_impl(anchor: object, leaf_index: uint64,
when fieldVar is object:
prefix_len = log2trunc(leaf_index) - tree_depth
proof_len -= prefix_len
let
let
bottom_bits = leaf_index and not (uint64.high shl prefix_len)
prefix_leaf_index = (1'u64 shl prefix_len) + bottom_bits
build_proof_impl(fieldVar, prefix_leaf_index, proof)
@ -114,12 +346,12 @@ func build_proof_impl(anchor: object, leaf_index: uint64,
proof[proof_index] = if b: cache[i shl 1]
else: cache[i shl 1 + 1]
func build_proof*(anchor: object, leaf_index: uint64,
func build_proof*(anchor: object, leaf_index: uint64,
proof: var openArray[Eth2Digest]) =
doAssert leaf_index > 0
doAssert proof.len == log2trunc(leaf_index)
build_proof_impl(anchor, leaf_index, proof)
const SLOTS_PER_SYNC_COMMITTEE_PERIOD* =
EPOCHS_PER_SYNC_COMMITTEE_PERIOD * SLOTS_PER_EPOCH

74
tests/test_helpers.nim
View File

@ -1,5 +1,5 @@
# beacon_chain
# Copyright (c) 2018 Status Research & Development GmbH
# Copyright (c) 2018, 2021 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@ -8,6 +8,8 @@
{.used.}
import
# Standard library
sequtils,
# Status libraries
stew/bitops2,
# Beacon chain internals
@ -55,3 +57,73 @@ suite "Spec helpers":
process(fieldVar, i shl childDepth)
i += 1
process(state, state.numLeaves)
test "get_branch_indices":
check:
toSeq(get_branch_indices(1.GeneralizedIndex)) == []
toSeq(get_branch_indices(0b101010.GeneralizedIndex)) ==
[
0b101011.GeneralizedIndex,
0b10100.GeneralizedIndex,
0b1011.GeneralizedIndex,
0b100.GeneralizedIndex,
0b11.GeneralizedIndex
]
test "get_path_indices":
check:
toSeq(get_path_indices(1.GeneralizedIndex)) == []
toSeq(get_path_indices(0b101010.GeneralizedIndex)) ==
[
0b101010.GeneralizedIndex,
0b10101.GeneralizedIndex,
0b1010.GeneralizedIndex,
0b101.GeneralizedIndex,
0b10.GeneralizedIndex
]
test "get_helper_indices":
check:
get_helper_indices(
[
8.GeneralizedIndex,
9.GeneralizedIndex,
14.GeneralizedIndex]) ==
[
15.GeneralizedIndex,
6.GeneralizedIndex,
5.GeneralizedIndex
]
test "verify_merkle_multiproof":
var nodes: array[16, Eth2Digest]
for i in countdown(15, 8):
nodes[i] = eth2digest([i.byte])
for i in countdown(7, 1):
nodes[i] = withEth2Hash:
h.update nodes[2 * i + 0].data
h.update nodes[2 * i + 1].data
proc verify(indices_int: openArray[int]) =
let
indices = indices_int.mapIt(it.GeneralizedIndex)
helper_indices = get_helper_indices(indices)
leaves = indices.mapIt(nodes[it])
proof = helper_indices.mapIt(nodes[it])
root = nodes[1]
checkpoint "Verifying " & $indices & "---" & $helper_indices
check:
verify_merkle_multiproof(leaves, proof, indices, root)
verify([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15])
for a in 1 .. 15:
verify([a])
for b in 1 .. 15:
verify([a, b])
for c in 1 .. 15:
verify([a, b, c])
for d in 8 .. 15:
verify([a, b, c, d])
for e in 1 .. 7:
verify([a, b, c, d, e])