From 711c08f804bef4fa5690516026841697855e94e0 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Wed, 29 Sep 2021 15:02:34 +0200 Subject: [PATCH] add function to build merkle proofs (#2874) Adds a function that constructs a Merkle proof for a generalized index. This will be used during light client sync to update light clients with a new state (see NEXT_SYNC_COMMITTEE_INDEX / FINALIZED_ROOT_INDEX). --- AllTests-mainnet.md | 5 +- FixtureAll-mainnet.md | 6 +- FixtureAll-minimal.md | 6 +- FixtureSSZConsensus-mainnet.md | 6 +- FixtureSSZConsensus-minimal.md | 6 +- beacon_chain/spec/helpers.nim | 53 +++++++++++++++ .../all_altair_fixtures_require_ssz.nim | 1 + .../test_fixture_merkle_single_proof.nim | 68 +++++++++++++++++++ tests/test_helpers.nim | 39 ++++++++++- 9 files changed, 178 insertions(+), 12 deletions(-) create mode 100644 tests/official/altair/test_fixture_merkle_single_proof.nim diff --git a/AllTests-mainnet.md b/AllTests-mainnet.md index e0008e7e8..24ba9df77 100644 --- a/AllTests-mainnet.md +++ b/AllTests-mainnet.md @@ -217,9 +217,10 @@ OK: 12/12 Fail: 0/12 Skip: 0/12 OK: 1/1 Fail: 0/1 Skip: 0/1 ## Spec helpers ```diff ++ build_proof - BeaconState OK + integer_squareroot OK ``` -OK: 1/1 Fail: 0/1 Skip: 0/1 +OK: 2/2 Fail: 0/2 Skip: 0/2 ## Specific field types ```diff + root update OK @@ -359,4 +360,4 @@ OK: 1/1 Fail: 0/1 Skip: 0/1 OK: 36/48 Fail: 0/48 Skip: 12/48 ---TOTAL--- -OK: 193/205 Fail: 0/205 Skip: 12/205 +OK: 194/206 Fail: 0/206 Skip: 12/206 diff --git a/FixtureAll-mainnet.md b/FixtureAll-mainnet.md index 86eaba027..c8e113dc6 100644 --- a/FixtureAll-mainnet.md +++ b/FixtureAll-mainnet.md @@ -381,14 +381,16 @@ FixtureAll-mainnet + altair_fork_random_low_balances OK + altair_fork_random_misc_balances OK + altair_fork_random_mismatched_attestations OK ++ finality_root_merkle_proof OK + fork_base_state OK + fork_many_next_epoch OK + fork_next_epoch OK + fork_next_epoch_with_block OK + fork_random_low_balances OK + fork_random_misc_balances OK ++ next_sync_committee_merkle_proof OK ``` -OK: 385/385 Fail: 0/385 Skip: 0/385 +OK: 387/387 Fail: 0/387 Skip: 0/387 ## Ethereum Foundation - Altair - Epoch Processing - Effective balance updates [Preset: mainnet] ```diff + Effective balance updates - effective_balance_hysteresis [Preset: mainnet] OK @@ -773,4 +775,4 @@ OK: 1/1 Fail: 0/1 Skip: 0/1 OK: 27/27 Fail: 0/27 Skip: 0/27 ---TOTAL--- -OK: 639/639 Fail: 0/639 Skip: 0/639 +OK: 641/641 Fail: 0/641 Skip: 0/641 diff --git a/FixtureAll-minimal.md b/FixtureAll-minimal.md index 76956f18e..c58a8a5d3 100644 --- a/FixtureAll-minimal.md +++ b/FixtureAll-minimal.md @@ -383,8 +383,10 @@ FixtureAll-minimal + [Valid] sync_committee_with_participating_exited_member OK + [Valid] sync_committee_with_participating_withdrawable_member OK + [Valid] valid_signature_future_committee OK ++ finality_root_merkle_proof OK ++ next_sync_committee_merkle_proof OK ``` -OK: 381/381 Fail: 0/381 Skip: 0/381 +OK: 383/383 Fail: 0/383 Skip: 0/383 ## Ethereum Foundation - Altair - Epoch Processing - Effective balance updates [Preset: minimal] ```diff + Effective balance updates - effective_balance_hysteresis [Preset: minimal] OK @@ -803,4 +805,4 @@ OK: 1/1 Fail: 0/1 Skip: 0/1 OK: 27/27 Fail: 0/27 Skip: 0/27 ---TOTAL--- -OK: 661/661 Fail: 0/661 Skip: 0/661 +OK: 663/663 Fail: 0/663 Skip: 0/663 diff --git a/FixtureSSZConsensus-mainnet.md b/FixtureSSZConsensus-mainnet.md index 5665948d7..397ff4b7a 100644 --- a/FixtureSSZConsensus-mainnet.md +++ b/FixtureSSZConsensus-mainnet.md @@ -381,14 +381,16 @@ FixtureSSZConsensus-mainnet + altair_fork_random_low_balances OK + altair_fork_random_misc_balances OK + altair_fork_random_mismatched_attestations OK ++ finality_root_merkle_proof OK + fork_base_state OK + fork_many_next_epoch OK + fork_next_epoch OK + fork_next_epoch_with_block OK + fork_random_low_balances OK + fork_random_misc_balances OK ++ next_sync_committee_merkle_proof OK ``` -OK: 385/385 Fail: 0/385 Skip: 0/385 +OK: 387/387 Fail: 0/387 Skip: 0/387 ## Ethereum Foundation - Altair - Epoch Processing - Effective balance updates [Preset: mainnet] ```diff + Effective balance updates - effective_balance_hysteresis [Preset: mainnet] OK @@ -673,4 +675,4 @@ OK: 1/1 Fail: 0/1 Skip: 0/1 OK: 27/27 Fail: 0/27 Skip: 0/27 ---TOTAL--- -OK: 579/579 Fail: 0/579 Skip: 0/579 +OK: 581/581 Fail: 0/581 Skip: 0/581 diff --git a/FixtureSSZConsensus-minimal.md b/FixtureSSZConsensus-minimal.md index 570f2e465..547465cde 100644 --- a/FixtureSSZConsensus-minimal.md +++ b/FixtureSSZConsensus-minimal.md @@ -383,8 +383,10 @@ FixtureSSZConsensus-minimal + [Valid] sync_committee_with_participating_exited_member OK + [Valid] sync_committee_with_participating_withdrawable_member OK + [Valid] valid_signature_future_committee OK ++ finality_root_merkle_proof OK ++ next_sync_committee_merkle_proof OK ``` -OK: 381/381 Fail: 0/381 Skip: 0/381 +OK: 383/383 Fail: 0/383 Skip: 0/383 ## Ethereum Foundation - Altair - Epoch Processing - Effective balance updates [Preset: minimal] ```diff + Effective balance updates - effective_balance_hysteresis [Preset: minimal] OK @@ -688,4 +690,4 @@ OK: 1/1 Fail: 0/1 Skip: 0/1 OK: 27/27 Fail: 0/27 Skip: 0/27 ---TOTAL--- -OK: 590/590 Fail: 0/590 Skip: 0/590 +OK: 592/592 Fail: 0/592 Skip: 0/592 diff --git a/beacon_chain/spec/helpers.nim b/beacon_chain/spec/helpers.nim index 218c521ee..5d3b1da1b 100644 --- a/beacon_chain/spec/helpers.nim +++ b/beacon_chain/spec/helpers.nim @@ -67,6 +67,59 @@ func is_valid_merkle_branch*(leaf: Eth2Digest, branch: openArray[Eth2Digest], value = eth2digest(buf) value == root +# https://github.com/ethereum/consensus-specs/blob/v1.1.0-beta.4/tests/core/pyspec/eth2spec/test/helpers/merkle.py#L4-L21 +func build_proof_impl(anchor: object, leaf_index: uint64, + proof: var openArray[Eth2Digest]) = + let + bottom_length = nextPow2(typeof(anchor).totalSerializedFields.uint64) + tree_depth = log2trunc(bottom_length) + parent_index = + if leaf_index < bottom_length shl 1: + 0'u64 + else: + var i = leaf_index + while i >= bottom_length shl 1: + i = i shr 1 + i + + var + prefix_len = 0 + proof_len = log2trunc(leaf_index) + cache = newSeq[Eth2Digest](bottom_length shl 1) + block: + var i = bottom_length + anchor.enumInstanceSerializedFields(fieldNameVar, fieldVar): + if i == parent_index: + when fieldVar is object: + prefix_len = log2trunc(leaf_index) - tree_depth + proof_len -= prefix_len + let + bottom_bits = leaf_index and not (uint64.high shl prefix_len) + prefix_leaf_index = (1'u64 shl prefix_len) + bottom_bits + build_proof_impl(fieldVar, prefix_leaf_index, proof) + else: raiseAssert "Invalid leaf_index" + cache[i] = hash_tree_root(fieldVar) + i += 1 + for i in countdown(bottom_length - 1, 1): + cache[i] = withEth2Hash: + h.update cache[i shl 1].data + h.update cache[i shl 1 + 1].data + + var i = if parent_index != 0: parent_index + else: leaf_index + doAssert i > 0 and i < bottom_length shl 1 + for proof_index in prefix_len ..< prefix_len + proof_len: + let b = (i and 1) != 0 + i = i shr 1 + proof[proof_index] = if b: cache[i shl 1] + else: cache[i shl 1 + 1] + +func build_proof*(anchor: object, leaf_index: uint64, + proof: var openArray[Eth2Digest]) = + doAssert leaf_index > 0 + doAssert proof.len == log2trunc(leaf_index) + build_proof_impl(anchor, leaf_index, proof) + const SLOTS_PER_SYNC_COMMITTEE_PERIOD* = EPOCHS_PER_SYNC_COMMITTEE_PERIOD * SLOTS_PER_EPOCH diff --git a/tests/official/altair/all_altair_fixtures_require_ssz.nim b/tests/official/altair/all_altair_fixtures_require_ssz.nim index 5128564e0..3599e6a0f 100644 --- a/tests/official/altair/all_altair_fixtures_require_ssz.nim +++ b/tests/official/altair/all_altair_fixtures_require_ssz.nim @@ -12,6 +12,7 @@ {.used.} import + ./test_fixture_merkle_single_proof, ./test_fixture_ssz_consensus_objects, ./test_fixture_sanity_slots, ./test_fixture_sanity_blocks, diff --git a/tests/official/altair/test_fixture_merkle_single_proof.nim b/tests/official/altair/test_fixture_merkle_single_proof.nim new file mode 100644 index 000000000..30cb47d11 --- /dev/null +++ b/tests/official/altair/test_fixture_merkle_single_proof.nim @@ -0,0 +1,68 @@ +# beacon_chain +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.used.} + +import + # Standard library + std/[os, sequtils, streams], + # Status libraries + stew/bitops2, + # Third-party + yaml, + # Beacon chain internals + ../../../beacon_chain/spec/datatypes/altair, + ../../../beacon_chain/spec/helpers, + # Test utilities + ../../testutil, + ../fixtures_utils + +const TestsDir = + SszTestsDir/const_preset/"altair"/"merkle"/"single_proof"/"pyspec_tests" + +proc runTest(identifier: string) = + # We wrap the tests in a proc to avoid running out of globals + # in the future: Nim supports up to 3500 globals + # but unittest with the macro/templates put everything as globals + # https://github.com/nim-lang/Nim/issues/12084#issue-486866402 + + let testDir = TestsDir / identifier + + proc `testImpl _ merkle_single_proof _ identifier`() = + test identifier: + type + TestProof = object + leaf: string + leaf_index: GeneralizedIndex + branch: seq[string] + + let + proof = block: + var s = openFileStream(testDir/"proof.yaml") + defer: close(s) + var res: TestProof + yaml.load(s, res) + res + + state = newClone(parseTest(testDir/"state.ssz_snappy", SSZ, + altair.BeaconState)) + + var computedProof = newSeq[Eth2Digest](log2trunc(proof.leaf_index)) + build_proof(state[], proof.leaf_index, computedProof) + + check: + computedProof == proof.branch.mapIt(Eth2Digest.fromHex(it)) + is_valid_merkle_branch(Eth2Digest.fromHex(proof.leaf), computedProof, + log2trunc(proof.leaf_index), + get_subtree_index(proof.leaf_index), + hash_tree_root(state[])) + + `testImpl _ merkle_single_proof _ identifier`() + +suite "Ethereum Foundation - Altair - Merkle - Single proof" & preset(): + for kind, path in walkDir(TestsDir, relative = true, checkDir = true): + runTest(path) diff --git a/tests/test_helpers.nim b/tests/test_helpers.nim index c2a749124..d59b6c074 100644 --- a/tests/test_helpers.nim +++ b/tests/test_helpers.nim @@ -8,8 +8,12 @@ {.used.} import - ./unittest2, - ../beacon_chain/spec/[helpers] + # Status libraries + stew/bitops2, + # Beacon chain internals + ../beacon_chain/spec/[helpers, state_transition], + # Test utilities + ./unittest2, mocking/mock_genesis suite "Spec helpers": test "integer_squareroot": @@ -20,3 +24,34 @@ suite "Spec helpers": integer_squareroot(3'u64) == 1'u64 integer_squareroot(4'u64) == 2'u64 integer_squareroot(5'u64) == 2'u64 + + test "build_proof - BeaconState": + var + forked = newClone(initGenesisState()) + cache = StateCache() + rewards = RewardInfo() + doAssert process_slots(defaultRuntimeConfig, forked[], + Slot(100), cache, rewards, flags = {}) + + let + state = forked[].hbsPhase0.data + root = state.hash_tree_root() + + func numLeaves(obj: object): GeneralizedIndex = + nextPow2(typeof(obj).totalSerializedFields.uint64).GeneralizedIndex + + proc process(anchor: object, index: GeneralizedIndex) = + var i = index + anchor.enumInstanceSerializedFields(fieldNameVar, fieldVar): + let depth = log2trunc(i) + var proof = newSeq[Eth2Digest](depth) + build_proof(state, i, proof) + check: is_valid_merkle_branch(hash_tree_root(fieldVar), proof, + depth, get_subtree_index(i), root) + when fieldVar is object and not (fieldVar is Eth2Digest): + let + numChildLeaves = fieldVar.numLeaves + childDepth = log2trunc(numChildLeaves) + process(fieldVar, i shl childDepth) + i += 1 + process(state, state.numLeaves)