From e7317e228346d2eb5f3d7414b20f9485b2fa687c Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 24 Sep 2021 11:10:57 +0200 Subject: [PATCH 1/2] merkle proof test generator Building merkle proofs is required functionality for implementing light client sync. Although the spec currently only defines a function to verify merkle proofs (`is_valid_merkle_branch`) there are still a few PySpec unit tests that produce merkle proofs. This patch adds a new generator to extract test vectors from those static unit tests, so that light client implementations can validate their merkle proof logic. --- .../test/altair/unittests/test_helpers.py | 8 +++++ tests/formats/merkle/README.md | 36 +++++++++++++++++++ tests/generators/merkle/README.md | 6 ++++ tests/generators/merkle/__init__.py | 0 tests/generators/merkle/main.py | 14 ++++++++ tests/generators/merkle/requirements.txt | 2 ++ 6 files changed, 66 insertions(+) create mode 100644 tests/formats/merkle/README.md create mode 100644 tests/generators/merkle/README.md create mode 100644 tests/generators/merkle/__init__.py create mode 100644 tests/generators/merkle/main.py create mode 100644 tests/generators/merkle/requirements.txt diff --git a/tests/core/pyspec/eth2spec/test/altair/unittests/test_helpers.py b/tests/core/pyspec/eth2spec/test/altair/unittests/test_helpers.py index 137539113..7eefc458b 100644 --- a/tests/core/pyspec/eth2spec/test/altair/unittests/test_helpers.py +++ b/tests/core/pyspec/eth2spec/test/altair/unittests/test_helpers.py @@ -9,7 +9,11 @@ from eth2spec.test.helpers.merkle import build_proof @with_phases([ALTAIR]) @spec_state_test def test_next_sync_committee_tree(spec, state): + yield "state", state + yield "leaf", state.next_sync_committee.hash_tree_root() + yield "leaf_index", "meta", spec.NEXT_SYNC_COMMITTEE_INDEX next_sync_committee_branch = build_proof(state.get_backing(), spec.NEXT_SYNC_COMMITTEE_INDEX) + yield "proof", next_sync_committee_branch assert spec.is_valid_merkle_branch( leaf=state.next_sync_committee.hash_tree_root(), branch=next_sync_committee_branch, @@ -22,7 +26,11 @@ def test_next_sync_committee_tree(spec, state): @with_phases([ALTAIR]) @spec_state_test def test_finality_root_tree(spec, state): + yield "state", state + yield "leaf", state.finalized_checkpoint.root + yield "leaf_index", "meta", spec.FINALIZED_ROOT_INDEX finality_branch = build_proof(state.get_backing(), spec.FINALIZED_ROOT_INDEX) + yield "proof", finality_branch assert spec.is_valid_merkle_branch( leaf=state.finalized_checkpoint.root, branch=finality_branch, diff --git a/tests/formats/merkle/README.md b/tests/formats/merkle/README.md new file mode 100644 index 000000000..a1810d667 --- /dev/null +++ b/tests/formats/merkle/README.md @@ -0,0 +1,36 @@ +# Merkle tests + +This series of tests provides reference test vectors for validating correct +generation and verification of merkle proofs based on static data. + +## Test case format + +### `meta.yaml` + +```yaml +leaf_index: int -- Generalized leaf index, verifying against the proof. +proof_count: int -- Amount of proof elements. +``` + +### `state.ssz_snappy` + +An SSZ-snappy encoded `BeaconState` object from which other data is generated. + +### `leaf.ssz_snappy` + +An SSZ-snappy encoded `Bytes32` reflecting the merkle root of `leaf_index` at +the given `state`. + +### `proof_.ssz_snappy` + +A series of files, with `` in range `[0, proof_count)`. Each file is an +SSZ-snappy encoded `Bytes32` and represents one element of the merkle proof for +`leaf_index` at the given `state`. + +## Condition + +A test-runner can implement the following assertions: +- Check that `is_valid_merkle_branch` confirms `leaf` at `leaf_index` to verify + against `has_tree_root(state)` and `proof`. +- If the implementation supports generating merkle proofs, check that the + self-generated proof matches the `proof` provided with the test. diff --git a/tests/generators/merkle/README.md b/tests/generators/merkle/README.md new file mode 100644 index 000000000..a19a67d9e --- /dev/null +++ b/tests/generators/merkle/README.md @@ -0,0 +1,6 @@ +# Merkle + +The purpose of this test-generator is to provide test-vectors for validating the +correct merkleization of objects and corresponding merkle proofs. + +Test-format documentation can be found [here](../../formats/merkle/README.md). diff --git a/tests/generators/merkle/__init__.py b/tests/generators/merkle/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/generators/merkle/main.py b/tests/generators/merkle/main.py new file mode 100644 index 000000000..53cb59de9 --- /dev/null +++ b/tests/generators/merkle/main.py @@ -0,0 +1,14 @@ +from eth2spec.test.helpers.constants import ALTAIR +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators + + +if __name__ == "__main__": + altair_mods = {key: 'eth2spec.test.altair.unittests.test_' + key for key in [ + 'helpers', + ]} + + all_mods = { + ALTAIR: altair_mods + } + + run_state_test_generators(runner_name="merkle", all_mods=all_mods) diff --git a/tests/generators/merkle/requirements.txt b/tests/generators/merkle/requirements.txt new file mode 100644 index 000000000..182248686 --- /dev/null +++ b/tests/generators/merkle/requirements.txt @@ -0,0 +1,2 @@ +pytest>=4.4 +../../../[generator] From bd8c97896511680a750afb53919bb57c67e710e1 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 Sep 2021 18:07:59 +0200 Subject: [PATCH 2/2] move merkle tests, output proof.yaml, update format --- .../eth2spec/test/altair/merkle/__init__.py | 0 .../test_single_proof.py} | 21 +++++++----- tests/formats/merkle/README.md | 34 ++----------------- tests/formats/merkle/single_proof.md | 28 +++++++++++++++ tests/generators/merkle/main.py | 4 +-- 5 files changed, 46 insertions(+), 41 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/altair/merkle/__init__.py rename tests/core/pyspec/eth2spec/test/altair/{unittests/test_helpers.py => merkle/test_single_proof.py} (66%) create mode 100644 tests/formats/merkle/single_proof.md diff --git a/tests/core/pyspec/eth2spec/test/altair/merkle/__init__.py b/tests/core/pyspec/eth2spec/test/altair/merkle/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/altair/unittests/test_helpers.py b/tests/core/pyspec/eth2spec/test/altair/merkle/test_single_proof.py similarity index 66% rename from tests/core/pyspec/eth2spec/test/altair/unittests/test_helpers.py rename to tests/core/pyspec/eth2spec/test/altair/merkle/test_single_proof.py index 7eefc458b..f9aa68add 100644 --- a/tests/core/pyspec/eth2spec/test/altair/unittests/test_helpers.py +++ b/tests/core/pyspec/eth2spec/test/altair/merkle/test_single_proof.py @@ -8,12 +8,14 @@ from eth2spec.test.helpers.merkle import build_proof @with_phases([ALTAIR]) @spec_state_test -def test_next_sync_committee_tree(spec, state): +def test_next_sync_committee_merkle_proof(spec, state): yield "state", state - yield "leaf", state.next_sync_committee.hash_tree_root() - yield "leaf_index", "meta", spec.NEXT_SYNC_COMMITTEE_INDEX next_sync_committee_branch = build_proof(state.get_backing(), spec.NEXT_SYNC_COMMITTEE_INDEX) - yield "proof", next_sync_committee_branch + yield "proof", { + "leaf": "0x" + state.next_sync_committee.hash_tree_root().hex(), + "leaf_index": spec.NEXT_SYNC_COMMITTEE_INDEX, + "branch": ['0x' + root.hex() for root in next_sync_committee_branch] + } assert spec.is_valid_merkle_branch( leaf=state.next_sync_committee.hash_tree_root(), branch=next_sync_committee_branch, @@ -25,12 +27,15 @@ def test_next_sync_committee_tree(spec, state): @with_phases([ALTAIR]) @spec_state_test -def test_finality_root_tree(spec, state): +def test_finality_root_merkle_proof(spec, state): yield "state", state - yield "leaf", state.finalized_checkpoint.root - yield "leaf_index", "meta", spec.FINALIZED_ROOT_INDEX finality_branch = build_proof(state.get_backing(), spec.FINALIZED_ROOT_INDEX) - yield "proof", finality_branch + yield "proof", { + "leaf": "0x" + state.finalized_checkpoint.root.hex(), + "leaf_index": spec.FINALIZED_ROOT_INDEX, + "branch": ['0x' + root.hex() for root in finality_branch] + } + assert spec.is_valid_merkle_branch( leaf=state.finalized_checkpoint.root, branch=finality_branch, diff --git a/tests/formats/merkle/README.md b/tests/formats/merkle/README.md index a1810d667..c0f0a205b 100644 --- a/tests/formats/merkle/README.md +++ b/tests/formats/merkle/README.md @@ -3,34 +3,6 @@ This series of tests provides reference test vectors for validating correct generation and verification of merkle proofs based on static data. -## Test case format - -### `meta.yaml` - -```yaml -leaf_index: int -- Generalized leaf index, verifying against the proof. -proof_count: int -- Amount of proof elements. -``` - -### `state.ssz_snappy` - -An SSZ-snappy encoded `BeaconState` object from which other data is generated. - -### `leaf.ssz_snappy` - -An SSZ-snappy encoded `Bytes32` reflecting the merkle root of `leaf_index` at -the given `state`. - -### `proof_.ssz_snappy` - -A series of files, with `` in range `[0, proof_count)`. Each file is an -SSZ-snappy encoded `Bytes32` and represents one element of the merkle proof for -`leaf_index` at the given `state`. - -## Condition - -A test-runner can implement the following assertions: -- Check that `is_valid_merkle_branch` confirms `leaf` at `leaf_index` to verify - against `has_tree_root(state)` and `proof`. -- If the implementation supports generating merkle proofs, check that the - self-generated proof matches the `proof` provided with the test. +Handlers: +- `single_proof`: see [Single leaf proof test format](./single_proof.md) +- Different types of merkle proofs may be supported in the future. diff --git a/tests/formats/merkle/single_proof.md b/tests/formats/merkle/single_proof.md new file mode 100644 index 000000000..65fe7c988 --- /dev/null +++ b/tests/formats/merkle/single_proof.md @@ -0,0 +1,28 @@ +# Single leaf merkle proof tests + +This series of tests provides reference test vectors for validating correct +generation and verification of merkle proofs based on static data. + +## Test case format + +### `state.ssz_snappy` + +An SSZ-snappy encoded `BeaconState` object from which other data is generated. + +### `proof.yaml` + +A proof of the leaf value (a merkle root) at generalized-index `leaf_index` in the given `state`. + +```yaml +leaf: Bytes32 # string, hex encoded, with 0x prefix +leaf_index: int # integer, decimal +branch: list of Bytes32 # list, each element is a string, hex encoded, with 0x prefix +``` + +## Condition + +A test-runner can implement the following assertions: +- Check that `is_valid_merkle_branch` confirms `leaf` at `leaf_index` to verify + against `has_tree_root(state)` and `proof`. +- If the implementation supports generating merkle proofs, check that the + self-generated proof matches the `proof` provided with the test. diff --git a/tests/generators/merkle/main.py b/tests/generators/merkle/main.py index 53cb59de9..7203c5f19 100644 --- a/tests/generators/merkle/main.py +++ b/tests/generators/merkle/main.py @@ -3,8 +3,8 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators if __name__ == "__main__": - altair_mods = {key: 'eth2spec.test.altair.unittests.test_' + key for key in [ - 'helpers', + altair_mods = {key: 'eth2spec.test.altair.merkle.test_' + key for key in [ + 'single_proof', ]} all_mods = {