Fix depth calculation...again(!) and add unittests

setup.py

@@ -209,6 +209,12 @@ def ceillog2(x: int) -> uint64:
     if x < 1:
         raise ValueError(f"ceillog2 accepts only positive values, x={x}")
     return uint64((x - 1).bit_length())
+
+
+def floorlog2(x: int) -> uint64:
+    if x < 1:
+        raise ValueError(f"floorlog2 accepts only positive values, x={x}")
+    return uint64(x.bit_length() - 1)
 '''
 PHASE0_SUNDRY_FUNCTIONS = '''
 def get_eth1_data(block: Eth1Block) -> Eth1Data:
@@ -321,7 +327,7 @@ def objects_to_spec(spec_object: SpecObject, imports: str, fork: str, ordered_cl
         )
     )
     for k in list(spec_object.functions):
-        if "ceillog2" in k:
+        if "ceillog2" in k or "floorlog2" in k:
             del spec_object.functions[k]
     functions_spec = '\n\n'.join(spec_object.functions.values())
     for k in list(spec_object.constants.keys()):
@@ -379,7 +385,7 @@ ignored_dependencies = [
     'Bytes1', 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector',
     'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256',
     'bytes', 'byte', 'ByteList', 'ByteVector',
-    'Dict', 'dict', 'field', 'ceillog2',
+    'Dict', 'dict', 'field', 'ceillog2', 'floorlog2',
 ]
 
 

specs/lightclient/sync-protocol.md

@@ -80,10 +80,10 @@ class LightClientUpdate(Container):
     header: BeaconBlockHeader
     # Next sync committee corresponding to the header
     next_sync_committee: SyncCommittee
-    next_sync_committee_branch: Vector[Bytes32, NEXT_SYNC_COMMITTEE_INDEX.bit_length()]
+    next_sync_committee_branch: Vector[Bytes32, floorlog2(NEXT_SYNC_COMMITTEE_INDEX)]
     # Finality proof for the update header
     finality_header: BeaconBlockHeader
-    finality_branch: Vector[Bytes32, FINALIZED_ROOT_INDEX.bit_length()]
+    finality_branch: Vector[Bytes32, floorlog2(FINALIZED_ROOT_INDEX)]
     # Sync committee aggregate signature
     sync_committee_bits: Bitvector[SYNC_COMMITTEE_SIZE]
     sync_committee_signature: BLSSignature
@@ -105,7 +105,7 @@ class LightClientStore(Container):
 
 ```python
 def get_subtree_index(generalized_index: GeneralizedIndex) -> uint64:
-    return uint64(generalized_index % 2**((generalized_index).bit_length()))
+    return uint64(generalized_index % 2**(floorlog2(generalized_index)))
 ```
 
 ## Light client state updates
@@ -127,13 +127,13 @@ def is_valid_light_client_update(snapshot: LightClientSnapshot, update: LightCli
     # Verify update header root is the finalized root of the finality header, if specified
     if update.finality_header == BeaconBlockHeader():
         signed_header = update.header
-        assert update.finality_branch == [Bytes32() for _ in range(FINALIZED_ROOT_INDEX.bit_length())]
+        assert update.finality_branch == [Bytes32() for _ in range(floorlog2(FINALIZED_ROOT_INDEX))]
     else:
         signed_header = update.finality_header
         assert is_valid_merkle_branch(
             leaf=hash_tree_root(update.header),
             branch=update.finality_branch,
-            depth=FINALIZED_ROOT_INDEX.bit_length(),
+            depth=floorlog2(FINALIZED_ROOT_INDEX),
             index=get_subtree_index(FINALIZED_ROOT_INDEX),
             root=update.finality_header.state_root,
         )
@@ -141,13 +141,13 @@ def is_valid_light_client_update(snapshot: LightClientSnapshot, update: LightCli
     # Verify update next sync committee if the update period incremented
     if update_period == snapshot_period:
         sync_committee = snapshot.current_sync_committee
-        assert update.next_sync_committee_branch == [Bytes32() for _ in range(NEXT_SYNC_COMMITTEE_INDEX.bit_length())]
+        assert update.next_sync_committee_branch == [Bytes32() for _ in range(floorlog2(NEXT_SYNC_COMMITTEE_INDEX))]
     else:
         sync_committee = snapshot.next_sync_committee
         assert is_valid_merkle_branch(
             leaf=hash_tree_root(update.next_sync_committee),
             branch=update.next_sync_committee_branch,
-            depth=NEXT_SYNC_COMMITTEE_INDEX.bit_length(),
+            depth=floorlog2(NEXT_SYNC_COMMITTEE_INDEX),
             index=get_subtree_index(NEXT_SYNC_COMMITTEE_INDEX),
             root=update.header.state_root,
         )

tests/core/pyspec/eth2spec/test/helpers/custody.py

@@ -1,7 +1,7 @@
 from eth2spec.test.helpers.keys import privkeys
+from eth2spec.test.helpers.merkle import build_proof
 from eth2spec.utils import bls
 from eth2spec.utils.ssz.ssz_typing import Bitlist, ByteVector, ByteList
-from remerkleable.tree import gindex_bit_iter
 
 BYTES_PER_CHUNK = 32
 
@@ -116,26 +116,6 @@ def custody_chunkify(spec, x):
     return [ByteVector[spec.BYTES_PER_CUSTODY_CHUNK](c) for c in chunks]
 
 
-def build_proof(anchor, leaf_index):
-    if leaf_index <= 1:
-        return []  # Nothing to prove / invalid index
-    node = anchor
-    proof = []
-    # Walk down, top to bottom to the leaf
-    bit_iter, _ = gindex_bit_iter(leaf_index)
-    for bit in bit_iter:
-        # Always take the opposite hand for the proof.
-        # 1 = right as leaf, thus get left
-        if bit:
-            proof.append(node.get_left().merkle_root())
-            node = node.get_right()
-        else:
-            proof.append(node.get_right().merkle_root())
-            node = node.get_left()
-
-    return list(reversed(proof))
-
-
 def get_valid_custody_chunk_response(spec, state, chunk_challenge, challenge_index,
                                      block_length_or_custody_data,
                                      invalid_chunk_data=False):

tests/core/pyspec/eth2spec/test/helpers/merkle.py

@@ -0,0 +1,21 @@
+from remerkleable.tree import gindex_bit_iter
+
+
+def build_proof(anchor, leaf_index):
+    if leaf_index <= 1:
+        return []  # Nothing to prove / invalid index
+    node = anchor
+    proof = []
+    # Walk down, top to bottom to the leaf
+    bit_iter, _ = gindex_bit_iter(leaf_index)
+    for bit in bit_iter:
+        # Always take the opposite hand for the proof.
+        # 1 = right as leaf, thus get left
+        if bit:
+            proof.append(node.get_left().merkle_root())
+            node = node.get_right()
+        else:
+            proof.append(node.get_right().merkle_root())
+            node = node.get_left()
+
+    return list(reversed(proof))

tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_helpers.py

@@ -0,0 +1,35 @@
+from eth2spec.test.context import (
+    spec_state_test,
+    with_phases,
+    LIGHTCLIENT_PATCH,
+)
+from eth2spec.test.helpers.merkle import build_proof
+
+
+@with_phases([LIGHTCLIENT_PATCH])
+@spec_state_test
+def test_next_sync_committee_tree(spec, state):
+    state.next_sync_committee = spec.SyncCommittee(
+        pubkeys=[state.validators[i]for i in range(spec.SYNC_COMMITTEE_SIZE)]
+    )
+    next_sync_committee_branch = build_proof(state.get_backing(), spec.NEXT_SYNC_COMMITTEE_INDEX)
+    assert spec.is_valid_merkle_branch(
+        leaf=state.next_sync_committee.hash_tree_root(),
+        branch=next_sync_committee_branch,
+        depth=spec.floorlog2(spec.NEXT_SYNC_COMMITTEE_INDEX),
+        index=spec.get_subtree_index(spec.NEXT_SYNC_COMMITTEE_INDEX),
+        root=state.hash_tree_root(),
+    )
+
+
+@with_phases([LIGHTCLIENT_PATCH])
+@spec_state_test
+def test_finality_root_tree(spec, state):
+    finality_branch = build_proof(state.get_backing(), spec.FINALIZED_ROOT_INDEX)
+    assert spec.is_valid_merkle_branch(
+        leaf=state.finalized_checkpoint.root,
+        branch=finality_branch,
+        depth=spec.floorlog2(spec.FINALIZED_ROOT_INDEX),
+        index=spec.get_subtree_index(spec.FINALIZED_ROOT_INDEX),
+        root=state.hash_tree_root(),
+    )