overhaul shuffling tests, focus on swap-or-not shuffle

configs/constant_presets/minimal.yaml

@@ -16,7 +16,7 @@ MAX_ATTESTATION_PARTICIPANTS: 4096
 # 2**2 ` (= 4)
 MAX_EXIT_DEQUEUES_PER_EPOCH: 4
 # See issue 563
-SHUFFLE_ROUND_COUNT: 90
+SHUFFLE_ROUND_COUNT: 10
 
 
 # Deposit contract

test_generators/shuffling/README.md

@@ -1,16 +1,16 @@
-# Shuffling Test Generator
+# Shuffling Tests
 
-```
+Tests for the swap-or-not shuffling in ETH 2.0.
-2018 Status Research & Development GmbH
-Copyright and related rights waived via [CC0](https://creativecommons.org/publicdomain/zero/1.0/).
 
-This work uses public domain work under CC0 from the Ethereum Foundation
+For implementers, possible test runners implementing testing can include:
-https://github.com/ethereum/eth2.0-specs
+1) just test permute-index, run it for each index `i` in `range(count)`, and check against expected `output[i]` (default spec implementation)
-```
+2) test un-permute-index (the reverse lookup. Implemented by running the shuffling rounds in reverse: from `round_count-1` to `0`)
+3) test the optimized complete shuffle, where all indices are shuffled at once, test output in one go.
+4) test complete shuffle in reverse (reverse rounds, same as 2)
 
-
+Tips for initial shuffling write:
-This file implements a test vectors generator for the shuffling algorithm described in the Ethereum
+- run with `round_count = 1` first, do the same with pyspec.
-[specs](https://github.com/ethereum/eth2.0-specs/blob/2983e68f0305551083fac7fcf9330c1fc9da3411/specs/core/0_beacon-chain.md#get_new_shuffling)
+- start with permute index
-
+- optimized shuffling implementations:
-Utilizes 'swap or not' shuffling found in [An Enciphering Scheme Based on a Card Shuffle](https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf).  
+  - vitalik, Python: https://github.com/ethereum/eth2.0-specs/pull/576#issue-250741806
-See the `Generalized domain` algorithm on page 3.
+  - protolambda, Go: https://github.com/protolambda/eth2-shuffle

test_generators/shuffling/main.py

@@ -1,6 +1,4 @@
-import random
+from eth2spec.phase0 import spec
-
-from eth2spec.phase0.spec import *
 from eth_utils import (
     to_dict, to_tuple
 )
@@ -9,118 +7,45 @@ from preset_loader import loader
 
 
 @to_dict
-def active_exited_validator_case(idx_max: int):
+def shuffling_case(seed: spec.Bytes32, count: int):
-    validators = []
+    yield 'seed', '0x' + seed.hex()
-
+    yield 'count', count
-    # Standard deviation, around 8% validators will activate or exit within
+    yield 'shuffled', [spec.get_permuted_index(i, count, seed) for i in range(count)]
-    # ENTRY_EXIT_DELAY inclusive from EPOCH thus creating an edge case for validator
-    # shuffling
-    RAND_EPOCH_STD = 35
-
-    # TODO: fix epoch numbers
-    
-    slot = 1000 * SLOTS_PER_EPOCH
-    # The epoch, also a mean for the normal distribution
-    epoch = slot_to_epoch(slot)
-    MAX_EXIT_EPOCH = epoch + 5000  # Maximum exit_epoch for easier reading
-
-    for idx in range(idx_max):
-        v = Validator(
-            pubkey=bytes(random.randint(0, 255) for _ in range(48)),
-            withdrawal_credentials=bytes(random.randint(0, 255) for _ in range(32)),
-            activation_epoch=FAR_FUTURE_EPOCH,
-            exit_epoch=FAR_FUTURE_EPOCH,
-            withdrawable_epoch=FAR_FUTURE_EPOCH,
-            initiated_exit=False,
-            slashed=False,
-            high_balance=0
-        )
-        # 4/5 of all validators are active
-        if random.random() < 0.8:
-            # Choose a normally distributed epoch number
-            rand_epoch = round(random.gauss(epoch, RAND_EPOCH_STD))
-
-            # for 1/2 of *active* validators rand_epoch is the activation epoch
-            if random.random() < 0.5:
-                v.activation_epoch = rand_epoch
-
-                # 1/4 of active validators will exit in forseeable future
-                if random.random() < 0.5:
-                    v.exit_epoch = random.randint(
-                        rand_epoch + ACTIVATION_EXIT_DELAY + 1, MAX_EXIT_EPOCH)
-                # 1/4 of active validators in theory remain in the set indefinitely
-                else:
-                    v.exit_epoch = FAR_FUTURE_EPOCH
-            # for the other active 1/2 rand_epoch is the exit epoch
-            else:
-                v.activation_epoch = random.randint(
-                    0, rand_epoch - ACTIVATION_EXIT_DELAY)
-                v.exit_epoch = rand_epoch
-
-        # The remaining 1/5 of all validators is not activated
-        else:
-            v.activation_epoch = FAR_FUTURE_EPOCH
-            v.exit_epoch = FAR_FUTURE_EPOCH
-
-        validators.append(v)
-
-    query_slot = slot + random.randint(-1, 1)
-    state = get_genesis_beacon_state([], 0, None)
-    state.validator_registry = validators
-    state.latest_randao_mixes = [b'\xde\xad\xbe\xef' * 8 for _ in range(LATEST_RANDAO_MIXES_LENGTH)]
-    state.slot = slot
-    state.latest_start_shard = random.randint(0, SHARD_COUNT - 1)
-    randao_mix = bytes(random.randint(0, 255) for _ in range(32))
-    state.latest_randao_mixes[slot_to_epoch(query_slot) % LATEST_RANDAO_MIXES_LENGTH] = randao_mix
-
-    committees = get_crosslink_committees_at_slot(state, query_slot)
-    yield 'validator_registry', [
-        {
-            'activation_epoch': v.activation_epoch,
-            'exit_epoch': v.exit_epoch
-        } for v in state.validator_registry
-    ]
-    yield 'randao_mix', '0x'+randao_mix.hex()
-    yield 'state_slot', state.slot
-    yield 'query_slot', query_slot
-    yield 'latest_start_shard', state.latest_start_shard
-    yield 'crosslink_committees', committees
 
 
 @to_tuple
-def active_exited_validator_cases():
+def shuffling_test_cases():
-    for i in range(3):
+    for seed in [spec.hash(spec.int_to_bytes4(seed_init_value)) for seed_init_value in range(30)]:
-        yield active_exited_validator_case(random.randint(100, min(200, SHARD_COUNT * 2)))
+        for count in [0, 1, 2, 3, 5, 10, 33, 100, 1000]:
+            yield shuffling_case(seed, count)
 
 
 def mini_shuffling_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
     presets = loader.load_presets(configs_path, 'minimal')
-    apply_constants_preset(presets)
+    spec.apply_constants_preset(presets)
 
     return ("shuffling_minimal", "core", gen_suite.render_suite(
-        title="Shuffling Algorithm Tests with minimal config",
+        title="Swap-or-Not Shuffling tests with minimal config",
-        summary="Test vectors for validator shuffling with different validator registry activity status and set size."
+        summary="Swap or not shuffling, with minimally configured testing round-count",
-                " Note: only relevant fields are defined.",
         forks_timeline="testing",
         forks=["phase0"],
         config="minimal",
         handler="core",
-        test_cases=active_exited_validator_cases()))
+        test_cases=shuffling_test_cases()))
 
 
 def full_shuffling_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
     presets = loader.load_presets(configs_path, 'mainnet')
-    apply_constants_preset(presets)
+    spec.apply_constants_preset(presets)
 
     return ("shuffling_full", "core", gen_suite.render_suite(
-        title="Shuffling Algorithm Tests with mainnet config",
+        title="Swap-or-Not Shuffling tests with mainnet config",
-        summary="Test vectors for validator shuffling with different validator registry activity status and set size."
+        summary="Swap or not shuffling, with normal configured (secure) mainnet round-count",
-                " Note: only relevant fields are defined.",
         forks_timeline="mainnet",
         forks=["phase0"],
         config="mainnet",
         handler="core",
-        test_cases=active_exited_validator_cases()))
+        test_cases=shuffling_test_cases()))
 
 
 if __name__ == "__main__":