ssz-static suite

test_generators/ssz_static/main.py

@@ -1,55 +1,79 @@
+from random import Random
+
+from eth2spec.debug import random_value, encode
+from eth2spec.phase0 import spec
+from eth2spec.utils.minimal_ssz import hash_tree_root, serialize
 from eth_utils import (
     to_tuple, to_dict
 )
-from preset_loader import loader
-from eth2spec.phase0 import spec
-from eth2spec.utils.minimal_ssz import hash_tree_root, serialize
-from eth2spec.debug import random_value, encode
-
 from gen_base import gen_runner, gen_suite, gen_typing
-from random import Random
+from preset_loader import loader
+
+MAX_BYTES_LENGTH = 100
+MAX_LIST_LENGTH = 10
 
 
 @to_dict
-def render_test_case(rng: Random, name):
+def create_test_case(rng: Random, name: str, mode: random_value.RandomizationMode, chaos: bool):
     typ = spec.get_ssz_type_by_name(name)
-    # TODO: vary randomization args
-    value = random_value.get_random_ssz_object(rng, typ, 100, 10, random_value.RandomizationMode.mode_random, False)
+    value = random_value.get_random_ssz_object(rng, typ, MAX_BYTES_LENGTH, MAX_LIST_LENGTH, mode, chaos)
     yield "type_name", name
     yield "value", encode.encode(value, typ)
-    yield "serialized", serialize(value)
+    yield "serialized", serialize(value).hex()
     yield "root", '0x' + hash_tree_root(value).hex()
 
 
 @to_tuple
-def ssz_static_cases(rng: Random):
+def ssz_static_cases(rng: Random, mode: random_value.RandomizationMode, chaos: bool, count: int):
     for type_name in spec.ssz_types:
-        # TODO more types
-        for i in range(10):
-            render_test_case(rng, type_name)
+        for i in range(count):
+            yield create_test_case(rng, type_name, mode, chaos)
 
 
-def min_ssz_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
-    presets = loader.load_presets(configs_path, 'minimal')
+def get_ssz_suite(seed: int, config_name: str, mode: random_value.RandomizationMode, chaos: bool, cases_if_random: int):
+    def ssz_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
+        # Apply changes to presets, this affects some of the vector types.
+        presets = loader.load_presets(configs_path, config_name)
         spec.apply_constants_preset(presets)
-    rng = Random(123)
 
-    return ("ssz_min_values_minimal", "core", gen_suite.render_suite(
-        title="ssz testing, with minimal config",
+        # Reproducible RNG
+        rng = Random(seed)
+
+        random_mode_name = mode.to_name()
+
+        suite_name = f"ssz_{config_name}_{random_mode_name}{'_chaos' if chaos else ''}"
+
+        count = cases_if_random if chaos or mode.is_changing() else 1
+        print(f"generating SSZ-static suite ({count} cases per ssz type): {suite_name}")
+
+        return (suite_name, "core", gen_suite.render_suite(
+            title=f"ssz testing, with {config_name} config, randomized with mode {random_mode_name}{' and with chaos applied' if chaos else ''}",
             summary="Test suite for ssz serialization and hash-tree-root",
             forks_timeline="testing",
             forks=["phase0"],
             config="minimal",
             runner="ssz",
             handler="static",
-        test_cases=ssz_static_cases(rng)))
+            test_cases=ssz_static_cases(rng, mode, chaos, count)))
 
-# TODO more suites
-
-# Variation in: randomization-mode, chaos mode, configuration
+    return ssz_suite
 
 
 if __name__ == "__main__":
+    # [(seed, config name, randomization mode, chaos on/off, cases_if_random)]
+    settings = []
+    seed = 1
+    for mode in random_value.RandomizationMode:
+        settings.append((seed, "minimal", mode, False, 30))
+        seed += 1
+    settings.append((seed, "minimal", random_value.RandomizationMode.mode_random, True, 30))
+    seed += 1
+    settings.append((seed, "mainnet", random_value.RandomizationMode.mode_random, False, 5))
+    seed += 1
+
+    print("Settings: %d, SSZ-types: %d" % (len(settings), len(spec.ssz_types)))
+
     gen_runner.run_generator("ssz_static", [
-        min_ssz_suite
+        get_ssz_suite(seed, config_name, mode, chaos, cases_if_random)
+            for (seed, config_name, mode, chaos, cases_if_random) in settings
     ])

test_libs/pyspec/eth2spec/debug/random_value.py

@@ -2,23 +2,34 @@ from random import Random
 from typing import Any
 from enum import Enum
 
+
 UINT_SIZES = [8, 16, 32, 64, 128, 256]
 
 basic_types = ["uint%d" % v for v in UINT_SIZES] + ['bool', 'byte']
 
+random_mode_names = ["random", "zero", "max", "nil", "one", "lengthy"]
+
+
 class RandomizationMode(Enum):
     # random content / length
     mode_random = 0
     # Zero-value
-    mode_zero = 2
+    mode_zero = 1
     # Maximum value, limited to count 1 however
-    mode_max = 3
+    mode_max = 2
     # Return 0 values, i.e. empty
-    mode_nil_count = 4
+    mode_nil_count = 3
     # Return 1 value, random content
-    mode_one_count = 5
+    mode_one_count = 4
     # Return max amount of values, random content
-    mode_max_count = 6
+    mode_max_count = 5
+
+    def to_name(self):
+        return random_mode_names[self.value]
+
+    def is_changing(self):
+        return self.value in [0, 4, 5]
+
 
 def get_random_ssz_object(rng: Random, typ: Any, max_bytes_length: int, max_list_length: int, mode: RandomizationMode, chaos: bool) -> Any:
     """
@@ -66,7 +77,7 @@ def get_random_ssz_object(rng: Random, typ: Any, max_bytes_length: int, max_list
             return get_random_basic_value(rng, typ)
     # Vector:
     elif isinstance(typ, list) and len(typ) == 2:
-        return [get_random_ssz_object(rng, typ[0], max_bytes_length, max_list_length, mode) for _ in range(typ[1])]
+        return [get_random_ssz_object(rng, typ[0], max_bytes_length, max_list_length, mode, chaos) for _ in range(typ[1])]
     # List:
     elif isinstance(typ, list) and len(typ) == 1:
         length = rng.randint(0, max_list_length)
@@ -74,10 +85,10 @@ def get_random_ssz_object(rng: Random, typ: Any, max_bytes_length: int, max_list
             length = 1
         if mode == RandomizationMode.mode_max_count:
             length = max_list_length
-        return [get_random_ssz_object(rng, typ[0], max_bytes_length, max_list_length, mode) for _ in range(length)]
+        return [get_random_ssz_object(rng, typ[0], max_bytes_length, max_list_length, mode, chaos) for _ in range(length)]
     # Container:
     elif hasattr(typ, 'fields'):
-        return typ({field: get_random_ssz_object(rng, subtype, max_bytes_length, max_list_length, mode) for field, subtype in typ.fields.items()})
+        return typ(**{field: get_random_ssz_object(rng, subtype, max_bytes_length, max_list_length, mode, chaos) for field, subtype in typ.fields.items()})
     else:
         print(typ)
         raise Exception("Type not recognized")