Work towards testing all edge-cases of SSZ, for known (static) object types

scripts/phase0/function_puller.py

@@ -62,4 +62,9 @@ def get_spec(file_name: str) -> List[str]:
         code_lines.append('')
         for type_line in ssz_type:
             code_lines.append('    ' + type_line)
+    code_lines.append('')
+    code_lines.append('ssz_types = [' + ', '.join([f'\'{ssz_type_name}\'' for (ssz_type_name, _) in type_defs]) + ']')
+    code_lines.append('')
+    code_lines.append('def get_ssz_type_by_name(name: str) -> SSZType: return globals()[name]')
+    code_lines.append('')
     return code_lines

test_generators/ssz_generic/main.py

@@ -44,4 +44,4 @@ def ssz_uint_bounds_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
 
 
 if __name__ == "__main__":
-    gen_runner.run_generator("ssz", [ssz_random_uint_suite, ssz_wrong_uint_suite, ssz_uint_bounds_suite])
+    gen_runner.run_generator("ssz_generic", [ssz_random_uint_suite, ssz_wrong_uint_suite, ssz_uint_bounds_suite])

test_generators/ssz_static/README.md

@@ -0,0 +1,4 @@
+# SSZ-static
+
+The purpose of this test-generator is to provide test-vectors for the most important applications of SSZ:
+ the serialization and hashing of ETH 2.0 data types

test_generators/ssz_static/main.py

@@ -0,0 +1,55 @@
+from eth_utils import (
+    to_tuple, to_dict
+)
+from preset_loader import loader
+from eth2spec.phase0 import spec
+from eth2spec.utils.minimal_ssz import hash_tree_root, serialize
+from eth2spec.debug import random_value, encode
+
+from gen_base import gen_runner, gen_suite, gen_typing
+from random import Random
+
+
+@to_dict
+def render_test_case(rng: Random, name):
+    typ = spec.get_ssz_type_by_name(name)
+    # TODO: vary randomization args
+    value = random_value.get_random_ssz_object(rng, typ, 100, 10, random_value.RandomizationMode.mode_random, False)
+    yield "type_name", name
+    yield "value", encode.encode(value, typ)
+    yield "serialized", serialize(value)
+    yield "root", '0x' + hash_tree_root(value).hex()
+
+
+@to_tuple
+def ssz_static_cases(rng: Random):
+    for type_name in spec.ssz_types:
+        # TODO more types
+        for i in range(10):
+            render_test_case(rng, type_name)
+
+
+def min_ssz_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
+    presets = loader.load_presets(configs_path, 'minimal')
+    spec.apply_constants_preset(presets)
+    rng = Random(123)
+
+    return ("ssz_min_values_minimal", "core", gen_suite.render_suite(
+        title="ssz testing, with minimal config",
+        summary="Test suite for ssz serialization and hash-tree-root",
+        forks_timeline="testing",
+        forks=["phase0"],
+        config="minimal",
+        runner="ssz",
+        handler="static",
+        test_cases=ssz_static_cases(rng)))
+
+# TODO more suites
+
+# Variation in: randomization-mode, chaos mode, configuration
+
+
+if __name__ == "__main__":
+    gen_runner.run_generator("ssz_static", [
+        min_ssz_suite
+    ])

test_generators/ssz_static/requirements.txt

@@ -0,0 +1,4 @@
+eth-utils==1.4.1
+../../test_libs/gen_helpers
+../../test_libs/config_helpers
+../../test_libs/pyspec

test_libs/pyspec/eth2spec/debug/random_value.py

@@ -0,0 +1,126 @@
+from random import Random
+from typing import Any
+from enum import Enum
+
+UINT_SIZES = [8, 16, 32, 64, 128, 256]
+
+basic_types = ["uint%d" % v for v in UINT_SIZES] + ['bool', 'byte']
+
+class RandomizationMode(Enum):
+    # random content / length
+    mode_random = 0
+    # Zero-value
+    mode_zero = 2
+    # Maximum value, limited to count 1 however
+    mode_max = 3
+    # Return 0 values, i.e. empty
+    mode_nil_count = 4
+    # Return 1 value, random content
+    mode_one_count = 5
+    # Return max amount of values, random content
+    mode_max_count = 6
+
+def get_random_ssz_object(rng: Random, typ: Any, max_bytes_length: int, max_list_length: int, mode: RandomizationMode, chaos: bool) -> Any:
+    """
+    Create an object for a given type, filled with random data.
+    :param rng: The random number generator to use.
+    :param typ: The type to instantiate
+    :param max_bytes_length: the max. length for a random bytes array
+    :param max_list_length: the max. length for a random list
+    :param mode: how to randomize
+    :param chaos: if true, the randomization-mode will be randomly changed
+    :return: the random object instance, of the given type.
+    """
+    if chaos:
+        mode = rng.choice(list(RandomizationMode))
+    if isinstance(typ, str):
+        # Bytes array
+        if typ == 'bytes':
+            if mode == RandomizationMode.mode_nil_count:
+                return b''
+            if mode == RandomizationMode.mode_max_count:
+                return get_random_bytes_list(rng, max_bytes_length)
+            if mode == RandomizationMode.mode_one_count:
+                return get_random_bytes_list(rng, 1)
+            if mode == RandomizationMode.mode_zero:
+                return b'\x00'
+            if mode == RandomizationMode.mode_max:
+                return b'\xff'
+            return get_random_bytes_list(rng, rng.randint(0, max_bytes_length))
+        elif typ[:5] == 'bytes' and len(typ) > 5:
+            length = int(typ[5:])
+            # Sanity, don't generate absurdly big random values
+            # If a client is aiming to performance-test, they should create a benchmark suite.
+            assert length <= max_bytes_length
+            if mode == RandomizationMode.mode_zero:
+                return b'\x00' * length
+            if mode == RandomizationMode.mode_max:
+                return b'\xff' * length
+            return get_random_bytes_list(rng, length)
+        # Basic types
+        else:
+            if mode == RandomizationMode.mode_zero:
+                return get_min_basic_value(typ)
+            if mode == RandomizationMode.mode_max:
+                return get_max_basic_value(typ)
+            return get_random_basic_value(rng, typ)
+    # Vector:
+    elif isinstance(typ, list) and len(typ) == 2:
+        return [get_random_ssz_object(rng, typ[0], max_bytes_length, max_list_length, mode) for _ in range(typ[1])]
+    # List:
+    elif isinstance(typ, list) and len(typ) == 1:
+        length = rng.randint(0, max_list_length)
+        if mode == RandomizationMode.mode_one_count:
+            length = 1
+        if mode == RandomizationMode.mode_max_count:
+            length = max_list_length
+        return [get_random_ssz_object(rng, typ[0], max_bytes_length, max_list_length, mode) for _ in range(length)]
+    # Container:
+    elif hasattr(typ, 'fields'):
+        return typ({field: get_random_ssz_object(rng, subtype, max_bytes_length, max_list_length, mode) for field, subtype in typ.fields.items()})
+    else:
+        print(typ)
+        raise Exception("Type not recognized")
+
+
+def get_random_bytes_list(rng: Random, length: int) -> bytes:
+    return bytes(rng.getrandbits(8) for _ in range(length))
+
+
+def get_random_basic_value(rng: Random, typ: str) -> Any:
+    if typ == 'bool':
+        return rng.choice((True, False))
+    if typ[:4] == 'uint':
+        size = int(typ[4:])
+        assert size in (8, 16, 32, 64, 128, 256)
+        return rng.randint(0, 2**size - 1)
+    if typ == 'byte':
+        return rng.randint(0, 8)
+    else:
+        raise ValueError("Not a basic type")
+
+
+def get_min_basic_value(typ: str) -> Any:
+    if typ == 'bool':
+        return False
+    if typ[:4] == 'uint':
+        size = int(typ[4:])
+        assert size in (8, 16, 32, 64, 128, 256)
+        return 0
+    if typ == 'byte':
+        return 0x00
+    else:
+        raise ValueError("Not a basic type")
+
+
+def get_max_basic_value(typ: str) -> Any:
+    if typ == 'bool':
+        return True
+    if typ[:4] == 'uint':
+        size = int(typ[4:])
+        assert size in (8, 16, 32, 64, 128, 256)
+        return 2**size - 1
+    if typ == 'byte':
+        return 0xff
+    else:
+        raise ValueError("Not a basic type")