Merge pull request #962 from ethereum/ssz-static-testing

SSZ static testing [blocked by #960]

scripts/phase0/function_puller.py

@@ -62,4 +62,9 @@ def get_spec(file_name: str) -> List[str]:
         code_lines.append('')
         for type_line in ssz_type:
             code_lines.append('    ' + type_line)
+    code_lines.append('')
+    code_lines.append('ssz_types = [' + ', '.join([f'\'{ssz_type_name}\'' for (ssz_type_name, _) in type_defs]) + ']')
+    code_lines.append('')
+    code_lines.append('def get_ssz_type_by_name(name: str) -> SSZType: return globals()[name]')
+    code_lines.append('')
     return code_lines

specs/test_formats/README.md

@@ -175,3 +175,24 @@ To prevent parsing of hundreds of different YAML files to test a specific test t
 │   ...                       <--- more handlers
 ...                           <--- more test types
 ```
+
+
+## Note for implementers
+
+The basic pattern for test-suite loading and running is:
+
+Iterate suites for given test-type, or sub-type (e.g. `operations > deposits`):
+1. Filter test-suite, options:
+    - Config: Load first few lines, load into YAML, and check `config`, either:
+        - Pass the suite to the correct compiled target
+        - Ignore the suite if running tests as part of a compiled target with different configuration
+        - Load the correct configuration for the suite dynamically before running the suite
+    - Select by file name
+    - Filter for specific suites (e.g. for a specific fork)
+2. Load the YAML
+    - Optionally translate the data into applicable naming, e.g. `snake_case` to `PascalCase`
+3. Iterate through the `test_cases`
+4. Ask test-runner to allocate a new test-case (i.e. objectify the test-case, generalize it with a `TestCase` interface) 
+    Optionally pass raw test-case data to enable dynamic test-case allocation.
+    1. Load test-case data into it.
+    2. Make the test-case run.

specs/test_formats/ssz/README.md

@@ -1,15 +0,0 @@
-# SSZ tests
-
-SSZ has changed throughout the development of ETH 2.0.
-
-## Contents
-
-A minimal but useful series of tests covering `uint` encoding and decoding is provided.
-This is a direct port of the older SSZ `uint` tests (minus outdated test cases).
-
-[uint test format](./uint.md).
-
-Note: the current phase-0 spec does not use larger uints, and uses byte vectors (fixed length) instead to represent roots etc.
-The exact uint lengths to support may be redefined in the future.
-
-Extension of the SSZ tests collection is planned, see CI/testing issues for progress tracking.

specs/test_formats/ssz_generic/README.md

@@ -0,0 +1,20 @@
+# SSZ, generic tests
+
+This set of test-suites provides general testing for SSZ:
+ to instantiate any container/list/vector/other type from binary data.
+
+Since SSZ is in a development-phase, not the full suite of features is covered yet.
+Note that these tests are based on the older SSZ package.
+The tests are still relevant, but limited in scope:
+ more complex object encodings have changed since the original SSZ testing.
+
+A minimal but useful series of tests covering `uint` encoding and decoding is provided.
+This is a direct port of the older SSZ `uint` tests (minus outdated test cases).
+
+[uint test format](./uint.md).
+
+Note: the current phase-0 spec does not use larger uints, and uses byte vectors (fixed length) instead to represent roots etc.
+The exact uint lengths to support may be redefined in the future.
+
+Extension of the SSZ tests collection is planned, with an update to the new spec-maintained `minimal_ssz.py`,
+ see CI/testing issues for progress tracking.

specs/test_formats/ssz_static/README.md

@@ -0,0 +1,8 @@
+# SSZ, static tests
+
+This set of test-suites provides static testing for SSZ:
+ to instantiate just the known ETH-2.0 SSZ types from binary data.
+
+This series of tests is based on the spec-maintained `minimal_ssz.py`, i.e. fully consistent with the SSZ spec. 
+
+Test format documentation can be found here: [core test format](./core.md).

specs/test_formats/ssz_static/core.md

@@ -0,0 +1,23 @@
+# Test format: SSZ static types
+
+The goal of this type is to provide clients with a solid reference how the known SSZ objects should be encoded.
+Each object described in the Phase-0 spec is covered.
+This is important, as many of the clients aiming to serialize/deserialize objects directly into structs/classes
+do not support (or have alternatives for) generic SSZ encoding/decoding.
+This test-format ensures these direct serializations are covered.
+
+## Test case format
+
+```yaml
+type_name: string  -- string, object name, formatted as in spec. E.g. "BeaconBlock"
+value: dynamic     -- the YAML-encoded value, of the type specified by type_name.
+serialized: bytes  -- string, SSZ-serialized data, hex encoded, with prefix 0x
+root: bytes32      -- string, hash-tree-root of the value, hex encoded, with prefix 0x
+```
+
+## Condition
+
+A test-runner can implement the following assertions:
+- Serialization: After parsing the `value`, SSZ-serialize it: the output should match `serialized`
+- Hash-tree-root: After parsing the `value`, Hash-tree-root it: the output should match `root`
+- Deserialization: SSZ-deserialize the `serialized` value, and see if it matches the parsed `value`

test_generators/ssz_generic/main.py

@@ -44,4 +44,4 @@ def ssz_uint_bounds_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
 
 
 if __name__ == "__main__":
-    gen_runner.run_generator("ssz", [ssz_random_uint_suite, ssz_wrong_uint_suite, ssz_uint_bounds_suite])
+    gen_runner.run_generator("ssz_generic", [ssz_random_uint_suite, ssz_wrong_uint_suite, ssz_uint_bounds_suite])

test_generators/ssz_static/README.md

@@ -0,0 +1,4 @@
+# SSZ-static
+
+The purpose of this test-generator is to provide test-vectors for the most important applications of SSZ:
+ the serialization and hashing of ETH 2.0 data types

test_generators/ssz_static/main.py

@@ -0,0 +1,79 @@
+from random import Random
+
+from eth2spec.debug import random_value, encode
+from eth2spec.phase0 import spec
+from eth2spec.utils.minimal_ssz import hash_tree_root, serialize
+from eth_utils import (
+    to_tuple, to_dict
+)
+from gen_base import gen_runner, gen_suite, gen_typing
+from preset_loader import loader
+
+MAX_BYTES_LENGTH = 100
+MAX_LIST_LENGTH = 10
+
+
+@to_dict
+def create_test_case(rng: Random, name: str, mode: random_value.RandomizationMode, chaos: bool):
+    typ = spec.get_ssz_type_by_name(name)
+    value = random_value.get_random_ssz_object(rng, typ, MAX_BYTES_LENGTH, MAX_LIST_LENGTH, mode, chaos)
+    yield "type_name", name
+    yield "value", encode.encode(value, typ)
+    yield "serialized", '0x' + serialize(value).hex()
+    yield "root", '0x' + hash_tree_root(value).hex()
+
+
+@to_tuple
+def ssz_static_cases(rng: Random, mode: random_value.RandomizationMode, chaos: bool, count: int):
+    for type_name in spec.ssz_types:
+        for i in range(count):
+            yield create_test_case(rng, type_name, mode, chaos)
+
+
+def get_ssz_suite(seed: int, config_name: str, mode: random_value.RandomizationMode, chaos: bool, cases_if_random: int):
+    def ssz_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
+        # Apply changes to presets, this affects some of the vector types.
+        presets = loader.load_presets(configs_path, config_name)
+        spec.apply_constants_preset(presets)
+
+        # Reproducible RNG
+        rng = Random(seed)
+
+        random_mode_name = mode.to_name()
+
+        suite_name = f"ssz_{config_name}_{random_mode_name}{'_chaos' if chaos else ''}"
+
+        count = cases_if_random if chaos or mode.is_changing() else 1
+        print(f"generating SSZ-static suite ({count} cases per ssz type): {suite_name}")
+
+        return (suite_name, "core", gen_suite.render_suite(
+            title=f"ssz testing, with {config_name} config, randomized with mode {random_mode_name}{' and with chaos applied' if chaos else ''}",
+            summary="Test suite for ssz serialization and hash-tree-root",
+            forks_timeline="testing",
+            forks=["phase0"],
+            config=config_name,
+            runner="ssz",
+            handler="static",
+            test_cases=ssz_static_cases(rng, mode, chaos, count)))
+
+    return ssz_suite
+
+
+if __name__ == "__main__":
+    # [(seed, config name, randomization mode, chaos on/off, cases_if_random)]
+    settings = []
+    seed = 1
+    for mode in random_value.RandomizationMode:
+        settings.append((seed, "minimal", mode, False, 30))
+        seed += 1
+    settings.append((seed, "minimal", random_value.RandomizationMode.mode_random, True, 30))
+    seed += 1
+    settings.append((seed, "mainnet", random_value.RandomizationMode.mode_random, False, 5))
+    seed += 1
+
+    print("Settings: %d, SSZ-types: %d" % (len(settings), len(spec.ssz_types)))
+
+    gen_runner.run_generator("ssz_static", [
+        get_ssz_suite(seed, config_name, mode, chaos, cases_if_random)
+            for (seed, config_name, mode, chaos, cases_if_random) in settings
+    ])

test_generators/ssz_static/requirements.txt

@@ -0,0 +1,4 @@
+eth-utils==1.4.1
+../../test_libs/gen_helpers
+../../test_libs/config_helpers
+../../test_libs/pyspec

test_libs/pyspec/eth2spec/debug/encode.py

@@ -3,6 +3,8 @@ from eth2spec.utils.minimal_ssz import hash_tree_root
 
 def encode(value, typ, include_hash_tree_roots=False):
     if isinstance(typ, str) and typ[:4] == 'uint':
+        if typ[4:] == '128' or typ[4:] == '256':
+            return str(value)
         return value
     elif typ == 'bool':
         assert value in (True, False)

test_libs/pyspec/eth2spec/debug/random_value.py

@@ -0,0 +1,137 @@
+from random import Random
+from typing import Any
+from enum import Enum
+
+
+UINT_SIZES = [8, 16, 32, 64, 128, 256]
+
+basic_types = ["uint%d" % v for v in UINT_SIZES] + ['bool', 'byte']
+
+random_mode_names = ["random", "zero", "max", "nil", "one", "lengthy"]
+
+
+class RandomizationMode(Enum):
+    # random content / length
+    mode_random = 0
+    # Zero-value
+    mode_zero = 1
+    # Maximum value, limited to count 1 however
+    mode_max = 2
+    # Return 0 values, i.e. empty
+    mode_nil_count = 3
+    # Return 1 value, random content
+    mode_one_count = 4
+    # Return max amount of values, random content
+    mode_max_count = 5
+
+    def to_name(self):
+        return random_mode_names[self.value]
+
+    def is_changing(self):
+        return self.value in [0, 4, 5]
+
+
+def get_random_ssz_object(rng: Random, typ: Any, max_bytes_length: int, max_list_length: int, mode: RandomizationMode, chaos: bool) -> Any:
+    """
+    Create an object for a given type, filled with random data.
+    :param rng: The random number generator to use.
+    :param typ: The type to instantiate
+    :param max_bytes_length: the max. length for a random bytes array
+    :param max_list_length: the max. length for a random list
+    :param mode: how to randomize
+    :param chaos: if true, the randomization-mode will be randomly changed
+    :return: the random object instance, of the given type.
+    """
+    if chaos:
+        mode = rng.choice(list(RandomizationMode))
+    if isinstance(typ, str):
+        # Bytes array
+        if typ == 'bytes':
+            if mode == RandomizationMode.mode_nil_count:
+                return b''
+            if mode == RandomizationMode.mode_max_count:
+                return get_random_bytes_list(rng, max_bytes_length)
+            if mode == RandomizationMode.mode_one_count:
+                return get_random_bytes_list(rng, 1)
+            if mode == RandomizationMode.mode_zero:
+                return b'\x00'
+            if mode == RandomizationMode.mode_max:
+                return b'\xff'
+            return get_random_bytes_list(rng, rng.randint(0, max_bytes_length))
+        elif typ[:5] == 'bytes' and len(typ) > 5:
+            length = int(typ[5:])
+            # Sanity, don't generate absurdly big random values
+            # If a client is aiming to performance-test, they should create a benchmark suite.
+            assert length <= max_bytes_length
+            if mode == RandomizationMode.mode_zero:
+                return b'\x00' * length
+            if mode == RandomizationMode.mode_max:
+                return b'\xff' * length
+            return get_random_bytes_list(rng, length)
+        # Basic types
+        else:
+            if mode == RandomizationMode.mode_zero:
+                return get_min_basic_value(typ)
+            if mode == RandomizationMode.mode_max:
+                return get_max_basic_value(typ)
+            return get_random_basic_value(rng, typ)
+    # Vector:
+    elif isinstance(typ, list) and len(typ) == 2:
+        return [get_random_ssz_object(rng, typ[0], max_bytes_length, max_list_length, mode, chaos) for _ in range(typ[1])]
+    # List:
+    elif isinstance(typ, list) and len(typ) == 1:
+        length = rng.randint(0, max_list_length)
+        if mode == RandomizationMode.mode_one_count:
+            length = 1
+        if mode == RandomizationMode.mode_max_count:
+            length = max_list_length
+        return [get_random_ssz_object(rng, typ[0], max_bytes_length, max_list_length, mode, chaos) for _ in range(length)]
+    # Container:
+    elif hasattr(typ, 'fields'):
+        return typ(**{field: get_random_ssz_object(rng, subtype, max_bytes_length, max_list_length, mode, chaos) for field, subtype in typ.fields.items()})
+    else:
+        print(typ)
+        raise Exception("Type not recognized")
+
+
+def get_random_bytes_list(rng: Random, length: int) -> bytes:
+    return bytes(rng.getrandbits(8) for _ in range(length))
+
+
+def get_random_basic_value(rng: Random, typ: str) -> Any:
+    if typ == 'bool':
+        return rng.choice((True, False))
+    if typ[:4] == 'uint':
+        size = int(typ[4:])
+        assert size in UINT_SIZES
+        return rng.randint(0, 2**size - 1)
+    if typ == 'byte':
+        return rng.randint(0, 8)
+    else:
+        raise ValueError("Not a basic type")
+
+
+def get_min_basic_value(typ: str) -> Any:
+    if typ == 'bool':
+        return False
+    if typ[:4] == 'uint':
+        size = int(typ[4:])
+        assert size in UINT_SIZES
+        return 0
+    if typ == 'byte':
+        return 0x00
+    else:
+        raise ValueError("Not a basic type")
+
+
+def get_max_basic_value(typ: str) -> Any:
+    if typ == 'bool':
+        return True
+    if typ[:4] == 'uint':
+        size = int(typ[4:])
+        assert size in UINT_SIZES
+        return 2**size - 1
+    if typ == 'byte':
+        return 0xff
+    else:
+        raise ValueError("Not a basic type")

test_libs/pyspec/eth2spec/utils/minimal_ssz.py

@@ -1,7 +1,7 @@
-from .hash_function import hash
-
 from typing import Any
 
+from .hash_function import hash
+
 BYTES_PER_CHUNK = 32
 BYTES_PER_LENGTH_PREFIX = 4
 ZERO_CHUNK = b'\x00' * BYTES_PER_CHUNK
@@ -17,10 +17,7 @@ def SSZType(fields):
                     setattr(self, f, kwargs[f])
 
         def __eq__(self, other):
-            return (
-                    self.fields == other.fields and
-                    self.serialize() == other.serialize()
-            )
+            return self.fields == other.fields and self.serialize() == other.serialize()
 
         def __hash__(self):
             return int.from_bytes(self.hash_tree_root(), byteorder="little")