From b73625fbf16330787ac586030c96de54a33b9816 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 25 Jul 2019 23:13:33 +0200 Subject: [PATCH] update test generation code (work in progress), improve the simplicity of configuration in context of forks, and update docs --- configs/README.md | 36 +++ configs/constant_presets/README.md | 20 -- configs/fork_timelines/README.md | 19 -- configs/fork_timelines/mainnet.yaml | 12 - configs/fork_timelines/testing.yaml | 6 - configs/{constant_presets => }/mainnet.yaml | 0 configs/{constant_presets => }/minimal.yaml | 0 specs/test_formats/README.md | 250 +++++++++---------- test_generators/epoch_processing/main.py | 50 ++-- test_libs/gen_helpers/gen_base/gen_runner.py | 53 ++-- test_libs/gen_helpers/gen_base/gen_suite.py | 22 -- test_libs/gen_helpers/gen_base/gen_typing.py | 32 ++- test_libs/gen_helpers/gen_from_tests/gen.py | 30 ++- 13 files changed, 257 insertions(+), 273 deletions(-) create mode 100644 configs/README.md delete mode 100644 configs/constant_presets/README.md delete mode 100644 configs/fork_timelines/README.md delete mode 100644 configs/fork_timelines/mainnet.yaml delete mode 100644 configs/fork_timelines/testing.yaml rename configs/{constant_presets => }/mainnet.yaml (100%) rename configs/{constant_presets => }/minimal.yaml (100%) delete mode 100644 test_libs/gen_helpers/gen_base/gen_suite.py diff --git a/configs/README.md b/configs/README.md new file mode 100644 index 000000000..8adb939c8 --- /dev/null +++ b/configs/README.md @@ -0,0 +1,36 @@ +# Configs + +This directory contains a set of constants presets used for testing, testnets, and mainnet. + +A preset file contains all the constants known for its target. +Later-fork constants can be ignored, e.g. ignore phase1 constants as a client that only supports phase 0 currently. + + +## Forking + +Configs are not replaced, but extended with forks. This is to support syncing from one state to the other over a fork boundary, without hot-swapping a config. +Instead, for forks that introduce changes in a constant, the constant name is prefixed with a short abbreviation of the fork. + +Over time, the need to sync an older state may be deprecated. +In this case, the prefix on the new constant may be removed, and the old constant will keep a special name before completely being removed. + +A previous iteration of forking made use of "timelines", but this collides with the definitions used in the spec (constants for special forking slots etc.), + and was not integrated sufficiently in any of the spec tools or implementations. +Instead, the config essentially doubles as fork definition now, changing the value for e.g. `PHASE_1_GENESIS_SLOT` changes the fork. + +Another reason to prefer forking through constants is the ability to program a forking moment based on context, instead of being limited to a static slot number. + + +## Format + +Each preset is a key-value mapping. + +**Key**: an `UPPER_SNAKE_CASE` (a.k.a. "macro case") formatted string, name of the constant. + +**Value** can be either: + - an unsigned integer number, can be up to 64 bits (incl.) + - a hexadecimal string, prefixed with `0x` + +Presets may contain comments to describe the values. + +See [`mainnet.yaml`](./mainnet.yaml) for a complete example. diff --git a/configs/constant_presets/README.md b/configs/constant_presets/README.md deleted file mode 100644 index 61c9a3a63..000000000 --- a/configs/constant_presets/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# Constant Presets - -This directory contains a set of constants presets used for testing, testnets, and mainnet. - -A preset file contains all the constants known for its target. -Later-fork constants can be ignored, e.g. ignore phase1 constants as a client that only supports phase 0 currently. - -## Format - -Each preset is a key-value mapping. - -**Key**: an `UPPER_SNAKE_CASE` (a.k.a. "macro case") formatted string, name of the constant. - -**Value** can be either: - - an unsigned integer number, can be up to 64 bits (incl.) - - a hexadecimal string, prefixed with `0x` - -Presets may contain comments to describe the values. - -See [`mainnet.yaml`](./mainnet.yaml) for a complete example. diff --git a/configs/fork_timelines/README.md b/configs/fork_timelines/README.md deleted file mode 100644 index da7445767..000000000 --- a/configs/fork_timelines/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# Fork timelines - -This directory contains a set of fork timelines used for testing, testnets, and mainnet. - -A timeline file contains all the forks known for its target. -Later forks can be ignored, e.g. ignore fork `phase1` as a client that only supports Phase 0 currently. - -## Format - -Each preset is a key-value mapping. - -**Key**: an `lower_snake_case` (a.k.a. "python case") formatted string, name of the fork. - -**Value**: an unsigned integer number, epoch number of activation of the fork. - -Timelines may contain comments to describe the values. - -See [`mainnet.yaml`](./mainnet.yaml) for a complete example. - diff --git a/configs/fork_timelines/mainnet.yaml b/configs/fork_timelines/mainnet.yaml deleted file mode 100644 index 0bb3c9db1..000000000 --- a/configs/fork_timelines/mainnet.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Mainnet fork timeline - -# Equal to GENESIS_EPOCH -phase0: 67108864 - -# Example 1: -# phase0_funny_fork_name: 67116000 - -# Example 2: -# Should be equal to PHASE_1_FORK_EPOCH -# (placeholder in example value here) -# phase1: 67163000 diff --git a/configs/fork_timelines/testing.yaml b/configs/fork_timelines/testing.yaml deleted file mode 100644 index 957a53b8c..000000000 --- a/configs/fork_timelines/testing.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# Testing fork timeline - -# Equal to GENESIS_EPOCH -phase0: 536870912 - -# No other forks considered in testing yet (to be implemented) diff --git a/configs/constant_presets/mainnet.yaml b/configs/mainnet.yaml similarity index 100% rename from configs/constant_presets/mainnet.yaml rename to configs/mainnet.yaml diff --git a/configs/constant_presets/minimal.yaml b/configs/minimal.yaml similarity index 100% rename from configs/constant_presets/minimal.yaml rename to configs/minimal.yaml diff --git a/specs/test_formats/README.md b/specs/test_formats/README.md index e4f013d8b..196315185 100644 --- a/specs/test_formats/README.md +++ b/specs/test_formats/README.md @@ -5,21 +5,25 @@ This document defines the YAML format and structure used for Eth 2.0 testing. ## Table of contents -- [General test format](#general-test-format) - - [Table of contents](#table-of-contents) - - [About](#about) - - [Test-case formats](#test-case-formats) - - [Glossary](#glossary) - - [Test format philosophy](#test-format-philosophy) - - [Config design](#config-design) - - [Fork config design](#fork-config-design) - - [Test completeness](#test-completeness) - - [Test suite](#test-suite) - - [Config](#config) - - [Fork-timeline](#fork-timeline) - - [Config sourcing](#config-sourcing) - - [Test structure](#test-structure) - - [Note for implementers](#note-for-implementers) +* [About](#about) + + [Test-case formats](#test-case-formats) +* [Glossary](#glossary) +* [Test format philosophy](#test-format-philosophy) + + [Config design](#config-design) + + [Test completeness](#test-completeness) +* [Test structure](#test-structure) + + [`/`](#--config-name---) + + [`/`](#--fork-or-phase-name---) + + [`/`](#--test-runner-name---) + + [`/`](#--test-handler-name---) + + [`/`](#--test-suite-name---) + + [`/`](#--test-case---) + + [``](#--output-part--) + - [Special output parts](#special-output-parts) + * [`meta.yaml`](#-metayaml-) +* [Config](#config) +* [Config sourcing](#config-sourcing) +* [Note for implementers](#note-for-implementers) @@ -42,6 +46,7 @@ Test formats: - [`ssz_static`](./ssz_static/README.md) - More formats are planned, see tracking issues for CI/testing + ## Glossary - `generator`: a program that outputs one or more `suite` files. @@ -59,13 +64,13 @@ Test formats: - `case`: a test case, an entry in the `test_cases` list of a `suite`. A case can be anything in general, but its format should be well-defined in the documentation corresponding to the `type` (and `handler`).\ A test has the same exact configuration and fork context as the other entries in the `case` list of its `suite`. -- `forks_timeline`: a fork timeline definition, a YAML file containing a key for each fork-name, and an epoch number as value. + ## Test format philosophy ### Config design -After long discussion, the following types of configured constants were identified: +The configuration constant types are: - Never changing: genesis data. - Changing, but reliant on old value: e.g. an epoch time may change, but if you want to do the conversion `(genesis data, timestamp) -> epoch number`, you end up needing both constants. @@ -75,26 +80,12 @@ After long discussion, the following types of configured constants were identifi - Changing: there is a very small chance some constant may really be *replaced*. In this off-chance, it is likely better to include it as an additional variable, and some clients may simply stop supporting the old one if they do not want to sync from genesis. + The change of functionality goes through a phase of deprecation of the old constant, and eventually only the new constant is kept around in the config (when old state is not supported anymore). Based on these types of changes, we model the config as a list of key value pairs, that only grows with every fork (they may change in development versions of forks, however; git manages this). With this approach, configurations are backwards compatible (older clients ignore unknown variables) and easy to maintain. -### Fork config design - -There are two types of fork-data: -1) Timeline: When does a fork take place? -2) Coverage: What forks are covered by a test? - -The first is neat to have as a separate form: we prevent duplication, and can run with different presets - (e.g. fork timeline for a minimal local test, for a public testnet, or for mainnet). - -The second does not affect the result of the tests, it just states what is covered by the tests, - so that the right suites can be executed to see coverage for a certain fork. -For some types of tests, it may be beneficial to ensure it runs exactly the same, with any given fork "active". -Test-formats can be explicit on the need to repeat a test with different forks being "active", - but generally tests run only once. - ### Test completeness Tests should be independent of any sync-data. If one wants to run a test, the input data should be available from the YAML. @@ -104,93 +95,66 @@ The aim is to provide clients with a well-defined scope of work to run a particu - Clients that are not complete in functionality can choose to ignore suites that use certain test-runners, or specific handlers of these test-runners. - Clients that are on older versions can test their work based on older releases of the generated tests, and catch up with newer releases when possible. -## Test suite - -``` -title: -- Display name for the test suite -summary: -- Summarizes the test suite -forks_timeline: -- Used to determine the forking timeline -forks: -- Defines the coverage. Test-runner code may decide to re-run with the different forks "activated", when applicable. -config: -- Used to determine which set of constants to run (possibly compile time) with -runner: *MUST be consistent with folder structure* -handler: *MUST be consistent with folder structure* - -test_cases: - ... - -``` - -## Config - -A configuration is a separate YAML file. -Separation of configuration and tests aims to: -- Prevent duplication of configuration -- Make all tests easy to upgrade (e.g. when a new config constant is introduced) -- Clearly define which constants to use -- Shareable between clients, for cross-client short- or long-lived testnets -- Minimize the amounts of different constants permutations to compile as a client. - *Note*: Some clients prefer compile-time constants and optimizations. - They should compile for each configuration once, and run the corresponding tests per build target. - -The format is described in [`configs/constant_presets`](../../configs/constant_presets/README.md#format). - - -## Fork-timeline - -A fork timeline is (preferably) loaded in as a configuration object into a client, as opposed to the constants configuration: - - We do not allocate or optimize any code based on epoch numbers. - - When we transition from one fork to the other, it is preferred to stay online. - - We may decide on an epoch number for a fork based on external events (e.g. Eth1 log event); - a client should be able to activate a fork dynamically. - -The format is described in [`configs/fork_timelines`](../../configs/fork_timelines/README.md#format). - -## Config sourcing - -The constants configurations are located in: - -``` -/configs/constant_presets/.yaml -``` - -And copied by CI for testing purposes to: - -``` -/configs/constant_presets/.yaml -``` - - -The fork timelines are located in: - -``` -/configs/fork_timelines/.yaml -``` - -And copied by CI for testing purposes to: - -``` -/configs/fork_timelines/.yaml -``` ## Test structure -To prevent parsing of hundreds of different YAML files to test a specific test type, - or even more specific, just a handler, tests should be structured in the following nested form: - ``` -. <--- root of eth2.0 tests repository -├── bls <--- collection of handler for a specific test-runner, example runner: "bls" -│   ├── verify_msg <--- collection of test suites for a specific handler, example handler: "verify_msg". If no multiple handlers, use a dummy folder (e.g. "core"), and specify that in the yaml. -│   │   ├── verify_valid.yml . -│   │   ├── special_cases.yml . a list of test suites -│   │   ├── domains.yml . -│   │   ├── invalid.yml . -│   │   ... <--- more suite files (optional) -│   ... <--- more handlers -... <--- more test types +File path structure: +tests/////// ``` -## Common test-case properties +### `/` + +Configs are upper level. Some clients want to run minimal first, and useful for sanity checks during development too. +As a top level dir, it is not duplicated, and the used config can be copied right into this directory as reference. + +### `/` + +This would be: "phase0", "transferparty", "phase1", etc. Each introduces new tests, but does not copy tests that do not change. +If you like to test phase 1, you run phase 0 tests, with the configuration that includes phase 1 changes. Out of scope for now however. + +### `/` + +The well known bls/shuffling/ssz_static/operations/epoch_processing/etc. Handlers can change the format, but there is a general target to test. + + +### `/` + +Specialization within category. All suites in here will have the same test case format. + +### `/` + +Suites are split up. Suite size does not change memory bounds, and makes lookups of particular tests fast to find and load. + +### `/` + +Cases are split up too. This enables diffing of parts of the test case, tracking changes per part, while still using LFS. Also enables different formats for some parts. + +### `` + +E.g. `pre.yaml`, `deposit.yaml`, `post.yaml`. + +Diffing a `pre.yaml` and `post.yaml` provides all the information for testing, good for readability of the change. +Then the difference between pre and post can be compared to anything that changes the pre state, e.g. `deposit.yaml` + +These files allow for custom formats for some parts of the test. E.g. something encoded in SSZ. + +Some yaml files have copies, but formatted as raw SSZ bytes: `pre.ssz`, `deposit.ssz`, `post.ssz`. +The yaml files are intended to be deprecated, and clients should shift to ssz inputs for efficiency. +Deprecation will start once a viewer of SSZ test-cases is in place, to maintain a standard of readable test cases. +This also means that some clients can drop legacy YAML -> JSON/other -> SSZ work-arounds. +(These were implemented to support the uint64 YAML, hex strings, etc. Things that were not idiomatic to their language.) + +Yaml will not be deprecated for tests that do not use SSZ: e.g. shuffling and BLS tests. +In this case, there is no work around for loading necessary anyway, and the size and efficiency of yaml is acceptable. + +#### Special output parts + +##### `meta.yaml` + +If present (it is optional), the test is enhanced with extra data to describe usage. Specialized data is described in the documentation of the specific test format. + +Common data is documented here: Some test-case formats share some common key-value pair patterns, and these are documented here: @@ -203,22 +167,52 @@ bls_setting: int -- optional, can have 3 different values: 2: known as "BLS ignored" - if the test validity is strictly dependent on BLS being OFF ``` + +## Config + +A configuration is a separate YAML file. +Separation of configuration and tests aims to: +- Prevent duplication of configuration +- Make all tests easy to upgrade (e.g. when a new config constant is introduced) +- Clearly define which constants to use +- Shareable between clients, for cross-client short- or long-lived testnets +- Minimize the amounts of different constants permutations to compile as a client. + *Note*: Some clients prefer compile-time constants and optimizations. + They should compile for each configuration once, and run the corresponding tests per build target. +- Includes constants to coordinate forking with. + +The format is described in [`/configs`](../../configs/README.md#format). + + +## Config sourcing + +The constants configurations are located in: + +``` +/configs/.yaml +``` + +And copied by CI for testing purposes to: + +``` +/tests//.yaml +``` + +The first `` is a directory, which contains exactly all tests that make use of the given config. + + ## Note for implementers The basic pattern for test-suite loading and running is: -Iterate suites for given test-type, or sub-type (e.g. `operations > deposits`): -1. Filter test-suite, options: - - Config: Load first few lines, load into YAML, and check `config`, either: - - Pass the suite to the correct compiled target - - Ignore the suite if running tests as part of a compiled target with different configuration - - Load the correct configuration for the suite dynamically before running the suite - - Select by file name - - Filter for specific suites (e.g. for a specific fork) -2. Load the YAML - - Optionally translate the data into applicable naming, e.g. `snake_case` to `PascalCase` -3. Iterate through the `test_cases` -4. Ask test-runner to allocate a new test-case (i.e. objectify the test-case, generalize it with a `TestCase` interface) - Optionally pass raw test-case data to enable dynamic test-case allocation. - 1. Load test-case data into it. - 2. Make the test-case run. +1. For a specific config, load it first (and only need to do so once), + then continue with the tests defined in the config folder. +2. Select a fork. Repeat for each fork if running tests for multiple forks. +3. Select the category and specialization of interest (e.g. `operations > deposits`). Again, repeat for each if running all. +4. Select a test suite. Or repeat for each. +5. Select a test case. Or repeat for each. +6. Load the parts of the case. And `meta.yaml` if present. +7. Run the test, as defined by the test format. + +Step 1 may be a step with compile time selection of a configuration, if desired for optimization. +The base requirement is just to use the same set of constants, independent of the loading process. diff --git a/test_generators/epoch_processing/main.py b/test_generators/epoch_processing/main.py index 6a578c598..da41c9e95 100644 --- a/test_generators/epoch_processing/main.py +++ b/test_generators/epoch_processing/main.py @@ -14,40 +14,36 @@ from gen_from_tests.gen import generate_from_tests from preset_loader import loader -def create_suite(transition_name: str, config_name: str, get_cases: Callable[[], Iterable[gen_typing.TestCase]]) \ - -> Callable[[str], gen_typing.TestSuiteOutput]: - def suite_definition(configs_path: str) -> gen_typing.TestSuiteOutput: +def create_suite(handler_name: str, tests_src, config_name: str) \ + -> Callable[[str], gen_typing.TestProvider]: + + def prepare_fn(configs_path: str) -> str: presets = loader.load_presets(configs_path, config_name) spec_phase0.apply_constants_preset(presets) spec_phase1.apply_constants_preset(presets) + return config_name - return ("%s_%s" % (transition_name, config_name), transition_name, gen_suite.render_suite( - title="%s epoch processing" % transition_name, - summary="Test suite for %s type epoch processing" % transition_name, - forks_timeline="testing", - forks=["phase0"], - config=config_name, - runner="epoch_processing", - handler=transition_name, - test_cases=get_cases())) + def cases_fn() -> Iterable[gen_typing.TestCase]: + return generate_from_tests( + runner_name='epoch_processing', + handler_name=handler_name, + src=tests_src, + fork_name='phase0' + ) - return suite_definition + return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) if __name__ == "__main__": gen_runner.run_generator("epoch_processing", [ - create_suite('crosslinks', 'minimal', lambda: generate_from_tests(test_process_crosslinks, 'phase0')), - create_suite('crosslinks', 'mainnet', lambda: generate_from_tests(test_process_crosslinks, 'phase0')), - create_suite('final_updates', 'minimal', lambda: generate_from_tests(test_process_final_updates, 'phase0')), - create_suite('final_updates', 'mainnet', lambda: generate_from_tests(test_process_final_updates, 'phase0')), - create_suite('justification_and_finalization', 'minimal', - lambda: generate_from_tests(test_process_justification_and_finalization, 'phase0')), - create_suite('justification_and_finalization', 'mainnet', - lambda: generate_from_tests(test_process_justification_and_finalization, 'phase0')), - create_suite('registry_updates', 'minimal', - lambda: generate_from_tests(test_process_registry_updates, 'phase0')), - create_suite('registry_updates', 'mainnet', - lambda: generate_from_tests(test_process_registry_updates, 'phase0')), - create_suite('slashings', 'minimal', lambda: generate_from_tests(test_process_slashings, 'phase0')), - create_suite('slashings', 'mainnet', lambda: generate_from_tests(test_process_slashings, 'phase0')), + create_suite('crosslinks', test_process_crosslinks, 'minimal'), + create_suite('crosslinks', test_process_crosslinks, 'mainnet'), + create_suite('final_updates', test_process_final_updates, 'minimal'), + create_suite('final_updates', test_process_final_updates, 'mainnet'), + create_suite('justification_and_finalization', test_process_justification_and_finalization, 'minimal'), + create_suite('justification_and_finalization', test_process_justification_and_finalization, 'mainnet'), + create_suite('registry_updates', test_process_registry_updates, 'minimal'), + create_suite('registry_updates', test_process_registry_updates, 'mainnet'), + create_suite('slashings', test_process_slashings, 'minimal'), + create_suite('slashings', test_process_slashings, 'mainnet'), ]) diff --git a/test_libs/gen_helpers/gen_base/gen_runner.py b/test_libs/gen_helpers/gen_base/gen_runner.py index e36d48b8b..b118f48d9 100644 --- a/test_libs/gen_helpers/gen_base/gen_runner.py +++ b/test_libs/gen_helpers/gen_base/gen_runner.py @@ -7,7 +7,7 @@ from ruamel.yaml import ( YAML, ) -from gen_base.gen_typing import TestSuiteCreator +from gen_base.gen_typing import TestProvider def validate_output_dir(path_str): @@ -46,14 +46,17 @@ def validate_configs_dir(path_str): return path -def run_generator(generator_name, suite_creators: List[TestSuiteCreator]): +def run_generator(generator_name, test_providers: Iterable[TestProvider]): """ Implementation for a general test generator. :param generator_name: The name of the generator. (lowercase snake_case) - :param suite_creators: A list of suite creators, each of these builds a list of test cases. + :param test_providers: A list of test provider, + each of these returns a callable that returns an iterable of test cases. + The call to get the iterable may set global configuration, + and the iterable should not be resumed after a pause with a change of that configuration. :return: """ - + parser = argparse.ArgumentParser( prog="gen-" + generator_name, description=f"Generate YAML test suite files for {generator_name}", @@ -92,24 +95,32 @@ def run_generator(generator_name, suite_creators: List[TestSuiteCreator]): yaml = YAML(pure=True) yaml.default_flow_style = None - print(f"Generating tests for {generator_name}, creating {len(suite_creators)} test suite files...") + print(f"Generating tests into {output_dir}...") print(f"Reading config presets and fork timelines from {args.configs_path}") - for suite_creator in suite_creators: - (output_name, handler, suite) = suite_creator(args.configs_path) - handler_output_dir = Path(output_dir) / Path(handler) - try: - if not handler_output_dir.exists(): - handler_output_dir.mkdir() - except FileNotFoundError as e: - sys.exit(f'Error when creating handler dir {handler} for test "{suite["title"]}" ({e})') + for tprov in test_providers: + # loads configuration etc. + config_name = tprov.prepare(args.configs_path) + for test_case in tprov.make_cases(): + case_dir = Path(output_dir) / Path(config_name) / Path(test_case.fork_name) \ + / Path(test_case.runner_name) / Path(test_case.handler_name) \ + / Path(test_case.suite_name) / Path(test_case.case_name) + print(f'Generating test: {case_dir}') - out_path = handler_output_dir / Path(output_name + '.yaml') + case_dir.mkdir(parents=True, exist_ok=True) - try: - with out_path.open(file_mode) as f: - yaml.dump(suite, f) - except IOError as e: - sys.exit(f'Error when dumping test "{suite["title"]}" ({e})') - - print("done.") + try: + for case_part in test_case.case_fn(): + if case_part.out_kind == "data" or case_part.out_kind == "ssz": + try: + out_path = case_dir / Path(case_part.name + '.yaml') + with out_path.open(file_mode) as f: + yaml.dump(case_part.data, f) + except IOError as e: + sys.exit(f'Error when dumping test "{case_dir}", part "{case_part.name}": {e}') + # if out_kind == "ssz": + # # TODO write SSZ as binary file too. + # out_path = case_dir / Path(name + '.ssz') + except Exception as e: + print(f"ERROR: failed to generate vector(s) for test {case_dir}: {e}") + print(f"completed {generator_name}") diff --git a/test_libs/gen_helpers/gen_base/gen_suite.py b/test_libs/gen_helpers/gen_base/gen_suite.py deleted file mode 100644 index a3f88791f..000000000 --- a/test_libs/gen_helpers/gen_base/gen_suite.py +++ /dev/null @@ -1,22 +0,0 @@ -from typing import Iterable - -from eth_utils import to_dict -from gen_base.gen_typing import TestCase - - -@to_dict -def render_suite(*, - title: str, summary: str, - forks_timeline: str, forks: Iterable[str], - config: str, - runner: str, - handler: str, - test_cases: Iterable[TestCase]): - yield "title", title - yield "summary", summary - yield "forks_timeline", forks_timeline, - yield "forks", forks - yield "config", config - yield "runner", runner - yield "handler", handler - yield "test_cases", test_cases diff --git a/test_libs/gen_helpers/gen_base/gen_typing.py b/test_libs/gen_helpers/gen_base/gen_typing.py index 011326a69..91c4be74a 100644 --- a/test_libs/gen_helpers/gen_base/gen_typing.py +++ b/test_libs/gen_helpers/gen_base/gen_typing.py @@ -1,14 +1,34 @@ from typing import ( Any, Callable, + Iterable, Dict, Tuple, ) +from collections import namedtuple -TestCase = Dict[str, Any] -TestSuite = Dict[str, Any] -# Tuple: (output name, handler name, suite) -- output name excl. ".yaml" -TestSuiteOutput = Tuple[str, str, TestSuite] -# Args: -TestSuiteCreator = Callable[[str], TestSuiteOutput] +@dataclass +class TestCasePart(object): + name: str # name of the file + out_kind: str # type of data ("data" for generic, "ssz" for SSZ encoded bytes) + data: Any + + +@dataclass +class TestCase(object): + fork_name: str + runner_name: str + handler_name: str + suite_name: str + case_name: str + case_fn: Callable[[], Iterable[TestCasePart]] + + +@dataclass +class TestProvider(object): + # Prepares the context with a configuration, loaded from the given config path. + # fn(config path) => chosen config name + prepare: Callable[[str], str] + # Retrieves an iterable of cases, called after prepare() + make_cases: Callable[[], Iterable[TestCase]] diff --git a/test_libs/gen_helpers/gen_from_tests/gen.py b/test_libs/gen_helpers/gen_from_tests/gen.py index 3810c385e..cc64fbf41 100644 --- a/test_libs/gen_helpers/gen_from_tests/gen.py +++ b/test_libs/gen_helpers/gen_from_tests/gen.py @@ -1,26 +1,32 @@ from inspect import getmembers, isfunction -def generate_from_tests(src, phase, bls_active=True): +from gen_base.gen_typing import TestCase + + +def generate_from_tests(runner_name: str, handler_name: str, src: Any, + fork_name: str, bls_active: bool = True) -> Iterable[TestCase]: """ Generate a list of test cases by running tests from the given src in generator-mode. + :param runner_name: to categorize the test in general as. + :param handler_name: to categorize the test specialization as. :param src: to retrieve tests from (discovered using inspect.getmembers). - :param phase: to run tests against particular phase. + :param fork_name: to run tests against particular phase and/or fork. + (if multiple forks are applicable, indicate the last fork) :param bls_active: optional, to override BLS switch preference. Defaults to True. - :return: the list of test cases. + :return: an iterable of test cases. """ fn_names = [ name for (name, _) in getmembers(src, isfunction) if name.startswith('test_') ] - out = [] print("generating test vectors from tests source: %s" % src.__name__) for name in fn_names: tfn = getattr(src, name) - try: - test_case = tfn(generator_mode=True, phase=phase, bls_active=bls_active) - # If no test case data is returned, the test is ignored. - if test_case is not None: - out.append(test_case) - except AssertionError: - print("ERROR: failed to generate vector from test: %s (src: %s)" % (name, src.__name__)) - return out + yield TestCase( + fork_name=fork_name, + runner_name=runner_name, + handler_name=handler_name, + suite_name='pyspec_tests', + case_name=name, + case_fn=lambda: tfn(generator_mode=True, phase=phase, bls_active=bls_active) + )