diff --git a/configs/fork_timelines/testing.yaml b/configs/fork_timelines/testing.yaml new file mode 100644 index 000000000..957a53b8c --- /dev/null +++ b/configs/fork_timelines/testing.yaml @@ -0,0 +1,6 @@ +# Testing fork timeline + +# Equal to GENESIS_EPOCH +phase0: 536870912 + +# No other forks considered in testing yet (to be implemented) diff --git a/specs/test_formats/README.md b/specs/test_formats/README.md index 371c489b6..f5d78193d 100644 --- a/specs/test_formats/README.md +++ b/specs/test_formats/README.md @@ -89,14 +89,14 @@ The aim is to provide clients with a well-defined scope of work to run a particu ## Test Suite ``` -title: -- Display name for the test suite -summary: -- Summarizes the test suite -forks_timeline: -- Used to determine the forking timeline -forks: -- Runner decides what to do: run for each fork, or run for all at once, each fork transition, etc. - - ... -config: -- Used to determine which set of constants to run (possibly compile time) with -runner: *MUST be consistent with folder structure* -handler: *MUST be consistent with folder structure* +title: -- Display name for the test suite +summary: -- Summarizes the test suite +forks_timeline: -- Used to determine the forking timeline +forks: -- Runner decides what to do: run for each fork, or run for all at once, each fork transition, etc. + - ... +config: -- Used to determine which set of constants to run (possibly compile time) with +runner: *MUST be consistent with folder structure* +handler: *MUST be consistent with folder structure* test_cases: ... @@ -163,7 +163,7 @@ To prevent parsing of hundreds of different YAML files to test a specific test t ``` . <--- root of eth2.0 tests repository ├── bls <--- collection of handler for a specific test-runner, example runner: "bls" -│   ├── signing <--- collection of test suites for a specific handler, example handler: "signing". If no handler, use a dummy folder "main" +│   ├── signing <--- collection of test suites for a specific handler, example handler: "signing". If no multiple handlers, use a dummy folder (e.g. "main"), and specify that in the yaml. │   │   ├── sign_msg.yml <--- an entry list of test suites │   │   ... <--- more suite files (optional) │   ... <--- more handlers diff --git a/test_generators/README.md b/test_generators/README.md index bacb7229a..c7c77c961 100644 --- a/test_generators/README.md +++ b/test_generators/README.md @@ -59,11 +59,12 @@ Create a `requirements.txt` in the root of your generator directory: ``` eth-utils==1.4.1 ../../test_libs/gen_helpers -``` -And optionally, to include pyspec, add: -``` +../../test_libs/config_helpers ../../test_libs/pyspec ``` +The config helper and pyspec is optional, but preferred. We encourage generators to derive tests from the spec itself, to prevent code duplication and outdated tests. +Applying configurations to the spec is easy, and enables you to create test suites with different contexts. + Note: make sure to run `make pyspec` from the root of the specs repository, to build the pyspec requirement. Install all the necessary requirements (re-run when you add more): @@ -82,45 +83,60 @@ from eth_utils import ( to_dict, to_tuple ) +from preset_loader import loader +from eth2spec.phase0 import spec @to_dict -def bar_test_case(v: int): - yield "bar_v", v - yield "bar_v_plus_1", v + 1 - yield "bar_list", list(range(v)) +def example_test_case(v: int): + yield "spec_SHARD_COUNT", spec.SHARD_COUNT + yield "example", v @to_tuple -def generate_bar_test_cases(): +def generate_example_test_cases(): for i in range(10): - yield bar_test_case(i) + yield example_test_case(i) -def bar_test_suite() -> gen_typing.TestSuite: +def example_minimal_suite(configs_path: str) -> gen_typing.TestSuite: + presets = loader.load_presets(configs_path, 'minimal') + spec.apply_constants_preset(presets) + return gen_suite.render_suite( - title="bar_minimal", + title="example_minimal", summary="Minimal example suite, testing bar.", - fork="v0.5.1", + forks_timeline="testing", + forks=["phase0"], config="minimal", - test_cases=generate_bar_test_cases()) + handler="main", + test_cases=generate_example_test_cases()) + + +def example_mainnet_suite(configs_path: str) -> gen_typing.TestSuite: + presets = loader.load_presets(configs_path, 'mainnet') + spec.apply_constants_preset(presets) + + return gen_suite.render_suite( + title="example_main_net", + summary="Main net based example suite.", + forks_timeline= "mainnet", + forks=["phase0"], + config="testing", + handler="main", + test_cases=generate_example_test_cases()) if __name__ == "__main__": - gen_runner.run_generator("foo", [bar_test_suite]) - -``` - -And to use the pyspec: - -``` -from eth2spec.phase0 import spec + gen_runner.run_generator("example", [example_minimal_suite, example_mainnet_suite]) ``` Recommendations: -- you can have more than just 1 generator, e.g. ` gen_runner.run_generator("foo", [bar_test_suite, abc_test_suite, example_test_suite])` +- you can have more than just 1 suite creator, e.g. ` gen_runner.run_generator("foo", [bar_test_suite, abc_test_suite, example_test_suite])` - you can concatenate lists of test cases, if you don't want to split it up in suites. -- you can split your suite generators into different python files/packages, good for code organization. -- use config "minimal" for performance. But also implement a suite with the default config where necessary +- you can split your suite creators into different python files/packages, good for code organization. +- use config "minimal" for performance. But also implement a suite with the default config where necessary. +- you may be able to write your test suite creator in a way where it does not make assumptions on constants. + If so, you can generate test suites with different configurations for the same scenario (see example). - the test-generator accepts `--output` and `--force` (overwrite output) ## How to add a new test generator @@ -133,13 +149,13 @@ In order to add a new test generator that builds `New Tests`: with any dependencies it may need. Leave it empty if your generator has none. 3. Your generator is assumed to have a `main.py` file in its root. By adding the base generator to your requirements, you can make a generator really easily. See docs below. -4. Your generator is called with `-o some/file/path/for_testing/can/be_anything`. +4. Your generator is called with `-o some/file/path/for_testing/can/be_anything -c some/other/path/to_configs/`. The base generator helps you handle this; you only have to define suite headers, and a list of tests for each suite you generate. 5. Finally, add any linting or testing commands to the [circleci config file](https://github.com/ethereum/eth2.0-test-generators/blob/master/.circleci/config.yml) if desired to increase code quality. - + Note: you do not have to change the makefile. However, if necessary (e.g. not using python, or mixing in other languages), submit an issue, and it can be a special case. Do note that generators should be easy to maintain, lean, and based on the spec. diff --git a/test_libs/gen_helpers/gen_base/gen_suite.py b/test_libs/gen_helpers/gen_base/gen_suite.py index fdfac8292..3459d9ae3 100644 --- a/test_libs/gen_helpers/gen_base/gen_suite.py +++ b/test_libs/gen_helpers/gen_base/gen_suite.py @@ -1,17 +1,20 @@ from typing import Iterable -from eth_utils import ( - to_dict, -) - +from eth_utils import to_dict from gen_base.gen_typing import TestCase @to_dict -def render_suite(*, title: str, summary: str, fork: str, config: str, test_cases: Iterable[TestCase]): +def render_suite(*, + title: str, summary: str, + forks_timeline: str, forks: Iterable[str], + config: str, + handler: str, + test_cases: Iterable[TestCase]): yield "title", title - if summary is not None: - yield "summary", summary - yield "fork", fork + yield "summary", summary + yield "forks_timeline", forks_timeline, + yield "forks", forks yield "config", config + yield "handler", handler yield "test_cases", test_cases