Add new check_mods function for generators (#3970)

* Add new check_mods function for generators

* Use single qoutes for consistency

* Add [ERROR] to the exception & update readme

* Fix typos

* Fix lint

tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py

@@ -1,5 +1,6 @@
 from importlib import import_module
 from inspect import getmembers, isfunction
+from pkgutil import walk_packages
 from typing import Any, Callable, Dict, Iterable, Optional, List, Union
 
 from eth2spec.utils import bls
@@ -134,3 +135,69 @@ def combine_mods(dict_1, dict_2):
             dict_3[key].append(dict_1[key])
 
     return dict_3
+
+
+def check_mods(all_mods, pkg):
+    """
+    Raise an exception if there is a missing/unexpected module in all_mods.
+    """
+    def get_expected_modules(package, absolute=False):
+        """
+        Return all modules (which are not packages) inside the given package.
+        """
+        modules = []
+        eth2spec = import_module('eth2spec')
+        prefix = eth2spec.__name__ + '.'
+        for _, modname, ispkg in walk_packages(eth2spec.__path__, prefix):
+            s = package if absolute else f'.{package}.'
+            if s in modname and not ispkg:
+                modules.append(modname)
+        return modules
+
+    mods = []
+    for fork in all_mods:
+        for mod in all_mods[fork].values():
+            # If this key has a single value, normalize to list.
+            if isinstance(mod, str):
+                mod = [mod]
+
+            # For each submodule, check if it is package.
+            # This is a "trick" we do to reuse a test format.
+            for sub in mod:
+                is_package = '.test_' not in sub
+                if is_package:
+                    mods.extend(get_expected_modules(sub, absolute=True))
+                else:
+                    mods.append(sub)
+
+    problems = []
+    expected_mods = get_expected_modules(pkg)
+    if mods != expected_mods:
+        for e in expected_mods:
+            # Skip forks which are not in all_mods.
+            # The fork name is the 3rd item in the path.
+            fork = e.split('.')[2]
+            if fork not in all_mods:
+                continue
+            # Skip modules in the unittests package.
+            # These are not associated with generators.
+            if '.unittests.' in e:
+                continue
+            # The expected module is not in our list of modules.
+            # Add it to our list of problems.
+            if e not in mods:
+                problems.append('missing: ' + e)
+
+        for t in mods:
+            # Skip helper modules.
+            # These do not define test functions.
+            if t.startswith('eth2spec.test.helpers'):
+                continue
+            # There is a module not defined in eth2spec.
+            # Add it to our list of problems.
+            if t not in expected_mods:
+                print('unexpected:', t)
+                problems.append('unexpected: ' + t)
+
+    if problems:
+        raise Exception('[ERROR] module problems:\n ' + '\n '.join(problems))

tests/generators/README.md

@@ -185,6 +185,7 @@ if __name__ == "__main__":
         PHASE0: phase_0_mods,
         ALTAIR: altair_mods,
     }
+    check_mods(all_mods, "sanity")
 
     run_state_test_generators(runner_name="sanity", all_mods=all_mods)
 ```

tests/generators/epoch_processing/main.py

@@ -1,4 +1,4 @@
-from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods
+from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods, check_mods
 from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA
 
 
@@ -63,5 +63,6 @@ if __name__ == "__main__":
         DENEB: deneb_mods,
         ELECTRA: electra_mods,
     }
+    check_mods(all_mods, "epoch_processing")
 
     run_state_test_generators(runner_name="epoch_processing", all_mods=all_mods)

tests/generators/finality/main.py

@@ -1,4 +1,4 @@
-from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
+from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, check_mods
 from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA
 
 
@@ -18,5 +18,6 @@ if __name__ == "__main__":
         DENEB: deneb_mods,
         ELECTRA: electra_mods,
     }
+    check_mods(all_mods, "finality")
 
     run_state_test_generators(runner_name="finality", all_mods=all_mods)

tests/generators/fork_choice/main.py

@@ -1,4 +1,4 @@
-from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods
+from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods, check_mods
 from eth2spec.test.helpers.constants import ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA
 
 
@@ -37,5 +37,6 @@ if __name__ == "__main__":
         DENEB: deneb_mods,
         ELECTRA: electra_mods,
     }
+    check_mods(all_mods, "fork_choice")
 
     run_state_test_generators(runner_name="fork_choice", all_mods=all_mods)

tests/generators/genesis/main.py

@@ -1,4 +1,4 @@
-from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods
+from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods, check_mods
 from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA
 
 
@@ -26,5 +26,6 @@ if __name__ == "__main__":
         DENEB: deneb_mods,
         ELECTRA: electra_mods,
     }
+    check_mods(all_mods, "genesis")
 
     run_state_test_generators(runner_name="genesis", all_mods=all_mods)

tests/generators/light_client/main.py

@@ -1,5 +1,5 @@
 from eth2spec.test.helpers.constants import ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA
-from eth2spec.gen_helpers.gen_from_tests.gen import combine_mods, run_state_test_generators
+from eth2spec.gen_helpers.gen_from_tests.gen import combine_mods, run_state_test_generators, check_mods
 
 
 if __name__ == "__main__":
@@ -24,5 +24,6 @@ if __name__ == "__main__":
         DENEB: deneb_mods,
         ELECTRA: electra_mods,
     }
+    check_mods(all_mods, "light_client")
 
     run_state_test_generators(runner_name="light_client", all_mods=all_mods)

tests/generators/merkle_proof/main.py

@@ -1,5 +1,5 @@
 from eth2spec.test.helpers.constants import DENEB, ELECTRA, EIP7594
-from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods
+from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods, check_mods
 
 
 if __name__ == "__main__":
@@ -17,5 +17,6 @@ if __name__ == "__main__":
         ELECTRA: electra_mods,
         EIP7594: eip_7594_mods,
     }
+    check_mods(all_mods, "merkle_proof")
 
     run_state_test_generators(runner_name="merkle_proof", all_mods=all_mods)

tests/generators/networking/main.py

@@ -1,6 +1,6 @@
 
 from eth2spec.test.helpers.constants import EIP7594
-from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
+from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, check_mods
 
 
 if __name__ == "__main__":
@@ -10,5 +10,6 @@ if __name__ == "__main__":
     all_mods = {
         EIP7594: eip7594_mods
     }
+    check_mods(all_mods, "networking")
 
     run_state_test_generators(runner_name="networking", all_mods=all_mods)

tests/generators/operations/main.py

@@ -1,4 +1,4 @@
-from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods
+from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods, check_mods
 from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA
 
 
@@ -61,5 +61,6 @@ if __name__ == "__main__":
         DENEB: deneb_mods,
         ELECTRA: electra_mods,
     }
+    check_mods(all_mods, "block_processing")
 
     run_state_test_generators(runner_name="operations", all_mods=all_mods)

tests/generators/random/main.py

@@ -1,7 +1,7 @@
 from eth2spec.test.helpers.constants import (
     PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA,
 )
-from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
+from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, check_mods
 
 
 if __name__ == "__main__":
@@ -32,5 +32,6 @@ if __name__ == "__main__":
         DENEB: deneb_mods,
         ELECTRA: electra_mods,
     }
+    check_mods(all_mods, "random")
 
     run_state_test_generators(runner_name="random", all_mods=all_mods)

tests/generators/rewards/main.py

@@ -1,4 +1,4 @@
-from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
+from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, check_mods
 from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA
 
 
@@ -27,5 +27,6 @@ if __name__ == "__main__":
         DENEB: deneb_mods,
         ELECTRA: electra_mods,
     }
+    check_mods(all_mods, "rewards")
 
     run_state_test_generators(runner_name="rewards", all_mods=all_mods)

tests/generators/sanity/main.py

@@ -1,5 +1,5 @@
 from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA
-from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods
+from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods, check_mods
 
 
 if __name__ == "__main__":
@@ -44,5 +44,6 @@ if __name__ == "__main__":
         DENEB: deneb_mods,
         ELECTRA: electra_mods,
     }
+    check_mods(all_mods, "sanity")
 
     run_state_test_generators(runner_name="sanity", all_mods=all_mods)

tests/generators/sync/main.py

@@ -1,4 +1,4 @@
-from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
+from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, check_mods
 from eth2spec.test.helpers.constants import BELLATRIX, CAPELLA, DENEB, ELECTRA
 
 
@@ -16,5 +16,6 @@ if __name__ == "__main__":
         DENEB: deneb_mods,
         ELECTRA: electra_mods,
     }
+    check_mods(all_mods, "sync")
 
     run_state_test_generators(runner_name="sync", all_mods=all_mods)