Update python to 3.12.4 (#3880)

* update python to 3.12 * drop down to 3.12.4 for compatibility * setup rust toolchain for deps * fix typo * fix linter * fixing linter * adding rust dep * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying new image * trying new image * trying new image * trying new image * invalidate cache * bump reqs * remove cache invalidation * fix cache name * update to use latest image
2024-08-21 20:15:57 +02:00 · 2024-08-21 20:15:57 +02:00 · ad91a835c0
parent 01aab85412
commit ad91a835c0
11 changed files with 62 additions and 59 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -60,7 +60,7 @@ commands:
 jobs:
  checkout_specs:
    docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
    working_directory: ~/specs-repo
    steps:
      # Restore git repo at point close to target branch/revision, to speed up checkout
@ -80,7 +80,7 @@ jobs:
            - ~/specs-repo
  install_pyspec_test:
    docker:
-      - image: circleci/python:3.9
+      - image: ethpandaops/circleci-python-rust:latest
    working_directory: ~/specs-repo
    steps:
      - restore_cache:
@ -88,11 +88,11 @@ jobs:
      - restore_pyspec_cached_venv
      - run:
          name: Install pyspec requirements
-          command: make install_test
+          command: make install_test && make preinstallation
      - save_pyspec_cached_venv
  test-phase0:
    docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
    working_directory: ~/specs-repo
    steps:
      - restore_cache:
@ -105,7 +105,7 @@ jobs:
          path: tests/core/pyspec/test-reports
  test-altair:
    docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
    working_directory: ~/specs-repo
    steps:
      - restore_cache:
@ -118,7 +118,7 @@ jobs:
          path: tests/core/pyspec/test-reports
  test-bellatrix:
    docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
    working_directory: ~/specs-repo
    steps:
      - restore_cache:
@ -131,7 +131,7 @@ jobs:
          path: tests/core/pyspec/test-reports
  test-capella:
    docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
    working_directory: ~/specs-repo
    steps:
      - restore_cache:
@ -144,7 +144,7 @@ jobs:
          path: tests/core/pyspec/test-reports
  test-deneb:
    docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
    working_directory: ~/specs-repo
    steps:
      - restore_cache:
@ -157,7 +157,7 @@ jobs:
          path: tests/core/pyspec/test-reports
  test-electra:
    docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
    working_directory: ~/specs-repo
    steps:
      - restore_cache:
@ -170,7 +170,7 @@ jobs:
          path: tests/core/pyspec/test-reports
  test-whisk:
    docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
    working_directory: ~/specs-repo
    steps:
      - restore_cache:
@ -183,7 +183,7 @@ jobs:
          path: tests/core/pyspec/test-reports
  test-eip7594:
    docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
    working_directory: ~/specs-repo
    steps:
      - restore_cache:
@ -205,7 +205,7 @@ jobs:
          command: sudo npm install -g doctoc@2.2.0 && make check_toc
  codespell:
    docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
    working_directory: ~/specs-repo
    steps:
      - checkout
@ -214,7 +214,7 @@ jobs:
          command: pip install 'codespell<3.0.0,>=2.0.0' --user &&  make codespell
  lint:
    docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
    working_directory: ~/specs-repo
    steps:
      - restore_cache:
@ -270,7 +270,7 @@ jobs:
            - /nix
  install_deposit_contract_web3_tester:
    docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
    working_directory: ~/specs-repo
    steps:
      - restore_cache:
@ -282,7 +282,7 @@ jobs:
      - save_deposit_contract_tester_cached_venv
  test_deposit_contract_web3_tests:
    docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
    working_directory: ~/specs-repo
    steps:
      - restore_cache:
--- a/.github/workflows/generate_vectors.yml
+++ b/.github/workflows/generate_vectors.yml
@ -32,7 +32,7 @@ jobs:
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
-          python-version: '3.10'
+          python-version: '3.12.4'
          cache: ''
      - name: Clean up Spec Repository
        run: |
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@ -45,7 +45,7 @@ jobs:
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
-          python-version: '3.10'
+          python-version: '3.12.4'
          cache: ''
      - name: Check codespell
        run: make codespell
@ -55,10 +55,12 @@ jobs:
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
+      - name: Setup Rust for dependencies
+        uses: actions-rust-lang/setup-rust-toolchain@v1
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
-          python-version: '3.10'
+          python-version: '3.12.4'
          cache: ''
      - name: Install pyspec requirements
        run: make install_test
@ -76,10 +78,12 @@ jobs:
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
+      - name: Setup Rust for dependencies
+        uses: actions-rust-lang/setup-rust-toolchain@v1
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
-          python-version: '3.10'
+          python-version: '3.12.4'
          cache: ''
      - name: set TEST_PRESET_TYPE
        if: github.event.inputs.test_preset_type != ''
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@ -1,5 +1,5 @@
 # Rename the build stage from 'base' to 'builder' for clarification and code readability
-FROM python:3.11.0-slim-bullseye as builder
+FROM python:3.12.4-slim-bullseye as builder

 ENV DEBIAN_FRONTEND=noninteractive \
    WORKDIR=/consensus-specs  \
--- a/requirements_preinstallation.txt
+++ b/requirements_preinstallation.txt
@ -1,4 +1,4 @@
-pip>=23.1.2
-wheel>=0.40.0
-setuptools>=68.0.0
-pylint>=3.0.0
+pip>=24.0.0
+wheel>=0.44.0
+setuptools>=72.0.0
+pylint>=3.2.0
--- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py
+++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py
@ -132,7 +132,7 @@ def should_skip_case_dir(case_dir, is_force, diagnostics_obj):
            print(f'Skipping already existing test: {case_dir}')
            is_skip = True
        else:
-            print(f'Warning, output directory {case_dir} already exist,'
+            print(f'Warning, output directory {case_dir} already exist, '
                  ' old files will be deleted and it will generate test vector files with the latest version')
            # Clear the existing case_dir folder
            shutil.rmtree(case_dir)
@ -356,7 +356,7 @@ def generate_test_vector(test_case, case_dir, log_file, file_mode):
    test_end = time.time()
    span = round(test_end - test_start, 2)
    if span > TIME_THRESHOLD_TO_PRINT:
-        print(f'    - generated in {span} seconds')
+        print(f'- generated in {span} seconds')

    return result

--- a/tests/generators/ssz_generic/ssz_basic_vector.py
+++ b/tests/generators/ssz_generic/ssz_basic_vector.py
@ -33,7 +33,7 @@ def valid_cases():
        for length in [1, 2, 3, 4, 5, 8, 16, 31, 512, 513]:
            for mode in random_modes:
                yield f'vec_{name}_{length}_{mode.to_name()}', \
-                      valid_test_case(lambda: basic_vector_case_fn(rng, mode, typ, length))
+                    valid_test_case(lambda: basic_vector_case_fn(rng, mode, typ, length))


 def invalid_cases():
@ -52,14 +52,14 @@ def invalid_cases():
                if length == 1:
                    # empty bytes, no elements. It may seem valid, but empty fixed-size elements are not valid SSZ.
                    yield f'vec_{name}_{length}_{mode.to_name()}_one_less', \
-                          invalid_test_case(lambda: b"")
+                        invalid_test_case(lambda: b"")
                else:
                    yield f'vec_{name}_{length}_{mode.to_name()}_one_less', \
-                          invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length - 1)))
+                        invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length - 1)))
                yield f'vec_{name}_{length}_{mode.to_name()}_one_more', \
-                      invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length + 1)))
+                    invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length + 1)))
                yield f'vec_{name}_{length}_{mode.to_name()}_one_byte_less', \
-                      invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length))[:-1])
+                    invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length))[:-1])
                yield f'vec_{name}_{length}_{mode.to_name()}_one_byte_more', \
-                      invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length))
-                                        + serialize(basic_vector_case_fn(rng, mode, uint8, 1)))
+                    invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length))
+                                      + serialize(basic_vector_case_fn(rng, mode, uint8, 1)))
--- a/tests/generators/ssz_generic/ssz_bitlist.py
+++ b/tests/generators/ssz_generic/ssz_bitlist.py
@ -22,7 +22,7 @@ def valid_cases():
                         RandomizationMode.mode_zero,
                         RandomizationMode.mode_max]:
                yield f'bitlist_{size}_{mode.to_name()}_{variation}', \
-                      valid_test_case(lambda: bitlist_case_fn(rng, mode, size))
+                    valid_test_case(lambda: bitlist_case_fn(rng, mode, size))


 def invalid_cases():
@ -33,5 +33,5 @@ def invalid_cases():
    for (typ_limit, test_limit) in [(1, 2), (1, 8), (1, 9), (2, 3), (3, 4), (4, 5),
                                    (5, 6), (8, 9), (32, 64), (32, 33), (512, 513)]:
        yield f'bitlist_{typ_limit}_but_{test_limit}', \
-              invalid_test_case(lambda: serialize(
-                  bitlist_case_fn(rng, RandomizationMode.mode_max_count, test_limit)))
+            invalid_test_case(lambda: serialize(
+                bitlist_case_fn(rng, RandomizationMode.mode_max_count, test_limit)))
--- a/tests/generators/ssz_generic/ssz_bitvector.py
+++ b/tests/generators/ssz_generic/ssz_bitvector.py
@ -38,5 +38,5 @@ def invalid_cases():
                                  (5, 6), (8, 9), (9, 8), (16, 8), (32, 33), (512, 513)]:
        for mode in [RandomizationMode.mode_random, RandomizationMode.mode_zero, RandomizationMode.mode_max]:
            yield f'bitvec_{typ_size}_{mode.to_name()}_{test_size}', \
-                  invalid_test_case(lambda: serialize(bitvector_case_fn(rng, mode, test_size,
-                                                                        invalid_making_pos=typ_size)))
+                invalid_test_case(lambda: serialize(bitvector_case_fn(rng, mode, test_size,
+                                                    invalid_making_pos=typ_size)))
--- a/tests/generators/ssz_generic/ssz_container.py
+++ b/tests/generators/ssz_generic/ssz_container.py
@ -46,7 +46,7 @@ class BitsStruct(Container):
    E: Bitvector[8]


-def container_case_fn(rng: Random, mode: RandomizationMode, typ: Type[View], chaos: bool=False):
+def container_case_fn(rng: Random, mode: RandomizationMode, typ: Type[View], chaos: bool = False):
    return get_random_ssz_object(rng, typ,
                                 max_bytes_length=2000,
                                 max_list_length=2000,
@ -77,7 +77,7 @@ def valid_cases():
        for mode in modes:
            for variation in range(3):
                yield f'{name}_{mode.to_name()}_chaos_{variation}', \
-                      valid_test_case(lambda: container_case_fn(rng, mode, typ, chaos=True))
+                    valid_test_case(lambda: container_case_fn(rng, mode, typ, chaos=True))
        # Notes: Below is the second wave of iteration, and only the random mode is selected
        # for container without offset since ``RandomizationMode.mode_zero`` and ``RandomizationMode.mode_max``
        # are deterministic.
@ -85,7 +85,7 @@ def valid_cases():
        for mode in modes:
            for variation in range(10):
                yield f'{name}_{mode.to_name()}_{variation}', \
-                      valid_test_case(lambda: container_case_fn(rng, mode, typ))
+                    valid_test_case(lambda: container_case_fn(rng, mode, typ))


 def mod_offset(b: bytes, offset_index: int, change: Callable[[int], int]):
@ -100,8 +100,8 @@ def invalid_cases():
    for (name, (typ, offsets)) in PRESET_CONTAINERS.items():
        # using mode_max_count, so that the extra byte cannot be picked up as normal list content
        yield f'{name}_extra_byte', \
-              invalid_test_case(lambda: serialize(
-                  container_case_fn(rng, RandomizationMode.mode_max_count, typ)) + b'\xff')
+            invalid_test_case(lambda: serialize(
+                container_case_fn(rng, RandomizationMode.mode_max_count, typ)) + b'\xff')

        if len(offsets) != 0:
            # Note: there are many more ways to have invalid offsets,
@ -112,23 +112,22 @@ def invalid_cases():
                         RandomizationMode.mode_max_count]:
                for index, offset_index in enumerate(offsets):
                    yield f'{name}_{mode.to_name()}_offset_{offset_index}_plus_one', \
-                          invalid_test_case(lambda: mod_offset(
-                              b=serialize(container_case_fn(rng, mode, typ)),
-                              offset_index=offset_index,
-                              change=lambda x: x + 1
-                          ))
+                        invalid_test_case(lambda: mod_offset(
+                            b=serialize(container_case_fn(rng, mode, typ)),
+                            offset_index=offset_index,
+                            change=lambda x: x + 1))
                    yield f'{name}_{mode.to_name()}_offset_{offset_index}_zeroed', \
-                          invalid_test_case(lambda: mod_offset(
-                              b=serialize(container_case_fn(rng, mode, typ)),
-                              offset_index=offset_index,
-                              change=lambda x: 0
-                          ))
+                        invalid_test_case(lambda: mod_offset(
+                            b=serialize(container_case_fn(rng, mode, typ)),
+                            offset_index=offset_index,
+                            change=lambda x: 0
+                        ))
                    if index == 0:
                        yield f'{name}_{mode.to_name()}_offset_{offset_index}_minus_one', \
                            invalid_test_case(lambda: mod_offset(
-                              b=serialize(container_case_fn(rng, mode, typ)),
-                              offset_index=offset_index,
-                              change=lambda x: x - 1
+                                b=serialize(container_case_fn(rng, mode, typ)),
+                                offset_index=offset_index,
+                                change=lambda x: x - 1
                            ))
                    if mode == RandomizationMode.mode_max_count:
                        serialized = serialize(container_case_fn(rng, mode, typ))
--- a/tests/generators/ssz_generic/ssz_uints.py
+++ b/tests/generators/ssz_generic/ssz_uints.py
@ -21,7 +21,7 @@ def valid_cases():
        mode = RandomizationMode.mode_random
        byte_len = uint_type.type_byte_length()
        yield f'uint_{byte_len * 8}_last_byte_empty', \
-              valid_test_case(lambda: uint_type((2 ** ((byte_len - 1) * 8)) - 1))
+            valid_test_case(lambda: uint_type((2 ** ((byte_len - 1) * 8)) - 1))
        for variation in range(5):
            yield f'uint_{byte_len * 8}_{mode.to_name()}_{variation}', \
                valid_test_case(lambda: uint_case_fn(rng, mode, uint_type))
@ -38,8 +38,8 @@ def invalid_cases():
    for uint_type in [uint8, uint16, uint32, uint64, uint128, uint256]:
        byte_len = uint_type.type_byte_length()
        yield f'uint_{byte_len * 8}_one_byte_longer', \
-              invalid_test_case(lambda: (2 ** (byte_len * 8) - 1).to_bytes(byte_len + 1, 'little'))
+            invalid_test_case(lambda: (2 ** (byte_len * 8) - 1).to_bytes(byte_len + 1, 'little'))
    for uint_type in [uint8, uint16, uint32, uint64, uint128, uint256]:
        byte_len = uint_type.type_byte_length()
        yield f'uint_{byte_len * 8}_one_byte_shorter', \
-              invalid_test_case(lambda: (2 ** ((byte_len - 1) * 8) - 1).to_bytes(byte_len - 1, 'little'))
+            invalid_test_case(lambda: (2 ** ((byte_len - 1) * 8) - 1).to_bytes(byte_len - 1, 'little'))