Merge branch 'dev' into deposits

Makefile

@@ -23,16 +23,18 @@ GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENER
 # To check generator matching:
 #$(info $$GENERATOR_TARGETS is [${GENERATOR_TARGETS}])
 
-MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/phase0/*.md) \
+MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/*/*.md) \
-                 $(wildcard $(SPEC_DIR)/altair/*.md) $(wildcard $(SPEC_DIR)/altair/**/*.md) \
+                 $(wildcard $(SPEC_DIR)/*/*/*.md) \
-                 $(wildcard $(SPEC_DIR)/bellatrix/*.md) \
+                 $(wildcard $(SPEC_DIR)/_features/*/*.md) \
-                 $(wildcard $(SPEC_DIR)/capella/*.md) $(wildcard $(SPEC_DIR)/capella/**/*.md) \
+                 $(wildcard $(SPEC_DIR)/_features/*/*/*.md) \
-                 $(wildcard $(SPEC_DIR)/deneb/*.md) $(wildcard $(SPEC_DIR)/deneb/**/*.md) \
-                 $(wildcard $(SPEC_DIR)/_features/custody/*.md) \
-                 $(wildcard $(SPEC_DIR)/_features/das/*.md) \
-                 $(wildcard $(SPEC_DIR)/_features/sharding/*.md) \
                  $(wildcard $(SSZ_DIR)/*.md)
 
+ALL_EXECUTABLE_SPECS = phase0 altair bellatrix capella deneb eip6110
+# The parameters for commands. Use `foreach` to avoid listing specs again.
+COVERAGE_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPECS), --cov=eth2spec.$S.$(TEST_PRESET_TYPE))
+PYLINT_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPECS), ./eth2spec/$S)
+MYPY_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPECS), -p eth2spec.$S)
+
 COV_HTML_OUT=.htmlcov
 COV_HTML_OUT_DIR=$(PY_SPEC_DIR)/$(COV_HTML_OUT)
 COV_INDEX_FILE=$(COV_HTML_OUT_DIR)/index.html
@@ -63,15 +65,14 @@ partial_clean:
 	rm -f .coverage
 	rm -rf $(PY_SPEC_DIR)/.pytest_cache
 	rm -rf $(DEPOSIT_CONTRACT_TESTER_DIR)/.pytest_cache
-	rm -rf $(ETH2SPEC_MODULE_DIR)/phase0
-	rm -rf $(ETH2SPEC_MODULE_DIR)/altair
-	rm -rf $(ETH2SPEC_MODULE_DIR)/bellatrix
-	rm -rf $(ETH2SPEC_MODULE_DIR)/capella
-	rm -rf $(ETH2SPEC_MODULE_DIR)/deneb
 	rm -rf $(COV_HTML_OUT_DIR)
 	rm -rf $(TEST_REPORT_DIR)
 	rm -rf eth2spec.egg-info dist build
-	rm -rf build
+	rm -rf build;
+	@for spec_name in $(ALL_EXECUTABLE_SPECS) ; do \
+		echo $$spec_name; \
+		rm -rf $(ETH2SPEC_MODULE_DIR)/$$spec_name; \
+	done
 
 clean: partial_clean
 	rm -rf venv
@@ -105,12 +106,12 @@ install_test:
 # Testing against `minimal` or `mainnet` config by default
 test: pyspec
 	. venv/bin/activate; cd $(PY_SPEC_DIR); \
-	python3 -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.$(TEST_PRESET_TYPE) --cov=eth2spec.altair.$(TEST_PRESET_TYPE) --cov=eth2spec.bellatrix.$(TEST_PRESET_TYPE) --cov=eth2spec.capella.$(TEST_PRESET_TYPE) --cov=eth2spec.deneb.$(TEST_PRESET_TYPE) --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
+	python3 -m pytest -n 4 --disable-bls $(COVERAGE_SCOPE) --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
 
 # Testing against `minimal` or `mainnet` config by default
 find_test: pyspec
 	. venv/bin/activate; cd $(PY_SPEC_DIR); \
-	python3 -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.$(TEST_PRESET_TYPE) --cov=eth2spec.altair.$(TEST_PRESET_TYPE) --cov=eth2spec.bellatrix.$(TEST_PRESET_TYPE) --cov=eth2spec.capella.$(TEST_PRESET_TYPE) --cov=eth2spec.deneb.$(TEST_PRESET_TYPE) --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
+	python3 -m pytest -k=$(K) --disable-bls $(COVERAGE_SCOPE) --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
 
 citest: pyspec
 	mkdir -p $(TEST_REPORT_DIR);
@@ -119,7 +120,7 @@ ifdef fork
 	python3 -m pytest -n 16 --bls-type=milagro --preset=$(TEST_PRESET_TYPE) --fork=$(fork) --junitxml=test-reports/test_results.xml eth2spec
 else
 	. venv/bin/activate; cd $(PY_SPEC_DIR); \
-	python3 -m pytest -n 16 --bls-type=milagro  --preset=$(TEST_PRESET_TYPE) --junitxml=test-reports/test_results.xml eth2spec
+	python3 -m pytest -n 16 --bls-type=milagro --preset=$(TEST_PRESET_TYPE) --junitxml=test-reports/test_results.xml eth2spec
 endif
 
 
@@ -137,13 +138,11 @@ check_toc: $(MARKDOWN_FILES:=.toc)
 codespell:
 	codespell . --skip "./.git,./venv,$(PY_SPEC_DIR)/.mypy_cache" -I .codespell-whitelist
 
-# TODO: add future protocol upgrade patch packages to linting.
-# NOTE: we use `pylint` just for catching unused arguments in spec code
 lint: pyspec
 	. venv/bin/activate; cd $(PY_SPEC_DIR); \
 	flake8  --config $(LINTER_CONFIG_FILE) ./eth2spec \
-	&& pylint --rcfile $(LINTER_CONFIG_FILE) ./eth2spec/phase0 ./eth2spec/altair ./eth2spec/bellatrix ./eth2spec/capella ./eth2spec/deneb ./eth2spec/eip6110 \
+	&& pylint --rcfile $(LINTER_CONFIG_FILE) $(PYLINT_SCOPE) \
-	&& mypy --config-file $(LINTER_CONFIG_FILE) -p eth2spec.phase0 -p eth2spec.altair -p eth2spec.bellatrix -p eth2spec.capella -p eth2spec.deneb -p eth2spec.eip6110
+	&& mypy --config-file $(LINTER_CONFIG_FILE) $(MYPY_SCOPE)
 
 lint_generators: pyspec
 	. venv/bin/activate; cd $(TEST_GENERATORS_DIR); \

specs/deneb/fork-choice.md

@@ -63,7 +63,7 @@ def is_data_available(beacon_block_root: Root, blob_kzg_commitments: Sequence[KZ
 
 ### `on_block`
 
-*Note*: The only modification is the addition of the verification of transition block conditions.
+*Note*: The only modification is the addition of the blob data availability check.
 
 ```python
 def on_block(store: Store, signed_block: SignedBeaconBlock) -> None:

specs/deneb/fork.md

@@ -64,8 +64,6 @@ Note that for the pure Deneb networks, we don't apply `upgrade_to_deneb` since i
 
 ### Upgrading the state
 
-Since the `deneb.BeaconState` format is equal to the `capella.BeaconState` format, we only have to update `BeaconState.fork`.
-
 ```python
 def upgrade_to_deneb(pre: capella.BeaconState) -> BeaconState:
     epoch = capella.get_current_epoch(pre)
@@ -82,10 +80,10 @@ def upgrade_to_deneb(pre: capella.BeaconState) -> BeaconState:
         timestamp=pre.latest_execution_payload_header.timestamp,
         extra_data=pre.latest_execution_payload_header.extra_data,
         base_fee_per_gas=pre.latest_execution_payload_header.base_fee_per_gas,
-        excess_data_gas=uint256(0),  # [New in Deneb]
         block_hash=pre.latest_execution_payload_header.block_hash,
         transactions_root=pre.latest_execution_payload_header.transactions_root,
         withdrawals_root=pre.latest_execution_payload_header.withdrawals_root,
+        excess_data_gas=uint256(0),  # [New in Deneb]
     )
     post = BeaconState(
         # Versioning

specs/deneb/p2p-interface.md

@@ -38,7 +38,7 @@ The specification of these changes continues in the same format as the network s
 | Name                                     | Value                             | Description                                                         |
 |------------------------------------------|-----------------------------------|---------------------------------------------------------------------|
 | `MAX_REQUEST_BLOCKS_DENEB`               | `2**7` (= 128)                    | Maximum number of blocks in a single request                        |
-| `MAX_REQUEST_BLOB_SIDECARS`              | `2**7` (= 128)                    | Maximum number of blob sidecars in a single request                 |
+| `MAX_REQUEST_BLOB_SIDECARS`              | `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK`      | Maximum number of blob sidecars in a single request                 |
 | `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS` | `2**12` (= 4096 epochs, ~18 days) | The minimum epoch range over which a node must serve blob sidecars |
 
 ## Containers
@@ -108,10 +108,10 @@ This topic is used to propagate signed blob sidecars, one for each sidecar index
 The following validations MUST pass before forwarding the `sidecar` on the network, assuming the alias `sidecar = signed_blob_sidecar.message`:
 
 - _[REJECT]_ The sidecar is for the correct topic -- i.e. `sidecar.index` matches the topic `{index}`.
-- _[IGNORE]_ The sidecar is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `sidecar.slot <= current_slot` (a client MAY queue future blocks for processing at the appropriate slot).
+- _[IGNORE]_ The sidecar is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `sidecar.slot <= current_slot` (a client MAY queue future sidecars for processing at the appropriate slot).
 - _[IGNORE]_ The sidecar is from a slot greater than the latest finalized slot -- i.e. validate that `sidecar.slot > compute_start_slot_at_epoch(state.finalized_checkpoint.epoch)`
-- _[IGNORE]_ The blob's block's parent (defined by `sidecar.block_parent_root`) has been seen (via both gossip and non-gossip sources) (a client MAY queue blocks for processing once the parent block is retrieved).
+- _[IGNORE]_ The sidecar's block's parent (defined by `sidecar.block_parent_root`) has been seen (via both gossip and non-gossip sources) (a client MAY queue sidecars for processing once the parent block is retrieved).
-- _[REJECT]_ The blob's block's parent (defined by `sidecar.block_parent_root`) passes validation.
+- _[REJECT]_ The sidecar's block's parent (defined by `sidecar.block_parent_root`) passes validation.
 - _[REJECT]_ The proposer signature, `signed_blob_sidecar.signature`, is valid with respect to the `sidecar.proposer_index` pubkey.
 - _[IGNORE]_ The sidecar is the only sidecar with valid signature received for the tuple `(sidecar.block_root, sidecar.index)`.
 - _[REJECT]_ The sidecar is proposed by the expected `proposer_index` for the block's slot in the context of the current shuffling (defined by `block_parent_root`/`slot`).
@@ -183,7 +183,7 @@ Request Content:
 
 ```
 (
-  List[BlobIdentifier, MAX_REQUEST_BLOBS_SIDECARS * MAX_BLOBS_PER_BLOCK]
+  List[BlobIdentifier, MAX_REQUEST_BLOBS_SIDECARS]
 )
 ```
 
@@ -191,7 +191,7 @@ Response Content:
 
 ```
 (
-  List[BlobSidecar, MAX_REQUEST_BLOBS_SIDECARS * MAX_BLOBS_PER_BLOCK]
+  List[BlobSidecar, MAX_REQUEST_BLOBS_SIDECARS]
 )
 ```
 
@@ -202,7 +202,7 @@ It may be less in the case that the responding peer is missing blocks or sidecar
 The response is unsigned, i.e. `BlobSidecar`, as the signature of the beacon block proposer
 may not be available beyond the initial distribution via gossip.
 
-No more than `MAX_REQUEST_BLOBS_SIDECARS * MAX_BLOBS_PER_BLOCK` may be requested at a time.
+No more than `MAX_REQUEST_BLOBS_SIDECARS` may be requested at a time.
 
 `BlobSidecarsByRoot` is primarily used to recover recent blobs (e.g. when receiving a block with a transaction whose corresponding blob is missing).
 
@@ -239,7 +239,7 @@ Request Content:
 Response Content:
 ```
 (
-  List[BlobSidecar, MAX_REQUEST_BLOB_SIDECARS * MAX_BLOBS_PER_BLOCK]
+  List[BlobSidecar, MAX_REQUEST_BLOB_SIDECARS]
 )
 ```
 
@@ -274,7 +274,7 @@ to be fully compliant with `BlobSidecarsByRange` requests.
 participating in the networking immediately, other peers MAY
 disconnect and/or temporarily ban such an un-synced or semi-synced client.
 
-Clients MUST respond with at least the blob sidecars of the first blob-carrying block that exists in the range, if they have it, and no more than `MAX_REQUEST_BLOB_SIDECARS * MAX_BLOBS_PER_BLOCK` sidecars.
+Clients MUST respond with at least the blob sidecars of the first blob-carrying block that exists in the range, if they have it, and no more than `MAX_REQUEST_BLOB_SIDECARS` sidecars.
 
 Clients MUST include all blob sidecars of each block from which they include blob sidecars.
 

specs/phase0/fork-choice.md

@@ -18,7 +18,7 @@
     - [`get_current_slot`](#get_current_slot)
     - [`compute_slots_since_epoch_start`](#compute_slots_since_epoch_start)
     - [`get_ancestor`](#get_ancestor)
-    - [`get_latest_attesting_balance`](#get_latest_attesting_balance)
+    - [`get_weight`](#get_weight)
     - [`filter_block_tree`](#filter_block_tree)
     - [`get_filtered_block_tree`](#get_filtered_block_tree)
     - [`get_head`](#get_head)
@@ -174,10 +174,10 @@ def get_ancestor(store: Store, root: Root, slot: Slot) -> Root:
         return root
 ```
 
-#### `get_latest_attesting_balance`
+#### `get_weight`
 
 ```python
-def get_latest_attesting_balance(store: Store, root: Root) -> Gwei:
+def get_weight(store: Store, root: Root) -> Gwei:
     state = store.checkpoint_states[store.justified_checkpoint]
     active_indices = get_active_validator_indices(state, get_current_epoch(state))
     attestation_score = Gwei(sum(
@@ -197,7 +197,6 @@ def get_latest_attesting_balance(store: Store, root: Root) -> Gwei:
         committee_weight = get_total_active_balance(state) // SLOTS_PER_EPOCH
         proposer_score = (committee_weight * PROPOSER_SCORE_BOOST) // 100
     return attestation_score + proposer_score
-
 ```
 
 #### `filter_block_tree`
@@ -270,7 +269,7 @@ def get_head(store: Store) -> Root:
             return head
         # Sort by latest attesting balance with ties broken lexicographically
         # Ties broken by favoring block with lexicographically higher root
-        head = max(children, key=lambda root: (get_latest_attesting_balance(store, root), root))
+        head = max(children, key=lambda root: (get_weight(store, root), root))
 ```
 
 #### `should_update_justified_checkpoint`

tests/core/pyspec/eth2spec/test/helpers/optimistic_sync.py

@@ -177,7 +177,7 @@ def get_opt_head_block_root(spec, mega_store):
             return head
         # Sort by latest attesting balance with ties broken lexicographically
         # Ties broken by favoring block with lexicographically higher root
-        head = max(children, key=lambda root: (spec.get_latest_attesting_balance(store, root), root))
+        head = max(children, key=lambda root: (spec.get_weight(store, root), root))
 
 
 def is_invalidated(mega_store, block_root):

tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_on_block.py

@@ -729,14 +729,14 @@ def test_proposer_boost(spec, state):
     on_tick_and_append_step(spec, store, time, test_steps)
     yield from add_block(spec, store, signed_block, test_steps)
     assert store.proposer_boost_root == spec.hash_tree_root(block)
-    assert spec.get_latest_attesting_balance(store, spec.hash_tree_root(block)) > 0
+    assert spec.get_weight(store, spec.hash_tree_root(block)) > 0
 
     # Ensure that boost is removed after slot is over
     time = (store.genesis_time + block.slot * spec.config.SECONDS_PER_SLOT +
             spec.config.SECONDS_PER_SLOT)
     on_tick_and_append_step(spec, store, time, test_steps)
     assert store.proposer_boost_root == spec.Root()
-    assert spec.get_latest_attesting_balance(store, spec.hash_tree_root(block)) == 0
+    assert spec.get_weight(store, spec.hash_tree_root(block)) == 0
 
     next_slots(spec, state, 3)
     block = build_empty_block_for_next_slot(spec, state)
@@ -747,14 +747,14 @@ def test_proposer_boost(spec, state):
     on_tick_and_append_step(spec, store, time, test_steps)
     yield from add_block(spec, store, signed_block, test_steps)
     assert store.proposer_boost_root == spec.hash_tree_root(block)
-    assert spec.get_latest_attesting_balance(store, spec.hash_tree_root(block)) > 0
+    assert spec.get_weight(store, spec.hash_tree_root(block)) > 0
 
     # Ensure that boost is removed after slot is over
     time = (store.genesis_time + block.slot * spec.config.SECONDS_PER_SLOT +
             spec.config.SECONDS_PER_SLOT)
     on_tick_and_append_step(spec, store, time, test_steps)
     assert store.proposer_boost_root == spec.Root()
-    assert spec.get_latest_attesting_balance(store, spec.hash_tree_root(block)) == 0
+    assert spec.get_weight(store, spec.hash_tree_root(block)) == 0
 
     test_steps.append({
         'checks': {