2021-11-25 18:41:39 +00:00
|
|
|
# beacon_chain
|
2023-03-04 13:35:39 +00:00
|
|
|
# Copyright (c) 2018-2023 Status Research & Development GmbH
|
2021-11-25 18:41:39 +00:00
|
|
|
# Licensed and distributed under either of
|
|
|
|
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
|
|
|
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
|
|
|
# at your option. This file may not be copied, modified, or distributed except according to those terms.
|
|
|
|
|
2021-12-01 12:55:42 +00:00
|
|
|
{.used.}
|
|
|
|
|
2021-11-25 18:41:39 +00:00
|
|
|
import
|
|
|
|
# Status libraries
|
2023-09-20 01:14:49 +00:00
|
|
|
stew/[byteutils, results], chronicles,
|
2023-06-19 22:43:50 +00:00
|
|
|
taskpools,
|
2021-11-25 18:41:39 +00:00
|
|
|
# Internals
|
2022-07-06 10:33:02 +00:00
|
|
|
../../beacon_chain/spec/[helpers, forks, state_transition_block],
|
2021-11-25 18:41:39 +00:00
|
|
|
../../beacon_chain/spec/datatypes/[
|
|
|
|
base,
|
2022-01-12 14:50:30 +00:00
|
|
|
phase0, altair, bellatrix],
|
2021-11-25 18:41:39 +00:00
|
|
|
../../beacon_chain/fork_choice/[fork_choice, fork_choice_types],
|
2021-12-21 18:56:08 +00:00
|
|
|
../../beacon_chain/[beacon_chain_db, beacon_clock],
|
2021-11-25 18:41:39 +00:00
|
|
|
../../beacon_chain/consensus_object_pools/[
|
2022-07-06 10:33:02 +00:00
|
|
|
blockchain_dag, block_clearance, block_quarantine, spec_cache],
|
2021-11-25 18:41:39 +00:00
|
|
|
# Third-party
|
|
|
|
yaml,
|
|
|
|
# Test
|
2022-10-03 13:10:08 +00:00
|
|
|
../testutil, ../testdbutil,
|
2023-02-10 20:59:38 +00:00
|
|
|
./fixtures_utils, ./os_ops
|
2021-11-25 18:41:39 +00:00
|
|
|
|
2023-05-25 07:55:00 +00:00
|
|
|
from std/json import
|
|
|
|
JsonNode, getBool, getInt, getStr, hasKey, items, len, pairs, `$`, `[]`
|
2023-09-20 01:14:49 +00:00
|
|
|
from std/sequtils import mapIt, toSeq
|
2023-05-25 07:55:00 +00:00
|
|
|
from std/strutils import contains
|
|
|
|
|
2023-05-11 09:54:29 +00:00
|
|
|
# Test format described at https://github.com/ethereum/consensus-specs/tree/v1.3.0/tests/formats/fork_choice
|
2021-11-25 18:41:39 +00:00
|
|
|
# Note that our implementation has been optimized with "ProtoArray"
|
|
|
|
# instead of following the spec (in particular the "store").
|
|
|
|
|
|
|
|
type
|
|
|
|
OpKind = enum
|
|
|
|
opOnTick
|
|
|
|
opOnAttestation
|
|
|
|
opOnBlock
|
|
|
|
opOnMergeBlock
|
2022-07-06 10:33:02 +00:00
|
|
|
opOnAttesterSlashing
|
2022-09-27 12:11:47 +00:00
|
|
|
opInvalidateRoot
|
2021-11-25 18:41:39 +00:00
|
|
|
opChecks
|
|
|
|
|
2023-09-20 01:14:49 +00:00
|
|
|
BlobData = object
|
|
|
|
blobs: seq[KzgBlob]
|
|
|
|
proofs: seq[KzgProof]
|
|
|
|
|
2021-11-25 18:41:39 +00:00
|
|
|
Operation = object
|
|
|
|
valid: bool
|
|
|
|
# variant specific fields
|
2022-07-06 10:33:02 +00:00
|
|
|
case kind: OpKind
|
2021-11-25 18:41:39 +00:00
|
|
|
of opOnTick:
|
|
|
|
tick: int
|
|
|
|
of opOnAttestation:
|
|
|
|
att: Attestation
|
|
|
|
of opOnBlock:
|
2022-07-06 10:33:02 +00:00
|
|
|
blck: ForkedSignedBeaconBlock
|
2023-09-20 01:14:49 +00:00
|
|
|
blobData: Opt[BlobData]
|
2021-11-25 18:41:39 +00:00
|
|
|
of opOnMergeBlock:
|
|
|
|
powBlock: PowBlock
|
2022-07-06 10:33:02 +00:00
|
|
|
of opOnAttesterSlashing:
|
|
|
|
attesterSlashing: AttesterSlashing
|
2022-09-27 12:11:47 +00:00
|
|
|
of opInvalidateRoot:
|
|
|
|
invalidatedRoot: Eth2Digest
|
|
|
|
latestValidHash: Eth2Digest
|
2021-11-25 18:41:39 +00:00
|
|
|
of opChecks:
|
|
|
|
checks: JsonNode
|
|
|
|
|
2022-11-09 17:32:10 +00:00
|
|
|
from ../../beacon_chain/spec/datatypes/capella import
|
|
|
|
BeaconBlock, BeaconState, SignedBeaconBlock
|
|
|
|
|
2023-03-02 20:09:24 +00:00
|
|
|
from ../../beacon_chain/spec/datatypes/deneb import
|
2023-09-20 01:14:49 +00:00
|
|
|
KzgBlob, KzgProof, BeaconBlock, BeaconState, SignedBeaconBlock
|
2022-12-08 16:21:53 +00:00
|
|
|
|
2021-11-25 18:41:39 +00:00
|
|
|
proc initialLoad(
|
2022-07-06 10:33:02 +00:00
|
|
|
path: string, db: BeaconChainDB,
|
|
|
|
StateType, BlockType: typedesc
|
|
|
|
): tuple[dag: ChainDAGRef, fkChoice: ref ForkChoice] =
|
2022-07-23 05:54:01 +00:00
|
|
|
let
|
|
|
|
forkedState = loadForkedState(
|
|
|
|
path/"anchor_state.ssz_snappy",
|
|
|
|
StateType.toFork)
|
2022-07-06 10:33:02 +00:00
|
|
|
|
2022-07-23 05:54:01 +00:00
|
|
|
blck = parseTest(
|
|
|
|
path/"anchor_block.ssz_snappy",
|
|
|
|
SSZ, BlockType)
|
2021-11-25 18:41:39 +00:00
|
|
|
|
2023-03-02 20:09:24 +00:00
|
|
|
when BlockType is deneb.BeaconBlock:
|
|
|
|
let signedBlock = ForkedSignedBeaconBlock.init(deneb.SignedBeaconBlock(
|
2022-12-08 16:21:53 +00:00
|
|
|
message: blck,
|
|
|
|
# signature: - unused as it's trusted
|
|
|
|
root: hash_tree_root(blck)
|
|
|
|
))
|
|
|
|
elif BlockType is capella.BeaconBlock:
|
2022-11-09 17:32:10 +00:00
|
|
|
let signedBlock = ForkedSignedBeaconBlock.init(capella.SignedBeaconBlock(
|
|
|
|
message: blck,
|
|
|
|
# signature: - unused as it's trusted
|
|
|
|
root: hash_tree_root(blck)
|
|
|
|
))
|
|
|
|
elif BlockType is bellatrix.BeaconBlock:
|
2022-07-06 10:33:02 +00:00
|
|
|
let signedBlock = ForkedSignedBeaconBlock.init(bellatrix.SignedBeaconBlock(
|
|
|
|
message: blck,
|
|
|
|
# signature: - unused as it's trusted
|
|
|
|
root: hash_tree_root(blck)
|
|
|
|
))
|
|
|
|
elif BlockType is altair.BeaconBlock:
|
|
|
|
let signedBlock = ForkedSignedBeaconBlock.init(altair.SignedBeaconBlock(
|
|
|
|
message: blck,
|
|
|
|
# signature: - unused as it's trusted
|
|
|
|
root: hash_tree_root(blck)
|
|
|
|
))
|
|
|
|
elif BlockType is phase0.BeaconBlock:
|
|
|
|
let signedBlock = ForkedSignedBeaconBlock.init(phase0.SignedBeaconBlock(
|
|
|
|
message: blck,
|
|
|
|
# signature: - unused as it's trusted
|
|
|
|
root: hash_tree_root(blck)
|
|
|
|
))
|
|
|
|
else: {.error: "Unknown block fork: " & name(BlockType).}
|
2021-11-25 18:41:39 +00:00
|
|
|
|
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
|
|
|
ChainDAGRef.preInit(db, forkedState[])
|
2021-12-20 19:20:31 +00:00
|
|
|
|
|
|
|
let
|
|
|
|
validatorMonitor = newClone(ValidatorMonitor.init())
|
|
|
|
dag = ChainDAGRef.init(
|
2023-04-18 19:26:36 +00:00
|
|
|
forkedState[].kind.genesisTestRuntimeConfig, db, validatorMonitor, {})
|
2021-12-20 19:20:31 +00:00
|
|
|
fkChoice = newClone(ForkChoice.init(
|
2023-09-12 07:52:51 +00:00
|
|
|
dag.getFinalizedEpochRef(), dag.finalizedHead.blck,
|
|
|
|
ForkChoiceVersion.Stable))
|
2021-11-25 18:41:39 +00:00
|
|
|
|
|
|
|
(dag, fkChoice)
|
|
|
|
|
2023-01-28 19:53:41 +00:00
|
|
|
proc loadOps(path: string, fork: ConsensusFork): seq[Operation] =
|
2023-02-10 20:59:38 +00:00
|
|
|
let stepsYAML = os_ops.readFile(path/"steps.yaml")
|
2021-11-25 18:41:39 +00:00
|
|
|
let steps = yaml.loadToJson(stepsYAML)
|
|
|
|
|
|
|
|
result = @[]
|
|
|
|
for step in steps[0]:
|
2023-09-20 01:14:49 +00:00
|
|
|
var numExtraFields = 0
|
|
|
|
|
2021-11-25 18:41:39 +00:00
|
|
|
if step.hasKey"tick":
|
2022-07-06 10:33:02 +00:00
|
|
|
result.add Operation(kind: opOnTick,
|
|
|
|
tick: step["tick"].getInt())
|
|
|
|
elif step.hasKey"attestation":
|
|
|
|
let filename = step["attestation"].getStr()
|
|
|
|
let att = parseTest(
|
|
|
|
path/filename & ".ssz_snappy",
|
|
|
|
SSZ, Attestation
|
|
|
|
)
|
|
|
|
result.add Operation(kind: opOnAttestation,
|
|
|
|
att: att)
|
2021-11-25 18:41:39 +00:00
|
|
|
elif step.hasKey"block":
|
|
|
|
let filename = step["block"].getStr()
|
2023-09-20 01:14:49 +00:00
|
|
|
doAssert step.hasKey"blobs" == step.hasKey"proofs"
|
2021-11-25 18:41:39 +00:00
|
|
|
case fork
|
2023-01-28 19:53:41 +00:00
|
|
|
of ConsensusFork.Phase0:
|
2022-07-06 10:33:02 +00:00
|
|
|
let blck = parseTest(
|
2021-11-25 18:41:39 +00:00
|
|
|
path/filename & ".ssz_snappy",
|
|
|
|
SSZ, phase0.SignedBeaconBlock
|
|
|
|
)
|
2023-09-20 01:14:49 +00:00
|
|
|
doAssert not step.hasKey"blobs"
|
2021-11-25 18:41:39 +00:00
|
|
|
result.add Operation(kind: opOnBlock,
|
2022-07-06 10:33:02 +00:00
|
|
|
blck: ForkedSignedBeaconBlock.init(blck))
|
2023-01-28 19:53:41 +00:00
|
|
|
of ConsensusFork.Altair:
|
2022-07-06 10:33:02 +00:00
|
|
|
let blck = parseTest(
|
2021-11-25 18:41:39 +00:00
|
|
|
path/filename & ".ssz_snappy",
|
|
|
|
SSZ, altair.SignedBeaconBlock
|
|
|
|
)
|
2023-09-20 01:14:49 +00:00
|
|
|
doAssert not step.hasKey"blobs"
|
2021-11-25 18:41:39 +00:00
|
|
|
result.add Operation(kind: opOnBlock,
|
2022-07-06 10:33:02 +00:00
|
|
|
blck: ForkedSignedBeaconBlock.init(blck))
|
2023-01-28 19:53:41 +00:00
|
|
|
of ConsensusFork.Bellatrix:
|
2022-07-06 10:33:02 +00:00
|
|
|
let blck = parseTest(
|
2021-11-25 18:41:39 +00:00
|
|
|
path/filename & ".ssz_snappy",
|
2022-01-12 14:50:30 +00:00
|
|
|
SSZ, bellatrix.SignedBeaconBlock
|
2021-11-25 18:41:39 +00:00
|
|
|
)
|
2023-09-20 01:14:49 +00:00
|
|
|
doAssert not step.hasKey"blobs"
|
2021-11-25 18:41:39 +00:00
|
|
|
result.add Operation(kind: opOnBlock,
|
2022-07-06 10:33:02 +00:00
|
|
|
blck: ForkedSignedBeaconBlock.init(blck))
|
2023-01-28 19:53:41 +00:00
|
|
|
of ConsensusFork.Capella:
|
2022-11-09 17:32:10 +00:00
|
|
|
let blck = parseTest(
|
|
|
|
path/filename & ".ssz_snappy",
|
|
|
|
SSZ, capella.SignedBeaconBlock
|
|
|
|
)
|
2023-09-20 01:14:49 +00:00
|
|
|
doAssert not step.hasKey"blobs"
|
2022-11-09 17:32:10 +00:00
|
|
|
result.add Operation(kind: opOnBlock,
|
|
|
|
blck: ForkedSignedBeaconBlock.init(blck))
|
2023-03-04 13:35:39 +00:00
|
|
|
of ConsensusFork.Deneb:
|
2023-09-20 01:14:49 +00:00
|
|
|
let
|
|
|
|
blck = parseTest(
|
|
|
|
path/filename & ".ssz_snappy",
|
|
|
|
SSZ, deneb.SignedBeaconBlock)
|
|
|
|
blobData =
|
|
|
|
if step.hasKey"blobs":
|
|
|
|
numExtraFields += 2
|
|
|
|
Opt.some BlobData(
|
|
|
|
blobs: distinctBase(parseTest(
|
|
|
|
path/(step["blobs"].getStr()) & ".ssz_snappy",
|
|
|
|
SSZ, List[KzgBlob, Limit MAX_BLOBS_PER_BLOCK])),
|
|
|
|
proofs: step["proofs"].mapIt(KzgProof.fromHex(it.getStr())))
|
|
|
|
else:
|
|
|
|
Opt.none(BlobData)
|
2022-12-08 16:21:53 +00:00
|
|
|
result.add Operation(kind: opOnBlock,
|
2023-09-20 01:14:49 +00:00
|
|
|
blck: ForkedSignedBeaconBlock.init(blck),
|
|
|
|
blobData: blobData)
|
2022-07-06 10:33:02 +00:00
|
|
|
elif step.hasKey"attester_slashing":
|
|
|
|
let filename = step["attester_slashing"].getStr()
|
|
|
|
let attesterSlashing = parseTest(
|
|
|
|
path/filename & ".ssz_snappy",
|
|
|
|
SSZ, AttesterSlashing
|
2021-11-25 18:41:39 +00:00
|
|
|
)
|
2022-07-06 10:33:02 +00:00
|
|
|
result.add Operation(kind: opOnAttesterSlashing,
|
|
|
|
attesterSlashing: attesterSlashing)
|
2022-09-27 12:11:47 +00:00
|
|
|
elif step.hasKey"payload_status":
|
|
|
|
if step["payload_status"]["status"].getStr() == "INVALID":
|
|
|
|
result.add Operation(kind: opInvalidateRoot,
|
|
|
|
valid: true,
|
|
|
|
invalidatedRoot: Eth2Digest.fromHex(step["block_hash"].getStr()),
|
|
|
|
latestValidHash: Eth2Digest.fromHex(
|
|
|
|
step["payload_status"]["latest_valid_hash"].getStr()))
|
2021-11-25 18:41:39 +00:00
|
|
|
elif step.hasKey"checks":
|
|
|
|
result.add Operation(kind: opChecks,
|
|
|
|
checks: step["checks"])
|
|
|
|
else:
|
2022-07-06 10:33:02 +00:00
|
|
|
doAssert false, "Unknown test step: " & $step
|
2021-11-25 18:41:39 +00:00
|
|
|
|
|
|
|
if step.hasKey"valid":
|
2023-09-20 01:14:49 +00:00
|
|
|
doAssert step.len == 2 + numExtraFields
|
2021-11-25 18:41:39 +00:00
|
|
|
result[^1].valid = step["valid"].getBool()
|
2022-09-27 12:11:47 +00:00
|
|
|
elif not step.hasKey"checks" and not step.hasKey"payload_status":
|
2023-09-20 01:14:49 +00:00
|
|
|
doAssert step.len == 1 + numExtraFields
|
2021-11-25 18:41:39 +00:00
|
|
|
result[^1].valid = true
|
|
|
|
|
|
|
|
proc stepOnBlock(
|
2022-04-14 10:47:14 +00:00
|
|
|
dag: ChainDAGRef,
|
2021-11-25 18:41:39 +00:00
|
|
|
fkChoice: ref ForkChoice,
|
2021-12-06 09:49:01 +00:00
|
|
|
verifier: var BatchVerifier,
|
2022-03-16 07:20:40 +00:00
|
|
|
state: var ForkedHashedBeaconState,
|
2021-11-25 18:41:39 +00:00
|
|
|
stateCache: var StateCache,
|
2021-12-06 09:49:01 +00:00
|
|
|
signedBlock: ForkySignedBeaconBlock,
|
2023-09-20 01:14:49 +00:00
|
|
|
blobData: Opt[BlobData],
|
2022-09-27 12:11:47 +00:00
|
|
|
time: BeaconTime,
|
|
|
|
invalidatedRoots: Table[Eth2Digest, Eth2Digest]):
|
2022-11-10 17:40:27 +00:00
|
|
|
Result[BlockRef, VerifierError] =
|
2023-09-20 01:14:49 +00:00
|
|
|
# 1. Validate blobs
|
|
|
|
when typeof(signedBlock).toFork() >= ConsensusFork.Deneb:
|
|
|
|
let kzgCommits = signedBlock.message.body.blob_kzg_commitments.asSeq
|
|
|
|
if kzgCommits.len > 0 or blobData.isSome:
|
|
|
|
if blobData.isNone or kzgCommits.validate_blobs(
|
|
|
|
blobData.get.blobs, blobData.get.proofs).isErr:
|
|
|
|
return err(VerifierError.Invalid)
|
|
|
|
else:
|
|
|
|
doAssert blobData.isNone, "Pre-Deneb test with specified blob data"
|
|
|
|
|
|
|
|
# 2. Move state to proper slot
|
2022-03-16 07:20:40 +00:00
|
|
|
doAssert dag.updateState(
|
2021-11-25 18:41:39 +00:00
|
|
|
state,
|
Prune `BlockRef` on finalization (#3513)
Up til now, the block dag has been using `BlockRef`, a structure adapted
for a full DAG, to represent all of chain history. This is a correct and
simple design, but does not exploit the linearity of the chain once
parts of it finalize.
By pruning the in-memory `BlockRef` structure at finalization, we save,
at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory
landing us at a steady state of ~750mb normal memory usage for a
validating node.
Above all though, we prevent memory usage from growing proportionally
with the length of the chain, something that would not be sustainable
over time - instead, the steady state memory usage is roughly
determined by the validator set size which grows much more slowly. With
these changes, the core should remain sustainable memory-wise post-merge
all the way to withdrawals (when the validator set is expected to grow).
In-memory indices are still used for the "hot" unfinalized portion of
the chain - this ensure that consensus performance remains unchanged.
What changes is that for historical access, we use a db-based linear
slot index which is cache-and-disk-friendly, keeping the cost for
accessing historical data at a similar level as before, achieving the
savings at no percievable cost to functionality or performance.
A nice collateral benefit is the almost-instant startup since we no
longer load any large indicies at dag init.
The cost of this functionality instead can be found in the complexity of
having to deal with two ways of traversing the chain - by `BlockRef` and
by slot.
* use `BlockId` instead of `BlockRef` where finalized / historical data
may be required
* simplify clearance pre-advancement
* remove dag.finalizedBlocks (~50:ish mb)
* remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead
* `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef`
instance, unlike `BlockRef` traversal
* prune `BlockRef` parents on finality (~200:ish mb)
* speed up ChainDAG init by not loading finalized history index
* mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
|
|
|
dag.getBlockIdAtSlot(time.slotOrZero).expect("block exists"),
|
2021-11-25 18:41:39 +00:00
|
|
|
save = false,
|
|
|
|
stateCache
|
|
|
|
)
|
|
|
|
|
2023-09-20 01:14:49 +00:00
|
|
|
# 3. Add block to DAG
|
2021-11-25 18:41:39 +00:00
|
|
|
when signedBlock is phase0.SignedBeaconBlock:
|
|
|
|
type TrustedBlock = phase0.TrustedSignedBeaconBlock
|
|
|
|
elif signedBlock is altair.SignedBeaconBlock:
|
|
|
|
type TrustedBlock = altair.TrustedSignedBeaconBlock
|
2022-11-09 17:32:10 +00:00
|
|
|
elif signedBlock is bellatrix.SignedBeaconBlock:
|
2022-01-12 14:50:30 +00:00
|
|
|
type TrustedBlock = bellatrix.TrustedSignedBeaconBlock
|
2022-11-09 17:32:10 +00:00
|
|
|
elif signedBlock is capella.SignedBeaconBlock:
|
|
|
|
type TrustedBlock = capella.TrustedSignedBeaconBlock
|
2023-03-02 20:09:24 +00:00
|
|
|
elif signedBlock is deneb.SignedBeaconBlock:
|
|
|
|
type TrustedBlock = deneb.TrustedSignedBeaconBlock
|
2022-11-09 17:32:10 +00:00
|
|
|
else:
|
|
|
|
doAssert false, "Unknown TrustedSignedBeaconBlock fork"
|
|
|
|
|
2021-11-25 18:41:39 +00:00
|
|
|
|
2022-09-27 12:11:47 +00:00
|
|
|
# In normal Nimbus flow, for this (effectively) newPayload-based INVALID, it
|
|
|
|
# is checked even before entering the DAG, by the block processor. Currently
|
|
|
|
# the optimistic sync test(s) don't include a later-fcU-INVALID case. Whilst
|
|
|
|
# this wouldn't be part of this check, presumably, their FC test vector step
|
|
|
|
# would also have `true` validity because it'd not be known they weren't, so
|
|
|
|
# adding this mock of the block processor is realistic and sufficient.
|
|
|
|
when not (
|
|
|
|
signedBlock is phase0.SignedBeaconBlock or
|
|
|
|
signedBlock is altair.SignedBeaconBlock):
|
|
|
|
let executionPayloadHash =
|
|
|
|
signedBlock.message.body.execution_payload.block_hash
|
|
|
|
if executionPayloadHash in invalidatedRoots:
|
|
|
|
# Mocks fork choice INVALID list application. These tests sequence this
|
|
|
|
# in a way the block processor does not, specifying each payload_status
|
|
|
|
# before the block itself, while Nimbus fork choice treats invalidating
|
|
|
|
# a non-existent block root as a no-op and does not remember it for the
|
|
|
|
# future.
|
|
|
|
let lvh = invalidatedRoots.getOrDefault(
|
|
|
|
executionPayloadHash, static(default(Eth2Digest)))
|
|
|
|
fkChoice[].mark_root_invalid(dag.getEarliestInvalidBlockRoot(
|
|
|
|
signedBlock.message.parent_root, lvh, executionPayloadHash))
|
|
|
|
|
2022-11-10 17:40:27 +00:00
|
|
|
return err VerifierError.Invalid
|
2022-09-27 12:11:47 +00:00
|
|
|
|
Backfill support for ChainDAG (#3171)
In the ChainDAG, 3 block pointers are kept: genesis, tail and head. This
PR adds one more block pointer: the backfill block which represents the
block that has been backfilled so far.
When doing a checkpoint sync, a random block is given as starting point
- this is the tail block, and we require that the tail block has a
corresponding state.
When backfilling, we end up with blocks without corresponding states,
hence we cannot use `tail` as a backfill pointer - there is no state.
Nonetheless, we need to keep track of where we are in the backfill
process between restarts, such that we can answer GetBeaconBlocksByRange
requests.
This PR adds the basic support for backfill handling - it needs to be
integrated with backfill sync, and the REST API needs to be adjusted to
take advantage of the new backfilled blocks when responding to certain
requests.
Future work will also enable moving the tail in either direction:
* pruning means moving the tail forward in time and removing states
* backwards means recreating past states from genesis, such that
intermediate states are recreated step by step all the way to the tail -
at that point, tail, genesis and backfill will match up.
* backfilling is done when backfill != genesis - later, this will be the
WSS checkpoint instead
2021-12-13 13:36:06 +00:00
|
|
|
let blockAdded = dag.addHeadBlock(verifier, signedBlock) do (
|
2022-07-06 10:33:02 +00:00
|
|
|
blckRef: BlockRef, signedBlock: TrustedBlock,
|
|
|
|
epochRef: EpochRef, unrealized: FinalityCheckpoints):
|
2021-11-25 18:41:39 +00:00
|
|
|
|
2023-09-20 01:14:49 +00:00
|
|
|
# 4. Update fork choice if valid
|
2021-11-25 18:41:39 +00:00
|
|
|
let status = fkChoice[].process_block(
|
2022-07-06 10:33:02 +00:00
|
|
|
dag, epochRef, blckRef, unrealized, signedBlock.message, time)
|
2021-11-25 18:41:39 +00:00
|
|
|
doAssert status.isOk()
|
2021-12-06 09:49:01 +00:00
|
|
|
|
2023-09-20 01:14:49 +00:00
|
|
|
# 5. Update DAG with new head
|
2022-07-06 10:33:02 +00:00
|
|
|
var quarantine = Quarantine.init()
|
|
|
|
let newHead = fkChoice[].get_head(dag, time).get()
|
2023-03-02 16:13:35 +00:00
|
|
|
dag.updateHead(dag.getBlockRef(newHead).get(), quarantine, [])
|
2022-07-06 10:33:02 +00:00
|
|
|
if dag.needStateCachesAndForkChoicePruning():
|
|
|
|
dag.pruneStateCachesDAG()
|
|
|
|
let pruneRes = fkChoice[].prune()
|
|
|
|
doAssert pruneRes.isOk()
|
2021-11-25 18:41:39 +00:00
|
|
|
|
2022-07-06 10:33:02 +00:00
|
|
|
blockAdded
|
2021-11-25 18:41:39 +00:00
|
|
|
|
|
|
|
proc stepChecks(
|
|
|
|
checks: JsonNode,
|
2022-04-14 10:47:14 +00:00
|
|
|
dag: ChainDAGRef,
|
2021-11-25 18:41:39 +00:00
|
|
|
fkChoice: ref ForkChoice,
|
2021-12-21 18:56:08 +00:00
|
|
|
time: BeaconTime
|
2021-11-25 18:41:39 +00:00
|
|
|
) =
|
|
|
|
doAssert checks.len >= 1, "No checks found"
|
|
|
|
for check, val in checks:
|
|
|
|
if check == "time":
|
2022-01-11 10:01:54 +00:00
|
|
|
doAssert time.ns_since_genesis == val.getInt().seconds.nanoseconds()
|
2021-12-21 18:56:08 +00:00
|
|
|
doAssert fkChoice.checkpoints.time.slotOrZero == time.slotOrZero
|
2021-11-25 18:41:39 +00:00
|
|
|
elif check == "head":
|
|
|
|
let headRoot = fkChoice[].get_head(dag, time).get()
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
let headRef = dag.getBlockRef(headRoot).get()
|
2021-11-25 18:41:39 +00:00
|
|
|
doAssert headRef.slot == Slot(val["slot"].getInt())
|
|
|
|
doAssert headRef.root == Eth2Digest.fromHex(val["root"].getStr())
|
|
|
|
elif check == "justified_checkpoint":
|
|
|
|
let checkpointRoot = fkChoice.checkpoints.justified.checkpoint.root
|
|
|
|
let checkpointEpoch = fkChoice.checkpoints.justified.checkpoint.epoch
|
|
|
|
doAssert checkpointEpoch == Epoch(val["epoch"].getInt())
|
|
|
|
doAssert checkpointRoot == Eth2Digest.fromHex(val["root"].getStr())
|
|
|
|
elif check == "justified_checkpoint_root": # undocumented check
|
|
|
|
let checkpointRoot = fkChoice.checkpoints.justified.checkpoint.root
|
|
|
|
doAssert checkpointRoot == Eth2Digest.fromHex(val.getStr())
|
|
|
|
elif check == "finalized_checkpoint":
|
|
|
|
let checkpointRoot = fkChoice.checkpoints.finalized.root
|
|
|
|
let checkpointEpoch = fkChoice.checkpoints.finalized.epoch
|
|
|
|
doAssert checkpointEpoch == Epoch(val["epoch"].getInt())
|
|
|
|
doAssert checkpointRoot == Eth2Digest.fromHex(val["root"].getStr())
|
|
|
|
elif check == "best_justified_checkpoint":
|
|
|
|
let checkpointRoot = fkChoice.checkpoints.best_justified.root
|
|
|
|
let checkpointEpoch = fkChoice.checkpoints.best_justified.epoch
|
|
|
|
doAssert checkpointEpoch == Epoch(val["epoch"].getInt())
|
|
|
|
doAssert checkpointRoot == Eth2Digest.fromHex(val["root"].getStr())
|
2021-12-01 12:55:42 +00:00
|
|
|
elif check == "proposer_boost_root":
|
2021-12-21 18:56:08 +00:00
|
|
|
doAssert fkChoice.checkpoints.proposer_boost_root ==
|
|
|
|
Eth2Digest.fromHex(val.getStr())
|
2021-11-25 18:41:39 +00:00
|
|
|
elif check == "genesis_time":
|
2022-10-14 19:40:10 +00:00
|
|
|
# We do not store genesis in fork choice..
|
|
|
|
discard
|
2021-11-25 18:41:39 +00:00
|
|
|
else:
|
|
|
|
doAssert false, "Unsupported check '" & $check & "'"
|
|
|
|
|
2023-01-28 19:53:41 +00:00
|
|
|
proc doRunTest(path: string, fork: ConsensusFork) =
|
2021-11-25 18:41:39 +00:00
|
|
|
let db = BeaconChainDB.new("", inMemory = true)
|
|
|
|
defer:
|
|
|
|
db.close()
|
|
|
|
|
2022-07-06 10:33:02 +00:00
|
|
|
let stores =
|
|
|
|
case fork
|
2023-03-04 13:35:39 +00:00
|
|
|
of ConsensusFork.Deneb:
|
2023-03-02 20:09:24 +00:00
|
|
|
initialLoad(path, db, deneb.BeaconState, deneb.BeaconBlock)
|
2023-01-28 19:53:41 +00:00
|
|
|
of ConsensusFork.Capella:
|
2022-11-09 17:32:10 +00:00
|
|
|
initialLoad(path, db, capella.BeaconState, capella.BeaconBlock)
|
2023-01-28 19:53:41 +00:00
|
|
|
of ConsensusFork.Bellatrix:
|
2022-07-06 10:33:02 +00:00
|
|
|
initialLoad(path, db, bellatrix.BeaconState, bellatrix.BeaconBlock)
|
2023-01-28 19:53:41 +00:00
|
|
|
of ConsensusFork.Altair:
|
2022-07-06 10:33:02 +00:00
|
|
|
initialLoad(path, db, altair.BeaconState, altair.BeaconBlock)
|
2023-01-28 19:53:41 +00:00
|
|
|
of ConsensusFork.Phase0:
|
2022-07-06 10:33:02 +00:00
|
|
|
initialLoad(path, db, phase0.BeaconState, phase0.BeaconBlock)
|
|
|
|
|
2023-06-19 22:43:50 +00:00
|
|
|
let
|
|
|
|
rng = HmacDrbgContext.new()
|
2021-12-06 09:49:01 +00:00
|
|
|
taskpool = Taskpool.new()
|
2023-08-03 08:36:45 +00:00
|
|
|
var verifier = BatchVerifier.init(rng, taskpool)
|
2021-11-25 18:41:39 +00:00
|
|
|
|
|
|
|
let steps = loadOps(path, fork)
|
|
|
|
var time = stores.fkChoice.checkpoints.time
|
2022-09-27 12:11:47 +00:00
|
|
|
var invalidatedRoots: Table[Eth2Digest, Eth2Digest]
|
2021-11-25 18:41:39 +00:00
|
|
|
|
|
|
|
let state = newClone(stores.dag.headState)
|
|
|
|
var stateCache = StateCache()
|
|
|
|
|
|
|
|
for step in steps:
|
|
|
|
case step.kind
|
|
|
|
of opOnTick:
|
2022-01-11 10:01:54 +00:00
|
|
|
time = BeaconTime(ns_since_genesis: step.tick.seconds.nanoseconds)
|
2022-07-06 10:33:02 +00:00
|
|
|
let status = stores.fkChoice[].update_time(stores.dag, time)
|
|
|
|
doAssert status.isOk == step.valid
|
|
|
|
of opOnAttestation:
|
|
|
|
let status = stores.fkChoice[].on_attestation(
|
|
|
|
stores.dag, step.att.data.slot, step.att.data.beacon_block_root,
|
|
|
|
toSeq(stores.dag.get_attesting_indices(step.att.asTrusted)), time)
|
|
|
|
doAssert status.isOk == step.valid
|
2021-11-25 18:41:39 +00:00
|
|
|
of opOnBlock:
|
2022-07-06 10:33:02 +00:00
|
|
|
withBlck(step.blck):
|
2022-11-09 17:32:10 +00:00
|
|
|
let status = stepOnBlock(
|
|
|
|
stores.dag, stores.fkChoice,
|
|
|
|
verifier, state[], stateCache,
|
2023-09-21 10:49:14 +00:00
|
|
|
forkyBlck, step.blobData, time, invalidatedRoots)
|
2022-11-09 17:32:10 +00:00
|
|
|
doAssert status.isOk == step.valid
|
2022-07-06 10:33:02 +00:00
|
|
|
of opOnAttesterSlashing:
|
|
|
|
let indices =
|
|
|
|
check_attester_slashing(state[], step.attesterSlashing, flags = {})
|
|
|
|
if indices.isOk:
|
|
|
|
for idx in indices.get:
|
|
|
|
stores.fkChoice[].process_equivocation(idx)
|
|
|
|
doAssert indices.isOk == step.valid
|
2022-09-27 12:11:47 +00:00
|
|
|
of opInvalidateRoot:
|
|
|
|
invalidatedRoots[step.invalidatedRoot] = step.latestValidHash
|
2021-11-25 18:41:39 +00:00
|
|
|
of opChecks:
|
|
|
|
stepChecks(step.checks, stores.dag, stores.fkChoice, time)
|
|
|
|
else:
|
|
|
|
doAssert false, "Unsupported"
|
|
|
|
|
2023-09-04 13:05:11 +00:00
|
|
|
proc runTest(suiteName: static[string], path: string, fork: ConsensusFork) =
|
2021-11-25 18:41:39 +00:00
|
|
|
const SKIP = [
|
|
|
|
# protoArray can handle blocks in the future gracefully
|
|
|
|
# spec: https://github.com/ethereum/consensus-specs/blame/v1.1.3/specs/phase0/fork-choice.md#L349
|
|
|
|
# test: tests/fork_choice/scenarios/no_votes.nim
|
|
|
|
# "Ensure the head is still 4 whilst the justified epoch is 0."
|
|
|
|
"on_block_future_block",
|
|
|
|
|
2022-07-06 10:33:02 +00:00
|
|
|
# TODO on_merge_block
|
|
|
|
"too_early_for_merge",
|
|
|
|
"too_late_for_merge",
|
|
|
|
"block_lookup_failed",
|
|
|
|
"all_valid",
|
|
|
|
]
|
2021-11-25 18:41:39 +00:00
|
|
|
|
2023-09-04 13:05:11 +00:00
|
|
|
test suiteName & " - " & path.relativePath(SszTestsDir):
|
2022-07-06 10:33:02 +00:00
|
|
|
when defined(windows):
|
|
|
|
# Some test files have very long paths
|
|
|
|
skip()
|
|
|
|
else:
|
2023-02-10 20:59:38 +00:00
|
|
|
if os_ops.splitPath(path).tail in SKIP:
|
2022-07-06 10:33:02 +00:00
|
|
|
skip()
|
|
|
|
else:
|
|
|
|
doRunTest(path, fork)
|
2021-11-25 18:41:39 +00:00
|
|
|
|
2022-09-27 12:11:47 +00:00
|
|
|
template fcSuite(suiteName: static[string], testPathElem: static[string]) =
|
|
|
|
suite "EF - " & suiteName & preset():
|
|
|
|
const presetPath = SszTestsDir/const_preset
|
|
|
|
for kind, path in walkDir(presetPath, relative = true, checkDir = true):
|
|
|
|
let testsPath = presetPath/path/testPathElem
|
2023-02-10 20:59:38 +00:00
|
|
|
if kind != pcDir or not os_ops.dirExists(testsPath):
|
2022-07-06 10:33:02 +00:00
|
|
|
continue
|
2023-05-25 07:55:00 +00:00
|
|
|
if testsPath.contains("/eip6110/") or testsPath.contains("\\eip6110\\"):
|
|
|
|
continue
|
2022-09-27 12:11:47 +00:00
|
|
|
let fork = forkForPathComponent(path).valueOr:
|
|
|
|
raiseAssert "Unknown test fork: " & testsPath
|
|
|
|
for kind, path in walkDir(testsPath, relative = true, checkDir = true):
|
|
|
|
let basePath = testsPath/path/"pyspec_tests"
|
|
|
|
if kind != pcDir:
|
|
|
|
continue
|
|
|
|
for kind, path in walkDir(basePath, relative = true, checkDir = true):
|
|
|
|
runTest(suiteName, basePath/path, fork)
|
|
|
|
|
2023-09-20 01:14:49 +00:00
|
|
|
from ../../beacon_chain/conf import loadKzgTrustedSetup
|
|
|
|
discard loadKzgTrustedSetup() # Required for Deneb tests
|
|
|
|
|
2022-09-27 12:11:47 +00:00
|
|
|
fcSuite("ForkChoice", "fork_choice")
|
|
|
|
fcSuite("Sync", "sync")
|