2020-05-01 15:51:24 +00:00
|
|
|
# beacon_chain
|
2022-01-04 09:45:38 +00:00
|
|
|
# Copyright (c) 2019-2022 Status Research & Development GmbH
|
2020-05-01 15:51:24 +00:00
|
|
|
# Licensed and distributed under either of
|
|
|
|
|
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
|
|
|
|
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
|
|
|
|
# at your option. This file may not be copied, modified, or distributed except according to those terms.
|
|
|
|
|
|
|
|
|
|
# `block_sim` is a block and attestation simulator similar to `state_sim` whose
|
|
|
|
|
# task is to run the beacon chain without considering the network or the
|
|
|
|
|
# wall clock. Functionally, it achieves the same as the distributed beacon chain
|
|
|
|
|
# by producing blocks and attestations as if they were created by separate
|
|
|
|
|
# nodes, just like a set of `beacon_node` instances would.
|
|
|
|
|
#
|
|
|
|
|
# Similar to `state_sim`, but uses the block and attestation pools along with
|
|
|
|
|
# a database, as if a real node was running.
|
|
|
|
|
|
|
|
|
|
import
|
2020-12-03 04:30:35 +00:00
|
|
|
math, stats, times, strformat,
|
2021-08-29 14:50:21 +00:00
|
|
|
tables, options, random, tables, os,
|
2020-12-03 04:30:35 +00:00
|
|
|
confutils, chronicles, eth/db/kvstore_sqlite3,
|
2021-09-17 00:13:52 +00:00
|
|
|
chronos/timer, eth/keys, taskpools,
|
2021-06-11 17:51:46 +00:00
|
|
|
../tests/testblockutil,
|
2021-08-12 13:08:20 +00:00
|
|
|
../beacon_chain/spec/[
|
|
|
|
|
beaconstate, forks, helpers, signatures, state_transition],
|
2022-01-18 13:36:52 +00:00
|
|
|
../beacon_chain/spec/datatypes/[phase0, altair, bellatrix],
|
2021-10-19 14:09:26 +00:00
|
|
|
../beacon_chain/[beacon_chain_db, beacon_clock],
|
2021-03-03 06:23:05 +00:00
|
|
|
../beacon_chain/eth1/eth1_monitor,
|
2021-03-02 10:27:45 +00:00
|
|
|
../beacon_chain/validators/validator_pool,
|
2021-12-09 12:56:54 +00:00
|
|
|
../beacon_chain/gossip_processing/[batch_validation, gossip_validation],
|
2021-05-21 09:23:28 +00:00
|
|
|
../beacon_chain/consensus_object_pools/[blockchain_dag, block_quarantine,
|
2021-08-29 14:50:21 +00:00
|
|
|
block_clearance, attestation_pool,
|
|
|
|
|
sync_committee_msg_pool],
|
2020-05-01 15:51:24 +00:00
|
|
|
./simutils
|
|
|
|
|
|
|
|
|
|
type Timers = enum
|
|
|
|
|
tBlock = "Process non-epoch slot with block"
|
|
|
|
|
tEpoch = "Process epoch slot with block"
|
|
|
|
|
tHashBlock = "Tree-hash block"
|
|
|
|
|
tSignBlock = "Sign block"
|
2020-05-03 17:44:04 +00:00
|
|
|
tAttest = "Have committee attest to block"
|
2021-08-29 14:50:21 +00:00
|
|
|
tSyncCommittees = "Produce sync committee actions"
|
2020-05-03 17:44:04 +00:00
|
|
|
tReplay = "Replay all produced blocks"
|
2020-05-01 15:51:24 +00:00
|
|
|
|
2021-08-29 14:50:21 +00:00
|
|
|
template seconds(x: uint64): timer.Duration =
|
|
|
|
|
timer.seconds(int(x))
|
|
|
|
|
|
2021-08-17 09:51:39 +00:00
|
|
|
func gauss(r: var Rand; mu = 0.0; sigma = 1.0): float =
|
2020-12-03 04:30:35 +00:00
|
|
|
# TODO This is present in Nim 1.4
|
|
|
|
|
const K = sqrt(2 / E)
|
|
|
|
|
var
|
|
|
|
|
a = 0.0
|
|
|
|
|
b = 0.0
|
|
|
|
|
while true:
|
|
|
|
|
a = rand(r, 1.0)
|
|
|
|
|
b = (2.0 * rand(r, 1.0) - 1.0) * K
|
|
|
|
|
if b * b <= -4.0 * a * a * ln(a): break
|
2021-08-17 09:51:39 +00:00
|
|
|
mu + sigma * (b / a)
|
2020-12-03 04:30:35 +00:00
|
|
|
|
2020-05-01 15:51:24 +00:00
|
|
|
# TODO confutils is an impenetrable black box. how can a help text be added here?
|
2021-06-23 14:43:18 +00:00
|
|
|
cli do(slots = SLOTS_PER_EPOCH * 6,
|
2021-03-17 13:35:59 +00:00
|
|
|
validators = SLOTS_PER_EPOCH * 400, # One per shard is minimum
|
2020-10-13 17:21:25 +00:00
|
|
|
attesterRatio {.desc: "ratio of validators that attest in each round"} = 0.82,
|
2022-05-23 12:02:54 +00:00
|
|
|
syncCommitteeRatio {.desc: "ratio of validators that perform sync committee actions in each round"} = 0.82,
|
2020-05-03 17:44:04 +00:00
|
|
|
blockRatio {.desc: "ratio of slots with blocks"} = 1.0,
|
|
|
|
|
replay = true):
|
2020-05-01 15:51:24 +00:00
|
|
|
let
|
2021-11-10 11:39:08 +00:00
|
|
|
(genesisState, depositContractSnapshot) = loadGenesis(validators, false)
|
|
|
|
|
genesisTime = float getStateField(genesisState[], genesis_time)
|
2020-05-01 15:51:24 +00:00
|
|
|
|
2021-08-29 14:50:21 +00:00
|
|
|
var
|
|
|
|
|
cfg = defaultRuntimeConfig
|
Implement split preset/config support (#2710)
* Implement split preset/config support
This is the initial bulk refactor to introduce runtime config values in
a number of places, somewhat replacing the existing mechanism of loading
network metadata.
It still needs more work, this is the initial refactor that introduces
runtime configuration in some of the places that need it.
The PR changes the way presets and constants work, to match the spec. In
particular, a "preset" now refers to the compile-time configuration
while a "cfg" or "RuntimeConfig" is the dynamic part.
A single binary can support either mainnet or minimal, but not both.
Support for other presets has been removed completely (can be readded,
in case there's need).
There's a number of outstanding tasks:
* `SECONDS_PER_SLOT` still needs fixing
* loading custom runtime configs needs redoing
* checking constants against YAML file
* yeerongpilly support
`build/nimbus_beacon_node --network=yeerongpilly --discv5:no --log-level=DEBUG`
* load fork epoch from config
* fix fork digest sent in status
* nicer error string for request failures
* fix tools
* one more
* fixup
* fixup
* fixup
* use "standard" network definition folder in local testnet
Files are loaded from their standard locations, including genesis etc,
to conform to the format used in the `eth2-networks` repo.
* fix launch scripts, allow unknown config values
* fix base config of rest test
* cleanups
* bundle mainnet config using common loader
* fix spec links and names
* only include supported preset in binary
* drop yeerongpilly, add altair-devnet-0, support boot_enr.yaml
2021-07-12 13:01:38 +00:00
|
|
|
|
2022-01-05 13:41:39 +00:00
|
|
|
cfg.ALTAIR_FORK_EPOCH = 32.Slot.epoch
|
2022-02-02 13:06:55 +00:00
|
|
|
cfg.BELLATRIX_FORK_EPOCH = 96.Slot.epoch
|
2021-06-23 14:43:18 +00:00
|
|
|
|
2020-05-01 15:51:24 +00:00
|
|
|
echo "Starting simulation..."
|
|
|
|
|
|
2021-07-13 14:27:10 +00:00
|
|
|
let db = BeaconChainDB.new("block_sim_db")
|
2020-09-16 07:16:23 +00:00
|
|
|
defer: db.close()
|
2020-05-01 15:51:24 +00:00
|
|
|
|
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
|
|
|
ChainDAGRef.preInit(db, genesisState[])
|
2020-12-03 04:30:35 +00:00
|
|
|
putInitialDepositContractSnapshot(db, depositContractSnapshot)
|
2020-05-01 15:51:24 +00:00
|
|
|
|
|
|
|
|
var
|
2021-12-20 19:20:31 +00:00
|
|
|
validatorMonitor = newClone(ValidatorMonitor.init())
|
|
|
|
|
dag = ChainDAGRef.init(cfg, db, validatorMonitor, {})
|
2021-07-13 14:27:10 +00:00
|
|
|
eth1Chain = Eth1Chain.init(cfg, db)
|
2022-06-29 16:53:59 +00:00
|
|
|
merkleizer = DepositsMerkleizer.init(depositContractSnapshot.depositContractState)
|
2021-09-17 00:13:52 +00:00
|
|
|
taskpool = Taskpool.new()
|
2021-12-06 09:49:01 +00:00
|
|
|
verifier = BatchVerifier(rng: keys.newRng(), taskpool: taskpool)
|
|
|
|
|
quarantine = newClone(Quarantine.init())
|
2021-06-01 11:13:40 +00:00
|
|
|
attPool = AttestationPool.init(dag, quarantine)
|
2021-12-09 12:56:54 +00:00
|
|
|
batchCrypto = BatchCrypto.new(
|
2022-02-17 11:53:55 +00:00
|
|
|
keys.newRng(), eager = func(): bool = true, taskpool)
|
2022-01-24 20:40:59 +00:00
|
|
|
syncCommitteePool = newClone SyncCommitteeMsgPool.init(keys.newRng())
|
2020-05-01 15:51:24 +00:00
|
|
|
timers: array[Timers, RunningStat]
|
|
|
|
|
attesters: RunningStat
|
2020-05-03 17:44:04 +00:00
|
|
|
r = initRand(1)
|
2021-06-01 11:13:40 +00:00
|
|
|
tmpState = assignClone(dag.headState)
|
2020-05-01 15:51:24 +00:00
|
|
|
|
2020-12-03 04:30:35 +00:00
|
|
|
eth1Chain.addBlock Eth1Block(
|
|
|
|
|
number: Eth1BlockNumber 1,
|
2022-08-10 12:31:10 +00:00
|
|
|
timestamp: Eth1BlockTimestamp genesisTime)
|
2020-12-03 04:30:35 +00:00
|
|
|
|
2021-06-01 11:13:40 +00:00
|
|
|
let replayState = assignClone(dag.headState)
|
2020-05-03 17:44:04 +00:00
|
|
|
|
|
|
|
|
proc handleAttestations(slot: Slot) =
|
2020-05-01 15:51:24 +00:00
|
|
|
let
|
2021-06-01 11:13:40 +00:00
|
|
|
attestationHead = dag.head.atSlot(slot)
|
2020-05-01 15:51:24 +00:00
|
|
|
|
Prune `BlockRef` on finalization (#3513)
Up til now, the block dag has been using `BlockRef`, a structure adapted
for a full DAG, to represent all of chain history. This is a correct and
simple design, but does not exploit the linearity of the chain once
parts of it finalize.
By pruning the in-memory `BlockRef` structure at finalization, we save,
at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory
landing us at a steady state of ~750mb normal memory usage for a
validating node.
Above all though, we prevent memory usage from growing proportionally
with the length of the chain, something that would not be sustainable
over time - instead, the steady state memory usage is roughly
determined by the validator set size which grows much more slowly. With
these changes, the core should remain sustainable memory-wise post-merge
all the way to withdrawals (when the validator set is expected to grow).
In-memory indices are still used for the "hot" unfinalized portion of
the chain - this ensure that consensus performance remains unchanged.
What changes is that for historical access, we use a db-based linear
slot index which is cache-and-disk-friendly, keeping the cost for
accessing historical data at a similar level as before, achieving the
savings at no percievable cost to functionality or performance.
A nice collateral benefit is the almost-instant startup since we no
longer load any large indicies at dag init.
The cost of this functionality instead can be found in the complexity of
having to deal with two ways of traversing the chain - by `BlockRef` and
by slot.
* use `BlockId` instead of `BlockRef` where finalized / historical data
may be required
* simplify clearance pre-advancement
* remove dag.finalizedBlocks (~50:ish mb)
* remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead
* `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef`
instance, unlike `BlockRef` traversal
* prune `BlockRef` parents on finality (~200:ish mb)
* speed up ChainDAG init by not loading finalized history index
* mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
|
|
|
dag.withUpdatedState(tmpState[], attestationHead.toBlockSlotId.expect("not nil")) do:
|
2022-06-29 16:53:59 +00:00
|
|
|
let
|
|
|
|
|
fork = getStateField(state, fork)
|
|
|
|
|
genesis_validators_root = getStateField(state, genesis_validators_root)
|
|
|
|
|
committees_per_slot =
|
|
|
|
|
get_committee_count_per_slot(state, slot.epoch, cache)
|
2020-05-01 15:51:24 +00:00
|
|
|
|
2022-01-08 23:28:49 +00:00
|
|
|
for committee_index in get_committee_indices(committees_per_slot):
|
2020-05-01 15:51:24 +00:00
|
|
|
let committee = get_beacon_committee(
|
2022-03-16 07:20:40 +00:00
|
|
|
state, slot, committee_index, cache)
|
2020-05-01 15:51:24 +00:00
|
|
|
|
2022-01-08 23:28:49 +00:00
|
|
|
for index_in_committee, validator_index in committee:
|
2020-05-03 17:44:04 +00:00
|
|
|
if rand(r, 1.0) <= attesterRatio:
|
2020-05-01 15:51:24 +00:00
|
|
|
let
|
2020-11-04 21:52:47 +00:00
|
|
|
data = makeAttestationData(
|
Prune `BlockRef` on finalization (#3513)
Up til now, the block dag has been using `BlockRef`, a structure adapted
for a full DAG, to represent all of chain history. This is a correct and
simple design, but does not exploit the linearity of the chain once
parts of it finalize.
By pruning the in-memory `BlockRef` structure at finalization, we save,
at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory
landing us at a steady state of ~750mb normal memory usage for a
validating node.
Above all though, we prevent memory usage from growing proportionally
with the length of the chain, something that would not be sustainable
over time - instead, the steady state memory usage is roughly
determined by the validator set size which grows much more slowly. With
these changes, the core should remain sustainable memory-wise post-merge
all the way to withdrawals (when the validator set is expected to grow).
In-memory indices are still used for the "hot" unfinalized portion of
the chain - this ensure that consensus performance remains unchanged.
What changes is that for historical access, we use a db-based linear
slot index which is cache-and-disk-friendly, keeping the cost for
accessing historical data at a similar level as before, achieving the
savings at no percievable cost to functionality or performance.
A nice collateral benefit is the almost-instant startup since we no
longer load any large indicies at dag init.
The cost of this functionality instead can be found in the complexity of
having to deal with two ways of traversing the chain - by `BlockRef` and
by slot.
* use `BlockId` instead of `BlockRef` where finalized / historical data
may be required
* simplify clearance pre-advancement
* remove dag.finalizedBlocks (~50:ish mb)
* remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead
* `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef`
instance, unlike `BlockRef` traversal
* prune `BlockRef` parents on finality (~200:ish mb)
* speed up ChainDAG init by not loading finalized history index
* mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
|
|
|
state, slot, committee_index, bid.root)
|
2020-05-01 15:51:24 +00:00
|
|
|
sig =
|
2022-06-29 16:53:59 +00:00
|
|
|
get_attestation_signature(
|
|
|
|
|
fork, genesis_validators_root, data,
|
|
|
|
|
MockPrivKeys[validator_index])
|
|
|
|
|
attestation = Attestation.init(
|
|
|
|
|
[uint64 index_in_committee], committee.len, data,
|
|
|
|
|
sig.toValidatorSig()).expect("valid data")
|
2020-05-01 15:51:24 +00:00
|
|
|
|
2020-07-09 09:29:32 +00:00
|
|
|
attPool.addAttestation(
|
2022-06-29 16:53:59 +00:00
|
|
|
attestation, [validator_index], sig, data.slot.start_beacon_time)
|
2022-01-05 18:38:04 +00:00
|
|
|
do:
|
|
|
|
|
raiseAssert "withUpdatedState failed"
|
2020-05-01 15:51:24 +00:00
|
|
|
|
2021-08-29 14:50:21 +00:00
|
|
|
proc handleSyncCommitteeActions(slot: Slot) =
|
|
|
|
|
type
|
|
|
|
|
Aggregator = object
|
2021-11-05 15:39:47 +00:00
|
|
|
subcommitteeIdx: SyncSubcommitteeIndex
|
Speed up altair block processing 2x (#3115)
* Speed up altair block processing >2x
Like #3089, this PR drastially speeds up historical REST queries and
other long state replays.
* cache sync committee validator indices
* use ~80mb less memory for validator pubkey mappings
* batch-verify sync aggregate signature (fixes #2985)
* document sync committee hack with head block vs sync message block
* add batch signature verification failure tests
Before:
```
../env.sh nim c -d:release -r ncli_db --db:mainnet_0/db bench --start-slot:-1000
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
5830.675, 0.000, 5830.675, 5830.675, 1, Initialize DB
0.481, 1.878, 0.215, 59.167, 981, Load block from database
8422.566, 0.000, 8422.566, 8422.566, 1, Load state from database
6.996, 1.678, 0.042, 14.385, 969, Advance slot, non-epoch
93.217, 8.318, 84.192, 122.209, 32, Advance slot, epoch
20.513, 23.665, 11.510, 201.561, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
After:
```
7081.422, 0.000, 7081.422, 7081.422, 1, Initialize DB
0.553, 2.122, 0.175, 66.692, 981, Load block from database
5439.446, 0.000, 5439.446, 5439.446, 1, Load state from database
6.829, 1.575, 0.043, 12.156, 969, Advance slot, non-epoch
94.716, 2.749, 88.395, 100.026, 32, Advance slot, epoch
11.636, 23.766, 4.889, 205.250, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
* add comment
2021-11-24 12:43:50 +00:00
|
|
|
validatorIdx: ValidatorIndex
|
2021-08-29 14:50:21 +00:00
|
|
|
selectionProof: ValidatorSig
|
|
|
|
|
|
|
|
|
|
let
|
|
|
|
|
syncCommittee = @(dag.syncCommitteeParticipants(slot + 1))
|
2022-04-08 16:22:49 +00:00
|
|
|
genesis_validators_root = dag.genesis_validators_root
|
2021-08-29 14:50:21 +00:00
|
|
|
fork = dag.forkAtEpoch(slot.epoch)
|
2022-01-11 10:01:54 +00:00
|
|
|
messagesTime = slot.attestation_deadline()
|
|
|
|
|
contributionsTime = slot.sync_contribution_deadline()
|
2021-08-29 14:50:21 +00:00
|
|
|
|
|
|
|
|
var aggregators: seq[Aggregator]
|
|
|
|
|
|
2022-01-08 23:28:49 +00:00
|
|
|
for subcommitteeIdx in SyncSubcommitteeIndex:
|
Speed up altair block processing 2x (#3115)
* Speed up altair block processing >2x
Like #3089, this PR drastially speeds up historical REST queries and
other long state replays.
* cache sync committee validator indices
* use ~80mb less memory for validator pubkey mappings
* batch-verify sync aggregate signature (fixes #2985)
* document sync committee hack with head block vs sync message block
* add batch signature verification failure tests
Before:
```
../env.sh nim c -d:release -r ncli_db --db:mainnet_0/db bench --start-slot:-1000
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
5830.675, 0.000, 5830.675, 5830.675, 1, Initialize DB
0.481, 1.878, 0.215, 59.167, 981, Load block from database
8422.566, 0.000, 8422.566, 8422.566, 1, Load state from database
6.996, 1.678, 0.042, 14.385, 969, Advance slot, non-epoch
93.217, 8.318, 84.192, 122.209, 32, Advance slot, epoch
20.513, 23.665, 11.510, 201.561, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
After:
```
7081.422, 0.000, 7081.422, 7081.422, 1, Initialize DB
0.553, 2.122, 0.175, 66.692, 981, Load block from database
5439.446, 0.000, 5439.446, 5439.446, 1, Load state from database
6.829, 1.575, 0.043, 12.156, 969, Advance slot, non-epoch
94.716, 2.749, 88.395, 100.026, 32, Advance slot, epoch
11.636, 23.766, 4.889, 205.250, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
* add comment
2021-11-24 12:43:50 +00:00
|
|
|
for validatorIdx in syncSubcommittee(syncCommittee, subcommitteeIdx):
|
2021-08-29 14:50:21 +00:00
|
|
|
if rand(r, 1.0) > syncCommitteeRatio:
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
let
|
2021-12-09 12:56:54 +00:00
|
|
|
validatorPrivKey = MockPrivKeys[validatorIdx]
|
|
|
|
|
signature = get_sync_committee_message_signature(
|
|
|
|
|
fork, genesis_validators_root, slot, dag.head.root, validatorPrivKey)
|
2021-08-29 14:50:21 +00:00
|
|
|
msg = SyncCommitteeMessage(
|
|
|
|
|
slot: slot,
|
|
|
|
|
beacon_block_root: dag.head.root,
|
|
|
|
|
validator_index: uint64 validatorIdx,
|
|
|
|
|
signature: signature.toValidatorSig)
|
|
|
|
|
|
2021-12-09 12:56:54 +00:00
|
|
|
let res = waitFor dag.validateSyncCommitteeMessage(
|
|
|
|
|
batchCrypto,
|
2021-12-11 15:39:24 +00:00
|
|
|
syncCommitteePool,
|
2021-08-29 14:50:21 +00:00
|
|
|
msg,
|
2021-11-05 15:39:47 +00:00
|
|
|
subcommitteeIdx,
|
2021-08-29 14:50:21 +00:00
|
|
|
messagesTime,
|
|
|
|
|
false)
|
|
|
|
|
|
|
|
|
|
doAssert res.isOk
|
|
|
|
|
|
2021-11-05 15:39:47 +00:00
|
|
|
let (positions, cookedSig) = res.get()
|
|
|
|
|
|
2021-11-25 12:20:36 +00:00
|
|
|
syncCommitteePool[].addSyncCommitteeMessage(
|
2021-11-05 15:39:47 +00:00
|
|
|
msg.slot,
|
|
|
|
|
msg.beacon_block_root,
|
|
|
|
|
msg.validator_index,
|
|
|
|
|
cookedSig,
|
|
|
|
|
subcommitteeIdx,
|
|
|
|
|
positions)
|
|
|
|
|
|
2021-08-29 14:50:21 +00:00
|
|
|
let
|
2021-12-09 12:56:54 +00:00
|
|
|
selectionProofSig = get_sync_committee_selection_proof(
|
2022-05-10 10:03:40 +00:00
|
|
|
fork, genesis_validators_root, slot, subcommitteeIdx,
|
2021-12-09 12:56:54 +00:00
|
|
|
validatorPrivKey).toValidatorSig
|
2021-08-29 14:50:21 +00:00
|
|
|
|
|
|
|
|
if is_sync_committee_aggregator(selectionProofSig):
|
|
|
|
|
aggregators.add Aggregator(
|
2021-11-05 15:39:47 +00:00
|
|
|
subcommitteeIdx: subcommitteeIdx,
|
2021-08-29 14:50:21 +00:00
|
|
|
validatorIdx: validatorIdx,
|
|
|
|
|
selectionProof: selectionProofSig)
|
|
|
|
|
|
|
|
|
|
for aggregator in aggregators:
|
|
|
|
|
var contribution: SyncCommitteeContribution
|
|
|
|
|
let contributionWasProduced = syncCommitteePool[].produceContribution(
|
2021-11-05 15:39:47 +00:00
|
|
|
slot, dag.head.root, aggregator.subcommitteeIdx, contribution)
|
2021-08-29 14:50:21 +00:00
|
|
|
|
|
|
|
|
if contributionWasProduced:
|
|
|
|
|
let
|
|
|
|
|
contributionAndProof = ContributionAndProof(
|
|
|
|
|
aggregator_index: uint64 aggregator.validatorIdx,
|
|
|
|
|
contribution: contribution,
|
|
|
|
|
selection_proof: aggregator.selectionProof)
|
|
|
|
|
|
2021-12-09 12:56:54 +00:00
|
|
|
validatorPrivKey =
|
2021-10-01 11:35:16 +00:00
|
|
|
MockPrivKeys[aggregator.validatorIdx.ValidatorIndex]
|
2021-08-29 14:50:21 +00:00
|
|
|
|
|
|
|
|
signedContributionAndProof = SignedContributionAndProof(
|
|
|
|
|
message: contributionAndProof,
|
2021-12-09 12:56:54 +00:00
|
|
|
signature: get_contribution_and_proof_signature(
|
|
|
|
|
fork, genesis_validators_root, contributionAndProof,
|
|
|
|
|
validatorPrivKey).toValidatorSig)
|
2021-08-29 14:50:21 +00:00
|
|
|
|
2021-12-09 12:56:54 +00:00
|
|
|
res = waitFor dag.validateContribution(
|
|
|
|
|
batchCrypto,
|
|
|
|
|
syncCommitteePool,
|
2021-08-29 14:50:21 +00:00
|
|
|
signedContributionAndProof,
|
|
|
|
|
contributionsTime,
|
|
|
|
|
false)
|
|
|
|
|
|
|
|
|
|
doAssert res.isOk
|
|
|
|
|
|
2022-08-09 09:52:11 +00:00
|
|
|
discard syncCommitteePool[].addContribution(
|
2021-12-20 19:20:31 +00:00
|
|
|
signedContributionAndProof, res.get()[0])
|
2021-11-05 15:39:47 +00:00
|
|
|
|
2021-06-23 14:43:18 +00:00
|
|
|
proc getNewBlock[T](
|
2022-03-16 07:20:40 +00:00
|
|
|
state: var ForkedHashedBeaconState, slot: Slot, cache: var StateCache): T =
|
2021-06-23 14:43:18 +00:00
|
|
|
let
|
|
|
|
|
finalizedEpochRef = dag.getFinalizedEpochRef()
|
|
|
|
|
proposerIdx = get_beacon_proposer_index(
|
2022-03-16 07:20:40 +00:00
|
|
|
state, cache, getStateField(state, slot)).get()
|
2021-10-01 11:35:16 +00:00
|
|
|
privKey = MockPrivKeys[proposerIdx]
|
2021-06-23 14:43:18 +00:00
|
|
|
eth1ProposalData = eth1Chain.getBlockProposalData(
|
2022-03-16 07:20:40 +00:00
|
|
|
state,
|
2021-06-23 14:43:18 +00:00
|
|
|
finalizedEpochRef.eth1_data,
|
|
|
|
|
finalizedEpochRef.eth1_deposit_index)
|
2021-09-29 12:10:44 +00:00
|
|
|
sync_aggregate =
|
2021-06-23 14:43:18 +00:00
|
|
|
when T is phase0.SignedBeaconBlock:
|
2021-09-30 13:56:07 +00:00
|
|
|
SyncAggregate.init()
|
2022-01-18 13:36:52 +00:00
|
|
|
elif T is altair.SignedBeaconBlock or T is bellatrix.SignedBeaconBlock:
|
2021-09-29 12:10:44 +00:00
|
|
|
syncCommitteePool[].produceSyncAggregate(dag.head.root)
|
2021-06-23 14:43:18 +00:00
|
|
|
else:
|
|
|
|
|
static: doAssert false
|
2021-09-29 12:10:44 +00:00
|
|
|
hashedState =
|
2021-08-29 14:50:21 +00:00
|
|
|
when T is phase0.SignedBeaconBlock:
|
2022-03-16 07:20:40 +00:00
|
|
|
addr state.phase0Data
|
2021-08-29 14:50:21 +00:00
|
|
|
elif T is altair.SignedBeaconBlock:
|
2022-03-16 07:20:40 +00:00
|
|
|
addr state.altairData
|
2022-01-18 13:36:52 +00:00
|
|
|
elif T is bellatrix.SignedBeaconBlock:
|
2022-03-16 07:20:40 +00:00
|
|
|
addr state.bellatrixData
|
2021-08-29 14:50:21 +00:00
|
|
|
else:
|
|
|
|
|
static: doAssert false
|
2021-09-29 12:10:44 +00:00
|
|
|
message = makeBeaconBlock(
|
|
|
|
|
cfg,
|
|
|
|
|
hashedState[],
|
|
|
|
|
proposerIdx,
|
2022-06-29 16:53:59 +00:00
|
|
|
get_epoch_signature(
|
2022-03-16 07:20:40 +00:00
|
|
|
getStateField(state, fork),
|
|
|
|
|
getStateField(state, genesis_validators_root),
|
2022-06-29 16:53:59 +00:00
|
|
|
slot.epoch, privKey).toValidatorSig(),
|
2021-09-29 12:10:44 +00:00
|
|
|
eth1ProposalData.vote,
|
|
|
|
|
default(GraffitiBytes),
|
2022-03-16 07:20:40 +00:00
|
|
|
attPool.getAttestationsForBlock(state, cache),
|
2021-09-29 12:10:44 +00:00
|
|
|
eth1ProposalData.deposits,
|
2021-10-18 16:37:27 +00:00
|
|
|
BeaconBlockExits(),
|
2021-09-29 12:10:44 +00:00
|
|
|
sync_aggregate,
|
|
|
|
|
default(ExecutionPayload),
|
|
|
|
|
noRollback,
|
|
|
|
|
cache)
|
2021-06-23 14:43:18 +00:00
|
|
|
|
|
|
|
|
var
|
|
|
|
|
newBlock = T(
|
|
|
|
|
message: message.get()
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
let blockRoot = withTimerRet(timers[tHashBlock]):
|
|
|
|
|
hash_tree_root(newBlock.message)
|
|
|
|
|
newBlock.root = blockRoot
|
|
|
|
|
# Careful, state no longer valid after here because of the await..
|
|
|
|
|
newBlock.signature = withTimerRet(timers[tSignBlock]):
|
|
|
|
|
get_block_signature(
|
2022-03-16 07:20:40 +00:00
|
|
|
getStateField(state, fork),
|
|
|
|
|
getStateField(state, genesis_validators_root),
|
2021-06-23 14:43:18 +00:00
|
|
|
newBlock.message.slot,
|
|
|
|
|
blockRoot, privKey).toValidatorSig()
|
|
|
|
|
|
|
|
|
|
newBlock
|
|
|
|
|
|
|
|
|
|
proc proposePhase0Block(slot: Slot) =
|
2020-05-03 17:44:04 +00:00
|
|
|
if rand(r, 1.0) > blockRatio:
|
|
|
|
|
return
|
|
|
|
|
|
Prune `BlockRef` on finalization (#3513)
Up til now, the block dag has been using `BlockRef`, a structure adapted
for a full DAG, to represent all of chain history. This is a correct and
simple design, but does not exploit the linearity of the chain once
parts of it finalize.
By pruning the in-memory `BlockRef` structure at finalization, we save,
at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory
landing us at a steady state of ~750mb normal memory usage for a
validating node.
Above all though, we prevent memory usage from growing proportionally
with the length of the chain, something that would not be sustainable
over time - instead, the steady state memory usage is roughly
determined by the validator set size which grows much more slowly. With
these changes, the core should remain sustainable memory-wise post-merge
all the way to withdrawals (when the validator set is expected to grow).
In-memory indices are still used for the "hot" unfinalized portion of
the chain - this ensure that consensus performance remains unchanged.
What changes is that for historical access, we use a db-based linear
slot index which is cache-and-disk-friendly, keeping the cost for
accessing historical data at a similar level as before, achieving the
savings at no percievable cost to functionality or performance.
A nice collateral benefit is the almost-instant startup since we no
longer load any large indicies at dag init.
The cost of this functionality instead can be found in the complexity of
having to deal with two ways of traversing the chain - by `BlockRef` and
by slot.
* use `BlockId` instead of `BlockRef` where finalized / historical data
may be required
* simplify clearance pre-advancement
* remove dag.finalizedBlocks (~50:ish mb)
* remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead
* `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef`
instance, unlike `BlockRef` traversal
* prune `BlockRef` parents on finality (~200:ish mb)
* speed up ChainDAG init by not loading finalized history index
* mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
|
|
|
dag.withUpdatedState(tmpState[], dag.getBlockIdAtSlot(slot).expect("block")) do:
|
2020-05-01 15:51:24 +00:00
|
|
|
let
|
2022-03-16 07:20:40 +00:00
|
|
|
newBlock = getNewBlock[phase0.SignedBeaconBlock](state, slot, cache)
|
Backfill support for ChainDAG (#3171)
In the ChainDAG, 3 block pointers are kept: genesis, tail and head. This
PR adds one more block pointer: the backfill block which represents the
block that has been backfilled so far.
When doing a checkpoint sync, a random block is given as starting point
- this is the tail block, and we require that the tail block has a
corresponding state.
When backfilling, we end up with blocks without corresponding states,
hence we cannot use `tail` as a backfill pointer - there is no state.
Nonetheless, we need to keep track of where we are in the backfill
process between restarts, such that we can answer GetBeaconBlocksByRange
requests.
This PR adds the basic support for backfill handling - it needs to be
integrated with backfill sync, and the REST API needs to be adjusted to
take advantage of the new backfilled blocks when responding to certain
requests.
Future work will also enable moving the tail in either direction:
* pruning means moving the tail forward in time and removing states
* backwards means recreating past states from genesis, such that
intermediate states are recreated step by step all the way to the tail -
at that point, tail, genesis and backfill will match up.
* backfilling is done when backfill != genesis - later, this will be the
WSS checkpoint instead
2021-12-13 13:36:06 +00:00
|
|
|
added = dag.addHeadBlock(verifier, newBlock) do (
|
2021-06-23 14:43:18 +00:00
|
|
|
blckRef: BlockRef, signedBlock: phase0.TrustedSignedBeaconBlock,
|
2022-07-06 10:33:02 +00:00
|
|
|
epochRef: EpochRef, unrealized: FinalityCheckpoints):
|
2021-06-23 14:43:18 +00:00
|
|
|
# Callback add to fork choice if valid
|
|
|
|
|
attPool.addForkChoice(
|
2022-07-06 10:33:02 +00:00
|
|
|
epochRef, blckRef, unrealized, signedBlock.message,
|
|
|
|
|
blckRef.slot.start_beacon_time)
|
2021-06-23 14:43:18 +00:00
|
|
|
|
2021-12-06 09:49:01 +00:00
|
|
|
dag.updateHead(added[], quarantine[])
|
2021-06-23 14:43:18 +00:00
|
|
|
if dag.needStateCachesAndForkChoicePruning():
|
|
|
|
|
dag.pruneStateCachesDAG()
|
|
|
|
|
attPool.prune()
|
2022-01-05 18:38:04 +00:00
|
|
|
do:
|
|
|
|
|
raiseAssert "withUpdatedState failed"
|
2020-05-01 15:51:24 +00:00
|
|
|
|
2021-06-23 14:43:18 +00:00
|
|
|
proc proposeAltairBlock(slot: Slot) =
|
|
|
|
|
if rand(r, 1.0) > blockRatio:
|
|
|
|
|
return
|
|
|
|
|
|
Prune `BlockRef` on finalization (#3513)
Up til now, the block dag has been using `BlockRef`, a structure adapted
for a full DAG, to represent all of chain history. This is a correct and
simple design, but does not exploit the linearity of the chain once
parts of it finalize.
By pruning the in-memory `BlockRef` structure at finalization, we save,
at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory
landing us at a steady state of ~750mb normal memory usage for a
validating node.
Above all though, we prevent memory usage from growing proportionally
with the length of the chain, something that would not be sustainable
over time - instead, the steady state memory usage is roughly
determined by the validator set size which grows much more slowly. With
these changes, the core should remain sustainable memory-wise post-merge
all the way to withdrawals (when the validator set is expected to grow).
In-memory indices are still used for the "hot" unfinalized portion of
the chain - this ensure that consensus performance remains unchanged.
What changes is that for historical access, we use a db-based linear
slot index which is cache-and-disk-friendly, keeping the cost for
accessing historical data at a similar level as before, achieving the
savings at no percievable cost to functionality or performance.
A nice collateral benefit is the almost-instant startup since we no
longer load any large indicies at dag init.
The cost of this functionality instead can be found in the complexity of
having to deal with two ways of traversing the chain - by `BlockRef` and
by slot.
* use `BlockId` instead of `BlockRef` where finalized / historical data
may be required
* simplify clearance pre-advancement
* remove dag.finalizedBlocks (~50:ish mb)
* remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead
* `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef`
instance, unlike `BlockRef` traversal
* prune `BlockRef` parents on finality (~200:ish mb)
* speed up ChainDAG init by not loading finalized history index
* mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
|
|
|
dag.withUpdatedState(tmpState[], dag.getBlockIdAtSlot(slot).expect("block")) do:
|
2021-06-23 14:43:18 +00:00
|
|
|
let
|
2022-03-16 07:20:40 +00:00
|
|
|
newBlock = getNewBlock[altair.SignedBeaconBlock](state, slot, cache)
|
Backfill support for ChainDAG (#3171)
In the ChainDAG, 3 block pointers are kept: genesis, tail and head. This
PR adds one more block pointer: the backfill block which represents the
block that has been backfilled so far.
When doing a checkpoint sync, a random block is given as starting point
- this is the tail block, and we require that the tail block has a
corresponding state.
When backfilling, we end up with blocks without corresponding states,
hence we cannot use `tail` as a backfill pointer - there is no state.
Nonetheless, we need to keep track of where we are in the backfill
process between restarts, such that we can answer GetBeaconBlocksByRange
requests.
This PR adds the basic support for backfill handling - it needs to be
integrated with backfill sync, and the REST API needs to be adjusted to
take advantage of the new backfilled blocks when responding to certain
requests.
Future work will also enable moving the tail in either direction:
* pruning means moving the tail forward in time and removing states
* backwards means recreating past states from genesis, such that
intermediate states are recreated step by step all the way to the tail -
at that point, tail, genesis and backfill will match up.
* backfilling is done when backfill != genesis - later, this will be the
WSS checkpoint instead
2021-12-13 13:36:06 +00:00
|
|
|
added = dag.addHeadBlock(verifier, newBlock) do (
|
2021-06-23 14:43:18 +00:00
|
|
|
blckRef: BlockRef, signedBlock: altair.TrustedSignedBeaconBlock,
|
2022-07-06 10:33:02 +00:00
|
|
|
epochRef: EpochRef, unrealized: FinalityCheckpoints):
|
2021-06-23 14:43:18 +00:00
|
|
|
# Callback add to fork choice if valid
|
|
|
|
|
attPool.addForkChoice(
|
2022-07-06 10:33:02 +00:00
|
|
|
epochRef, blckRef, unrealized, signedBlock.message,
|
|
|
|
|
blckRef.slot.start_beacon_time)
|
2020-07-22 09:42:55 +00:00
|
|
|
|
2021-12-06 09:49:01 +00:00
|
|
|
dag.updateHead(added[], quarantine[])
|
2021-06-01 11:13:40 +00:00
|
|
|
if dag.needStateCachesAndForkChoicePruning():
|
|
|
|
|
dag.pruneStateCachesDAG()
|
2021-03-09 14:36:17 +00:00
|
|
|
attPool.prune()
|
2022-01-05 18:38:04 +00:00
|
|
|
do:
|
|
|
|
|
raiseAssert "withUpdatedState failed"
|
2020-05-01 15:51:24 +00:00
|
|
|
|
2022-01-04 09:45:38 +00:00
|
|
|
proc proposeBellatrixBlock(slot: Slot) =
|
2021-09-30 01:07:24 +00:00
|
|
|
if rand(r, 1.0) > blockRatio:
|
|
|
|
|
return
|
|
|
|
|
|
Prune `BlockRef` on finalization (#3513)
Up til now, the block dag has been using `BlockRef`, a structure adapted
for a full DAG, to represent all of chain history. This is a correct and
simple design, but does not exploit the linearity of the chain once
parts of it finalize.
By pruning the in-memory `BlockRef` structure at finalization, we save,
at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory
landing us at a steady state of ~750mb normal memory usage for a
validating node.
Above all though, we prevent memory usage from growing proportionally
with the length of the chain, something that would not be sustainable
over time - instead, the steady state memory usage is roughly
determined by the validator set size which grows much more slowly. With
these changes, the core should remain sustainable memory-wise post-merge
all the way to withdrawals (when the validator set is expected to grow).
In-memory indices are still used for the "hot" unfinalized portion of
the chain - this ensure that consensus performance remains unchanged.
What changes is that for historical access, we use a db-based linear
slot index which is cache-and-disk-friendly, keeping the cost for
accessing historical data at a similar level as before, achieving the
savings at no percievable cost to functionality or performance.
A nice collateral benefit is the almost-instant startup since we no
longer load any large indicies at dag init.
The cost of this functionality instead can be found in the complexity of
having to deal with two ways of traversing the chain - by `BlockRef` and
by slot.
* use `BlockId` instead of `BlockRef` where finalized / historical data
may be required
* simplify clearance pre-advancement
* remove dag.finalizedBlocks (~50:ish mb)
* remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead
* `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef`
instance, unlike `BlockRef` traversal
* prune `BlockRef` parents on finality (~200:ish mb)
* speed up ChainDAG init by not loading finalized history index
* mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
|
|
|
dag.withUpdatedState(tmpState[], dag.getBlockIdAtSlot(slot).expect("block")) do:
|
2021-09-30 01:07:24 +00:00
|
|
|
let
|
2022-03-16 07:20:40 +00:00
|
|
|
newBlock = getNewBlock[bellatrix.SignedBeaconBlock](state, slot, cache)
|
Backfill support for ChainDAG (#3171)
In the ChainDAG, 3 block pointers are kept: genesis, tail and head. This
PR adds one more block pointer: the backfill block which represents the
block that has been backfilled so far.
When doing a checkpoint sync, a random block is given as starting point
- this is the tail block, and we require that the tail block has a
corresponding state.
When backfilling, we end up with blocks without corresponding states,
hence we cannot use `tail` as a backfill pointer - there is no state.
Nonetheless, we need to keep track of where we are in the backfill
process between restarts, such that we can answer GetBeaconBlocksByRange
requests.
This PR adds the basic support for backfill handling - it needs to be
integrated with backfill sync, and the REST API needs to be adjusted to
take advantage of the new backfilled blocks when responding to certain
requests.
Future work will also enable moving the tail in either direction:
* pruning means moving the tail forward in time and removing states
* backwards means recreating past states from genesis, such that
intermediate states are recreated step by step all the way to the tail -
at that point, tail, genesis and backfill will match up.
* backfilling is done when backfill != genesis - later, this will be the
WSS checkpoint instead
2021-12-13 13:36:06 +00:00
|
|
|
added = dag.addHeadBlock(verifier, newBlock) do (
|
2022-01-18 13:36:52 +00:00
|
|
|
blckRef: BlockRef, signedBlock: bellatrix.TrustedSignedBeaconBlock,
|
2022-07-06 10:33:02 +00:00
|
|
|
epochRef: EpochRef, unrealized: FinalityCheckpoints):
|
2021-09-30 01:07:24 +00:00
|
|
|
# Callback add to fork choice if valid
|
|
|
|
|
attPool.addForkChoice(
|
2022-07-06 10:33:02 +00:00
|
|
|
epochRef, blckRef, unrealized, signedBlock.message,
|
|
|
|
|
blckRef.slot.start_beacon_time)
|
2021-09-30 01:07:24 +00:00
|
|
|
|
2021-12-06 09:49:01 +00:00
|
|
|
dag.updateHead(added[], quarantine[])
|
2021-09-30 01:07:24 +00:00
|
|
|
if dag.needStateCachesAndForkChoicePruning():
|
|
|
|
|
dag.pruneStateCachesDAG()
|
|
|
|
|
attPool.prune()
|
2022-01-05 18:38:04 +00:00
|
|
|
do:
|
|
|
|
|
raiseAssert "withUpdatedState failed"
|
2021-09-30 01:07:24 +00:00
|
|
|
|
2020-12-03 04:30:35 +00:00
|
|
|
var
|
|
|
|
|
lastEth1BlockAt = genesisTime
|
|
|
|
|
eth1BlockNum = 1000
|
|
|
|
|
|
2020-05-01 15:51:24 +00:00
|
|
|
for i in 0..<slots:
|
|
|
|
|
let
|
2020-05-03 17:44:04 +00:00
|
|
|
slot = Slot(i + 1)
|
2020-05-01 15:51:24 +00:00
|
|
|
t =
|
2022-01-11 10:01:54 +00:00
|
|
|
if slot.is_epoch: tEpoch
|
2020-05-01 15:51:24 +00:00
|
|
|
else: tBlock
|
2020-12-03 04:30:35 +00:00
|
|
|
now = genesisTime + float(slot * SECONDS_PER_SLOT)
|
|
|
|
|
|
|
|
|
|
while true:
|
|
|
|
|
let nextBlockTime = lastEth1BlockAt +
|
Implement split preset/config support (#2710)
* Implement split preset/config support
This is the initial bulk refactor to introduce runtime config values in
a number of places, somewhat replacing the existing mechanism of loading
network metadata.
It still needs more work, this is the initial refactor that introduces
runtime configuration in some of the places that need it.
The PR changes the way presets and constants work, to match the spec. In
particular, a "preset" now refers to the compile-time configuration
while a "cfg" or "RuntimeConfig" is the dynamic part.
A single binary can support either mainnet or minimal, but not both.
Support for other presets has been removed completely (can be readded,
in case there's need).
There's a number of outstanding tasks:
* `SECONDS_PER_SLOT` still needs fixing
* loading custom runtime configs needs redoing
* checking constants against YAML file
* yeerongpilly support
`build/nimbus_beacon_node --network=yeerongpilly --discv5:no --log-level=DEBUG`
* load fork epoch from config
* fix fork digest sent in status
* nicer error string for request failures
* fix tools
* one more
* fixup
* fixup
* fixup
* use "standard" network definition folder in local testnet
Files are loaded from their standard locations, including genesis etc,
to conform to the format used in the `eth2-networks` repo.
* fix launch scripts, allow unknown config values
* fix base config of rest test
* cleanups
* bundle mainnet config using common loader
* fix spec links and names
* only include supported preset in binary
* drop yeerongpilly, add altair-devnet-0, support boot_enr.yaml
2021-07-12 13:01:38 +00:00
|
|
|
max(1.0, gauss(r, float defaultRuntimeConfig.SECONDS_PER_ETH1_BLOCK, 3.0))
|
2020-12-03 04:30:35 +00:00
|
|
|
if nextBlockTime > now:
|
|
|
|
|
break
|
|
|
|
|
|
|
|
|
|
inc eth1BlockNum
|
|
|
|
|
var eth1Block = Eth1Block(
|
2022-08-10 12:31:10 +00:00
|
|
|
hash: makeFakeHash(eth1BlockNum),
|
2020-12-03 04:30:35 +00:00
|
|
|
number: Eth1BlockNumber eth1BlockNum,
|
2022-08-10 12:31:10 +00:00
|
|
|
timestamp: Eth1BlockTimestamp nextBlockTime)
|
2020-12-03 04:30:35 +00:00
|
|
|
|
|
|
|
|
let newDeposits = int clamp(gauss(r, 5.0, 8.0), 0.0, 1000.0)
|
|
|
|
|
for i in 0 ..< newDeposits:
|
2021-08-29 14:50:21 +00:00
|
|
|
let validatorIdx = merkleizer.getChunkCount.int
|
2022-04-14 15:39:37 +00:00
|
|
|
let d = makeDeposit(validatorIdx, {skipBlsValidation})
|
2020-12-03 04:30:35 +00:00
|
|
|
eth1Block.deposits.add d
|
|
|
|
|
merkleizer.addChunk hash_tree_root(d).data
|
|
|
|
|
|
2022-08-10 12:31:10 +00:00
|
|
|
eth1Block.depositRoot = merkleizer.getDepositsRoot
|
|
|
|
|
eth1Block.depositCount = merkleizer.getChunkCount
|
2020-12-03 04:30:35 +00:00
|
|
|
|
|
|
|
|
eth1Chain.addBlock eth1Block
|
|
|
|
|
lastEth1BlockAt = nextBlockTime
|
2020-05-01 15:51:24 +00:00
|
|
|
|
2020-05-03 17:44:04 +00:00
|
|
|
if blockRatio > 0.0:
|
|
|
|
|
withTimer(timers[t]):
|
2021-10-04 08:31:21 +00:00
|
|
|
case dag.cfg.stateForkAtEpoch(slot.epoch)
|
2022-11-02 16:23:30 +00:00
|
|
|
of BeaconStateFork.Capella: raiseAssert $capellaImplementationMissing
|
2022-01-04 09:45:38 +00:00
|
|
|
of BeaconStateFork.Bellatrix: proposeBellatrixBlock(slot)
|
|
|
|
|
of BeaconStateFork.Altair: proposeAltairBlock(slot)
|
|
|
|
|
of BeaconStateFork.Phase0: proposePhase0Block(slot)
|
2020-05-01 15:51:24 +00:00
|
|
|
if attesterRatio > 0.0:
|
|
|
|
|
withTimer(timers[tAttest]):
|
2020-05-03 17:44:04 +00:00
|
|
|
handleAttestations(slot)
|
2021-08-29 14:50:21 +00:00
|
|
|
if syncCommitteeRatio > 0.0:
|
|
|
|
|
withTimer(timers[tSyncCommittees]):
|
|
|
|
|
handleSyncCommitteeActions(slot)
|
|
|
|
|
|
2021-08-30 01:00:37 +00:00
|
|
|
syncCommitteePool[].pruneData(slot)
|
2020-05-01 15:51:24 +00:00
|
|
|
|
2020-05-03 17:44:04 +00:00
|
|
|
# TODO if attestation pool was smarter, it would include older attestations
|
|
|
|
|
# too!
|
2022-03-16 07:20:40 +00:00
|
|
|
verifyConsensus(dag.headState, attesterRatio * blockRatio)
|
2020-05-01 15:51:24 +00:00
|
|
|
|
|
|
|
|
if t == tEpoch:
|
|
|
|
|
echo &". slot: {shortLog(slot)} ",
|
2022-01-11 10:01:54 +00:00
|
|
|
&"epoch: {shortLog(slot.epoch)}"
|
2020-05-01 15:51:24 +00:00
|
|
|
else:
|
|
|
|
|
write(stdout, ".")
|
|
|
|
|
flushFile(stdout)
|
|
|
|
|
|
2020-05-03 17:44:04 +00:00
|
|
|
if replay:
|
|
|
|
|
withTimer(timers[tReplay]):
|
2020-08-18 20:29:33 +00:00
|
|
|
var cache = StateCache()
|
2022-03-16 07:20:40 +00:00
|
|
|
doAssert dag.updateState(
|
Prune `BlockRef` on finalization (#3513)
Up til now, the block dag has been using `BlockRef`, a structure adapted
for a full DAG, to represent all of chain history. This is a correct and
simple design, but does not exploit the linearity of the chain once
parts of it finalize.
By pruning the in-memory `BlockRef` structure at finalization, we save,
at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory
landing us at a steady state of ~750mb normal memory usage for a
validating node.
Above all though, we prevent memory usage from growing proportionally
with the length of the chain, something that would not be sustainable
over time - instead, the steady state memory usage is roughly
determined by the validator set size which grows much more slowly. With
these changes, the core should remain sustainable memory-wise post-merge
all the way to withdrawals (when the validator set is expected to grow).
In-memory indices are still used for the "hot" unfinalized portion of
the chain - this ensure that consensus performance remains unchanged.
What changes is that for historical access, we use a db-based linear
slot index which is cache-and-disk-friendly, keeping the cost for
accessing historical data at a similar level as before, achieving the
savings at no percievable cost to functionality or performance.
A nice collateral benefit is the almost-instant startup since we no
longer load any large indicies at dag init.
The cost of this functionality instead can be found in the complexity of
having to deal with two ways of traversing the chain - by `BlockRef` and
by slot.
* use `BlockId` instead of `BlockRef` where finalized / historical data
may be required
* simplify clearance pre-advancement
* remove dag.finalizedBlocks (~50:ish mb)
* remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead
* `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef`
instance, unlike `BlockRef` traversal
* prune `BlockRef` parents on finality (~200:ish mb)
* speed up ChainDAG init by not loading finalized history index
* mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
|
|
|
replayState[], dag.getBlockIdAtSlot(Slot(slots)).expect("block"),
|
|
|
|
|
false, cache)
|
2020-05-03 17:44:04 +00:00
|
|
|
|
2020-05-01 15:51:24 +00:00
|
|
|
echo "Done!"
|
|
|
|
|
|
2022-03-16 07:20:40 +00:00
|
|
|
printTimers(dag.headState, attesters, true, timers)
|
