2019-02-28 21:21:29 +00:00
|
|
|
# beacon_chain
|
2021-06-11 17:51:46 +00:00
|
|
|
# Copyright (c) 2018-2021 Status Research & Development GmbH
|
2019-02-28 21:21:29 +00:00
|
|
|
# Licensed and distributed under either of
|
2019-11-25 15:30:02 +00:00
|
|
|
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
|
|
|
|
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
2019-02-28 21:21:29 +00:00
|
|
|
# at your option. This file may not be copied, modified, or distributed except according to those terms.
|
|
|
|
|
|
2019-11-14 10:47:55 +00:00
|
|
|
{.used.}
|
|
|
|
|
|
2019-02-28 21:21:29 +00:00
|
|
|
import
|
2021-06-23 14:43:18 +00:00
|
|
|
chronicles,
|
2021-04-28 16:41:02 +00:00
|
|
|
unittest2,
|
2020-12-16 08:37:22 +00:00
|
|
|
stew/assign2,
|
2021-09-17 00:13:52 +00:00
|
|
|
eth/keys, taskpools,
|
2021-06-21 08:35:24 +00:00
|
|
|
../beacon_chain/spec/datatypes/base,
|
Speed up altair block processing 2x (#3115)
* Speed up altair block processing >2x
Like #3089, this PR drastially speeds up historical REST queries and
other long state replays.
* cache sync committee validator indices
* use ~80mb less memory for validator pubkey mappings
* batch-verify sync aggregate signature (fixes #2985)
* document sync committee hack with head block vs sync message block
* add batch signature verification failure tests
Before:
```
../env.sh nim c -d:release -r ncli_db --db:mainnet_0/db bench --start-slot:-1000
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
5830.675, 0.000, 5830.675, 5830.675, 1, Initialize DB
0.481, 1.878, 0.215, 59.167, 981, Load block from database
8422.566, 0.000, 8422.566, 8422.566, 1, Load state from database
6.996, 1.678, 0.042, 14.385, 969, Advance slot, non-epoch
93.217, 8.318, 84.192, 122.209, 32, Advance slot, epoch
20.513, 23.665, 11.510, 201.561, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
After:
```
7081.422, 0.000, 7081.422, 7081.422, 1, Initialize DB
0.553, 2.122, 0.175, 66.692, 981, Load block from database
5439.446, 0.000, 5439.446, 5439.446, 1, Load state from database
6.829, 1.575, 0.043, 12.156, 969, Advance slot, non-epoch
94.716, 2.749, 88.395, 100.026, 32, Advance slot, epoch
11.636, 23.766, 4.889, 205.250, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
* add comment
2021-11-24 12:43:50 +00:00
|
|
|
../beacon_chain/spec/[beaconstate, forks, helpers, signatures, state_transition],
|
2021-08-18 18:57:58 +00:00
|
|
|
../beacon_chain/[beacon_chain_db],
|
2021-04-16 08:49:37 +00:00
|
|
|
../beacon_chain/consensus_object_pools/[
|
2021-09-08 03:46:33 +00:00
|
|
|
attestation_pool, blockchain_dag, block_quarantine, block_clearance],
|
2021-04-28 16:41:02 +00:00
|
|
|
./testutil, ./testdbutil, ./testblockutil
|
2019-02-28 21:21:29 +00:00
|
|
|
|
2021-12-09 17:06:21 +00:00
|
|
|
func `$`(x: BlockRef): string = shortLog(x)
|
2020-10-14 20:23:04 +00:00
|
|
|
|
Speed up altair block processing 2x (#3115)
* Speed up altair block processing >2x
Like #3089, this PR drastially speeds up historical REST queries and
other long state replays.
* cache sync committee validator indices
* use ~80mb less memory for validator pubkey mappings
* batch-verify sync aggregate signature (fixes #2985)
* document sync committee hack with head block vs sync message block
* add batch signature verification failure tests
Before:
```
../env.sh nim c -d:release -r ncli_db --db:mainnet_0/db bench --start-slot:-1000
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
5830.675, 0.000, 5830.675, 5830.675, 1, Initialize DB
0.481, 1.878, 0.215, 59.167, 981, Load block from database
8422.566, 0.000, 8422.566, 8422.566, 1, Load state from database
6.996, 1.678, 0.042, 14.385, 969, Advance slot, non-epoch
93.217, 8.318, 84.192, 122.209, 32, Advance slot, epoch
20.513, 23.665, 11.510, 201.561, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
After:
```
7081.422, 0.000, 7081.422, 7081.422, 1, Initialize DB
0.553, 2.122, 0.175, 66.692, 981, Load block from database
5439.446, 0.000, 5439.446, 5439.446, 1, Load state from database
6.829, 1.575, 0.043, 12.156, 969, Advance slot, non-epoch
94.716, 2.749, 88.395, 100.026, 32, Advance slot, epoch
11.636, 23.766, 4.889, 205.250, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
* add comment
2021-11-24 12:43:50 +00:00
|
|
|
const
|
|
|
|
|
nilPhase0Callback = OnPhase0BlockAdded(nil)
|
|
|
|
|
nilAltairCallback = OnAltairBlockAdded(nil)
|
|
|
|
|
|
2021-03-09 14:36:17 +00:00
|
|
|
proc pruneAtFinalization(dag: ChainDAGRef) =
|
|
|
|
|
if dag.needStateCachesAndForkChoicePruning():
|
|
|
|
|
dag.pruneStateCachesDAG()
|
|
|
|
|
|
2021-12-09 17:06:21 +00:00
|
|
|
suite "ChainDAG helpers":
|
|
|
|
|
test "epochAncestor sanity" & preset():
|
2019-12-23 15:34:09 +00:00
|
|
|
let
|
2021-12-09 17:06:21 +00:00
|
|
|
s0 = BlockRef(bid: BlockId(slot: Slot(0)))
|
|
|
|
|
var cur = s0
|
|
|
|
|
for i in 1..SLOTS_PER_EPOCH * 2:
|
|
|
|
|
cur = BlockRef(bid: BlockId(slot: Slot(i)), parent: cur)
|
2019-12-23 15:34:09 +00:00
|
|
|
|
2021-12-09 17:06:21 +00:00
|
|
|
let ancestor = cur.epochAncestor(cur.slot.epoch)
|
2019-12-23 15:34:09 +00:00
|
|
|
|
|
|
|
|
check:
|
2021-12-09 17:06:21 +00:00
|
|
|
ancestor.epoch == cur.slot.epoch
|
|
|
|
|
ancestor.blck != cur # should have selected a parent
|
2019-12-23 15:34:09 +00:00
|
|
|
|
2021-12-09 17:06:21 +00:00
|
|
|
ancestor.blck.epochAncestor(cur.slot.epoch) == ancestor
|
|
|
|
|
ancestor.blck.epochAncestor(ancestor.blck.slot.epoch) != ancestor
|
2019-12-23 15:34:09 +00:00
|
|
|
|
2021-04-28 16:41:02 +00:00
|
|
|
suite "Block pool processing" & preset():
|
2020-05-05 09:18:44 +00:00
|
|
|
setup:
|
|
|
|
|
var
|
|
|
|
|
db = makeTestDB(SLOTS_PER_EPOCH)
|
Implement split preset/config support (#2710)
* Implement split preset/config support
This is the initial bulk refactor to introduce runtime config values in
a number of places, somewhat replacing the existing mechanism of loading
network metadata.
It still needs more work, this is the initial refactor that introduces
runtime configuration in some of the places that need it.
The PR changes the way presets and constants work, to match the spec. In
particular, a "preset" now refers to the compile-time configuration
while a "cfg" or "RuntimeConfig" is the dynamic part.
A single binary can support either mainnet or minimal, but not both.
Support for other presets has been removed completely (can be readded,
in case there's need).
There's a number of outstanding tasks:
* `SECONDS_PER_SLOT` still needs fixing
* loading custom runtime configs needs redoing
* checking constants against YAML file
* yeerongpilly support
`build/nimbus_beacon_node --network=yeerongpilly --discv5:no --log-level=DEBUG`
* load fork epoch from config
* fix fork digest sent in status
* nicer error string for request failures
* fix tools
* one more
* fixup
* fixup
* fixup
* use "standard" network definition folder in local testnet
Files are loaded from their standard locations, including genesis etc,
to conform to the format used in the `eth2-networks` repo.
* fix launch scripts, allow unknown config values
* fix base config of rest test
* cleanups
* bundle mainnet config using common loader
* fix spec links and names
* only include supported preset in binary
* drop yeerongpilly, add altair-devnet-0, support boot_enr.yaml
2021-07-12 13:01:38 +00:00
|
|
|
dag = init(ChainDAGRef, defaultRuntimeConfig, db, {})
|
2021-12-06 09:49:01 +00:00
|
|
|
verifier = BatchVerifier(rng: keys.newRng(), taskpool: Taskpool.new())
|
|
|
|
|
quarantine = Quarantine.init()
|
2021-05-21 09:23:28 +00:00
|
|
|
state = newClone(dag.headState.data)
|
2020-07-15 10:44:18 +00:00
|
|
|
cache = StateCache()
|
2021-10-13 14:24:36 +00:00
|
|
|
info = ForkedEpochInfo()
|
2021-05-21 09:23:28 +00:00
|
|
|
att0 = makeFullAttestations(state[], dag.tail.root, 0.Slot, cache)
|
2021-11-18 12:02:43 +00:00
|
|
|
b1 = addTestBlock(state[], cache, attestations = att0).phase0Data
|
|
|
|
|
b2 = addTestBlock(state[], cache).phase0Data
|
2021-04-28 16:41:02 +00:00
|
|
|
test "getRef returns nil for missing blocks":
|
2020-05-05 09:18:44 +00:00
|
|
|
check:
|
2020-07-30 19:18:17 +00:00
|
|
|
dag.getRef(default Eth2Digest) == nil
|
2020-04-10 14:06:24 +00:00
|
|
|
|
2021-04-28 16:41:02 +00:00
|
|
|
test "loading tail block works" & preset():
|
2020-05-05 09:18:44 +00:00
|
|
|
let
|
2020-07-30 19:18:17 +00:00
|
|
|
b0 = dag.get(dag.tail.root)
|
2019-12-19 14:13:35 +00:00
|
|
|
|
2020-05-05 09:18:44 +00:00
|
|
|
check:
|
|
|
|
|
b0.isSome()
|
2020-05-04 21:07:18 +00:00
|
|
|
|
2021-04-28 16:41:02 +00:00
|
|
|
test "Simple block add&get" & preset():
|
2020-05-05 09:18:44 +00:00
|
|
|
let
|
2021-12-06 09:49:01 +00:00
|
|
|
b1Add = dag.addRawBlock(verifier, b1, nilPhase0Callback)
|
2020-07-30 19:18:17 +00:00
|
|
|
b1Get = dag.get(b1.root)
|
2019-12-19 14:13:35 +00:00
|
|
|
|
2020-05-05 09:18:44 +00:00
|
|
|
check:
|
|
|
|
|
b1Get.isSome()
|
2021-04-12 20:25:09 +00:00
|
|
|
b1Get.get().refs.root == b1.root
|
2020-07-09 09:29:32 +00:00
|
|
|
b1Add[].root == b1Get.get().refs.root
|
2020-07-30 19:18:17 +00:00
|
|
|
dag.heads.len == 1
|
|
|
|
|
dag.heads[0] == b1Add[]
|
2019-12-19 14:13:35 +00:00
|
|
|
|
2020-05-05 09:18:44 +00:00
|
|
|
let
|
2021-12-06 09:49:01 +00:00
|
|
|
b2Add = dag.addRawBlock(verifier, b2, nilPhase0Callback)
|
2020-07-30 19:18:17 +00:00
|
|
|
b2Get = dag.get(b2.root)
|
2021-06-10 07:37:02 +00:00
|
|
|
er = dag.findEpochRef(b1Add[], b1Add[].slot.epoch)
|
2021-06-11 17:51:46 +00:00
|
|
|
validators = getStateField(dag.headState.data, validators).lenu64()
|
2019-12-19 14:13:35 +00:00
|
|
|
|
2020-05-05 09:18:44 +00:00
|
|
|
check:
|
|
|
|
|
b2Get.isSome()
|
2020-07-16 13:16:51 +00:00
|
|
|
b2Get.get().refs.root == b2.root
|
2020-07-09 09:29:32 +00:00
|
|
|
b2Add[].root == b2Get.get().refs.root
|
2020-07-30 19:18:17 +00:00
|
|
|
dag.heads.len == 1
|
|
|
|
|
dag.heads[0] == b2Add[]
|
2021-06-10 07:37:02 +00:00
|
|
|
not er.isNil
|
|
|
|
|
# Same epoch - same epochRef
|
|
|
|
|
er == dag.findEpochRef(b2Add[], b2Add[].slot.epoch)
|
|
|
|
|
# Different epoch that was never processed
|
2021-03-17 10:17:15 +00:00
|
|
|
dag.findEpochRef(b1Add[], b1Add[].slot.epoch + 1).isNil
|
2019-12-19 14:13:35 +00:00
|
|
|
|
2021-06-10 07:37:02 +00:00
|
|
|
er.validatorKey(0'u64).isSome()
|
|
|
|
|
er.validatorKey(validators - 1).isSome()
|
|
|
|
|
er.validatorKey(validators).isNone()
|
|
|
|
|
|
2020-05-05 09:18:44 +00:00
|
|
|
# Skip one slot to get a gap
|
2020-05-19 15:46:29 +00:00
|
|
|
check:
|
2021-06-11 17:51:46 +00:00
|
|
|
process_slots(
|
Implement split preset/config support (#2710)
* Implement split preset/config support
This is the initial bulk refactor to introduce runtime config values in
a number of places, somewhat replacing the existing mechanism of loading
network metadata.
It still needs more work, this is the initial refactor that introduces
runtime configuration in some of the places that need it.
The PR changes the way presets and constants work, to match the spec. In
particular, a "preset" now refers to the compile-time configuration
while a "cfg" or "RuntimeConfig" is the dynamic part.
A single binary can support either mainnet or minimal, but not both.
Support for other presets has been removed completely (can be readded,
in case there's need).
There's a number of outstanding tasks:
* `SECONDS_PER_SLOT` still needs fixing
* loading custom runtime configs needs redoing
* checking constants against YAML file
* yeerongpilly support
`build/nimbus_beacon_node --network=yeerongpilly --discv5:no --log-level=DEBUG`
* load fork epoch from config
* fix fork digest sent in status
* nicer error string for request failures
* fix tools
* one more
* fixup
* fixup
* fixup
* use "standard" network definition folder in local testnet
Files are loaded from their standard locations, including genesis etc,
to conform to the format used in the `eth2-networks` repo.
* fix launch scripts, allow unknown config values
* fix base config of rest test
* cleanups
* bundle mainnet config using common loader
* fix spec links and names
* only include supported preset in binary
* drop yeerongpilly, add altair-devnet-0, support boot_enr.yaml
2021-07-12 13:01:38 +00:00
|
|
|
defaultRuntimeConfig, state[], getStateField(state[], slot) + 1, cache,
|
2021-10-13 14:24:36 +00:00
|
|
|
info, {})
|
2019-12-19 14:13:35 +00:00
|
|
|
|
2020-05-05 09:18:44 +00:00
|
|
|
let
|
2021-11-18 12:02:43 +00:00
|
|
|
b4 = addTestBlock(state[], cache).phase0Data
|
2021-12-06 09:49:01 +00:00
|
|
|
b4Add = dag.addRawBlock(verifier, b4, nilPhase0Callback)
|
2019-12-19 14:13:35 +00:00
|
|
|
|
2020-05-05 09:18:44 +00:00
|
|
|
check:
|
2020-07-09 09:29:32 +00:00
|
|
|
b4Add[].parent == b2Add[]
|
2019-12-19 14:13:35 +00:00
|
|
|
|
2020-08-31 09:00:38 +00:00
|
|
|
dag.updateHead(b4Add[], quarantine)
|
2021-03-09 14:36:17 +00:00
|
|
|
dag.pruneAtFinalization()
|
2020-04-21 06:43:39 +00:00
|
|
|
|
2020-05-05 09:18:44 +00:00
|
|
|
var blocks: array[3, BlockRef]
|
2020-04-21 06:43:39 +00:00
|
|
|
|
2020-05-05 09:18:44 +00:00
|
|
|
check:
|
2020-07-30 19:18:17 +00:00
|
|
|
dag.getBlockRange(Slot(0), 1, blocks.toOpenArray(0, 0)) == 0
|
|
|
|
|
blocks[0..<1] == [dag.tail]
|
2020-04-21 06:43:39 +00:00
|
|
|
|
2020-07-30 19:18:17 +00:00
|
|
|
dag.getBlockRange(Slot(0), 1, blocks.toOpenArray(0, 1)) == 0
|
|
|
|
|
blocks[0..<2] == [dag.tail, b1Add[]]
|
2020-04-21 06:43:39 +00:00
|
|
|
|
2020-07-30 19:18:17 +00:00
|
|
|
dag.getBlockRange(Slot(0), 2, blocks.toOpenArray(0, 1)) == 0
|
|
|
|
|
blocks[0..<2] == [dag.tail, b2Add[]]
|
2020-04-21 06:43:39 +00:00
|
|
|
|
2020-07-30 19:18:17 +00:00
|
|
|
dag.getBlockRange(Slot(0), 3, blocks.toOpenArray(0, 1)) == 1
|
2020-08-05 23:22:12 +00:00
|
|
|
blocks[1..<2] == [dag.tail] # block 3 is missing!
|
2020-04-21 06:43:39 +00:00
|
|
|
|
2020-07-30 19:18:17 +00:00
|
|
|
dag.getBlockRange(Slot(2), 2, blocks.toOpenArray(0, 1)) == 0
|
2020-07-09 09:29:32 +00:00
|
|
|
blocks[0..<2] == [b2Add[], b4Add[]] # block 3 is missing!
|
2020-04-21 06:43:39 +00:00
|
|
|
|
2020-10-14 20:23:04 +00:00
|
|
|
# large skip step
|
|
|
|
|
dag.getBlockRange(Slot(0), uint64.high, blocks.toOpenArray(0, 2)) == 2
|
|
|
|
|
blocks[2..2] == [dag.tail]
|
|
|
|
|
|
|
|
|
|
# large skip step
|
|
|
|
|
dag.getBlockRange(Slot(2), uint64.high, blocks.toOpenArray(0, 1)) == 1
|
|
|
|
|
blocks[1..1] == [b2Add[]]
|
|
|
|
|
|
2020-05-05 09:18:44 +00:00
|
|
|
# empty length
|
2020-07-30 19:18:17 +00:00
|
|
|
dag.getBlockRange(Slot(2), 2, blocks.toOpenArray(0, -1)) == 0
|
2020-04-21 06:43:39 +00:00
|
|
|
|
2020-05-05 09:18:44 +00:00
|
|
|
# No blocks in sight
|
2020-07-30 19:18:17 +00:00
|
|
|
dag.getBlockRange(Slot(5), 1, blocks.toOpenArray(0, 1)) == 2
|
2020-04-21 06:43:39 +00:00
|
|
|
|
2020-10-14 20:23:04 +00:00
|
|
|
# No blocks in sight
|
|
|
|
|
dag.getBlockRange(Slot(uint64.high), 1, blocks.toOpenArray(0, 1)) == 2
|
|
|
|
|
|
2020-05-05 09:18:44 +00:00
|
|
|
# No blocks in sight either due to gaps
|
2020-07-30 19:18:17 +00:00
|
|
|
dag.getBlockRange(Slot(3), 2, blocks.toOpenArray(0, 1)) == 2
|
2020-08-05 23:22:12 +00:00
|
|
|
blocks[2..<2].len == 0
|
2020-04-21 06:43:39 +00:00
|
|
|
|
2021-04-28 16:41:02 +00:00
|
|
|
test "updateHead updates head and headState" & preset():
|
2020-05-05 09:18:44 +00:00
|
|
|
let
|
2021-12-06 09:49:01 +00:00
|
|
|
b1Add = dag.addRawBlock(verifier, b1, nilPhase0Callback)
|
2020-05-05 09:18:44 +00:00
|
|
|
|
2020-08-31 09:00:38 +00:00
|
|
|
dag.updateHead(b1Add[], quarantine)
|
2021-03-09 14:36:17 +00:00
|
|
|
dag.pruneAtFinalization()
|
2020-05-05 09:18:44 +00:00
|
|
|
|
|
|
|
|
check:
|
2020-07-30 19:18:17 +00:00
|
|
|
dag.head == b1Add[]
|
2021-06-11 17:51:46 +00:00
|
|
|
getStateField(dag.headState.data, slot) == b1Add[].slot
|
2020-05-05 09:18:44 +00:00
|
|
|
|
2021-04-28 16:41:02 +00:00
|
|
|
test "updateStateData sanity" & preset():
|
2020-05-05 09:18:44 +00:00
|
|
|
let
|
2021-12-06 09:49:01 +00:00
|
|
|
b1Add = dag.addRawBlock(verifier, b1, nilPhase0Callback)
|
|
|
|
|
b2Add = dag.addRawBlock(verifier, b2, nilPhase0Callback)
|
2020-07-09 09:29:32 +00:00
|
|
|
bs1 = BlockSlot(blck: b1Add[], slot: b1.message.slot)
|
|
|
|
|
bs1_3 = b1Add[].atSlot(3.Slot)
|
|
|
|
|
bs2_3 = b2Add[].atSlot(3.Slot)
|
2020-05-05 09:18:44 +00:00
|
|
|
|
2020-07-30 19:18:17 +00:00
|
|
|
var tmpState = assignClone(dag.headState)
|
2020-05-05 09:18:44 +00:00
|
|
|
|
|
|
|
|
# move to specific block
|
2020-08-18 20:29:33 +00:00
|
|
|
var cache = StateCache()
|
2020-10-18 15:47:39 +00:00
|
|
|
dag.updateStateData(tmpState[], bs1, false, cache)
|
2020-05-05 09:18:44 +00:00
|
|
|
|
|
|
|
|
check:
|
2020-07-09 09:29:32 +00:00
|
|
|
tmpState.blck == b1Add[]
|
2021-06-11 17:51:46 +00:00
|
|
|
getStateField(tmpState.data, slot) == bs1.slot
|
2020-05-05 09:18:44 +00:00
|
|
|
|
|
|
|
|
# Skip slots
|
2020-10-18 15:47:39 +00:00
|
|
|
dag.updateStateData(tmpState[], bs1_3, false, cache) # skip slots
|
2020-05-05 09:18:44 +00:00
|
|
|
|
|
|
|
|
check:
|
2020-07-09 09:29:32 +00:00
|
|
|
tmpState.blck == b1Add[]
|
2021-06-11 17:51:46 +00:00
|
|
|
getStateField(tmpState.data, slot) == bs1_3.slot
|
2020-05-05 09:18:44 +00:00
|
|
|
|
|
|
|
|
# Move back slots, but not blocks
|
2020-10-18 15:47:39 +00:00
|
|
|
dag.updateStateData(tmpState[], bs1_3.parent(), false, cache)
|
2020-05-05 09:18:44 +00:00
|
|
|
check:
|
2020-07-09 09:29:32 +00:00
|
|
|
tmpState.blck == b1Add[]
|
2021-06-11 17:51:46 +00:00
|
|
|
getStateField(tmpState.data, slot) == bs1_3.parent().slot
|
2020-05-05 09:18:44 +00:00
|
|
|
|
|
|
|
|
# Move to different block and slot
|
2020-10-18 15:47:39 +00:00
|
|
|
dag.updateStateData(tmpState[], bs2_3, false, cache)
|
2020-05-05 09:18:44 +00:00
|
|
|
check:
|
2020-07-09 09:29:32 +00:00
|
|
|
tmpState.blck == b2Add[]
|
2021-06-11 17:51:46 +00:00
|
|
|
getStateField(tmpState.data, slot) == bs2_3.slot
|
2020-05-05 09:18:44 +00:00
|
|
|
|
|
|
|
|
# Move back slot and block
|
2020-10-18 15:47:39 +00:00
|
|
|
dag.updateStateData(tmpState[], bs1, false, cache)
|
2020-05-05 09:18:44 +00:00
|
|
|
check:
|
2020-07-09 09:29:32 +00:00
|
|
|
tmpState.blck == b1Add[]
|
2021-06-11 17:51:46 +00:00
|
|
|
getStateField(tmpState.data, slot) == bs1.slot
|
2020-05-05 09:18:44 +00:00
|
|
|
|
|
|
|
|
# Move back to genesis
|
2020-10-18 15:47:39 +00:00
|
|
|
dag.updateStateData(tmpState[], bs1.parent(), false, cache)
|
2020-05-05 09:18:44 +00:00
|
|
|
check:
|
2020-07-09 09:29:32 +00:00
|
|
|
tmpState.blck == b1Add[].parent
|
2021-06-11 17:51:46 +00:00
|
|
|
getStateField(tmpState.data, slot) == bs1.parent.slot
|
2020-05-05 09:18:44 +00:00
|
|
|
|
Speed up altair block processing 2x (#3115)
* Speed up altair block processing >2x
Like #3089, this PR drastially speeds up historical REST queries and
other long state replays.
* cache sync committee validator indices
* use ~80mb less memory for validator pubkey mappings
* batch-verify sync aggregate signature (fixes #2985)
* document sync committee hack with head block vs sync message block
* add batch signature verification failure tests
Before:
```
../env.sh nim c -d:release -r ncli_db --db:mainnet_0/db bench --start-slot:-1000
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
5830.675, 0.000, 5830.675, 5830.675, 1, Initialize DB
0.481, 1.878, 0.215, 59.167, 981, Load block from database
8422.566, 0.000, 8422.566, 8422.566, 1, Load state from database
6.996, 1.678, 0.042, 14.385, 969, Advance slot, non-epoch
93.217, 8.318, 84.192, 122.209, 32, Advance slot, epoch
20.513, 23.665, 11.510, 201.561, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
After:
```
7081.422, 0.000, 7081.422, 7081.422, 1, Initialize DB
0.553, 2.122, 0.175, 66.692, 981, Load block from database
5439.446, 0.000, 5439.446, 5439.446, 1, Load state from database
6.829, 1.575, 0.043, 12.156, 969, Advance slot, non-epoch
94.716, 2.749, 88.395, 100.026, 32, Advance slot, epoch
11.636, 23.766, 4.889, 205.250, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
* add comment
2021-11-24 12:43:50 +00:00
|
|
|
suite "Block pool altair processing" & preset():
|
|
|
|
|
setup:
|
|
|
|
|
var
|
|
|
|
|
cfg = defaultRuntimeConfig
|
|
|
|
|
cfg.ALTAIR_FORK_EPOCH = Epoch(1)
|
|
|
|
|
|
|
|
|
|
var
|
|
|
|
|
db = makeTestDB(SLOTS_PER_EPOCH)
|
|
|
|
|
dag = init(ChainDAGRef, cfg, db, {})
|
2021-12-06 09:49:01 +00:00
|
|
|
verifier = BatchVerifier(rng: keys.newRng(), taskpool: Taskpool.new())
|
|
|
|
|
quarantine = Quarantine.init()
|
Speed up altair block processing 2x (#3115)
* Speed up altair block processing >2x
Like #3089, this PR drastially speeds up historical REST queries and
other long state replays.
* cache sync committee validator indices
* use ~80mb less memory for validator pubkey mappings
* batch-verify sync aggregate signature (fixes #2985)
* document sync committee hack with head block vs sync message block
* add batch signature verification failure tests
Before:
```
../env.sh nim c -d:release -r ncli_db --db:mainnet_0/db bench --start-slot:-1000
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
5830.675, 0.000, 5830.675, 5830.675, 1, Initialize DB
0.481, 1.878, 0.215, 59.167, 981, Load block from database
8422.566, 0.000, 8422.566, 8422.566, 1, Load state from database
6.996, 1.678, 0.042, 14.385, 969, Advance slot, non-epoch
93.217, 8.318, 84.192, 122.209, 32, Advance slot, epoch
20.513, 23.665, 11.510, 201.561, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
After:
```
7081.422, 0.000, 7081.422, 7081.422, 1, Initialize DB
0.553, 2.122, 0.175, 66.692, 981, Load block from database
5439.446, 0.000, 5439.446, 5439.446, 1, Load state from database
6.829, 1.575, 0.043, 12.156, 969, Advance slot, non-epoch
94.716, 2.749, 88.395, 100.026, 32, Advance slot, epoch
11.636, 23.766, 4.889, 205.250, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
* add comment
2021-11-24 12:43:50 +00:00
|
|
|
state = newClone(dag.headState.data)
|
|
|
|
|
cache = StateCache()
|
|
|
|
|
info = ForkedEpochInfo()
|
|
|
|
|
|
|
|
|
|
# Advance to altair
|
|
|
|
|
check:
|
|
|
|
|
process_slots(
|
|
|
|
|
cfg, state[], cfg.ALTAIR_FORK_EPOCH.compute_start_slot_at_epoch(), cache,
|
|
|
|
|
info, {})
|
|
|
|
|
|
|
|
|
|
state[].kind == BeaconStateFork.Altair
|
|
|
|
|
|
|
|
|
|
var
|
|
|
|
|
b1 = addTestBlock(state[], cache).altairData
|
|
|
|
|
att1 = makeFullAttestations(state[], b1.root, b1.message.slot, cache)
|
|
|
|
|
b2 = addTestBlock(state[], cache, attestations = att1).altairData
|
|
|
|
|
|
|
|
|
|
test "Invalid signatures" & preset():
|
|
|
|
|
let badSignature = get_slot_signature(
|
|
|
|
|
Fork(), Eth2Digest(), 42.Slot,
|
|
|
|
|
MockPrivKeys[ValidatorIndex(0)]).toValidatorSig()
|
|
|
|
|
|
|
|
|
|
check:
|
2021-12-06 09:49:01 +00:00
|
|
|
dag.addRawBlock(verifier, b1, nilAltairCallback).isOk()
|
Speed up altair block processing 2x (#3115)
* Speed up altair block processing >2x
Like #3089, this PR drastially speeds up historical REST queries and
other long state replays.
* cache sync committee validator indices
* use ~80mb less memory for validator pubkey mappings
* batch-verify sync aggregate signature (fixes #2985)
* document sync committee hack with head block vs sync message block
* add batch signature verification failure tests
Before:
```
../env.sh nim c -d:release -r ncli_db --db:mainnet_0/db bench --start-slot:-1000
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
5830.675, 0.000, 5830.675, 5830.675, 1, Initialize DB
0.481, 1.878, 0.215, 59.167, 981, Load block from database
8422.566, 0.000, 8422.566, 8422.566, 1, Load state from database
6.996, 1.678, 0.042, 14.385, 969, Advance slot, non-epoch
93.217, 8.318, 84.192, 122.209, 32, Advance slot, epoch
20.513, 23.665, 11.510, 201.561, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
After:
```
7081.422, 0.000, 7081.422, 7081.422, 1, Initialize DB
0.553, 2.122, 0.175, 66.692, 981, Load block from database
5439.446, 0.000, 5439.446, 5439.446, 1, Load state from database
6.829, 1.575, 0.043, 12.156, 969, Advance slot, non-epoch
94.716, 2.749, 88.395, 100.026, 32, Advance slot, epoch
11.636, 23.766, 4.889, 205.250, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
* add comment
2021-11-24 12:43:50 +00:00
|
|
|
|
|
|
|
|
block: # Main signature
|
|
|
|
|
var b = b2
|
|
|
|
|
b.signature = badSignature
|
|
|
|
|
let
|
2021-12-06 09:49:01 +00:00
|
|
|
bAdd = dag.addRawBlock(verifier, b, nilAltairCallback)
|
Speed up altair block processing 2x (#3115)
* Speed up altair block processing >2x
Like #3089, this PR drastially speeds up historical REST queries and
other long state replays.
* cache sync committee validator indices
* use ~80mb less memory for validator pubkey mappings
* batch-verify sync aggregate signature (fixes #2985)
* document sync committee hack with head block vs sync message block
* add batch signature verification failure tests
Before:
```
../env.sh nim c -d:release -r ncli_db --db:mainnet_0/db bench --start-slot:-1000
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
5830.675, 0.000, 5830.675, 5830.675, 1, Initialize DB
0.481, 1.878, 0.215, 59.167, 981, Load block from database
8422.566, 0.000, 8422.566, 8422.566, 1, Load state from database
6.996, 1.678, 0.042, 14.385, 969, Advance slot, non-epoch
93.217, 8.318, 84.192, 122.209, 32, Advance slot, epoch
20.513, 23.665, 11.510, 201.561, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
After:
```
7081.422, 0.000, 7081.422, 7081.422, 1, Initialize DB
0.553, 2.122, 0.175, 66.692, 981, Load block from database
5439.446, 0.000, 5439.446, 5439.446, 1, Load state from database
6.829, 1.575, 0.043, 12.156, 969, Advance slot, non-epoch
94.716, 2.749, 88.395, 100.026, 32, Advance slot, epoch
11.636, 23.766, 4.889, 205.250, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
* add comment
2021-11-24 12:43:50 +00:00
|
|
|
check:
|
2021-12-06 09:49:01 +00:00
|
|
|
bAdd.error() == BlockError.Invalid
|
Speed up altair block processing 2x (#3115)
* Speed up altair block processing >2x
Like #3089, this PR drastially speeds up historical REST queries and
other long state replays.
* cache sync committee validator indices
* use ~80mb less memory for validator pubkey mappings
* batch-verify sync aggregate signature (fixes #2985)
* document sync committee hack with head block vs sync message block
* add batch signature verification failure tests
Before:
```
../env.sh nim c -d:release -r ncli_db --db:mainnet_0/db bench --start-slot:-1000
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
5830.675, 0.000, 5830.675, 5830.675, 1, Initialize DB
0.481, 1.878, 0.215, 59.167, 981, Load block from database
8422.566, 0.000, 8422.566, 8422.566, 1, Load state from database
6.996, 1.678, 0.042, 14.385, 969, Advance slot, non-epoch
93.217, 8.318, 84.192, 122.209, 32, Advance slot, epoch
20.513, 23.665, 11.510, 201.561, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
After:
```
7081.422, 0.000, 7081.422, 7081.422, 1, Initialize DB
0.553, 2.122, 0.175, 66.692, 981, Load block from database
5439.446, 0.000, 5439.446, 5439.446, 1, Load state from database
6.829, 1.575, 0.043, 12.156, 969, Advance slot, non-epoch
94.716, 2.749, 88.395, 100.026, 32, Advance slot, epoch
11.636, 23.766, 4.889, 205.250, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
* add comment
2021-11-24 12:43:50 +00:00
|
|
|
|
|
|
|
|
block: # Randao reveal
|
|
|
|
|
var b = b2
|
|
|
|
|
b.message.body.randao_reveal = badSignature
|
|
|
|
|
let
|
2021-12-06 09:49:01 +00:00
|
|
|
bAdd = dag.addRawBlock(verifier, b, nilAltairCallback)
|
Speed up altair block processing 2x (#3115)
* Speed up altair block processing >2x
Like #3089, this PR drastially speeds up historical REST queries and
other long state replays.
* cache sync committee validator indices
* use ~80mb less memory for validator pubkey mappings
* batch-verify sync aggregate signature (fixes #2985)
* document sync committee hack with head block vs sync message block
* add batch signature verification failure tests
Before:
```
../env.sh nim c -d:release -r ncli_db --db:mainnet_0/db bench --start-slot:-1000
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
5830.675, 0.000, 5830.675, 5830.675, 1, Initialize DB
0.481, 1.878, 0.215, 59.167, 981, Load block from database
8422.566, 0.000, 8422.566, 8422.566, 1, Load state from database
6.996, 1.678, 0.042, 14.385, 969, Advance slot, non-epoch
93.217, 8.318, 84.192, 122.209, 32, Advance slot, epoch
20.513, 23.665, 11.510, 201.561, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
After:
```
7081.422, 0.000, 7081.422, 7081.422, 1, Initialize DB
0.553, 2.122, 0.175, 66.692, 981, Load block from database
5439.446, 0.000, 5439.446, 5439.446, 1, Load state from database
6.829, 1.575, 0.043, 12.156, 969, Advance slot, non-epoch
94.716, 2.749, 88.395, 100.026, 32, Advance slot, epoch
11.636, 23.766, 4.889, 205.250, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
* add comment
2021-11-24 12:43:50 +00:00
|
|
|
check:
|
2021-12-06 09:49:01 +00:00
|
|
|
bAdd.error() == BlockError.Invalid
|
Speed up altair block processing 2x (#3115)
* Speed up altair block processing >2x
Like #3089, this PR drastially speeds up historical REST queries and
other long state replays.
* cache sync committee validator indices
* use ~80mb less memory for validator pubkey mappings
* batch-verify sync aggregate signature (fixes #2985)
* document sync committee hack with head block vs sync message block
* add batch signature verification failure tests
Before:
```
../env.sh nim c -d:release -r ncli_db --db:mainnet_0/db bench --start-slot:-1000
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
5830.675, 0.000, 5830.675, 5830.675, 1, Initialize DB
0.481, 1.878, 0.215, 59.167, 981, Load block from database
8422.566, 0.000, 8422.566, 8422.566, 1, Load state from database
6.996, 1.678, 0.042, 14.385, 969, Advance slot, non-epoch
93.217, 8.318, 84.192, 122.209, 32, Advance slot, epoch
20.513, 23.665, 11.510, 201.561, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
After:
```
7081.422, 0.000, 7081.422, 7081.422, 1, Initialize DB
0.553, 2.122, 0.175, 66.692, 981, Load block from database
5439.446, 0.000, 5439.446, 5439.446, 1, Load state from database
6.829, 1.575, 0.043, 12.156, 969, Advance slot, non-epoch
94.716, 2.749, 88.395, 100.026, 32, Advance slot, epoch
11.636, 23.766, 4.889, 205.250, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
* add comment
2021-11-24 12:43:50 +00:00
|
|
|
|
|
|
|
|
block: # Attestations
|
|
|
|
|
var b = b2
|
|
|
|
|
b.message.body.attestations[0].signature = badSignature
|
|
|
|
|
let
|
2021-12-06 09:49:01 +00:00
|
|
|
bAdd = dag.addRawBlock(verifier, b, nilAltairCallback)
|
Speed up altair block processing 2x (#3115)
* Speed up altair block processing >2x
Like #3089, this PR drastially speeds up historical REST queries and
other long state replays.
* cache sync committee validator indices
* use ~80mb less memory for validator pubkey mappings
* batch-verify sync aggregate signature (fixes #2985)
* document sync committee hack with head block vs sync message block
* add batch signature verification failure tests
Before:
```
../env.sh nim c -d:release -r ncli_db --db:mainnet_0/db bench --start-slot:-1000
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
5830.675, 0.000, 5830.675, 5830.675, 1, Initialize DB
0.481, 1.878, 0.215, 59.167, 981, Load block from database
8422.566, 0.000, 8422.566, 8422.566, 1, Load state from database
6.996, 1.678, 0.042, 14.385, 969, Advance slot, non-epoch
93.217, 8.318, 84.192, 122.209, 32, Advance slot, epoch
20.513, 23.665, 11.510, 201.561, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
After:
```
7081.422, 0.000, 7081.422, 7081.422, 1, Initialize DB
0.553, 2.122, 0.175, 66.692, 981, Load block from database
5439.446, 0.000, 5439.446, 5439.446, 1, Load state from database
6.829, 1.575, 0.043, 12.156, 969, Advance slot, non-epoch
94.716, 2.749, 88.395, 100.026, 32, Advance slot, epoch
11.636, 23.766, 4.889, 205.250, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
* add comment
2021-11-24 12:43:50 +00:00
|
|
|
check:
|
2021-12-06 09:49:01 +00:00
|
|
|
bAdd.error() == BlockError.Invalid
|
Speed up altair block processing 2x (#3115)
* Speed up altair block processing >2x
Like #3089, this PR drastially speeds up historical REST queries and
other long state replays.
* cache sync committee validator indices
* use ~80mb less memory for validator pubkey mappings
* batch-verify sync aggregate signature (fixes #2985)
* document sync committee hack with head block vs sync message block
* add batch signature verification failure tests
Before:
```
../env.sh nim c -d:release -r ncli_db --db:mainnet_0/db bench --start-slot:-1000
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
5830.675, 0.000, 5830.675, 5830.675, 1, Initialize DB
0.481, 1.878, 0.215, 59.167, 981, Load block from database
8422.566, 0.000, 8422.566, 8422.566, 1, Load state from database
6.996, 1.678, 0.042, 14.385, 969, Advance slot, non-epoch
93.217, 8.318, 84.192, 122.209, 32, Advance slot, epoch
20.513, 23.665, 11.510, 201.561, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
After:
```
7081.422, 0.000, 7081.422, 7081.422, 1, Initialize DB
0.553, 2.122, 0.175, 66.692, 981, Load block from database
5439.446, 0.000, 5439.446, 5439.446, 1, Load state from database
6.829, 1.575, 0.043, 12.156, 969, Advance slot, non-epoch
94.716, 2.749, 88.395, 100.026, 32, Advance slot, epoch
11.636, 23.766, 4.889, 205.250, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
* add comment
2021-11-24 12:43:50 +00:00
|
|
|
|
|
|
|
|
block: # SyncAggregate empty
|
|
|
|
|
var b = b2
|
|
|
|
|
b.message.body.sync_aggregate.sync_committee_signature = badSignature
|
|
|
|
|
let
|
2021-12-06 09:49:01 +00:00
|
|
|
bAdd = dag.addRawBlock(verifier, b, nilAltairCallback)
|
Speed up altair block processing 2x (#3115)
* Speed up altair block processing >2x
Like #3089, this PR drastially speeds up historical REST queries and
other long state replays.
* cache sync committee validator indices
* use ~80mb less memory for validator pubkey mappings
* batch-verify sync aggregate signature (fixes #2985)
* document sync committee hack with head block vs sync message block
* add batch signature verification failure tests
Before:
```
../env.sh nim c -d:release -r ncli_db --db:mainnet_0/db bench --start-slot:-1000
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
5830.675, 0.000, 5830.675, 5830.675, 1, Initialize DB
0.481, 1.878, 0.215, 59.167, 981, Load block from database
8422.566, 0.000, 8422.566, 8422.566, 1, Load state from database
6.996, 1.678, 0.042, 14.385, 969, Advance slot, non-epoch
93.217, 8.318, 84.192, 122.209, 32, Advance slot, epoch
20.513, 23.665, 11.510, 201.561, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
After:
```
7081.422, 0.000, 7081.422, 7081.422, 1, Initialize DB
0.553, 2.122, 0.175, 66.692, 981, Load block from database
5439.446, 0.000, 5439.446, 5439.446, 1, Load state from database
6.829, 1.575, 0.043, 12.156, 969, Advance slot, non-epoch
94.716, 2.749, 88.395, 100.026, 32, Advance slot, epoch
11.636, 23.766, 4.889, 205.250, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
* add comment
2021-11-24 12:43:50 +00:00
|
|
|
check:
|
2021-12-06 09:49:01 +00:00
|
|
|
bAdd.error() == BlockError.Invalid
|
Speed up altair block processing 2x (#3115)
* Speed up altair block processing >2x
Like #3089, this PR drastially speeds up historical REST queries and
other long state replays.
* cache sync committee validator indices
* use ~80mb less memory for validator pubkey mappings
* batch-verify sync aggregate signature (fixes #2985)
* document sync committee hack with head block vs sync message block
* add batch signature verification failure tests
Before:
```
../env.sh nim c -d:release -r ncli_db --db:mainnet_0/db bench --start-slot:-1000
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
5830.675, 0.000, 5830.675, 5830.675, 1, Initialize DB
0.481, 1.878, 0.215, 59.167, 981, Load block from database
8422.566, 0.000, 8422.566, 8422.566, 1, Load state from database
6.996, 1.678, 0.042, 14.385, 969, Advance slot, non-epoch
93.217, 8.318, 84.192, 122.209, 32, Advance slot, epoch
20.513, 23.665, 11.510, 201.561, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
After:
```
7081.422, 0.000, 7081.422, 7081.422, 1, Initialize DB
0.553, 2.122, 0.175, 66.692, 981, Load block from database
5439.446, 0.000, 5439.446, 5439.446, 1, Load state from database
6.829, 1.575, 0.043, 12.156, 969, Advance slot, non-epoch
94.716, 2.749, 88.395, 100.026, 32, Advance slot, epoch
11.636, 23.766, 4.889, 205.250, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
* add comment
2021-11-24 12:43:50 +00:00
|
|
|
|
|
|
|
|
block: # SyncAggregate junk
|
|
|
|
|
var b = b2
|
|
|
|
|
b.message.body.sync_aggregate.sync_committee_signature = badSignature
|
|
|
|
|
b.message.body.sync_aggregate.sync_committee_bits[0] = true
|
|
|
|
|
|
|
|
|
|
let
|
2021-12-06 09:49:01 +00:00
|
|
|
bAdd = dag.addRawBlock(verifier, b, nilAltairCallback)
|
Speed up altair block processing 2x (#3115)
* Speed up altair block processing >2x
Like #3089, this PR drastially speeds up historical REST queries and
other long state replays.
* cache sync committee validator indices
* use ~80mb less memory for validator pubkey mappings
* batch-verify sync aggregate signature (fixes #2985)
* document sync committee hack with head block vs sync message block
* add batch signature verification failure tests
Before:
```
../env.sh nim c -d:release -r ncli_db --db:mainnet_0/db bench --start-slot:-1000
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
5830.675, 0.000, 5830.675, 5830.675, 1, Initialize DB
0.481, 1.878, 0.215, 59.167, 981, Load block from database
8422.566, 0.000, 8422.566, 8422.566, 1, Load state from database
6.996, 1.678, 0.042, 14.385, 969, Advance slot, non-epoch
93.217, 8.318, 84.192, 122.209, 32, Advance slot, epoch
20.513, 23.665, 11.510, 201.561, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
After:
```
7081.422, 0.000, 7081.422, 7081.422, 1, Initialize DB
0.553, 2.122, 0.175, 66.692, 981, Load block from database
5439.446, 0.000, 5439.446, 5439.446, 1, Load state from database
6.829, 1.575, 0.043, 12.156, 969, Advance slot, non-epoch
94.716, 2.749, 88.395, 100.026, 32, Advance slot, epoch
11.636, 23.766, 4.889, 205.250, 981, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database load
0.000, 0.000, 0.000, 0.000, 0, Database store
```
* add comment
2021-11-24 12:43:50 +00:00
|
|
|
check:
|
2021-12-06 09:49:01 +00:00
|
|
|
bAdd.error() == BlockError.Invalid
|
2020-05-05 09:18:44 +00:00
|
|
|
|
2021-04-28 16:41:02 +00:00
|
|
|
suite "chain DAG finalization tests" & preset():
|
2020-07-01 17:00:14 +00:00
|
|
|
setup:
|
|
|
|
|
var
|
|
|
|
|
db = makeTestDB(SLOTS_PER_EPOCH)
|
Implement split preset/config support (#2710)
* Implement split preset/config support
This is the initial bulk refactor to introduce runtime config values in
a number of places, somewhat replacing the existing mechanism of loading
network metadata.
It still needs more work, this is the initial refactor that introduces
runtime configuration in some of the places that need it.
The PR changes the way presets and constants work, to match the spec. In
particular, a "preset" now refers to the compile-time configuration
while a "cfg" or "RuntimeConfig" is the dynamic part.
A single binary can support either mainnet or minimal, but not both.
Support for other presets has been removed completely (can be readded,
in case there's need).
There's a number of outstanding tasks:
* `SECONDS_PER_SLOT` still needs fixing
* loading custom runtime configs needs redoing
* checking constants against YAML file
* yeerongpilly support
`build/nimbus_beacon_node --network=yeerongpilly --discv5:no --log-level=DEBUG`
* load fork epoch from config
* fix fork digest sent in status
* nicer error string for request failures
* fix tools
* one more
* fixup
* fixup
* fixup
* use "standard" network definition folder in local testnet
Files are loaded from their standard locations, including genesis etc,
to conform to the format used in the `eth2-networks` repo.
* fix launch scripts, allow unknown config values
* fix base config of rest test
* cleanups
* bundle mainnet config using common loader
* fix spec links and names
* only include supported preset in binary
* drop yeerongpilly, add altair-devnet-0, support boot_enr.yaml
2021-07-12 13:01:38 +00:00
|
|
|
dag = init(ChainDAGRef, defaultRuntimeConfig, db, {})
|
2021-12-06 09:49:01 +00:00
|
|
|
verifier = BatchVerifier(rng: keys.newRng(), taskpool: Taskpool.new())
|
|
|
|
|
quarantine = Quarantine.init()
|
2020-07-15 10:44:18 +00:00
|
|
|
cache = StateCache()
|
2021-10-13 14:24:36 +00:00
|
|
|
info = ForkedEpochInfo()
|
2020-02-05 11:41:46 +00:00
|
|
|
|
2021-04-28 16:41:02 +00:00
|
|
|
test "prune heads on finalization" & preset():
|
2020-07-01 17:00:14 +00:00
|
|
|
# Create a fork that will not be taken
|
|
|
|
|
var
|
2021-11-18 12:02:43 +00:00
|
|
|
blck = makeTestBlock(dag.headState.data, cache).phase0Data
|
2020-07-30 19:18:17 +00:00
|
|
|
tmpState = assignClone(dag.headState.data)
|
2020-07-01 17:00:14 +00:00
|
|
|
check:
|
|
|
|
|
process_slots(
|
Implement split preset/config support (#2710)
* Implement split preset/config support
This is the initial bulk refactor to introduce runtime config values in
a number of places, somewhat replacing the existing mechanism of loading
network metadata.
It still needs more work, this is the initial refactor that introduces
runtime configuration in some of the places that need it.
The PR changes the way presets and constants work, to match the spec. In
particular, a "preset" now refers to the compile-time configuration
while a "cfg" or "RuntimeConfig" is the dynamic part.
A single binary can support either mainnet or minimal, but not both.
Support for other presets has been removed completely (can be readded,
in case there's need).
There's a number of outstanding tasks:
* `SECONDS_PER_SLOT` still needs fixing
* loading custom runtime configs needs redoing
* checking constants against YAML file
* yeerongpilly support
`build/nimbus_beacon_node --network=yeerongpilly --discv5:no --log-level=DEBUG`
* load fork epoch from config
* fix fork digest sent in status
* nicer error string for request failures
* fix tools
* one more
* fixup
* fixup
* fixup
* use "standard" network definition folder in local testnet
Files are loaded from their standard locations, including genesis etc,
to conform to the format used in the `eth2-networks` repo.
* fix launch scripts, allow unknown config values
* fix base config of rest test
* cleanups
* bundle mainnet config using common loader
* fix spec links and names
* only include supported preset in binary
* drop yeerongpilly, add altair-devnet-0, support boot_enr.yaml
2021-07-12 13:01:38 +00:00
|
|
|
defaultRuntimeConfig, tmpState[],
|
|
|
|
|
getStateField(tmpState[], slot) + (5 * SLOTS_PER_EPOCH).uint64,
|
2021-10-13 14:24:36 +00:00
|
|
|
cache, info, {})
|
2020-07-01 17:00:14 +00:00
|
|
|
|
2021-11-18 12:02:43 +00:00
|
|
|
let lateBlock = addTestBlock(tmpState[], cache).phase0Data
|
2020-07-09 09:29:32 +00:00
|
|
|
block:
|
2021-12-06 09:49:01 +00:00
|
|
|
let status = dag.addRawBlock(verifier, blck, nilPhase0Callback)
|
2020-07-09 09:29:32 +00:00
|
|
|
check: status.isOk()
|
|
|
|
|
|
2020-08-27 07:34:12 +00:00
|
|
|
assign(tmpState[], dag.headState.data)
|
|
|
|
|
|
2020-07-01 17:00:14 +00:00
|
|
|
for i in 0 ..< (SLOTS_PER_EPOCH * 6):
|
|
|
|
|
if i == 1:
|
|
|
|
|
# There are 2 heads now because of the fork at slot 1
|
|
|
|
|
check:
|
2020-07-30 19:18:17 +00:00
|
|
|
dag.heads.len == 2
|
2020-07-09 09:29:32 +00:00
|
|
|
|
2020-08-27 07:34:12 +00:00
|
|
|
blck = addTestBlock(
|
2021-11-18 12:02:43 +00:00
|
|
|
tmpState[], cache,
|
2020-07-09 09:29:32 +00:00
|
|
|
attestations = makeFullAttestations(
|
2021-10-18 16:37:27 +00:00
|
|
|
tmpState[], dag.head.root, getStateField(tmpState[], slot), cache, {})).phase0Data
|
2021-12-06 09:49:01 +00:00
|
|
|
let added = dag.addRawBlock(verifier, blck, nilPhase0Callback)
|
2020-07-09 09:29:32 +00:00
|
|
|
check: added.isOk()
|
2020-08-31 09:00:38 +00:00
|
|
|
dag.updateHead(added[], quarantine)
|
2021-03-09 14:36:17 +00:00
|
|
|
dag.pruneAtFinalization()
|
2020-07-01 17:00:14 +00:00
|
|
|
|
|
|
|
|
check:
|
2020-07-30 19:18:17 +00:00
|
|
|
dag.heads.len() == 1
|
2021-12-01 09:52:40 +00:00
|
|
|
dag.getBlockBySlot(0.Slot) == BlockSlot(blck: dag.genesis, slot: 0.Slot)
|
|
|
|
|
dag.getBlockBySlot(dag.head.slot) == BlockSlot(
|
|
|
|
|
blck: dag.head, slot: dag.head.slot.Slot)
|
|
|
|
|
dag.getBlockBySlot(dag.head.slot + 1) == BlockSlot(
|
|
|
|
|
blck: dag.head, slot: dag.head.slot.Slot + 1)
|
2020-07-01 17:00:14 +00:00
|
|
|
|
2020-08-26 15:06:40 +00:00
|
|
|
check:
|
2021-06-11 17:51:46 +00:00
|
|
|
dag.db.immutableValidators.len() == getStateField(dag.headState.data, validators).len()
|
2020-08-26 15:06:40 +00:00
|
|
|
|
2021-06-01 11:13:40 +00:00
|
|
|
let
|
|
|
|
|
finalER = dag.findEpochRef(dag.finalizedHead.blck, dag.finalizedHead.slot.epoch)
|
2020-08-11 19:39:53 +00:00
|
|
|
|
2020-11-20 14:16:04 +00:00
|
|
|
# The EpochRef for the finalized block is needed for eth1 voting, so we
|
|
|
|
|
# should never drop it!
|
2021-06-01 11:13:40 +00:00
|
|
|
check:
|
2020-11-20 14:16:04 +00:00
|
|
|
not finalER.isNil
|
|
|
|
|
|
2020-08-26 15:06:40 +00:00
|
|
|
block:
|
2021-03-17 10:17:15 +00:00
|
|
|
for er in dag.epochRefs:
|
2021-06-10 22:07:16 +00:00
|
|
|
check: er == nil or er.epoch >= dag.finalizedHead.slot.epoch
|
2021-03-17 10:17:15 +00:00
|
|
|
|
2021-06-03 13:32:00 +00:00
|
|
|
block:
|
|
|
|
|
let tmpStateData = assignClone(dag.headState)
|
|
|
|
|
|
|
|
|
|
# Check that cached data is available after updateStateData - since we
|
|
|
|
|
# just processed the head the relevant epochrefs should not have been
|
|
|
|
|
# evicted yet
|
|
|
|
|
cache = StateCache()
|
|
|
|
|
updateStateData(
|
|
|
|
|
dag, tmpStateData[], dag.head.atSlot(dag.head.slot), false, cache)
|
|
|
|
|
|
|
|
|
|
check:
|
|
|
|
|
dag.head.slot.epoch in cache.shuffled_active_validator_indices
|
|
|
|
|
(dag.head.slot.epoch - 1) in cache.shuffled_active_validator_indices
|
|
|
|
|
|
|
|
|
|
dag.head.slot in cache.beacon_proposer_indices
|
|
|
|
|
|
2020-07-09 09:29:32 +00:00
|
|
|
block:
|
2020-07-01 17:00:14 +00:00
|
|
|
# The late block is a block whose parent was finalized long ago and thus
|
|
|
|
|
# is no longer a viable head candidate
|
2021-12-06 09:49:01 +00:00
|
|
|
let status = dag.addRawBlock(verifier, lateBlock, nilPhase0Callback)
|
|
|
|
|
check: status.error == BlockError.UnviableFork
|
2020-07-01 17:00:14 +00:00
|
|
|
|
2021-03-01 19:50:43 +00:00
|
|
|
block:
|
|
|
|
|
let
|
|
|
|
|
finalizedCheckpoint = dag.finalizedHead.stateCheckpoint
|
|
|
|
|
headCheckpoint = dag.head.atSlot(dag.head.slot).stateCheckpoint
|
|
|
|
|
check:
|
|
|
|
|
db.getStateRoot(headCheckpoint.blck.root, headCheckpoint.slot).isSome
|
|
|
|
|
db.getStateRoot(finalizedCheckpoint.blck.root, finalizedCheckpoint.slot).isSome
|
|
|
|
|
|
2020-07-01 17:00:14 +00:00
|
|
|
let
|
Implement split preset/config support (#2710)
* Implement split preset/config support
This is the initial bulk refactor to introduce runtime config values in
a number of places, somewhat replacing the existing mechanism of loading
network metadata.
It still needs more work, this is the initial refactor that introduces
runtime configuration in some of the places that need it.
The PR changes the way presets and constants work, to match the spec. In
particular, a "preset" now refers to the compile-time configuration
while a "cfg" or "RuntimeConfig" is the dynamic part.
A single binary can support either mainnet or minimal, but not both.
Support for other presets has been removed completely (can be readded,
in case there's need).
There's a number of outstanding tasks:
* `SECONDS_PER_SLOT` still needs fixing
* loading custom runtime configs needs redoing
* checking constants against YAML file
* yeerongpilly support
`build/nimbus_beacon_node --network=yeerongpilly --discv5:no --log-level=DEBUG`
* load fork epoch from config
* fix fork digest sent in status
* nicer error string for request failures
* fix tools
* one more
* fixup
* fixup
* fixup
* use "standard" network definition folder in local testnet
Files are loaded from their standard locations, including genesis etc,
to conform to the format used in the `eth2-networks` repo.
* fix launch scripts, allow unknown config values
* fix base config of rest test
* cleanups
* bundle mainnet config using common loader
* fix spec links and names
* only include supported preset in binary
* drop yeerongpilly, add altair-devnet-0, support boot_enr.yaml
2021-07-12 13:01:38 +00:00
|
|
|
dag2 = init(ChainDAGRef, defaultRuntimeConfig, db, {})
|
2020-07-01 17:00:14 +00:00
|
|
|
|
|
|
|
|
# check that the state reloaded from database resembles what we had before
|
|
|
|
|
check:
|
2020-07-30 19:18:17 +00:00
|
|
|
dag2.tail.root == dag.tail.root
|
|
|
|
|
dag2.head.root == dag.head.root
|
|
|
|
|
dag2.finalizedHead.blck.root == dag.finalizedHead.blck.root
|
|
|
|
|
dag2.finalizedHead.slot == dag.finalizedHead.slot
|
2021-11-05 07:34:34 +00:00
|
|
|
getStateRoot(dag2.headState.data) == getStateRoot(dag.headState.data)
|
2020-06-25 09:36:03 +00:00
|
|
|
|
2021-04-28 16:41:02 +00:00
|
|
|
test "orphaned epoch block" & preset():
|
2021-10-18 16:37:27 +00:00
|
|
|
var prestate = (ref ForkedHashedBeaconState)(kind: BeaconStateFork.Phase0)
|
2020-08-13 09:50:05 +00:00
|
|
|
for i in 0 ..< SLOTS_PER_EPOCH:
|
|
|
|
|
if i == SLOTS_PER_EPOCH - 1:
|
|
|
|
|
assign(prestate[], dag.headState.data)
|
|
|
|
|
|
2021-11-18 12:02:43 +00:00
|
|
|
let blck = makeTestBlock(dag.headState.data, cache).phase0Data
|
2021-12-06 09:49:01 +00:00
|
|
|
let added = dag.addRawBlock(verifier, blck, nilPhase0Callback)
|
2020-08-13 09:50:05 +00:00
|
|
|
check: added.isOk()
|
2020-08-31 09:00:38 +00:00
|
|
|
dag.updateHead(added[], quarantine)
|
2021-03-09 14:36:17 +00:00
|
|
|
dag.pruneAtFinalization()
|
2020-08-13 09:50:05 +00:00
|
|
|
|
|
|
|
|
check:
|
|
|
|
|
dag.heads.len() == 1
|
|
|
|
|
|
2020-09-07 15:04:33 +00:00
|
|
|
# The loop creates multiple branches, which StateCache isn't suitable for
|
|
|
|
|
cache = StateCache()
|
|
|
|
|
|
2021-06-11 17:51:46 +00:00
|
|
|
doAssert process_slots(
|
Implement split preset/config support (#2710)
* Implement split preset/config support
This is the initial bulk refactor to introduce runtime config values in
a number of places, somewhat replacing the existing mechanism of loading
network metadata.
It still needs more work, this is the initial refactor that introduces
runtime configuration in some of the places that need it.
The PR changes the way presets and constants work, to match the spec. In
particular, a "preset" now refers to the compile-time configuration
while a "cfg" or "RuntimeConfig" is the dynamic part.
A single binary can support either mainnet or minimal, but not both.
Support for other presets has been removed completely (can be readded,
in case there's need).
There's a number of outstanding tasks:
* `SECONDS_PER_SLOT` still needs fixing
* loading custom runtime configs needs redoing
* checking constants against YAML file
* yeerongpilly support
`build/nimbus_beacon_node --network=yeerongpilly --discv5:no --log-level=DEBUG`
* load fork epoch from config
* fix fork digest sent in status
* nicer error string for request failures
* fix tools
* one more
* fixup
* fixup
* fixup
* use "standard" network definition folder in local testnet
Files are loaded from their standard locations, including genesis etc,
to conform to the format used in the `eth2-networks` repo.
* fix launch scripts, allow unknown config values
* fix base config of rest test
* cleanups
* bundle mainnet config using common loader
* fix spec links and names
* only include supported preset in binary
* drop yeerongpilly, add altair-devnet-0, support boot_enr.yaml
2021-07-12 13:01:38 +00:00
|
|
|
defaultRuntimeConfig, prestate[], getStateField(prestate[], slot) + 1,
|
2021-10-13 14:24:36 +00:00
|
|
|
cache, info, {})
|
2020-08-13 09:50:05 +00:00
|
|
|
|
|
|
|
|
# create another block, orphaning the head
|
2021-11-18 12:02:43 +00:00
|
|
|
let blck = makeTestBlock(prestate[], cache).phase0Data
|
2020-08-13 09:50:05 +00:00
|
|
|
|
|
|
|
|
# Add block, but don't update head
|
2021-12-06 09:49:01 +00:00
|
|
|
let added = dag.addRawBlock(verifier, blck, nilPhase0Callback)
|
2020-08-13 09:50:05 +00:00
|
|
|
check: added.isOk()
|
|
|
|
|
|
|
|
|
|
var
|
Implement split preset/config support (#2710)
* Implement split preset/config support
This is the initial bulk refactor to introduce runtime config values in
a number of places, somewhat replacing the existing mechanism of loading
network metadata.
It still needs more work, this is the initial refactor that introduces
runtime configuration in some of the places that need it.
The PR changes the way presets and constants work, to match the spec. In
particular, a "preset" now refers to the compile-time configuration
while a "cfg" or "RuntimeConfig" is the dynamic part.
A single binary can support either mainnet or minimal, but not both.
Support for other presets has been removed completely (can be readded,
in case there's need).
There's a number of outstanding tasks:
* `SECONDS_PER_SLOT` still needs fixing
* loading custom runtime configs needs redoing
* checking constants against YAML file
* yeerongpilly support
`build/nimbus_beacon_node --network=yeerongpilly --discv5:no --log-level=DEBUG`
* load fork epoch from config
* fix fork digest sent in status
* nicer error string for request failures
* fix tools
* one more
* fixup
* fixup
* fixup
* use "standard" network definition folder in local testnet
Files are loaded from their standard locations, including genesis etc,
to conform to the format used in the `eth2-networks` repo.
* fix launch scripts, allow unknown config values
* fix base config of rest test
* cleanups
* bundle mainnet config using common loader
* fix spec links and names
* only include supported preset in binary
* drop yeerongpilly, add altair-devnet-0, support boot_enr.yaml
2021-07-12 13:01:38 +00:00
|
|
|
dag2 = init(ChainDAGRef, defaultRuntimeConfig, db, {})
|
2020-08-13 09:50:05 +00:00
|
|
|
|
|
|
|
|
# check that we can apply the block after the orphaning
|
2021-12-06 09:49:01 +00:00
|
|
|
let added2 = dag2.addRawBlock(verifier, blck, nilPhase0Callback)
|
2020-08-13 09:50:05 +00:00
|
|
|
check: added2.isOk()
|
|
|
|
|
|
2021-04-28 16:41:02 +00:00
|
|
|
test "init with gaps" & preset():
|
2020-08-27 07:34:12 +00:00
|
|
|
for blck in makeTestBlocks(
|
2021-11-18 12:02:43 +00:00
|
|
|
dag.headState.data, cache, int(SLOTS_PER_EPOCH * 6 - 2),
|
2020-08-27 07:34:12 +00:00
|
|
|
true):
|
2021-12-06 09:49:01 +00:00
|
|
|
let added = dag.addRawBlock(verifier, blck.phase0Data, nilPhase0Callback)
|
2020-08-13 09:50:05 +00:00
|
|
|
check: added.isOk()
|
2020-08-31 09:00:38 +00:00
|
|
|
dag.updateHead(added[], quarantine)
|
2021-03-09 14:36:17 +00:00
|
|
|
dag.pruneAtFinalization()
|
2020-08-13 09:50:05 +00:00
|
|
|
|
|
|
|
|
# Advance past epoch so that the epoch transition is gapped
|
|
|
|
|
check:
|
|
|
|
|
process_slots(
|
Implement split preset/config support (#2710)
* Implement split preset/config support
This is the initial bulk refactor to introduce runtime config values in
a number of places, somewhat replacing the existing mechanism of loading
network metadata.
It still needs more work, this is the initial refactor that introduces
runtime configuration in some of the places that need it.
The PR changes the way presets and constants work, to match the spec. In
particular, a "preset" now refers to the compile-time configuration
while a "cfg" or "RuntimeConfig" is the dynamic part.
A single binary can support either mainnet or minimal, but not both.
Support for other presets has been removed completely (can be readded,
in case there's need).
There's a number of outstanding tasks:
* `SECONDS_PER_SLOT` still needs fixing
* loading custom runtime configs needs redoing
* checking constants against YAML file
* yeerongpilly support
`build/nimbus_beacon_node --network=yeerongpilly --discv5:no --log-level=DEBUG`
* load fork epoch from config
* fix fork digest sent in status
* nicer error string for request failures
* fix tools
* one more
* fixup
* fixup
* fixup
* use "standard" network definition folder in local testnet
Files are loaded from their standard locations, including genesis etc,
to conform to the format used in the `eth2-networks` repo.
* fix launch scripts, allow unknown config values
* fix base config of rest test
* cleanups
* bundle mainnet config using common loader
* fix spec links and names
* only include supported preset in binary
* drop yeerongpilly, add altair-devnet-0, support boot_enr.yaml
2021-07-12 13:01:38 +00:00
|
|
|
defaultRuntimeConfig, dag.headState.data, Slot(SLOTS_PER_EPOCH * 6 + 2),
|
2021-10-13 14:24:36 +00:00
|
|
|
cache, info, {})
|
2020-08-13 09:50:05 +00:00
|
|
|
|
|
|
|
|
var blck = makeTestBlock(
|
2021-11-18 12:02:43 +00:00
|
|
|
dag.headState.data, cache,
|
2020-08-13 09:50:05 +00:00
|
|
|
attestations = makeFullAttestations(
|
2021-06-11 17:51:46 +00:00
|
|
|
dag.headState.data, dag.head.root, getStateField(dag.headState.data, slot),
|
2021-10-18 16:37:27 +00:00
|
|
|
cache, {})).phase0Data
|
2020-08-13 09:50:05 +00:00
|
|
|
|
2021-12-06 09:49:01 +00:00
|
|
|
let added = dag.addRawBlock(verifier, blck, nilPhase0Callback)
|
2020-08-13 09:50:05 +00:00
|
|
|
check: added.isOk()
|
2020-08-31 09:00:38 +00:00
|
|
|
dag.updateHead(added[], quarantine)
|
2021-03-09 14:36:17 +00:00
|
|
|
dag.pruneAtFinalization()
|
2020-08-13 09:50:05 +00:00
|
|
|
|
2021-05-17 16:37:26 +00:00
|
|
|
block:
|
|
|
|
|
# Check that we can rewind to every block from head to finalized
|
|
|
|
|
var
|
|
|
|
|
cur = dag.head
|
|
|
|
|
tmpStateData = assignClone(dag.headState)
|
|
|
|
|
while cur.slot >= dag.finalizedHead.slot:
|
|
|
|
|
assign(tmpStateData[], dag.headState)
|
|
|
|
|
dag.updateStateData(tmpStateData[], cur.atSlot(cur.slot), false, cache)
|
|
|
|
|
check:
|
2021-11-05 07:34:34 +00:00
|
|
|
dag.get(cur).data.phase0Data.message.state_root ==
|
|
|
|
|
getStateRoot(tmpStateData[].data)
|
|
|
|
|
getStateRoot(tmpStateData[].data) == hash_tree_root(
|
|
|
|
|
tmpStateData[].data.phase0Data.data)
|
2021-05-17 16:37:26 +00:00
|
|
|
cur = cur.parent
|
|
|
|
|
|
2020-08-13 09:50:05 +00:00
|
|
|
let
|
Implement split preset/config support (#2710)
* Implement split preset/config support
This is the initial bulk refactor to introduce runtime config values in
a number of places, somewhat replacing the existing mechanism of loading
network metadata.
It still needs more work, this is the initial refactor that introduces
runtime configuration in some of the places that need it.
The PR changes the way presets and constants work, to match the spec. In
particular, a "preset" now refers to the compile-time configuration
while a "cfg" or "RuntimeConfig" is the dynamic part.
A single binary can support either mainnet or minimal, but not both.
Support for other presets has been removed completely (can be readded,
in case there's need).
There's a number of outstanding tasks:
* `SECONDS_PER_SLOT` still needs fixing
* loading custom runtime configs needs redoing
* checking constants against YAML file
* yeerongpilly support
`build/nimbus_beacon_node --network=yeerongpilly --discv5:no --log-level=DEBUG`
* load fork epoch from config
* fix fork digest sent in status
* nicer error string for request failures
* fix tools
* one more
* fixup
* fixup
* fixup
* use "standard" network definition folder in local testnet
Files are loaded from their standard locations, including genesis etc,
to conform to the format used in the `eth2-networks` repo.
* fix launch scripts, allow unknown config values
* fix base config of rest test
* cleanups
* bundle mainnet config using common loader
* fix spec links and names
* only include supported preset in binary
* drop yeerongpilly, add altair-devnet-0, support boot_enr.yaml
2021-07-12 13:01:38 +00:00
|
|
|
dag2 = init(ChainDAGRef, defaultRuntimeConfig, db, {})
|
2020-08-13 09:50:05 +00:00
|
|
|
|
|
|
|
|
# check that the state reloaded from database resembles what we had before
|
|
|
|
|
check:
|
|
|
|
|
dag2.tail.root == dag.tail.root
|
|
|
|
|
dag2.head.root == dag.head.root
|
|
|
|
|
dag2.finalizedHead.blck.root == dag.finalizedHead.blck.root
|
|
|
|
|
dag2.finalizedHead.slot == dag.finalizedHead.slot
|
2021-11-05 07:34:34 +00:00
|
|
|
getStateRoot(dag2.headState.data) == getStateRoot(dag.headState.data)
|
2021-08-05 08:26:10 +00:00
|
|
|
|
|
|
|
|
suite "Old database versions" & preset():
|
|
|
|
|
setup:
|
|
|
|
|
let
|
2021-11-18 12:02:43 +00:00
|
|
|
genState = newClone(initialize_hashed_beacon_state_from_eth1(
|
2021-08-09 11:14:28 +00:00
|
|
|
defaultRuntimeConfig,
|
2021-08-05 08:26:10 +00:00
|
|
|
Eth2Digest(),
|
|
|
|
|
0,
|
|
|
|
|
makeInitialDeposits(SLOTS_PER_EPOCH.uint64, flags = {skipBlsValidation}),
|
2021-11-18 12:02:43 +00:00
|
|
|
{skipBlsValidation}))
|
2021-08-05 08:26:10 +00:00
|
|
|
genBlock = get_initial_beacon_block(genState[])
|
2021-12-06 09:49:01 +00:00
|
|
|
var
|
|
|
|
|
verifier = BatchVerifier(rng: keys.newRng(), taskpool: Taskpool.new())
|
|
|
|
|
quarantine = Quarantine.init()
|
2021-08-05 08:26:10 +00:00
|
|
|
|
|
|
|
|
test "pre-1.1.0":
|
|
|
|
|
# only kvstore, no immutable validator keys
|
|
|
|
|
|
2021-08-09 11:14:28 +00:00
|
|
|
let db = BeaconChainDB.new("", inMemory = true)
|
2021-08-05 08:26:10 +00:00
|
|
|
|
|
|
|
|
# preInit a database to a v1.0.12 state
|
2021-11-18 12:02:43 +00:00
|
|
|
db.putStateRoot(
|
|
|
|
|
genState[].latest_block_root(), genState[].data.slot, genState[].root)
|
|
|
|
|
db.putStateV0(genState[].root, genState[].data)
|
|
|
|
|
|
2021-08-05 08:26:10 +00:00
|
|
|
db.putBlockV0(genBlock)
|
|
|
|
|
db.putTailBlock(genBlock.root)
|
|
|
|
|
db.putHeadBlock(genBlock.root)
|
2021-11-05 07:34:34 +00:00
|
|
|
db.putGenesisBlock(genBlock.root)
|
2021-08-05 08:26:10 +00:00
|
|
|
|
|
|
|
|
var
|
2021-08-09 11:14:28 +00:00
|
|
|
dag = init(ChainDAGRef, defaultRuntimeConfig, db, {})
|
2021-08-05 08:26:10 +00:00
|
|
|
state = newClone(dag.headState.data)
|
|
|
|
|
cache = StateCache()
|
|
|
|
|
att0 = makeFullAttestations(state[], dag.tail.root, 0.Slot, cache)
|
2021-11-18 12:02:43 +00:00
|
|
|
b1 = addTestBlock(state[], cache, attestations = att0).phase0Data
|
2021-12-06 09:49:01 +00:00
|
|
|
b1Add = dag.addRawBlock(verifier, b1, nilPhase0Callback)
|
2021-08-05 08:26:10 +00:00
|
|
|
|
|
|
|
|
check:
|
|
|
|
|
b1Add.isOk()
|
2021-09-08 03:46:33 +00:00
|
|
|
|
|
|
|
|
suite "Diverging hardforks":
|
|
|
|
|
setup:
|
|
|
|
|
var
|
|
|
|
|
phase0RuntimeConfig = defaultRuntimeConfig
|
|
|
|
|
altairRuntimeConfig = defaultRuntimeConfig
|
|
|
|
|
|
|
|
|
|
phase0RuntimeConfig.ALTAIR_FORK_EPOCH = FAR_FUTURE_EPOCH
|
|
|
|
|
altairRuntimeConfig.ALTAIR_FORK_EPOCH = 2.Epoch
|
|
|
|
|
|
|
|
|
|
var
|
|
|
|
|
db = makeTestDB(SLOTS_PER_EPOCH)
|
|
|
|
|
dag = init(ChainDAGRef, phase0RuntimeConfig, db, {})
|
2021-12-06 09:49:01 +00:00
|
|
|
verifier = BatchVerifier(rng: keys.newRng(), taskpool: Taskpool.new())
|
|
|
|
|
quarantine = newClone(Quarantine.init())
|
2021-09-08 03:46:33 +00:00
|
|
|
cache = StateCache()
|
2021-10-13 14:24:36 +00:00
|
|
|
info = ForkedEpochInfo()
|
2021-09-08 03:46:33 +00:00
|
|
|
tmpState = assignClone(dag.headState.data)
|
|
|
|
|
|
|
|
|
|
test "Tail block only in common":
|
|
|
|
|
check:
|
|
|
|
|
process_slots(
|
|
|
|
|
phase0RuntimeConfig, tmpState[],
|
|
|
|
|
getStateField(tmpState[], slot) + (3 * SLOTS_PER_EPOCH).uint64,
|
2021-10-13 14:24:36 +00:00
|
|
|
cache, info, {})
|
2021-09-08 03:46:33 +00:00
|
|
|
|
|
|
|
|
# Because the first block is after the Altair transition, the only block in
|
|
|
|
|
# common is the tail block
|
|
|
|
|
var
|
2021-11-18 12:02:43 +00:00
|
|
|
b1 = addTestBlock(tmpState[], cache).phase0Data
|
2021-12-06 09:49:01 +00:00
|
|
|
b1Add = dag.addRawBlock(verifier, b1, nilPhase0Callback)
|
2021-09-08 03:46:33 +00:00
|
|
|
|
|
|
|
|
check b1Add.isOk()
|
2021-12-06 09:49:01 +00:00
|
|
|
dag.updateHead(b1Add[], quarantine[])
|
2021-09-08 03:46:33 +00:00
|
|
|
|
|
|
|
|
var dagAltair = init(ChainDAGRef, altairRuntimeConfig, db, {})
|
|
|
|
|
discard AttestationPool.init(dagAltair, quarantine)
|
|
|
|
|
|
|
|
|
|
test "Non-tail block in common":
|
|
|
|
|
check:
|
|
|
|
|
process_slots(
|
|
|
|
|
phase0RuntimeConfig, tmpState[],
|
|
|
|
|
getStateField(tmpState[], slot) + SLOTS_PER_EPOCH.uint64,
|
2021-10-13 14:24:36 +00:00
|
|
|
cache, info, {})
|
2021-09-08 03:46:33 +00:00
|
|
|
|
|
|
|
|
# There's a block in the shared-correct phase0 hardfork, before epoch 2
|
|
|
|
|
var
|
2021-11-18 12:02:43 +00:00
|
|
|
b1 = addTestBlock(tmpState[], cache).phase0Data
|
2021-12-06 09:49:01 +00:00
|
|
|
b1Add = dag.addRawBlock(verifier, b1, nilPhase0Callback)
|
2021-09-08 03:46:33 +00:00
|
|
|
|
|
|
|
|
check:
|
|
|
|
|
b1Add.isOk()
|
|
|
|
|
process_slots(
|
|
|
|
|
phase0RuntimeConfig, tmpState[],
|
|
|
|
|
getStateField(tmpState[], slot) + (3 * SLOTS_PER_EPOCH).uint64,
|
2021-10-13 14:24:36 +00:00
|
|
|
cache, info, {})
|
2021-09-08 03:46:33 +00:00
|
|
|
|
|
|
|
|
var
|
2021-11-18 12:02:43 +00:00
|
|
|
b2 = addTestBlock(tmpState[], cache).phase0Data
|
2021-12-06 09:49:01 +00:00
|
|
|
b2Add = dag.addRawBlock(verifier, b2, nilPhase0Callback)
|
2021-09-08 03:46:33 +00:00
|
|
|
|
|
|
|
|
check b2Add.isOk()
|
2021-12-06 09:49:01 +00:00
|
|
|
dag.updateHead(b2Add[], quarantine[])
|
2021-09-08 03:46:33 +00:00
|
|
|
|
|
|
|
|
var dagAltair = init(ChainDAGRef, altairRuntimeConfig, db, {})
|
|
|
|
|
discard AttestationPool.init(dagAltair, quarantine)
|
