status-im/nimbus-eth2
2
0
Fork 0
mirror of https://github.com/status-im/nimbus-eth2.git synced 2025-02-23 19:58:22 +00:00
Code Issues Projects Releases Wiki Activity

speed up epoch processing 6x+ (#3089)

Browse Source 
* speed up epoch processing 6x+

This change above all helps contain long replay times on epoch change, reorg
and deep history inspection via REST/RPC

* most effective balances don't actually change due to MAX_EFFECTIVE_BALANCE
* ditto for inactivity scores
* avoid signature check for trusted sync aggregates

pre:
```
./ncli_db --db:mainnet_0/db bench --start-slot=-3200
All time are ms
     Average,       StdDev,          Min,          Max,      Samples,         Test
Validation is turned off meaning that no BLS operations are performed

    3468.621,        0.000,     3468.621,     3468.621,            1, Initialize DB
       0.357,        0.938,        0.171,       52.752,         3155, Load block from database
   15691.471,        0.000,    15691.471,    15691.471,            1, Load state from database
       6.100,        9.469,        0.033,      526.816,         3101, Advance slot, non-epoch
     579.131,        9.523,      566.936,      610.328,          100, Advance slot, epoch
      18.551,       16.317,       12.664,      136.668,         3155, Apply block, no slot processing
       0.000,        0.000,        0.000,        0.000,            0, Database load
       0.000,        0.000,        0.000,        0.000,            0, Database store
```

post:
```
     Average,       StdDev,          Min,          Max,      Samples,         Test
Validation is turned off meaning that no BLS operations are performed
    3488.541,        0.000,     3488.541,     3488.541,            1, Initialize DB
       0.369,        1.123,        0.183,       63.208,         3155, Load block from database
   13430.642,        0.000,    13430.642,    13430.642,            1, Load state from database
       6.522,        1.721,        0.034,       36.708,         3101, Advance slot, non-epoch
      89.074,        3.162,       83.573,      101.436,          100, Advance slot, epoch
      18.325,       18.346,       13.005,      145.040,         3155, Apply block, no slot processing
       0.000,        0.000,        0.000,        0.000,            0, Database load
       0.000,        0.000,        0.000,        0.000,            0, Database store
```

* Update beacon_chain/spec/state_transition_block.nim

Co-authored-by: zah <zahary@gmail.com>

* avoid copying validator data in accessor

```
    5291.227,        0.000,     5291.227,     5291.227,            1, Initialize DB
       0.436,        0.928,        0.138,       51.438,         3155, Load block from database
   11962.826,        0.000,    11962.826,    11962.826,            1, Load state from database
       6.477,        1.675,        0.037,       34.174,         3101, Advance slot, non-epoch
      76.633,        3.705,       71.106,       98.085,          100, Advance slot, epoch
      18.301,       18.593,       13.208,      149.153,         3155, Apply block, no slot processing
       0.000,        0.000,        0.000,        0.000,            0, Database load
       0.000,        0.000,        0.000,        0.000,            0, Database store
```

* work around compiler bug

Co-authored-by: zah <zahary@gmail.com>
This commit is contained in:
Jacek Sieka 2021-11-11 20:24:29 +01:00 committed by GitHub
parent 9f4a55b82b
commit 00bbc8e0fe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 34 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
beacon_chain/spec/datatypes/altair.nim
View File

@ -83,6 +83,10 @@ type
sync_committee_bits*: BitArray[SYNC_COMMITTEE_SIZE] sync_committee_bits*: BitArray[SYNC_COMMITTEE_SIZE]
sync_committee_signature*: ValidatorSig sync_committee_signature*: ValidatorSig
TrustedSyncAggregate* = object
sync_committee_bits*: BitArray[SYNC_COMMITTEE_SIZE]
sync_committee_signature*: TrustedSig
# https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/altair/beacon-chain.md#synccommittee # https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/altair/beacon-chain.md#synccommittee
SyncCommittee* = object SyncCommittee* = object
pubkeys*: HashArray[Limit SYNC_COMMITTEE_SIZE, ValidatorPubKey] pubkeys*: HashArray[Limit SYNC_COMMITTEE_SIZE, ValidatorPubKey]
@ -369,7 +373,7 @@ type
voluntary_exits*: List[TrustedSignedVoluntaryExit, Limit MAX_VOLUNTARY_EXITS] voluntary_exits*: List[TrustedSignedVoluntaryExit, Limit MAX_VOLUNTARY_EXITS]
# [New in Altair] # [New in Altair]
sync_aggregate*: SyncAggregate sync_aggregate*: SyncAggregate # TODO TrustedSyncAggregate after batching
SyncnetBits* = BitArray[SYNC_COMMITTEE_SUBNET_COUNT] SyncnetBits* = BitArray[SYNC_COMMITTEE_SUBNET_COUNT]
@ -393,7 +397,7 @@ type
voluntary_exits*: List[TrustedSignedVoluntaryExit, Limit MAX_VOLUNTARY_EXITS] voluntary_exits*: List[TrustedSignedVoluntaryExit, Limit MAX_VOLUNTARY_EXITS]
# [New in Altair] # [New in Altair]
sync_aggregate*: SyncAggregate sync_aggregate*: TrustedSyncAggregate
# https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/phase0/beacon-chain.md#signedbeaconblock # https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/phase0/beacon-chain.md#signedbeaconblock
SignedBeaconBlock* = object SignedBeaconBlock* = object
@ -431,6 +435,8 @@ type
SomeBeaconBlock* = BeaconBlock | SigVerifiedBeaconBlock | TrustedBeaconBlock SomeBeaconBlock* = BeaconBlock | SigVerifiedBeaconBlock | TrustedBeaconBlock
SomeBeaconBlockBody* = BeaconBlockBody | SigVerifiedBeaconBlockBody | TrustedBeaconBlockBody SomeBeaconBlockBody* = BeaconBlockBody | SigVerifiedBeaconBlockBody | TrustedBeaconBlockBody
SomeSyncAggregate* = SyncAggregate | TrustedSyncAggregate
SyncSubcommitteeIndex* = distinct uint8 SyncSubcommitteeIndex* = distinct uint8
IndexInSyncCommittee* = distinct uint16 IndexInSyncCommittee* = distinct uint16

7
beacon_chain/spec/datatypes/base.nim
View File

@ -623,13 +623,10 @@ proc readValue*(reader: var JsonReader, value: var ForkDigest)
static: doAssert high(int) >= high(int32) static: doAssert high(int) >= high(int32)
# `ValidatorIndex` seq handling. # `ValidatorIndex` seq handling.
func `[]`*[T](a: var seq[T], b: ValidatorIndex): var T = template `[]=`*[T](a: var seq[T], b: ValidatorIndex, c: T) =
a[b.int]
func `[]=`*[T](a: var seq[T], b: ValidatorIndex, c: T) =
a[b.int] = c a[b.int] = c
func `[]`*[T](a: seq[T], b: ValidatorIndex): auto = template `[]`*[T](a: seq[T], b: ValidatorIndex): auto = # Also var seq (!)
a[b.int] a[b.int]
# `ValidatorIndex` Nim integration # `ValidatorIndex` Nim integration

4
beacon_chain/spec/datatypes/merge.nim
View File

@ -263,7 +263,7 @@ type
attestations*: List[Attestation, Limit MAX_ATTESTATIONS] attestations*: List[Attestation, Limit MAX_ATTESTATIONS]
deposits*: List[Deposit, Limit MAX_DEPOSITS] deposits*: List[Deposit, Limit MAX_DEPOSITS]
voluntary_exits*: List[SignedVoluntaryExit, Limit MAX_VOLUNTARY_EXITS] voluntary_exits*: List[SignedVoluntaryExit, Limit MAX_VOLUNTARY_EXITS]
sync_aggregate*: SyncAggregate sync_aggregate*: SyncAggregate # TODO TrustedSyncAggregate after batching
# Execution # Execution
execution_payload*: ExecutionPayload # [New in Merge] execution_payload*: ExecutionPayload # [New in Merge]
@ -283,7 +283,7 @@ type
attestations*: List[TrustedAttestation, Limit MAX_ATTESTATIONS] attestations*: List[TrustedAttestation, Limit MAX_ATTESTATIONS]
deposits*: List[Deposit, Limit MAX_DEPOSITS] deposits*: List[Deposit, Limit MAX_DEPOSITS]
voluntary_exits*: List[TrustedSignedVoluntaryExit, Limit MAX_VOLUNTARY_EXITS] voluntary_exits*: List[TrustedSignedVoluntaryExit, Limit MAX_VOLUNTARY_EXITS]
sync_aggregate*: SyncAggregate sync_aggregate*: TrustedSyncAggregate
# Execution # Execution
execution_payload*: ExecutionPayload # [New in Merge] execution_payload*: ExecutionPayload # [New in Merge]

3
beacon_chain/spec/state_transition_block.nim
View File

@ -429,7 +429,7 @@ proc process_operations(cfg: RuntimeConfig,
# https://github.com/ethereum/consensus-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#sync-committee-processing # https://github.com/ethereum/consensus-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#sync-committee-processing
proc process_sync_aggregate*( proc process_sync_aggregate*(
state: var (altair.BeaconState | merge.BeaconState), state: var (altair.BeaconState | merge.BeaconState),
aggregate: SyncAggregate, total_active_balance: Gwei, cache: var StateCache): aggregate: SomeSyncAggregate, total_active_balance: Gwei, cache: var StateCache):
Result[void, cstring] {.nbench.} = Result[void, cstring] {.nbench.} =
# Verify sync committee aggregate signature signing over the previous slot # Verify sync committee aggregate signature signing over the previous slot
# block root # block root
@ -439,6 +439,7 @@ proc process_sync_aggregate*(
domain = get_domain(state, DOMAIN_SYNC_COMMITTEE, compute_epoch_at_slot(previous_slot)) domain = get_domain(state, DOMAIN_SYNC_COMMITTEE, compute_epoch_at_slot(previous_slot))
signing_root = compute_signing_root(get_block_root_at_slot(state, previous_slot), domain) signing_root = compute_signing_root(get_block_root_at_slot(state, previous_slot), domain)
when aggregate.sync_committee_signature isnot TrustedSig:
var participant_pubkeys: seq[ValidatorPubKey] var participant_pubkeys: seq[ValidatorPubKey]
for i in 0 ..< committee_pubkeys.len: for i in 0 ..< committee_pubkeys.len:
if aggregate.sync_committee_bits[i]: if aggregate.sync_committee_bits[i]:

13
beacon_chain/spec/state_transition_epoch.nim
View File

@ -870,10 +870,13 @@ func process_effective_balance_updates*(state: var ForkyBeaconState) {.nbench.}
let effective_balance = state.validators.asSeq()[index].effective_balance let effective_balance = state.validators.asSeq()[index].effective_balance
if balance + DOWNWARD_THRESHOLD < effective_balance or if balance + DOWNWARD_THRESHOLD < effective_balance or
effective_balance + UPWARD_THRESHOLD < balance: effective_balance + UPWARD_THRESHOLD < balance:
state.validators[index].effective_balance = let new_effective_balance =
min( min(
balance - balance mod EFFECTIVE_BALANCE_INCREMENT, balance - balance mod EFFECTIVE_BALANCE_INCREMENT,
MAX_EFFECTIVE_BALANCE) MAX_EFFECTIVE_BALANCE)
# Protect against unnecessary cache invalidation
if new_effective_balance != effective_balance:
state.validators[index].effective_balance = new_effective_balance
# https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/phase0/beacon-chain.md#slashings-balances-updates # https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/phase0/beacon-chain.md#slashings-balances-updates
func process_slashings_reset*(state: var ForkyBeaconState) {.nbench.} = func process_slashings_reset*(state: var ForkyBeaconState) {.nbench.} =
@ -946,13 +949,13 @@ func process_inactivity_updates*(
previous_epoch = get_previous_epoch(state) # get_eligible_validator_indices() previous_epoch = get_previous_epoch(state) # get_eligible_validator_indices()
not_in_inactivity_leak = not is_in_inactivity_leak(state) not_in_inactivity_leak = not is_in_inactivity_leak(state)
state.inactivity_scores.clearCache()
for index in 0'u64 ..< state.validators.lenu64: for index in 0'u64 ..< state.validators.lenu64:
if not is_eligible_validator(info.validators[index]): if not is_eligible_validator(info.validators[index]):
continue continue
# Increase the inactivity score of inactive validators # Increase the inactivity score of inactive validators
var inactivity_score = state.inactivity_scores.asSeq()[index] let pre_inactivity_score = state.inactivity_scores.asSeq()[index]
var inactivity_score = pre_inactivity_score
# TODO activeness already checked; remove redundant checks between # TODO activeness already checked; remove redundant checks between
# is_active_validator and is_unslashed_participating_index # is_active_validator and is_unslashed_participating_index
if is_unslashed_participating_index( if is_unslashed_participating_index(
@ -964,7 +967,9 @@ func process_inactivity_updates*(
# leak-free epoch # leak-free epoch
if not_in_inactivity_leak: if not_in_inactivity_leak:
inactivity_score -= min(INACTIVITY_SCORE_RECOVERY_RATE.uint64, inactivity_score) inactivity_score -= min(INACTIVITY_SCORE_RECOVERY_RATE.uint64, inactivity_score)
state.inactivity_scores.asSeq()[index] = inactivity_score # Most inactivity scores remain at 0 - avoid invalidating cache
if pre_inactivity_score != inactivity_score:
state.inactivity_scores[index] = inactivity_score
# https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/phase0/beacon-chain.md#epoch-processing # https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/phase0/beacon-chain.md#epoch-processing
proc process_epoch*( proc process_epoch*(