Jacek Sieka 5713a3ce4c
performance fixes (#2259)
* performance fixes

* don't mark tree cache as dirty on read-only List accesses
* store only blob in memory for keys and signatures, parse blob lazily
* compare public keys by blob instead of parsing / converting to raw
* compare Eth2Digest using non-constant-time comparison
* avoid some unnecessary validator copying

This branch will in particular speed up deposit processing which has
been slowing down block replay.

Pre (mainnet, 1600 blocks):

```
All time are ms
     Average,       StdDev,          Min,          Max,      Samples,         Test
Validation is turned off meaning that no BLS operations are performed
    3450.269,        0.000,     3450.269,     3450.269,            1, Initialize DB
       0.417,        0.822,        0.036,       21.098,         1400, Load block from database
      16.521,        0.000,       16.521,       16.521,            1, Load state from database
      27.906,       50.846,        8.104,     1507.633,         1350, Apply block
      52.617,       37.029,       20.640,      135.938,           50, Apply epoch block
```

Post:

```
    3502.715,        0.000,     3502.715,     3502.715,            1, Initialize DB
       0.080,        0.560,        0.035,       21.015,         1400, Load block from database
      17.595,        0.000,       17.595,       17.595,            1, Load state from database
      15.706,       11.028,        8.300,      107.537,         1350, Apply block
      33.217,       12.622,       17.331,       60.580,           50, Apply epoch block
```

* more perf fixes

* load EpochRef cache into StateCache more aggressively
* point out security concern with public key cache
* reuse proposer index from state when processing block
* avoid genericAssign in a few more places
* don't parse key when signature is unparseable
* fix `==` overload for Eth2Digest
* preallocate validator list when getting active validators
* speed up proposer index calculation a little bit
* reuse cache when replaying blocks in ncli_db
* avoid a few more copying loops

```
     Average,       StdDev,          Min,          Max,      Samples,         Test
Validation is turned off meaning that no BLS operations are performed
    3279.158,        0.000,     3279.158,     3279.158,            1, Initialize DB
       0.072,        0.357,        0.035,       13.400,         1400, Load block from database
      17.295,        0.000,       17.295,       17.295,            1, Load state from database
       5.918,        9.896,        0.198,       98.028,         1350, Apply block
      15.888,       10.951,        7.902,       39.535,           50, Apply epoch block
       0.000,        0.000,        0.000,        0.000,            0, Database block store
```

* clear full balance cache before processing rewards and penalties

```
All time are ms
     Average,       StdDev,          Min,          Max,      Samples,         Test
Validation is turned off meaning that no BLS operations are performed
    3947.901,        0.000,     3947.901,     3947.901,            1, Initialize DB
       0.124,        0.506,        0.026,      202.370,       363345, Load block from database
      97.614,        0.000,       97.614,       97.614,            1, Load state from database
       0.186,        0.188,        0.012,       99.561,       357262, Advance slot, non-epoch
      14.161,        5.966,        1.099,      395.511,        11524, Advance slot, epoch
       1.372,        4.170,        0.017,      276.401,       363345, Apply block, no slot processing
       0.000,        0.000,        0.000,        0.000,            0, Database block store
```
2021-01-25 13:04:18 +01:00

185 lines
7.3 KiB
Nim

# beacon_chain
# Copyright (c) 2018-2020 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
# Uncategorized helper functions from the spec
{.push raises: [Defect].}
import
# Standard lib
std/[math, tables],
# Third-party
stew/endians2,
# Internal
./datatypes, ./digest, ./crypto, ../ssz/merkleization
type
# This solves an ambiguous identifier Error in some contexts
# (other candidate is nativesockets.Domain)
Domain = datatypes.Domain
# https://github.com/ethereum/eth2.0-specs/blob/v1.0.0/specs/phase0/beacon-chain.md#integer_squareroot
func integer_squareroot*(n: SomeInteger): SomeInteger =
## Return the largest integer ``x`` such that ``x**2 <= n``.
doAssert n >= 0'u64
var
x = n
y = (x + 1) div 2
while y < x:
x = y
y = (x + n div x) div 2
x
# https://github.com/ethereum/eth2.0-specs/blob/v1.0.0/specs/phase0/beacon-chain.md#compute_epoch_at_slot
func compute_epoch_at_slot*(slot: Slot|uint64): Epoch =
## Return the epoch number at ``slot``.
(slot div SLOTS_PER_EPOCH).Epoch
template epoch*(slot: Slot): Epoch =
compute_epoch_at_slot(slot)
template isEpoch*(slot: Slot): bool =
(slot mod SLOTS_PER_EPOCH) == 0
# https://github.com/ethereum/eth2.0-specs/blob/v1.0.0/specs/phase0/beacon-chain.md#compute_start_slot_at_epoch
func compute_start_slot_at_epoch*(epoch: Epoch): Slot =
## Return the start slot of ``epoch``.
(epoch * SLOTS_PER_EPOCH).Slot
# https://github.com/ethereum/eth2.0-specs/blob/v1.0.0/specs/phase0/beacon-chain.md#is_active_validator
func is_active_validator*(validator: Validator, epoch: Epoch): bool =
## Check if ``validator`` is active
validator.activation_epoch <= epoch and epoch < validator.exit_epoch
# https://github.com/ethereum/eth2.0-specs/blob/v1.0.0/specs/phase0/beacon-chain.md#get_active_validator_indices
func get_active_validator_indices*(state: BeaconState, epoch: Epoch):
seq[ValidatorIndex] =
## Return the sequence of active validator indices at ``epoch``.
result = newSeqOfCap[ValidatorIndex](state.validators.len)
for idx in 0..<state.validators.len:
if is_active_validator(state.validators[idx], epoch):
result.add idx.ValidatorIndex
func get_active_validator_indices_len*(state: BeaconState, epoch: Epoch): uint64 =
for idx in 0..<state.validators.len:
if is_active_validator(state.validators[idx], epoch):
inc result
# https://github.com/ethereum/eth2.0-specs/blob/v1.0.0/specs/phase0/beacon-chain.md#get_current_epoch
func get_current_epoch*(state: BeaconState): Epoch =
## Return the current epoch.
doAssert state.slot >= GENESIS_SLOT, $state.slot
compute_epoch_at_slot(state.slot)
# https://github.com/ethereum/eth2.0-specs/blob/v1.0.0/specs/phase0/beacon-chain.md#get_randao_mix
func get_randao_mix*(state: BeaconState,
epoch: Epoch): Eth2Digest =
## Returns the randao mix at a recent ``epoch``.
state.randao_mixes[epoch mod EPOCHS_PER_HISTORICAL_VECTOR]
func bytes_to_uint64*(data: openArray[byte]): uint64 =
doAssert data.len == 8
# Little-endian data representation
result = 0
for i in countdown(7, 0):
result = result * 256 + data[i]
# Have 1, 4, and 8-byte versions. Spec only defines 8-byte version, but useful
# to check invariants on rest.
func uint_to_bytes8*(x: uint64): array[8, byte] =
x.toBytesLE()
func uint_to_bytes4*(x: uint64): array[4, byte] =
doAssert x < 2'u64^32
# Little-endian data representation
result[0] = ((x shr 0) and 0xff).byte
result[1] = ((x shr 8) and 0xff).byte
result[2] = ((x shr 16) and 0xff).byte
result[3] = ((x shr 24) and 0xff).byte
# https://github.com/ethereum/eth2.0-specs/blob/v1.0.0/specs/phase0/beacon-chain.md#compute_fork_data_root
func compute_fork_data_root(current_version: Version,
genesis_validators_root: Eth2Digest): Eth2Digest =
## Return the 32-byte fork data root for the ``current_version`` and
## ``genesis_validators_root``.
## This is used primarily in signature domains to avoid collisions across
## forks/chains.
hash_tree_root(ForkData(
current_version: current_version,
genesis_validators_root: genesis_validators_root
))
# https://github.com/ethereum/eth2.0-specs/blob/v1.0.0/specs/phase0/beacon-chain.md#compute_fork_digest
func compute_fork_digest*(current_version: Version,
genesis_validators_root: Eth2Digest): ForkDigest =
## Return the 4-byte fork digest for the ``current_version`` and
## ``genesis_validators_root``.
## This is a digest primarily used for domain separation on the p2p layer.
## 4-bytes suffices for practical separation of forks/chains.
array[4, byte](result)[0..3] =
compute_fork_data_root(
current_version, genesis_validators_root).data.toOpenArray(0, 3)
# https://github.com/ethereum/eth2.0-specs/blob/v1.0.0/specs/phase0/beacon-chain.md#compute_domain
func compute_domain*(
domain_type: DomainType,
fork_version: Version,
genesis_validators_root: Eth2Digest = ZERO_HASH): Domain =
## Return the domain for the ``domain_type`` and ``fork_version``.
let fork_data_root =
compute_fork_data_root(fork_version, genesis_validators_root)
result[0..3] = uint_to_bytes4(domain_type.uint64)
result[4..31] = fork_data_root.data.toOpenArray(0, 27)
# https://github.com/ethereum/eth2.0-specs/blob/v1.0.0/specs/phase0/beacon-chain.md#get_domain
func get_domain*(
fork: Fork, domain_type: DomainType, epoch: Epoch, genesis_validators_root: Eth2Digest): Domain =
## Return the signature domain (fork version concatenated with domain type)
## of a message.
let fork_version =
if epoch < fork.epoch:
fork.previous_version
else:
fork.current_version
compute_domain(domain_type, fork_version, genesis_validators_root)
func get_domain*(
state: BeaconState, domain_type: DomainType, epoch: Epoch): Domain =
## Return the signature domain (fork version concatenated with domain type)
## of a message.
get_domain(state.fork, domain_type, epoch, state.genesis_validators_root)
# https://github.com/ethereum/eth2.0-specs/blob/v1.0.0/specs/phase0/beacon-chain.md#compute_signing_root
func compute_signing_root*(ssz_object: auto, domain: Domain): Eth2Digest =
## Return the signing root of an object by calculating the root of the
## object-domain tree.
let domain_wrapped_object = SigningData(
object_root: hash_tree_root(ssz_object),
domain: domain
)
hash_tree_root(domain_wrapped_object)
# https://github.com/ethereum/eth2.0-specs/blob/v1.0.0/specs/phase0/beacon-chain.md#get_seed
func get_seed*(state: BeaconState, epoch: Epoch, domain_type: DomainType): Eth2Digest =
## Return the seed at ``epoch``.
var seed_input : array[4+8+32, byte]
# Detect potential underflow
static:
doAssert EPOCHS_PER_HISTORICAL_VECTOR > MIN_SEED_LOOKAHEAD
seed_input[0..3] = uint_to_bytes4(domain_type.uint64)
seed_input[4..11] = uint_to_bytes8(epoch.uint64)
seed_input[12..43] =
get_randao_mix(state, # Avoid underflow
epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD - 1).data
eth2digest(seed_input)