database state storage benchmarking via ncli_db (#2312)
* database state storage benchmarking via ncli_db * more cleanups from immutable validator state branch * unexport some eth2_network constants and remove unused variables/templates * make two PeerScore constants public
This commit is contained in:
parent
946608b572
commit
5cab17dc1a
|
@ -2,11 +2,11 @@
|
|||
|
||||
import
|
||||
typetraits, tables,
|
||||
stew/[results, objects, endians2, io2],
|
||||
stew/[endians2, io2, objects, results],
|
||||
serialization, chronicles, snappy,
|
||||
eth/db/[kvstore, kvstore_sqlite3],
|
||||
./network_metadata,
|
||||
./spec/[datatypes, digest, crypto, state_transition],
|
||||
./spec/[crypto, datatypes, digest, state_transition],
|
||||
./ssz/[ssz_serialization, merkleization],
|
||||
merkle_minimal, filepath
|
||||
|
||||
|
@ -235,10 +235,8 @@ proc init*(T: type BeaconChainDB,
|
|||
if sqliteStore.exec("DROP TABLE IF EXISTS deposits;").isErr:
|
||||
debug "Failed to drop the deposits table"
|
||||
|
||||
var
|
||||
validatorKeyToIndex = initTable[ValidatorPubKey, ValidatorIndex]()
|
||||
genesisDepositsSeq = DbSeq[DepositData].init(sqliteStore, "genesis_deposits")
|
||||
|
||||
var genesisDepositsSeq =
|
||||
DbSeq[DepositData].init(sqliteStore, "genesis_deposits")
|
||||
|
||||
T(backend: kvStore sqliteStore,
|
||||
preset: preset,
|
||||
|
|
|
@ -210,9 +210,6 @@ const
|
|||
clientId* = "Nimbus beacon node " & fullVersionStr
|
||||
nodeMetadataFilename = "node-metadata.json"
|
||||
|
||||
TCP = net.Protocol.IPPROTO_TCP
|
||||
HandshakeTimeout = FaultOrError
|
||||
|
||||
NewPeerScore* = 200
|
||||
## Score which will be assigned to new connected Peer
|
||||
PeerScoreLowLimit* = 0
|
||||
|
@ -221,34 +218,29 @@ const
|
|||
## Max value of peer's score
|
||||
PeerScoreInvalidRequest* = -500
|
||||
## This peer is sending malformed or nonsensical data
|
||||
PeerScoreFlooder* = -250
|
||||
## This peer is sending too many expensive requests
|
||||
|
||||
ConcurrentConnections* = 10
|
||||
ConcurrentConnections = 10
|
||||
## Maximum number of active concurrent connection requests.
|
||||
|
||||
SeenTableTimeTimeout* =
|
||||
SeenTableTimeTimeout =
|
||||
when not defined(local_testnet): 5.minutes else: 10.seconds
|
||||
|
||||
## Seen period of time for timeout connections
|
||||
SeenTableTimeDeadPeer* =
|
||||
SeenTableTimeDeadPeer =
|
||||
when not defined(local_testnet): 5.minutes else: 10.seconds
|
||||
|
||||
## Period of time for dead peers.
|
||||
SeenTableTimeIrrelevantNetwork* = 24.hours
|
||||
SeenTableTimeIrrelevantNetwork = 24.hours
|
||||
## Period of time for `IrrelevantNetwork` error reason.
|
||||
SeenTableTimeClientShutDown* = 10.minutes
|
||||
SeenTableTimeClientShutDown = 10.minutes
|
||||
## Period of time for `ClientShutDown` error reason.
|
||||
SeenTableTimeFaultOrError* = 10.minutes
|
||||
SeenTableTimeFaultOrError = 10.minutes
|
||||
## Period of time for `FaultOnError` error reason.
|
||||
SeenTablePenaltyError* = 60.minutes
|
||||
SeenTablePenaltyError = 60.minutes
|
||||
## Period of time for peers which score below or equal to zero.
|
||||
SeenTableTimeReconnect* = 1.minutes
|
||||
SeenTableTimeReconnect = 1.minutes
|
||||
## Minimal time between disconnection and reconnection attempt
|
||||
|
||||
ResolvePeerTimeout* = 1.minutes
|
||||
## Maximum time allowed for peer resolve process.
|
||||
|
||||
template neterr(kindParam: Eth2NetworkingErrorKind): auto =
|
||||
err(type(result), Eth2NetworkingError(kind: kindParam))
|
||||
|
||||
|
|
|
@ -1368,7 +1368,7 @@ proc handleValidatorExitCommand(config: BeaconNodeConf) {.async.} =
|
|||
fatal "Failed to connect to the beacon node RPC service", err = err.msg
|
||||
quit 1
|
||||
|
||||
let (validator, validatorIdx, status, balance) = try:
|
||||
let (validator, validatorIdx, _, _) = try:
|
||||
await rpcClient.get_v1_beacon_states_stateId_validators_validatorId(
|
||||
"head", config.exitedValidator)
|
||||
except CatchableError as err:
|
||||
|
@ -1431,7 +1431,7 @@ proc handleValidatorExitCommand(config: BeaconNodeConf) {.async.} =
|
|||
try:
|
||||
stdout.write prompt, ": "
|
||||
stdin.readLine()
|
||||
except IOError as err:
|
||||
except IOError:
|
||||
fatal "Failed to read user input from stdin"
|
||||
quit 1
|
||||
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
# beacon_chain
|
||||
# Copyright (c) 2018-2020 Status Research & Development GmbH
|
||||
# Copyright (c) 2018-2021 Status Research & Development GmbH
|
||||
# Licensed and distributed under either of
|
||||
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
||||
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
||||
|
@ -31,8 +31,6 @@ import
|
|||
|
||||
logScope: topics = "vc"
|
||||
|
||||
template sourceDir: string = currentSourcePath.rsplit(DirSep, 1)[0]
|
||||
|
||||
type
|
||||
ValidatorClient = ref object
|
||||
config: ValidatorClientConf
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
# Copyright (c) 2018-2020 Status Research & Development GmbH
|
||||
# Copyright (c) 2018-2021 Status Research & Development GmbH
|
||||
# Licensed and distributed under either of
|
||||
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
||||
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
||||
|
@ -16,7 +16,7 @@ import
|
|||
../eth1_monitor, ../validator_duties,
|
||||
../spec/[digest, datatypes, presets],
|
||||
|
||||
libp2p/protocols/pubsub/[gossipsub, pubsubpeer]
|
||||
libp2p/protocols/pubsub/pubsubpeer
|
||||
|
||||
|
||||
logScope: topics = "nimbusapi"
|
||||
|
|
|
@ -656,6 +656,11 @@ type
|
|||
DoppelgangerProtection* = object
|
||||
broadcastStartEpoch*: Epoch
|
||||
|
||||
func getImmutableValidatorData*(validator: Validator): ImmutableValidatorData =
|
||||
ImmutableValidatorData(
|
||||
pubkey: validator.pubkey,
|
||||
withdrawal_credentials: validator.withdrawal_credentials)
|
||||
|
||||
func getDepositMessage*(depositData: DepositData): DepositMessage =
|
||||
result.pubkey = depositData.pubkey
|
||||
result.amount = depositData.amount
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
# beacon_chain
|
||||
# Copyright (c) 2020 Status Research & Development GmbH
|
||||
# Copyright (c) 2020-2021 Status Research & Development GmbH
|
||||
# Licensed and distributed under either of
|
||||
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
||||
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
||||
|
@ -26,11 +26,6 @@ func applyModIncrement[T, U](
|
|||
ha[indexSlot mod U.uint64] = item
|
||||
indexSlot += 1
|
||||
|
||||
func getImmutableValidatorData*(validator: Validator): ImmutableValidatorData =
|
||||
ImmutableValidatorData(
|
||||
pubkey: validator.pubkey,
|
||||
withdrawal_credentials: validator.withdrawal_credentials)
|
||||
|
||||
func applyValidatorIdentities(
|
||||
validators: var HashList[Validator, Limit VALIDATOR_REGISTRY_LIMIT],
|
||||
hl: auto) =
|
||||
|
@ -39,20 +34,6 @@ func applyValidatorIdentities(
|
|||
pubkey: item.pubkey,
|
||||
withdrawal_credentials: item.withdrawal_credentials)
|
||||
|
||||
func getValidatorStatus(validator: Validator): ValidatorStatus =
|
||||
ValidatorStatus(
|
||||
effective_balance: validator.effective_balance,
|
||||
slashed: validator.slashed,
|
||||
activation_eligibility_epoch: validator.activation_eligibility_epoch,
|
||||
activation_epoch: validator.activation_epoch,
|
||||
exit_epoch: validator.exit_epoch,
|
||||
withdrawable_epoch: validator.withdrawable_epoch)
|
||||
|
||||
func getValidatorStatuses(state: BeaconState):
|
||||
List[ValidatorStatus, Limit VALIDATOR_REGISTRY_LIMIT] =
|
||||
for validator in state.validators:
|
||||
result.add getValidatorStatus(validator)
|
||||
|
||||
func setValidatorStatuses(
|
||||
validators: var HashList[Validator, Limit VALIDATOR_REGISTRY_LIMIT],
|
||||
hl: List[ValidatorStatus, Limit VALIDATOR_REGISTRY_LIMIT]) =
|
||||
|
@ -115,6 +96,21 @@ func replaceOrAddDecodeEth1Votes[T, U](
|
|||
for item in votes1:
|
||||
votes0.add item
|
||||
|
||||
func getMutableValidatorStatus(validator: Validator): ValidatorStatus =
|
||||
ValidatorStatus(
|
||||
effective_balance: validator.effective_balance,
|
||||
slashed: validator.slashed,
|
||||
activation_eligibility_epoch: validator.activation_eligibility_epoch,
|
||||
activation_epoch: validator.activation_epoch,
|
||||
exit_epoch: validator.exit_epoch,
|
||||
withdrawable_epoch: validator.withdrawable_epoch)
|
||||
|
||||
func getMutableValidatorStatuses(state: BeaconState):
|
||||
List[ValidatorStatus, Limit VALIDATOR_REGISTRY_LIMIT] =
|
||||
# use mapIt + .init(foo)?
|
||||
for validator in state.validators:
|
||||
result.add getMutableValidatorStatus(validator)
|
||||
|
||||
func diffStates*(state0, state1: BeaconState): BeaconStateDiff =
|
||||
doAssert state1.slot > state0.slot
|
||||
doAssert state0.slot.isEpoch
|
||||
|
@ -149,7 +145,7 @@ func diffStates*(state0, state1: BeaconState): BeaconStateDiff =
|
|||
eth1_data_votes: eth1_data_votes,
|
||||
eth1_deposit_index: state1.eth1_deposit_index,
|
||||
|
||||
validatorStatuses: getValidatorStatuses(state1),
|
||||
validatorStatuses: getMutableValidatorStatuses(state1),
|
||||
balances: deltaEncodeBalances(state1.balances),
|
||||
|
||||
# RANDAO mixes gets updated every block, in place
|
||||
|
|
|
@ -3,7 +3,7 @@ import
|
|||
chronicles, confutils, stew/byteutils, eth/db/kvstore_sqlite3,
|
||||
../beacon_chain/network_metadata,
|
||||
../beacon_chain/[beacon_chain_db, extras],
|
||||
../beacon_chain/block_pools/[chain_dag],
|
||||
../beacon_chain/block_pools/chain_dag,
|
||||
../beacon_chain/spec/[crypto, datatypes, digest, helpers,
|
||||
state_transition, presets],
|
||||
../beacon_chain/[ssz, sszdump],
|
||||
|
@ -16,7 +16,7 @@ type Timers = enum
|
|||
tAdvanceSlot = "Advance slot, non-epoch"
|
||||
tAdvanceEpoch = "Advance slot, epoch"
|
||||
tApplyBlock = "Apply block, no slot processing"
|
||||
tDbStore = "Database block store"
|
||||
tDbStore = "Database store"
|
||||
|
||||
type
|
||||
DbCmd* = enum
|
||||
|
@ -50,6 +50,9 @@ type
|
|||
storeBlocks* {.
|
||||
defaultValue: false
|
||||
desc: "Store each read block back into a separate database".}: bool
|
||||
storeStates* {.
|
||||
defaultValue: false
|
||||
desc: "Store a state each epoch into a separate database".}: bool
|
||||
printTimes* {.
|
||||
defaultValue: true
|
||||
desc: "Print csv of block processing time".}: bool
|
||||
|
@ -93,7 +96,9 @@ proc cmdBench(conf: DbConf, runtimePreset: RuntimePreset) =
|
|||
let
|
||||
db = BeaconChainDB.init(runtimePreset, conf.databaseDir.string)
|
||||
dbBenchmark = BeaconChainDB.init(runtimePreset, "benchmark")
|
||||
defer: db.close()
|
||||
defer:
|
||||
db.close()
|
||||
dbBenchmark.close()
|
||||
|
||||
if not ChainDAGRef.isInitialized(db):
|
||||
echo "Database not initialized"
|
||||
|
@ -152,6 +157,12 @@ proc cmdBench(conf: DbConf, runtimePreset: RuntimePreset) =
|
|||
withTimer(timers[tDbStore]):
|
||||
dbBenchmark.putBlock(b)
|
||||
|
||||
if conf.storeStates:
|
||||
withTimer(timers[tDbStore]):
|
||||
if state[].data.slot mod SLOTS_PER_EPOCH == 0:
|
||||
dbBenchmark.putState(state[].root, state[].data)
|
||||
dbBenchmark.checkpoint()
|
||||
|
||||
printTimers(false, timers)
|
||||
|
||||
proc cmdDumpState(conf: DbConf, preset: RuntimePreset) =
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
# beacon_chain
|
||||
# Copyright (c) 2019-2020 Status Research & Development GmbH
|
||||
# Copyright (c) 2019-2021 Status Research & Development GmbH
|
||||
# Licensed and distributed under either of
|
||||
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
||||
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
||||
|
@ -42,7 +42,7 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
|
|||
validate = true):
|
||||
let
|
||||
flags = if validate: {} else: {skipBlsValidation}
|
||||
(state, depositContractState) = loadGenesis(validators, validate)
|
||||
(state, _) = loadGenesis(validators, validate)
|
||||
genesisBlock = get_initial_beacon_block(state.data)
|
||||
|
||||
echo "Starting simulation..."
|
||||
|
|
Loading…
Reference in New Issue