database state storage benchmarking via ncli_db (#2312)

* database state storage benchmarking via ncli_db

* more cleanups from immutable validator state branch

* unexport some eth2_network constants and remove unused variables/templates

* make two PeerScore constants public

beacon_chain/beacon_chain_db.nim

@@ -2,11 +2,11 @@
 
 import
   typetraits, tables,
-  stew/[results, objects, endians2, io2],
+  stew/[endians2, io2, objects, results],
   serialization, chronicles, snappy,
   eth/db/[kvstore, kvstore_sqlite3],
   ./network_metadata,
-  ./spec/[datatypes, digest, crypto, state_transition],
+  ./spec/[crypto, datatypes, digest, state_transition],
   ./ssz/[ssz_serialization, merkleization],
   merkle_minimal, filepath
 
@@ -235,10 +235,8 @@ proc init*(T: type BeaconChainDB,
     if sqliteStore.exec("DROP TABLE IF EXISTS deposits;").isErr:
       debug "Failed to drop the deposits table"
 
-    var
+    var genesisDepositsSeq =
-      validatorKeyToIndex = initTable[ValidatorPubKey, ValidatorIndex]()
+      DbSeq[DepositData].init(sqliteStore, "genesis_deposits")
-      genesisDepositsSeq = DbSeq[DepositData].init(sqliteStore, "genesis_deposits")
-
 
     T(backend: kvStore sqliteStore,
       preset: preset,

beacon_chain/eth2_network.nim

@@ -210,9 +210,6 @@ const
   clientId* = "Nimbus beacon node " & fullVersionStr
   nodeMetadataFilename = "node-metadata.json"
 
-  TCP = net.Protocol.IPPROTO_TCP
-  HandshakeTimeout = FaultOrError
-
   NewPeerScore* = 200
     ## Score which will be assigned to new connected Peer
   PeerScoreLowLimit* = 0
@@ -221,34 +218,29 @@ const
     ## Max value of peer's score
   PeerScoreInvalidRequest* = -500
     ## This peer is sending malformed or nonsensical data
-  PeerScoreFlooder* = -250
-    ## This peer is sending too many expensive requests
 
-  ConcurrentConnections* = 10
+  ConcurrentConnections = 10
     ## Maximum number of active concurrent connection requests.
 
-  SeenTableTimeTimeout* =
+  SeenTableTimeTimeout =
     when not defined(local_testnet): 5.minutes else: 10.seconds
 
     ## Seen period of time for timeout connections
-  SeenTableTimeDeadPeer* =
+  SeenTableTimeDeadPeer =
     when not defined(local_testnet): 5.minutes else: 10.seconds
 
     ## Period of time for dead peers.
-  SeenTableTimeIrrelevantNetwork* = 24.hours
+  SeenTableTimeIrrelevantNetwork = 24.hours
     ## Period of time for `IrrelevantNetwork` error reason.
-  SeenTableTimeClientShutDown* = 10.minutes
+  SeenTableTimeClientShutDown = 10.minutes
     ## Period of time for `ClientShutDown` error reason.
-  SeenTableTimeFaultOrError* = 10.minutes
+  SeenTableTimeFaultOrError = 10.minutes
     ## Period of time for `FaultOnError` error reason.
-  SeenTablePenaltyError* = 60.minutes
+  SeenTablePenaltyError = 60.minutes
     ## Period of time for peers which score below or equal to zero.
-  SeenTableTimeReconnect* = 1.minutes
+  SeenTableTimeReconnect = 1.minutes
     ## Minimal time between disconnection and reconnection attempt
 
-  ResolvePeerTimeout* = 1.minutes
-    ## Maximum time allowed for peer resolve process.
-
 template neterr(kindParam: Eth2NetworkingErrorKind): auto =
   err(type(result), Eth2NetworkingError(kind: kindParam))
 

beacon_chain/nimbus_beacon_node.nim

@@ -1368,7 +1368,7 @@ proc handleValidatorExitCommand(config: BeaconNodeConf) {.async.} =
     fatal "Failed to connect to the beacon node RPC service", err = err.msg
     quit 1
 
-  let (validator, validatorIdx, status, balance) = try:
+  let (validator, validatorIdx, _, _) = try:
     await rpcClient.get_v1_beacon_states_stateId_validators_validatorId(
       "head", config.exitedValidator)
   except CatchableError as err:
@@ -1431,7 +1431,7 @@ proc handleValidatorExitCommand(config: BeaconNodeConf) {.async.} =
     try:
       stdout.write prompt, ": "
       stdin.readLine()
-    except IOError as err:
+    except IOError:
       fatal "Failed to read user input from stdin"
       quit 1
 

beacon_chain/nimbus_validator_client.nim

@@ -1,5 +1,5 @@
 # beacon_chain
-# Copyright (c) 2018-2020 Status Research & Development GmbH
+# Copyright (c) 2018-2021 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -31,8 +31,6 @@ import
 
 logScope: topics = "vc"
 
-template sourceDir: string = currentSourcePath.rsplit(DirSep, 1)[0]
-
 type
   ValidatorClient = ref object
     config: ValidatorClientConf

beacon_chain/rpc/nimbus_api.nim

@@ -1,4 +1,4 @@
-# Copyright (c) 2018-2020 Status Research & Development GmbH
+# Copyright (c) 2018-2021 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -16,7 +16,7 @@ import
   ../eth1_monitor, ../validator_duties,
   ../spec/[digest, datatypes, presets],
 
-  libp2p/protocols/pubsub/[gossipsub, pubsubpeer]
+  libp2p/protocols/pubsub/pubsubpeer
 
 
 logScope: topics = "nimbusapi"

beacon_chain/spec/datatypes.nim

@@ -656,6 +656,11 @@ type
   DoppelgangerProtection* = object
     broadcastStartEpoch*: Epoch
 
+func getImmutableValidatorData*(validator: Validator): ImmutableValidatorData =
+  ImmutableValidatorData(
+    pubkey: validator.pubkey,
+    withdrawal_credentials: validator.withdrawal_credentials)
+
 func getDepositMessage*(depositData: DepositData): DepositMessage =
   result.pubkey = depositData.pubkey
   result.amount = depositData.amount

beacon_chain/statediff.nim

@@ -1,5 +1,5 @@
 # beacon_chain
-# Copyright (c) 2020 Status Research & Development GmbH
+# Copyright (c) 2020-2021 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -26,11 +26,6 @@ func applyModIncrement[T, U](
     ha[indexSlot mod U.uint64] = item
     indexSlot += 1
 
-func getImmutableValidatorData*(validator: Validator): ImmutableValidatorData =
-  ImmutableValidatorData(
-    pubkey: validator.pubkey,
-    withdrawal_credentials: validator.withdrawal_credentials)
-
 func applyValidatorIdentities(
     validators: var HashList[Validator, Limit VALIDATOR_REGISTRY_LIMIT],
     hl: auto) =
@@ -39,20 +34,6 @@ func applyValidatorIdentities(
       pubkey: item.pubkey,
       withdrawal_credentials: item.withdrawal_credentials)
 
-func getValidatorStatus(validator: Validator): ValidatorStatus =
-  ValidatorStatus(
-      effective_balance: validator.effective_balance,
-      slashed: validator.slashed,
-      activation_eligibility_epoch: validator.activation_eligibility_epoch,
-      activation_epoch: validator.activation_epoch,
-      exit_epoch: validator.exit_epoch,
-      withdrawable_epoch: validator.withdrawable_epoch)
-
-func getValidatorStatuses(state: BeaconState):
-    List[ValidatorStatus, Limit VALIDATOR_REGISTRY_LIMIT] =
-  for validator in state.validators:
-    result.add getValidatorStatus(validator)
-
 func setValidatorStatuses(
     validators: var HashList[Validator, Limit VALIDATOR_REGISTRY_LIMIT],
     hl: List[ValidatorStatus, Limit VALIDATOR_REGISTRY_LIMIT]) =
@@ -115,6 +96,21 @@ func replaceOrAddDecodeEth1Votes[T, U](
   for item in votes1:
     votes0.add item
 
+func getMutableValidatorStatus(validator: Validator): ValidatorStatus =
+  ValidatorStatus(
+      effective_balance: validator.effective_balance,
+      slashed: validator.slashed,
+      activation_eligibility_epoch: validator.activation_eligibility_epoch,
+      activation_epoch: validator.activation_epoch,
+      exit_epoch: validator.exit_epoch,
+      withdrawable_epoch: validator.withdrawable_epoch)
+
+func getMutableValidatorStatuses(state: BeaconState):
+    List[ValidatorStatus, Limit VALIDATOR_REGISTRY_LIMIT] =
+  # use mapIt + .init(foo)?
+  for validator in state.validators:
+    result.add getMutableValidatorStatus(validator)
+
 func diffStates*(state0, state1: BeaconState): BeaconStateDiff =
   doAssert state1.slot > state0.slot
   doAssert state0.slot.isEpoch
@@ -149,7 +145,7 @@ func diffStates*(state0, state1: BeaconState): BeaconStateDiff =
     eth1_data_votes: eth1_data_votes,
     eth1_deposit_index: state1.eth1_deposit_index,
 
-    validatorStatuses: getValidatorStatuses(state1),
+    validatorStatuses: getMutableValidatorStatuses(state1),
     balances: deltaEncodeBalances(state1.balances),
 
     # RANDAO mixes gets updated every block, in place

ncli/ncli_db.nim

@@ -3,7 +3,7 @@ import
   chronicles, confutils, stew/byteutils, eth/db/kvstore_sqlite3,
   ../beacon_chain/network_metadata,
   ../beacon_chain/[beacon_chain_db, extras],
-  ../beacon_chain/block_pools/[chain_dag],
+  ../beacon_chain/block_pools/chain_dag,
   ../beacon_chain/spec/[crypto, datatypes, digest, helpers,
                         state_transition, presets],
   ../beacon_chain/[ssz, sszdump],
@@ -16,7 +16,7 @@ type Timers = enum
   tAdvanceSlot = "Advance slot, non-epoch"
   tAdvanceEpoch = "Advance slot, epoch"
   tApplyBlock = "Apply block, no slot processing"
-  tDbStore = "Database block store"
+  tDbStore = "Database store"
 
 type
   DbCmd* = enum
@@ -50,6 +50,9 @@ type
       storeBlocks* {.
         defaultValue: false
         desc: "Store each read block back into a separate database".}: bool
+      storeStates* {.
+        defaultValue: false
+        desc: "Store a state each epoch into a separate database".}: bool
       printTimes* {.
         defaultValue: true
         desc: "Print csv of block processing time".}: bool
@@ -93,7 +96,9 @@ proc cmdBench(conf: DbConf, runtimePreset: RuntimePreset) =
   let
     db = BeaconChainDB.init(runtimePreset, conf.databaseDir.string)
     dbBenchmark = BeaconChainDB.init(runtimePreset, "benchmark")
-  defer: db.close()
+  defer:
+    db.close()
+    dbBenchmark.close()
 
   if not ChainDAGRef.isInitialized(db):
     echo "Database not initialized"
@@ -152,6 +157,12 @@ proc cmdBench(conf: DbConf, runtimePreset: RuntimePreset) =
       withTimer(timers[tDbStore]):
         dbBenchmark.putBlock(b)
 
+    if conf.storeStates:
+      withTimer(timers[tDbStore]):
+        if state[].data.slot mod SLOTS_PER_EPOCH == 0:
+          dbBenchmark.putState(state[].root, state[].data)
+        dbBenchmark.checkpoint()
+
   printTimers(false, timers)
 
 proc cmdDumpState(conf: DbConf, preset: RuntimePreset) =

research/state_sim.nim

@@ -1,5 +1,5 @@
 # beacon_chain
-# Copyright (c) 2019-2020 Status Research & Development GmbH
+# Copyright (c) 2019-2021 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -42,7 +42,7 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
        validate = true):
   let
     flags = if validate: {} else: {skipBlsValidation}
-    (state, depositContractState) = loadGenesis(validators, validate)
+    (state, _) = loadGenesis(validators, validate)
     genesisBlock = get_initial_beacon_block(state.data)
 
   echo "Starting simulation..."