diff --git a/beacon_chain/validators/slashing_protection_v2.nim b/beacon_chain/validators/slashing_protection_v2.nim index 32e327498..89dad240a 100644 --- a/beacon_chain/validators/slashing_protection_v2.nim +++ b/beacon_chain/validators/slashing_protection_v2.nim @@ -218,7 +218,7 @@ type internalIds: Table[ValidatorIndex, ValidatorInternalID] - ValidatorInternalID = int32 + ValidatorInternalID = int64 ## Validator internal ID in the DB ## This is cached to cost querying cost @@ -307,13 +307,13 @@ proc checkDB(db: SlashingProtectionDB_v2, genesis_validators_root: Eth2Digest) = ## Check the metadata of the DB let selectStmt = db.backend.prepareStmt( "SELECT * FROM metadata;", - NoParams, (int32, Hash32), + NoParams, (int64, Hash32), managed = false # manual memory management ).get() - var version: int32 + var version: int64 var root: Eth2Digest - let status = selectStmt.exec do (res: (int32, Hash32)): + let status = selectStmt.exec do (res: (int64, Hash32)): version = res[0] root.data = res[1] @@ -584,12 +584,12 @@ proc getMetadataTable_DbV2*(db: SlashingProtectionDB_v2): Option[Eth2Digest] = WHERE 1=1 and type='table' and name='metadata' - """, NoParams, int32, + """, NoParams, int64, managed = false # manual memory management ).get() - var hasV2: int32 - let v2exists = existenceStmt.exec do (res: int32): + var hasV2: int64 + let v2exists = existenceStmt.exec do (res: int64): hasV2 = res existenceStmt.dispose() @@ -602,13 +602,13 @@ proc getMetadataTable_DbV2*(db: SlashingProtectionDB_v2): Option[Eth2Digest] = let selectStmt = db.backend.prepareStmt( "SELECT * FROM metadata;", - NoParams, (int32, Hash32), + NoParams, (int64, Hash32), managed = false # manual memory management ).get() - var version: int32 + var version: int64 var root: Eth2Digest - let status = selectStmt.exec do (res: (int32, Hash32)): + let status = selectStmt.exec do (res: (int64, Hash32)): version = res[0] root.data = res[1] diff --git a/ncli/ncli_db.nim b/ncli/ncli_db.nim index 83393a682..e456a6b0e 100644 --- a/ncli/ncli_db.nim +++ b/ncli/ncli_db.nim @@ -5,7 +5,7 @@ import ../beacon_chain/[beacon_chain_db, extras], ../beacon_chain/consensus_object_pools/[blockchain_dag, statedata_helpers], ../beacon_chain/spec/[crypto, datatypes, digest, helpers, state_transition, - presets], + state_transition_epoch, presets], ../beacon_chain/ssz, ../beacon_chain/ssz/sszdump, ../research/simutils, ./e2store @@ -28,6 +28,7 @@ type rewindState exportEra validatorPerf + validatorDb = "Create or update attestation performance database" # TODO: # This should probably allow specifying a run-time preset @@ -115,9 +116,17 @@ type defaultValue: 0 name: "slots" desc: "Number of slots to run benchmark for, 0 = all the way to head".}: uint64 + of validatorDb: + outDir* {. + defaultValue: "" + name: "out-db" + desc: "Output database".}: string + perfect* {. + defaultValue: false + name: "perfect" + desc: "Include perfect records (full rewards)".}: bool -proc getBlockRange(dag: ChainDAGRef, startSlot: int64, count: uint64): seq[BlockRef] = - # Range of block in reverse order +proc getSlotRange(dag: ChainDAGRef, startSlot: int64, count: uint64): (Slot, Slot) = let start = if startSlot >= 0: Slot(startSlot) @@ -126,6 +135,10 @@ proc getBlockRange(dag: ChainDAGRef, startSlot: int64, count: uint64): seq[Block ends = if count == 0: dag.head.slot + 1 else: start + count + (start, ends) + +proc getBlockRange(dag: ChainDAGRef, start, ends: Slot): seq[BlockRef] = + # Range of block in reverse order var blockRefs: seq[BlockRef] cur = dag.head @@ -160,7 +173,8 @@ proc cmdBench(conf: DbConf, runtimePreset: RuntimePreset) = ChainDAGRef.init(runtimePreset, db, {}) var - blockRefs = dag.getBlockRange(conf.benchSlot, conf.benchSlots) + (start, ends) = dag.getSlotRange(conf.benchSlot, conf.benchSlots) + blockRefs = dag.getBlockRange(start, ends) blocks: seq[TrustedSignedBeaconBlock] echo &"Loaded {dag.blocks.len} blocks, head slot {dag.head.slot}, selected {blockRefs.len} blocks" @@ -438,7 +452,8 @@ proc cmdValidatorPerf(conf: DbConf, runtimePreset: RuntimePreset) = let dag = ChainDAGRef.init(runtimePreset, db, {}) var - blockRefs = dag.getBlockRange(conf.perfSlot, conf.perfSlots) + (start, ends) = dag.getSlotRange(conf.perfSlot, conf.perfSlots) + blockRefs = dag.getBlockRange(start, ends) perfs = newSeq[ValidatorPerformance]( getStateField(dag.headState, validators).len()) cache = StateCache() @@ -504,7 +519,6 @@ proc cmdValidatorPerf(conf: DbConf, runtimePreset: RuntimePreset) = else: perf.attestation_misses += 1; - for bi in 0.. ends: + echo "No (new) data found, database at ", minEpoch, ", finalized to ", ends.epoch + quit 1 + + let blockRefs = dag.getBlockRange(start, ends) + + echo "Analyzing performance for epochs ", + start.epoch, " - ", ends.epoch + + let state = newClone(dag.headState) + dag.updateStateData( + state[], blockRefs[^1].atSlot(if start > 0: start - 1 else: 0.Slot), + false, cache) + + proc processEpoch() = + echo getStateField(state[], slot).epoch + outDb.exec("BEGIN TRANSACTION;").expect("DB") + insertEpochInfo.exec( + (getStateField(state[], slot).epoch.int64, + rewards.total_balances.current_epoch_raw.int64, + rewards.total_balances.previous_epoch_raw.int64, + rewards.total_balances.current_epoch_attesters_raw.int64, + rewards.total_balances.current_epoch_target_attesters_raw.int64, + rewards.total_balances.previous_epoch_attesters_raw.int64, + rewards.total_balances.previous_epoch_target_attesters_raw.int64, + rewards.total_balances.previous_epoch_head_attesters_raw.int64) + ).expect("DB") + + for index, status in rewards.statuses.pairs(): + if not is_eligible_validator(status): + continue + let + notSlashed = (RewardFlags.isSlashed notin status.flags) + source_attester = + notSlashed and status.is_previous_epoch_attester.isSome() + target_attester = + notSlashed and RewardFlags.isPreviousEpochTargetAttester in status.flags + head_attester = + notSlashed and RewardFlags.isPreviousEpochHeadAttester in status.flags + delay = + if notSlashed and status.is_previous_epoch_attester.isSome(): + some(int64(status.is_previous_epoch_attester.get().delay)) + else: + none(int64) + + if conf.perfect or not + (source_attester and target_attester and head_attester and + delay.isSome() and delay.get() == 1): + insertValidatorInfo.exec( + (index.int64, + getStateField(state[], slot).epoch.int64, + status.delta.rewards.int64, + status.delta.penalties.int64, + int64(source_attester), # Source delta + int64(target_attester), # Target delta + int64(head_attester), # Head delta + delay)).expect("DB") + outDb.exec("COMMIT;").expect("DB") + + for bi in 0..