From b49003988a291626b32b94f65eca54ad0734314a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mamy=20Andr=C3=A9-Ratsimbazafy?= Date: Thu, 9 Jan 2020 16:29:41 +0100 Subject: [PATCH] Add Epoch Processing to nbench --- beacon_chain/spec/state_transition_epoch.nim | 2 +- nbench/README.md | 16 ++++- nbench/nbench.nim | 22 +++++++ nbench/nbench_official_fixtures.nim | 23 ++++++++ nbench/scenarios.nim | 62 ++++++++++++++++---- 5 files changed, 110 insertions(+), 15 deletions(-) diff --git a/beacon_chain/spec/state_transition_epoch.nim b/beacon_chain/spec/state_transition_epoch.nim index 5e30b2a12..2e2daa9a8 100644 --- a/beacon_chain/spec/state_transition_epoch.nim +++ b/beacon_chain/spec/state_transition_epoch.nim @@ -351,7 +351,7 @@ func process_rewards_and_penalties( decrease_balance(state, i.ValidatorIndex, penalties[i]) # https://github.com/ethereum/eth2.0-specs/blob/v0.9.4/specs/core/0_beacon-chain.md#slashings -func process_slashings*(state: var BeaconState) = +func process_slashings*(state: var BeaconState) {.nbench.}= let epoch = get_current_epoch(state) total_balance = get_total_active_balance(state) diff --git a/nbench/README.md b/nbench/README.md index d5cc7f1ae..37814af74 100644 --- a/nbench/README.md +++ b/nbench/README.md @@ -24,7 +24,7 @@ Features ``` nim c -d:const_preset=mainnet -d:nbench -d:release -o:build/nbench nbench/nbench.nim -export SCENARIOS=tests/official/fixtures/tests-v0.9.3/mainnet/phase0 +export SCENARIOS=tests/official/fixtures/tests-v0.9.4/mainnet/phase0 # Full state transition build/nbench cmdFullStateTransition -d="${SCENARIOS}"/sanity/blocks/pyspec_tests/voluntary_exit/ -q=2 @@ -32,6 +32,18 @@ build/nbench cmdFullStateTransition -d="${SCENARIOS}"/sanity/blocks/pyspec_tests # Slot processing build/nbench cmdSlotProcessing -d="${SCENARIOS}"/sanity/slots/pyspec_tests/slots_1 +# Justification-Finalisation +build/nbench cmdEpochProcessing --epochProcessingCat=catJustificationFinalization -d="${SCENARIOS}"/epoch_processing/justification_and_finalization/pyspec_tests/234_ok_support/ + +# Registry updates +build/nbench cmdEpochProcessing --epochProcessingCat=catRegistryUpdates -d="${SCENARIOS}"/epoch_processing/registry_updates/pyspec_tests/activation_queue_efficiency/ + +# Slashings +build/nbench cmdEpochProcessing --epochProcessingCat=catSlashings -d="${SCENARIOS}"/epoch_processing/slashings/pyspec_tests/max_penalties/ + +# Final updates +build/nbench cmdEpochProcessing --epochProcessingCat=catFinalUpdates -d="${SCENARIOS}"/epoch_processing/final_updates/pyspec_tests/effective_balance_hysteresis/ + # Block header processing build/nbench cmdBlockProcessing --blockProcessingCat=catBlockHeader -d="${SCENARIOS}"/operations/block_header/pyspec_tests/proposer_slashed/ @@ -59,7 +71,7 @@ Furthermore benchmarks are run in parallel and might interfere which each other. ``` nim c -d:const_preset=mainnet -d:nbench -d:release -o:build/nbench nbench/nbench.nim nim c -o:build/nbench_tests nbench/nbench_official_fixtures.nim -nbench_tests --nbench=build/nbench --tests=tests/official/fixtures/tests-v0.9.4/mainnet/ +build/nbench_tests --nbench=build/nbench --tests=tests/official/fixtures/tests-v0.9.4/mainnet/ ``` ## TODO Reporting diff --git a/nbench/nbench.nim b/nbench/nbench.nim index d3677cdd2..86bfc7821 100644 --- a/nbench/nbench.nim +++ b/nbench/nbench.nim @@ -99,6 +99,28 @@ proc main() = ) else: quit "Unsupported" + of cmdEpochProcessing: + case scenario.epochProcessingCat + of catJustificationFinalization: + runProcessJustificationFinalization( + scenario.scenarioDir.string, + scenario.preState + ) + of catRegistryUpdates: + runProcessRegistryUpdates( + scenario.scenarioDir.string, + scenario.preState + ) + of catSlashings: + runProcessSlashings( + scenario.scenarioDir.string, + scenario.preState + ) + of catFinalUpdates: + runProcessFinalUpdates( + scenario.scenarioDir.string, + scenario.preState + ) else: quit "Unsupported" diff --git a/nbench/nbench_official_fixtures.nim b/nbench/nbench_official_fixtures.nim index 5e0363273..66756f58e 100644 --- a/nbench/nbench_official_fixtures.nim +++ b/nbench/nbench_official_fixtures.nim @@ -28,9 +28,13 @@ proc collectTarget(cmds: var CmdLists, nbench, name, cmd, cat, path: string) = var cat = cat if cmd == "cmdBlockProcessing": cat = "--blockProcessingCat=" & cat + elif cmd == "cmdEpochProcessing": + cat = "--epochProcessingCat=" & cat cmds.add &"{nbench} {cmd} {cat} -d={path/folder}" proc collectBenchTargets(nbench, basePath: string): CmdLists = + # State processing + # ------------------------------------------------------------------------- block: # Full state transitions echo "----------------------------------------" echo "Collecting full state transitions" @@ -42,9 +46,27 @@ proc collectBenchTargets(nbench, basePath: string): CmdLists = inc countBlocks echo "Found: ", folder, " with ", countBlocks, " blocks" result.add &"{nbench} cmdFullStateTransition -d={path/folder} -q={$countBlocks}" + # Slot processing + # ------------------------------------------------------------------------- block: # Slot processing let path = basePath/"phase0"/"sanity"/"slots"/"pyspec_tests" result.collectTarget(nbench, "slot", "cmdSlotProcessing", "", path) + # Epoch processing + # ------------------------------------------------------------------------- + block: # Justification-Finalization + let path = basePath/"phase0"/"epoch_processing"/"justification_and_finalization"/"pyspec_tests" + result.collectTarget(nbench, "justification_and_finalization", "cmdEpochProcessing", "catJustificationFinalization", path) + block: # Registry updates + let path = basePath/"phase0"/"epoch_processing"/"justification_and_finalization"/"pyspec_tests" + result.collectTarget(nbench, "registry_updates", "cmdEpochProcessing", "catRegistryUpdates", path) + block: # Slashings + let path = basePath/"phase0"/"epoch_processing"/"slashings"/"pyspec_tests" + result.collectTarget(nbench, "slashings", "cmdEpochProcessing", "catSlashings", path) + block: # Justification-Finalization + let path = basePath/"phase0"/"epoch_processing"/"final_updates"/"pyspec_tests" + result.collectTarget(nbench, "final_updates", "cmdEpochProcessing", "catFinalUpdates", path) + # Block processing + # ------------------------------------------------------------------------- block: # Attestation let path = basePath/"phase0"/"operations"/"attestation"/"pyspec_tests" result.collectTarget(nbench, "attestation", "cmdBlockProcessing", "catAttestations", path) @@ -66,5 +88,6 @@ proc collectBenchTargets(nbench, basePath: string): CmdLists = cli do(nbench: string, tests: string): let cmdLists = collectBenchTargets(nbench, tests) + echo "\n========================================================\n" let err = execProcesses(cmdLists) quit err diff --git a/nbench/scenarios.nim b/nbench/scenarios.nim index 9bfa9e858..3f5bd4e8c 100644 --- a/nbench/scenarios.nim +++ b/nbench/scenarios.nim @@ -11,7 +11,7 @@ import # Status libraries confutils/defs, serialization, # Beacon-chain - ../beacon_chain/spec/[datatypes, crypto, beaconstate, validator, state_transition_block], + ../beacon_chain/spec/[datatypes, crypto, beaconstate, validator, state_transition_block, state_transition_epoch], ../beacon_chain/[ssz, state_transition, extras] # Nimbus Bench - Scenario configuration @@ -35,6 +35,13 @@ type catDeposits catVoluntaryExits + EpochProcessingCat* = enum + catFinalUpdates + catJustificationFinalization + catRegistryUpdates + catSlashings + # catRewardsPenalties # no upstream tests + ScenarioConf* = object scenarioDir* {. desc: "The directory of your benchmark scenario" @@ -114,7 +121,7 @@ type name: "voluntary_exit" defaultValue: "voluntary_exit".}: string of cmdEpochProcessing: - discard + epochProcessingCat*: EpochProcessingCat proc parseSSZ(path: string, T: typedesc): T = try: @@ -157,7 +164,33 @@ proc runProcessSlots*(dir, preState: string, numSlots: uint64) = process_slots(state[], state.slot + numSlots) -template processScenarioImpl( +template processEpochScenarioImpl( + dir, preState: string, + transitionFn: untyped, + needCache: static bool): untyped = + let prePath = dir/preState & ".ssz" + + var state: ref BeaconState + new state + echo "Running: ", prePath + state[] = parseSSZ(prePath, BeaconState) + + when needCache: + var cache = get_empty_per_epoch_cache() + + # Epoch transitions can't fail (TODO is this true?) + when needCache: + transitionFn(state[], cache) + else: + transitionFn(state[]) + + echo astToStr(transitionFn) & " status: ", "Done" # if success: "SUCCESS ✓" else: "FAILURE ⚠️" + +template genProcessEpochScenario(name, transitionFn: untyped, needCache: static bool): untyped = + proc `name`*(dir, preState: string) = + processEpochScenarioImpl(dir, preState, transitionFn, needCache) + +template processBlockScenarioImpl( dir, preState: string, skipBLS: bool, transitionFn, paramName: untyped, ConsensusObject: typedesc, @@ -192,18 +225,23 @@ template processScenarioImpl( echo astToStr(transitionFn) & " status: ", if success: "SUCCESS ✓" else: "FAILURE ⚠️" -template genProcessScenario(name, transitionFn, paramName: untyped, ConsensusObject: typedesc, needFlags, needCache: static bool): untyped = +template genProcessBlockScenario(name, transitionFn, paramName: untyped, ConsensusObject: typedesc, needFlags, needCache: static bool): untyped = when needFlags: proc `name`*(dir, preState, `paramName`: string, skipBLS: bool) = - processScenarioImpl(dir, preState, skipBLS, transitionFn, paramName, ConsensusObject, needFlags, needCache) + processBlockScenarioImpl(dir, preState, skipBLS, transitionFn, paramName, ConsensusObject, needFlags, needCache) else: proc `name`*(dir, preState, `paramName`: string) = # skipBLS is a dummy to avoid undeclared identifier - processScenarioImpl(dir, preState, skipBLS = false, transitionFn, paramName, ConsensusObject, needFlags, needCache) + processBlockScenarioImpl(dir, preState, skipBLS = false, transitionFn, paramName, ConsensusObject, needFlags, needCache) -genProcessScenario(runProcessBlockHeader, process_block_header, block_header, BeaconBlock, needFlags = true, needCache = true) -genProcessScenario(runProcessProposerSlashing, process_proposer_slashing, proposer_slashing, ProposerSlashing, needFlags = true, needCache = true) -genProcessScenario(runProcessAttestation, process_attestation, attestation, Attestation, needFlags = true, needCache = true) -genProcessScenario(runProcessAttesterSlashing, process_attester_slashing, att_slash, AttesterSlashing, needFlags = false, needCache = true) -genProcessScenario(runProcessDeposit, process_deposit, deposit, Deposit, needFlags = true, needCache = false) -genProcessScenario(runProcessVoluntaryExits, process_voluntary_exit, deposit, SignedVoluntaryExit, needFlags = true, needCache = false) +genProcessEpochScenario(runProcessJustificationFinalization, process_justification_and_finalization, needCache = true) +genProcessEpochScenario(runProcessRegistryUpdates, process_registry_updates, needCache = false) +genProcessEpochScenario(runProcessSlashings, process_slashings, needCache = false) +genProcessEpochScenario(runProcessFinalUpdates, process_final_updates, needCache = false) + +genProcessBlockScenario(runProcessBlockHeader, process_block_header, block_header, BeaconBlock, needFlags = true, needCache = true) +genProcessBlockScenario(runProcessProposerSlashing, process_proposer_slashing, proposer_slashing, ProposerSlashing, needFlags = true, needCache = true) +genProcessBlockScenario(runProcessAttestation, process_attestation, attestation, Attestation, needFlags = true, needCache = true) +genProcessBlockScenario(runProcessAttesterSlashing, process_attester_slashing, att_slash, AttesterSlashing, needFlags = false, needCache = true) +genProcessBlockScenario(runProcessDeposit, process_deposit, deposit, Deposit, needFlags = true, needCache = false) +genProcessBlockScenario(runProcessVoluntaryExits, process_voluntary_exit, deposit, SignedVoluntaryExit, needFlags = true, needCache = false)