status-im/nimbus-eth2
2
0
Fork 0
mirror of https://github.com/status-im/nimbus-eth2.git synced 2025-02-23 11:48:33 +00:00
Code Issues Projects Releases Wiki Activity

12x speedup on state sim with 100k validators sans BLS by caching get_beacon_proposer_index(...)

Browse Source
This commit is contained in:
Dustin Brody 2020-06-04 14:03:16 +02:00 committed by tersec
parent b22e5b6d1d
commit 3cb7896bab
11 changed files with 48 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
beacon_chain/spec/datatypes.nim
View File

@ -399,11 +399,11 @@ type
data*: BeaconState
root*: Eth2Digest # hash_tree_root(data)
# TODO remove some of these, or otherwise coordinate with EpochRef
StateCache* = object
shuffled_active_validator_indices*:
Table[Epoch, seq[ValidatorIndex]]
committee_count_cache*: Table[Epoch, uint64]
beacon_proposer_indices*: Table[Slot, Option[ValidatorIndex]]
func shortValidatorKey*(state: BeaconState, validatorIdx: int): string =
($state.validators[validatorIdx].pubkey)[0..7]

12
beacon_chain/spec/validator.nim
View File

@ -196,6 +196,12 @@ func compute_proposer_index(state: BeaconState, indices: seq[ValidatorIndex],
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#get_beacon_proposer_index
func get_beacon_proposer_index*(state: BeaconState, cache: var StateCache, slot: Slot):
Option[ValidatorIndex] =
try:
if slot in cache.beacon_proposer_indices:
return cache.beacon_proposer_indices[slot]
except KeyError:
raiseAssert("Cached entries are added before use")
# Return the beacon proposer index at the current slot.
let epoch = get_current_epoch(state)
@ -215,14 +221,16 @@ func get_beacon_proposer_index*(state: BeaconState, cache: var StateCache, slot:
indices =
sorted(cache.shuffled_active_validator_indices[epoch], system.cmp)
cache.beacon_proposer_indices[slot] =
compute_proposer_index(state, indices, seed)
cache.beacon_proposer_indices[slot]
except KeyError:
raiseAssert("Cached entries are added before use")
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#get_beacon_proposer_index
func get_beacon_proposer_index*(state: BeaconState, stateCache: var StateCache):
func get_beacon_proposer_index*(state: BeaconState, cache: var StateCache):
Option[ValidatorIndex] =
get_beacon_proposer_index(state, stateCache, state.slot)
get_beacon_proposer_index(state, cache, state.slot)
# Not from spec
# TODO: cache the results from this and reuse in subsequent calls to get_beacon_proposer_index

4
beacon_chain/state_transition.nim
View File

@ -273,11 +273,11 @@ proc makeBeaconBlock*(
graffiti: Eth2Digest,
attestations: seq[Attestation],
deposits: seq[Deposit],
rollback: RollbackHashedProc): Option[BeaconBlock] =
rollback: RollbackHashedProc,
cache: var StateCache): Option[BeaconBlock] =
## Create a block for the given state. The last block applied to it must be
## the one identified by parent_root and process_slots must be called up to
## the slot for which a block is to be created.
var cache = get_empty_per_epoch_cache()
# To create a block, we'll first apply a partial block to the state, skipping
# some validations.

4
beacon_chain/validator_duties.nim
View File

@ -176,6 +176,7 @@ proc makeBeaconBlockForHeadAndSlot*(node: BeaconNode,
doAssert v.addr == addr poolPtr.tmpState.data
poolPtr.tmpState = poolPtr.headState
var cache = get_empty_per_epoch_cache()
let message = makeBeaconBlock(
hashedState,
validator_index,
@ -185,7 +186,8 @@ proc makeBeaconBlockForHeadAndSlot*(node: BeaconNode,
graffiti,
node.attestationPool.getAttestationsForBlock(state),
deposits,
restore)
restore,
cache)
if message.isSome():
# TODO this restore is needed because otherwise tmpState will be internally

3
ncli/ncli_query.nim
View File

@ -1,8 +1,7 @@
import
confutils, os, strutils, json_serialization,
stew/byteutils,
../beacon_chain/spec/[crypto, datatypes],
../beacon_chain/ssz/dynamic_navigator
../beacon_chain/spec/[crypto, datatypes]
type
QueryCmd* = enum

3
research/block_sim.nim
View File

@ -117,7 +117,8 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
Eth2Digest(),
attPool.getAttestationsForBlock(state),
@[],
noRollback)
noRollback,
cache)
var
newBlock = SignedBeaconBlock(

3
research/state_sim.nim
View File

@ -95,7 +95,8 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
withTimer(timers[t]):
signedBlock = addTestBlock(
state[], latest_block_root, attestations = blockAttestations, flags = flags)
state[], latest_block_root, cache, attestations = blockAttestations,
flags = flags)
latest_block_root = withTimerRet(timers[tHashBlock]):
hash_tree_root(signedBlock.message)

9
tests/test_attestation_pool.nim
View File

@ -149,8 +149,9 @@ suiteReport "Attestation pool processing" & preset():
attestations.len == 1
timedTest "Fork choice returns latest block with no attestations":
var cache = get_empty_per_epoch_cache()
let
b1 = addTestBlock(state.data, blockPool.tail.root)
b1 = addTestBlock(state.data, blockPool.tail.root, cache)
b1Root = hash_tree_root(b1.message)
b1Add = blockPool.add(b1Root, b1)[]
head = pool.selectHead()
@ -159,7 +160,7 @@ suiteReport "Attestation pool processing" & preset():
head == b1Add
let
b2 = addTestBlock(state.data, b1Root)
b2 = addTestBlock(state.data, b1Root, cache)
b2Root = hash_tree_root(b2.message)
b2Add = blockPool.add(b2Root, b2)[]
head2 = pool.selectHead()
@ -170,7 +171,7 @@ suiteReport "Attestation pool processing" & preset():
timedTest "Fork choice returns block with attestation":
var cache = get_empty_per_epoch_cache()
let
b10 = makeTestBlock(state.data, blockPool.tail.root)
b10 = makeTestBlock(state.data, blockPool.tail.root, cache)
b10Root = hash_tree_root(b10.message)
b10Add = blockPool.add(b10Root, b10)[]
head = pool.selectHead()
@ -179,7 +180,7 @@ suiteReport "Attestation pool processing" & preset():
head == b10Add
let
b11 = makeTestBlock(state.data, blockPool.tail.root,
b11 = makeTestBlock(state.data, blockPool.tail.root, cache,
graffiti = Eth2Digest(data: [1'u8, 0, 0, 0 ,0 ,0 ,0 ,0 ,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
)
b11Root = hash_tree_root(b11.message)

19
tests/test_block_pool.nim
View File

@ -10,7 +10,7 @@
import
options, sequtils, unittest,
./testutil, ./testblockutil,
../beacon_chain/spec/[datatypes, digest, helpers],
../beacon_chain/spec/[datatypes, digest, helpers, validator],
../beacon_chain/[beacon_node_types, block_pool, state_transition, ssz]
suiteReport "BlockRef and helpers" & preset():
@ -88,9 +88,10 @@ suiteReport "Block pool processing" & preset():
db = makeTestDB(SLOTS_PER_EPOCH)
pool = BlockPool.init(db)
stateData = newClone(pool.loadTailState())
b1 = addTestBlock(stateData.data, pool.tail.root)
cache = get_empty_per_epoch_cache()
b1 = addTestBlock(stateData.data, pool.tail.root, cache)
b1Root = hash_tree_root(b1.message)
b2 = addTestBlock(stateData.data, b1Root)
b2 = addTestBlock(stateData.data, b1Root, cache)
b2Root {.used.} = hash_tree_root(b2.message)
timedTest "getRef returns nil for missing blocks":
@ -132,7 +133,7 @@ suiteReport "Block pool processing" & preset():
process_slots(stateData.data, stateData.data.data.slot + 1)
let
b4 = addTestBlock(stateData.data, b2Root)
b4 = addTestBlock(stateData.data, b2Root, cache)
b4Root = hash_tree_root(b4.message)
b4Add = pool.add(b4Root, b4)[]
@ -284,12 +285,13 @@ when const_preset == "minimal": # These require some minutes in mainnet
var
db = makeTestDB(SLOTS_PER_EPOCH)
pool = BlockPool.init(db)
cache = get_empty_per_epoch_cache()
timedTest "prune heads on finalization" & preset():
block:
# Create a fork that will not be taken
var
blck = makeTestBlock(pool.headState.data, pool.head.blck.root)
blck = makeTestBlock(pool.headState.data, pool.head.blck.root, cache)
check: pool.add(hash_tree_root(blck.message), blck).isOk
for i in 0 ..< (SLOTS_PER_EPOCH * 6):
@ -299,9 +301,8 @@ when const_preset == "minimal": # These require some minutes in mainnet
pool.tail.children.len == 2
pool.heads.len == 2
var
cache = get_empty_per_epoch_cache()
blck = makeTestBlock(
pool.headState.data, pool.head.blck.root,
pool.headState.data, pool.head.blck.root, cache,
attestations = makeFullAttestations(
pool.headState.data.data, pool.head.blck.root,
pool.headState.data.data.slot, cache, {}))
@ -332,7 +333,7 @@ when const_preset == "minimal": # These require some minutes in mainnet
for i in 0 ..< (SLOTS_PER_EPOCH * 6 - 2):
var
blck = makeTestBlock(
pool.headState.data, pool.head.blck.root,
pool.headState.data, pool.head.blck.root, cache,
attestations = makeFullAttestations(
pool.headState.data.data, pool.head.blck.root,
pool.headState.data.data.slot, cache, {}))
@ -345,7 +346,7 @@ when const_preset == "minimal": # These require some minutes in mainnet
pool.headState.data, Slot(SLOTS_PER_EPOCH * 6 + 2) )
var blck = makeTestBlock(
pool.headState.data, pool.head.blck.root,
pool.headState.data, pool.head.blck.root, cache,
attestations = makeFullAttestations(
pool.headState.data.data, pool.head.blck.root,
pool.headState.data.data.slot, cache, {}))

8
tests/test_state_transition.nim
View File

@ -36,7 +36,8 @@ suiteReport "Block processing" & preset():
timedTest "Passes from genesis state, empty block" & preset():
var
previous_block_root = hash_tree_root(genesisBlock.message)
new_block = makeTestBlock(state[], previous_block_root)
cache = get_empty_per_epoch_cache()
new_block = makeTestBlock(state[], previous_block_root, cache)
let block_ok = state_transition(state[], new_block, {}, noRollback)
@ -53,9 +54,10 @@ suiteReport "Block processing" & preset():
timedTest "Passes through epoch update, empty block" & preset():
var
previous_block_root = genesisRoot
cache = get_empty_per_epoch_cache()
for i in 1..SLOTS_PER_EPOCH.int:
let new_block = makeTestBlock(state[], previous_block_root)
let new_block = makeTestBlock(state[], previous_block_root, cache)
let block_ok = state_transition(state[], new_block, {}, noRollback)
@ -90,7 +92,7 @@ suiteReport "Block processing" & preset():
state[], GENESIS_SLOT + MIN_ATTESTATION_INCLUSION_DELAY + 1)
let
new_block = makeTestBlock(state[], previous_block_root,
new_block = makeTestBlock(state[], previous_block_root, cache,
attestations = @[attestation]
)
check state_transition(state[], new_block, {}, noRollback)

9
tests/testblockutil.nim
View File

@ -89,13 +89,13 @@ func signBlock*(
proc addTestBlock*(
state: var HashedBeaconState,
parent_root: Eth2Digest,
cache: var StateCache,
eth1_data = Eth1Data(),
attestations = newSeq[Attestation](),
deposits = newSeq[Deposit](),
graffiti = Eth2Digest(),
flags: set[UpdateFlag] = {}): SignedBeaconBlock =
# Create and add a block to state - state will advance by one slot!
var cache = get_empty_per_epoch_cache()
advance_slot(state, err(Opt[Eth2Digest]), flags, cache)
let
@ -122,7 +122,8 @@ proc addTestBlock*(
graffiti,
attestations,
deposits,
noRollback)
noRollback,
cache)
doAssert message.isSome(), "Should have created a valid block!"
@ -136,6 +137,7 @@ proc addTestBlock*(
proc makeTestBlock*(
state: HashedBeaconState,
parent_root: Eth2Digest,
cache: var StateCache,
eth1_data = Eth1Data(),
attestations = newSeq[Attestation](),
deposits = newSeq[Deposit](),
@ -147,7 +149,8 @@ proc makeTestBlock*(
# because the block includes the state root.
var tmpState = newClone(state)
addTestBlock(
tmpState[], parent_root, eth1_data, attestations, deposits, graffiti, flags)
tmpState[], parent_root, cache, eth1_data, attestations, deposits,
graffiti, flags)
proc makeAttestation*(
state: BeaconState, beacon_block_root: Eth2Digest,