12x speedup on state sim with 100k validators sans BLS by caching get_beacon_proposer_index(...)
This commit is contained in:
parent
b22e5b6d1d
commit
3cb7896bab
|
@ -399,11 +399,11 @@ type
|
||||||
data*: BeaconState
|
data*: BeaconState
|
||||||
root*: Eth2Digest # hash_tree_root(data)
|
root*: Eth2Digest # hash_tree_root(data)
|
||||||
|
|
||||||
# TODO remove some of these, or otherwise coordinate with EpochRef
|
|
||||||
StateCache* = object
|
StateCache* = object
|
||||||
shuffled_active_validator_indices*:
|
shuffled_active_validator_indices*:
|
||||||
Table[Epoch, seq[ValidatorIndex]]
|
Table[Epoch, seq[ValidatorIndex]]
|
||||||
committee_count_cache*: Table[Epoch, uint64]
|
committee_count_cache*: Table[Epoch, uint64]
|
||||||
|
beacon_proposer_indices*: Table[Slot, Option[ValidatorIndex]]
|
||||||
|
|
||||||
func shortValidatorKey*(state: BeaconState, validatorIdx: int): string =
|
func shortValidatorKey*(state: BeaconState, validatorIdx: int): string =
|
||||||
($state.validators[validatorIdx].pubkey)[0..7]
|
($state.validators[validatorIdx].pubkey)[0..7]
|
||||||
|
|
|
@ -196,6 +196,12 @@ func compute_proposer_index(state: BeaconState, indices: seq[ValidatorIndex],
|
||||||
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#get_beacon_proposer_index
|
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#get_beacon_proposer_index
|
||||||
func get_beacon_proposer_index*(state: BeaconState, cache: var StateCache, slot: Slot):
|
func get_beacon_proposer_index*(state: BeaconState, cache: var StateCache, slot: Slot):
|
||||||
Option[ValidatorIndex] =
|
Option[ValidatorIndex] =
|
||||||
|
try:
|
||||||
|
if slot in cache.beacon_proposer_indices:
|
||||||
|
return cache.beacon_proposer_indices[slot]
|
||||||
|
except KeyError:
|
||||||
|
raiseAssert("Cached entries are added before use")
|
||||||
|
|
||||||
# Return the beacon proposer index at the current slot.
|
# Return the beacon proposer index at the current slot.
|
||||||
let epoch = get_current_epoch(state)
|
let epoch = get_current_epoch(state)
|
||||||
|
|
||||||
|
@ -215,14 +221,16 @@ func get_beacon_proposer_index*(state: BeaconState, cache: var StateCache, slot:
|
||||||
indices =
|
indices =
|
||||||
sorted(cache.shuffled_active_validator_indices[epoch], system.cmp)
|
sorted(cache.shuffled_active_validator_indices[epoch], system.cmp)
|
||||||
|
|
||||||
|
cache.beacon_proposer_indices[slot] =
|
||||||
compute_proposer_index(state, indices, seed)
|
compute_proposer_index(state, indices, seed)
|
||||||
|
cache.beacon_proposer_indices[slot]
|
||||||
except KeyError:
|
except KeyError:
|
||||||
raiseAssert("Cached entries are added before use")
|
raiseAssert("Cached entries are added before use")
|
||||||
|
|
||||||
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#get_beacon_proposer_index
|
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#get_beacon_proposer_index
|
||||||
func get_beacon_proposer_index*(state: BeaconState, stateCache: var StateCache):
|
func get_beacon_proposer_index*(state: BeaconState, cache: var StateCache):
|
||||||
Option[ValidatorIndex] =
|
Option[ValidatorIndex] =
|
||||||
get_beacon_proposer_index(state, stateCache, state.slot)
|
get_beacon_proposer_index(state, cache, state.slot)
|
||||||
|
|
||||||
# Not from spec
|
# Not from spec
|
||||||
# TODO: cache the results from this and reuse in subsequent calls to get_beacon_proposer_index
|
# TODO: cache the results from this and reuse in subsequent calls to get_beacon_proposer_index
|
||||||
|
|
|
@ -273,11 +273,11 @@ proc makeBeaconBlock*(
|
||||||
graffiti: Eth2Digest,
|
graffiti: Eth2Digest,
|
||||||
attestations: seq[Attestation],
|
attestations: seq[Attestation],
|
||||||
deposits: seq[Deposit],
|
deposits: seq[Deposit],
|
||||||
rollback: RollbackHashedProc): Option[BeaconBlock] =
|
rollback: RollbackHashedProc,
|
||||||
|
cache: var StateCache): Option[BeaconBlock] =
|
||||||
## Create a block for the given state. The last block applied to it must be
|
## Create a block for the given state. The last block applied to it must be
|
||||||
## the one identified by parent_root and process_slots must be called up to
|
## the one identified by parent_root and process_slots must be called up to
|
||||||
## the slot for which a block is to be created.
|
## the slot for which a block is to be created.
|
||||||
var cache = get_empty_per_epoch_cache()
|
|
||||||
|
|
||||||
# To create a block, we'll first apply a partial block to the state, skipping
|
# To create a block, we'll first apply a partial block to the state, skipping
|
||||||
# some validations.
|
# some validations.
|
||||||
|
|
|
@ -176,6 +176,7 @@ proc makeBeaconBlockForHeadAndSlot*(node: BeaconNode,
|
||||||
doAssert v.addr == addr poolPtr.tmpState.data
|
doAssert v.addr == addr poolPtr.tmpState.data
|
||||||
poolPtr.tmpState = poolPtr.headState
|
poolPtr.tmpState = poolPtr.headState
|
||||||
|
|
||||||
|
var cache = get_empty_per_epoch_cache()
|
||||||
let message = makeBeaconBlock(
|
let message = makeBeaconBlock(
|
||||||
hashedState,
|
hashedState,
|
||||||
validator_index,
|
validator_index,
|
||||||
|
@ -185,7 +186,8 @@ proc makeBeaconBlockForHeadAndSlot*(node: BeaconNode,
|
||||||
graffiti,
|
graffiti,
|
||||||
node.attestationPool.getAttestationsForBlock(state),
|
node.attestationPool.getAttestationsForBlock(state),
|
||||||
deposits,
|
deposits,
|
||||||
restore)
|
restore,
|
||||||
|
cache)
|
||||||
|
|
||||||
if message.isSome():
|
if message.isSome():
|
||||||
# TODO this restore is needed because otherwise tmpState will be internally
|
# TODO this restore is needed because otherwise tmpState will be internally
|
||||||
|
|
|
@ -1,8 +1,7 @@
|
||||||
import
|
import
|
||||||
confutils, os, strutils, json_serialization,
|
confutils, os, strutils, json_serialization,
|
||||||
stew/byteutils,
|
stew/byteutils,
|
||||||
../beacon_chain/spec/[crypto, datatypes],
|
../beacon_chain/spec/[crypto, datatypes]
|
||||||
../beacon_chain/ssz/dynamic_navigator
|
|
||||||
|
|
||||||
type
|
type
|
||||||
QueryCmd* = enum
|
QueryCmd* = enum
|
||||||
|
|
|
@ -117,7 +117,8 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
|
||||||
Eth2Digest(),
|
Eth2Digest(),
|
||||||
attPool.getAttestationsForBlock(state),
|
attPool.getAttestationsForBlock(state),
|
||||||
@[],
|
@[],
|
||||||
noRollback)
|
noRollback,
|
||||||
|
cache)
|
||||||
|
|
||||||
var
|
var
|
||||||
newBlock = SignedBeaconBlock(
|
newBlock = SignedBeaconBlock(
|
||||||
|
|
|
@ -95,7 +95,8 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
|
||||||
|
|
||||||
withTimer(timers[t]):
|
withTimer(timers[t]):
|
||||||
signedBlock = addTestBlock(
|
signedBlock = addTestBlock(
|
||||||
state[], latest_block_root, attestations = blockAttestations, flags = flags)
|
state[], latest_block_root, cache, attestations = blockAttestations,
|
||||||
|
flags = flags)
|
||||||
latest_block_root = withTimerRet(timers[tHashBlock]):
|
latest_block_root = withTimerRet(timers[tHashBlock]):
|
||||||
hash_tree_root(signedBlock.message)
|
hash_tree_root(signedBlock.message)
|
||||||
|
|
||||||
|
|
|
@ -149,8 +149,9 @@ suiteReport "Attestation pool processing" & preset():
|
||||||
attestations.len == 1
|
attestations.len == 1
|
||||||
|
|
||||||
timedTest "Fork choice returns latest block with no attestations":
|
timedTest "Fork choice returns latest block with no attestations":
|
||||||
|
var cache = get_empty_per_epoch_cache()
|
||||||
let
|
let
|
||||||
b1 = addTestBlock(state.data, blockPool.tail.root)
|
b1 = addTestBlock(state.data, blockPool.tail.root, cache)
|
||||||
b1Root = hash_tree_root(b1.message)
|
b1Root = hash_tree_root(b1.message)
|
||||||
b1Add = blockPool.add(b1Root, b1)[]
|
b1Add = blockPool.add(b1Root, b1)[]
|
||||||
head = pool.selectHead()
|
head = pool.selectHead()
|
||||||
|
@ -159,7 +160,7 @@ suiteReport "Attestation pool processing" & preset():
|
||||||
head == b1Add
|
head == b1Add
|
||||||
|
|
||||||
let
|
let
|
||||||
b2 = addTestBlock(state.data, b1Root)
|
b2 = addTestBlock(state.data, b1Root, cache)
|
||||||
b2Root = hash_tree_root(b2.message)
|
b2Root = hash_tree_root(b2.message)
|
||||||
b2Add = blockPool.add(b2Root, b2)[]
|
b2Add = blockPool.add(b2Root, b2)[]
|
||||||
head2 = pool.selectHead()
|
head2 = pool.selectHead()
|
||||||
|
@ -170,7 +171,7 @@ suiteReport "Attestation pool processing" & preset():
|
||||||
timedTest "Fork choice returns block with attestation":
|
timedTest "Fork choice returns block with attestation":
|
||||||
var cache = get_empty_per_epoch_cache()
|
var cache = get_empty_per_epoch_cache()
|
||||||
let
|
let
|
||||||
b10 = makeTestBlock(state.data, blockPool.tail.root)
|
b10 = makeTestBlock(state.data, blockPool.tail.root, cache)
|
||||||
b10Root = hash_tree_root(b10.message)
|
b10Root = hash_tree_root(b10.message)
|
||||||
b10Add = blockPool.add(b10Root, b10)[]
|
b10Add = blockPool.add(b10Root, b10)[]
|
||||||
head = pool.selectHead()
|
head = pool.selectHead()
|
||||||
|
@ -179,7 +180,7 @@ suiteReport "Attestation pool processing" & preset():
|
||||||
head == b10Add
|
head == b10Add
|
||||||
|
|
||||||
let
|
let
|
||||||
b11 = makeTestBlock(state.data, blockPool.tail.root,
|
b11 = makeTestBlock(state.data, blockPool.tail.root, cache,
|
||||||
graffiti = Eth2Digest(data: [1'u8, 0, 0, 0 ,0 ,0 ,0 ,0 ,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
|
graffiti = Eth2Digest(data: [1'u8, 0, 0, 0 ,0 ,0 ,0 ,0 ,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
|
||||||
)
|
)
|
||||||
b11Root = hash_tree_root(b11.message)
|
b11Root = hash_tree_root(b11.message)
|
||||||
|
|
|
@ -10,7 +10,7 @@
|
||||||
import
|
import
|
||||||
options, sequtils, unittest,
|
options, sequtils, unittest,
|
||||||
./testutil, ./testblockutil,
|
./testutil, ./testblockutil,
|
||||||
../beacon_chain/spec/[datatypes, digest, helpers],
|
../beacon_chain/spec/[datatypes, digest, helpers, validator],
|
||||||
../beacon_chain/[beacon_node_types, block_pool, state_transition, ssz]
|
../beacon_chain/[beacon_node_types, block_pool, state_transition, ssz]
|
||||||
|
|
||||||
suiteReport "BlockRef and helpers" & preset():
|
suiteReport "BlockRef and helpers" & preset():
|
||||||
|
@ -88,9 +88,10 @@ suiteReport "Block pool processing" & preset():
|
||||||
db = makeTestDB(SLOTS_PER_EPOCH)
|
db = makeTestDB(SLOTS_PER_EPOCH)
|
||||||
pool = BlockPool.init(db)
|
pool = BlockPool.init(db)
|
||||||
stateData = newClone(pool.loadTailState())
|
stateData = newClone(pool.loadTailState())
|
||||||
b1 = addTestBlock(stateData.data, pool.tail.root)
|
cache = get_empty_per_epoch_cache()
|
||||||
|
b1 = addTestBlock(stateData.data, pool.tail.root, cache)
|
||||||
b1Root = hash_tree_root(b1.message)
|
b1Root = hash_tree_root(b1.message)
|
||||||
b2 = addTestBlock(stateData.data, b1Root)
|
b2 = addTestBlock(stateData.data, b1Root, cache)
|
||||||
b2Root {.used.} = hash_tree_root(b2.message)
|
b2Root {.used.} = hash_tree_root(b2.message)
|
||||||
|
|
||||||
timedTest "getRef returns nil for missing blocks":
|
timedTest "getRef returns nil for missing blocks":
|
||||||
|
@ -132,7 +133,7 @@ suiteReport "Block pool processing" & preset():
|
||||||
process_slots(stateData.data, stateData.data.data.slot + 1)
|
process_slots(stateData.data, stateData.data.data.slot + 1)
|
||||||
|
|
||||||
let
|
let
|
||||||
b4 = addTestBlock(stateData.data, b2Root)
|
b4 = addTestBlock(stateData.data, b2Root, cache)
|
||||||
b4Root = hash_tree_root(b4.message)
|
b4Root = hash_tree_root(b4.message)
|
||||||
b4Add = pool.add(b4Root, b4)[]
|
b4Add = pool.add(b4Root, b4)[]
|
||||||
|
|
||||||
|
@ -284,12 +285,13 @@ when const_preset == "minimal": # These require some minutes in mainnet
|
||||||
var
|
var
|
||||||
db = makeTestDB(SLOTS_PER_EPOCH)
|
db = makeTestDB(SLOTS_PER_EPOCH)
|
||||||
pool = BlockPool.init(db)
|
pool = BlockPool.init(db)
|
||||||
|
cache = get_empty_per_epoch_cache()
|
||||||
|
|
||||||
timedTest "prune heads on finalization" & preset():
|
timedTest "prune heads on finalization" & preset():
|
||||||
block:
|
block:
|
||||||
# Create a fork that will not be taken
|
# Create a fork that will not be taken
|
||||||
var
|
var
|
||||||
blck = makeTestBlock(pool.headState.data, pool.head.blck.root)
|
blck = makeTestBlock(pool.headState.data, pool.head.blck.root, cache)
|
||||||
check: pool.add(hash_tree_root(blck.message), blck).isOk
|
check: pool.add(hash_tree_root(blck.message), blck).isOk
|
||||||
|
|
||||||
for i in 0 ..< (SLOTS_PER_EPOCH * 6):
|
for i in 0 ..< (SLOTS_PER_EPOCH * 6):
|
||||||
|
@ -299,9 +301,8 @@ when const_preset == "minimal": # These require some minutes in mainnet
|
||||||
pool.tail.children.len == 2
|
pool.tail.children.len == 2
|
||||||
pool.heads.len == 2
|
pool.heads.len == 2
|
||||||
var
|
var
|
||||||
cache = get_empty_per_epoch_cache()
|
|
||||||
blck = makeTestBlock(
|
blck = makeTestBlock(
|
||||||
pool.headState.data, pool.head.blck.root,
|
pool.headState.data, pool.head.blck.root, cache,
|
||||||
attestations = makeFullAttestations(
|
attestations = makeFullAttestations(
|
||||||
pool.headState.data.data, pool.head.blck.root,
|
pool.headState.data.data, pool.head.blck.root,
|
||||||
pool.headState.data.data.slot, cache, {}))
|
pool.headState.data.data.slot, cache, {}))
|
||||||
|
@ -332,7 +333,7 @@ when const_preset == "minimal": # These require some minutes in mainnet
|
||||||
for i in 0 ..< (SLOTS_PER_EPOCH * 6 - 2):
|
for i in 0 ..< (SLOTS_PER_EPOCH * 6 - 2):
|
||||||
var
|
var
|
||||||
blck = makeTestBlock(
|
blck = makeTestBlock(
|
||||||
pool.headState.data, pool.head.blck.root,
|
pool.headState.data, pool.head.blck.root, cache,
|
||||||
attestations = makeFullAttestations(
|
attestations = makeFullAttestations(
|
||||||
pool.headState.data.data, pool.head.blck.root,
|
pool.headState.data.data, pool.head.blck.root,
|
||||||
pool.headState.data.data.slot, cache, {}))
|
pool.headState.data.data.slot, cache, {}))
|
||||||
|
@ -345,7 +346,7 @@ when const_preset == "minimal": # These require some minutes in mainnet
|
||||||
pool.headState.data, Slot(SLOTS_PER_EPOCH * 6 + 2) )
|
pool.headState.data, Slot(SLOTS_PER_EPOCH * 6 + 2) )
|
||||||
|
|
||||||
var blck = makeTestBlock(
|
var blck = makeTestBlock(
|
||||||
pool.headState.data, pool.head.blck.root,
|
pool.headState.data, pool.head.blck.root, cache,
|
||||||
attestations = makeFullAttestations(
|
attestations = makeFullAttestations(
|
||||||
pool.headState.data.data, pool.head.blck.root,
|
pool.headState.data.data, pool.head.blck.root,
|
||||||
pool.headState.data.data.slot, cache, {}))
|
pool.headState.data.data.slot, cache, {}))
|
||||||
|
|
|
@ -36,7 +36,8 @@ suiteReport "Block processing" & preset():
|
||||||
timedTest "Passes from genesis state, empty block" & preset():
|
timedTest "Passes from genesis state, empty block" & preset():
|
||||||
var
|
var
|
||||||
previous_block_root = hash_tree_root(genesisBlock.message)
|
previous_block_root = hash_tree_root(genesisBlock.message)
|
||||||
new_block = makeTestBlock(state[], previous_block_root)
|
cache = get_empty_per_epoch_cache()
|
||||||
|
new_block = makeTestBlock(state[], previous_block_root, cache)
|
||||||
|
|
||||||
let block_ok = state_transition(state[], new_block, {}, noRollback)
|
let block_ok = state_transition(state[], new_block, {}, noRollback)
|
||||||
|
|
||||||
|
@ -53,9 +54,10 @@ suiteReport "Block processing" & preset():
|
||||||
timedTest "Passes through epoch update, empty block" & preset():
|
timedTest "Passes through epoch update, empty block" & preset():
|
||||||
var
|
var
|
||||||
previous_block_root = genesisRoot
|
previous_block_root = genesisRoot
|
||||||
|
cache = get_empty_per_epoch_cache()
|
||||||
|
|
||||||
for i in 1..SLOTS_PER_EPOCH.int:
|
for i in 1..SLOTS_PER_EPOCH.int:
|
||||||
let new_block = makeTestBlock(state[], previous_block_root)
|
let new_block = makeTestBlock(state[], previous_block_root, cache)
|
||||||
|
|
||||||
let block_ok = state_transition(state[], new_block, {}, noRollback)
|
let block_ok = state_transition(state[], new_block, {}, noRollback)
|
||||||
|
|
||||||
|
@ -90,7 +92,7 @@ suiteReport "Block processing" & preset():
|
||||||
state[], GENESIS_SLOT + MIN_ATTESTATION_INCLUSION_DELAY + 1)
|
state[], GENESIS_SLOT + MIN_ATTESTATION_INCLUSION_DELAY + 1)
|
||||||
|
|
||||||
let
|
let
|
||||||
new_block = makeTestBlock(state[], previous_block_root,
|
new_block = makeTestBlock(state[], previous_block_root, cache,
|
||||||
attestations = @[attestation]
|
attestations = @[attestation]
|
||||||
)
|
)
|
||||||
check state_transition(state[], new_block, {}, noRollback)
|
check state_transition(state[], new_block, {}, noRollback)
|
||||||
|
|
|
@ -89,13 +89,13 @@ func signBlock*(
|
||||||
proc addTestBlock*(
|
proc addTestBlock*(
|
||||||
state: var HashedBeaconState,
|
state: var HashedBeaconState,
|
||||||
parent_root: Eth2Digest,
|
parent_root: Eth2Digest,
|
||||||
|
cache: var StateCache,
|
||||||
eth1_data = Eth1Data(),
|
eth1_data = Eth1Data(),
|
||||||
attestations = newSeq[Attestation](),
|
attestations = newSeq[Attestation](),
|
||||||
deposits = newSeq[Deposit](),
|
deposits = newSeq[Deposit](),
|
||||||
graffiti = Eth2Digest(),
|
graffiti = Eth2Digest(),
|
||||||
flags: set[UpdateFlag] = {}): SignedBeaconBlock =
|
flags: set[UpdateFlag] = {}): SignedBeaconBlock =
|
||||||
# Create and add a block to state - state will advance by one slot!
|
# Create and add a block to state - state will advance by one slot!
|
||||||
var cache = get_empty_per_epoch_cache()
|
|
||||||
advance_slot(state, err(Opt[Eth2Digest]), flags, cache)
|
advance_slot(state, err(Opt[Eth2Digest]), flags, cache)
|
||||||
|
|
||||||
let
|
let
|
||||||
|
@ -122,7 +122,8 @@ proc addTestBlock*(
|
||||||
graffiti,
|
graffiti,
|
||||||
attestations,
|
attestations,
|
||||||
deposits,
|
deposits,
|
||||||
noRollback)
|
noRollback,
|
||||||
|
cache)
|
||||||
|
|
||||||
doAssert message.isSome(), "Should have created a valid block!"
|
doAssert message.isSome(), "Should have created a valid block!"
|
||||||
|
|
||||||
|
@ -136,6 +137,7 @@ proc addTestBlock*(
|
||||||
proc makeTestBlock*(
|
proc makeTestBlock*(
|
||||||
state: HashedBeaconState,
|
state: HashedBeaconState,
|
||||||
parent_root: Eth2Digest,
|
parent_root: Eth2Digest,
|
||||||
|
cache: var StateCache,
|
||||||
eth1_data = Eth1Data(),
|
eth1_data = Eth1Data(),
|
||||||
attestations = newSeq[Attestation](),
|
attestations = newSeq[Attestation](),
|
||||||
deposits = newSeq[Deposit](),
|
deposits = newSeq[Deposit](),
|
||||||
|
@ -147,7 +149,8 @@ proc makeTestBlock*(
|
||||||
# because the block includes the state root.
|
# because the block includes the state root.
|
||||||
var tmpState = newClone(state)
|
var tmpState = newClone(state)
|
||||||
addTestBlock(
|
addTestBlock(
|
||||||
tmpState[], parent_root, eth1_data, attestations, deposits, graffiti, flags)
|
tmpState[], parent_root, cache, eth1_data, attestations, deposits,
|
||||||
|
graffiti, flags)
|
||||||
|
|
||||||
proc makeAttestation*(
|
proc makeAttestation*(
|
||||||
state: BeaconState, beacon_block_root: Eth2Digest,
|
state: BeaconState, beacon_block_root: Eth2Digest,
|
||||||
|
|
Loading…
Reference in New Issue