status-im
/
nimbus-eth2
mirror of https://github.com/status-im/nimbus-eth2.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Introduce slot->BlockRef mapping for finalized chain (#3144) 

* Introduce slot->BlockRef mapping for finalized chain

The finalized chain is linear, thus we can use a seq to lookup blocks by
slot number.

Here, we introduce such a seq, even though in the future, it should
likely be backed by a database structure instead, or, more likely, a
flat era file with a flat lookup index.

This dramatically speeds up requests by slot, such as those coming from
the REST interface or GetBlocksByRange, as these are currently served by
a linear iteration from head.

* fix REST block requests to not return blocks from an earlier slot when
the given slot is empty
* fix StateId interpretation such that it doesn't treat state roots as
block roots
* don't load full block from database just to return its root
This commit is contained in:
Jacek Sieka 2021-12-06 19:52:35 +01:00 committed by GitHub
parent 850eece949
commit 89d6a1b403
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 97 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
AllTests-mainnet.md
View File

@ -92,7 +92,6 @@ OK: 5/5 Fail: 0/5 Skip: 0/5
OK: 1/1 Fail: 0/1 Skip: 0/1
## BlockRef and helpers [Preset: mainnet]
```diff
+ epochAncestor sanity [Preset: mainnet] OK
+ get_ancestor sanity [Preset: mainnet] OK
+ isAncestorOf sanity [Preset: mainnet] OK
```

4
beacon_chain/consensus_object_pools/block_pools_types.nim
View File

@ -89,6 +89,10 @@ type
## Directed acyclic graph of blocks pointing back to a finalized block on the chain we're
## interested in - we call that block the tail
finalizedBlocks*: seq[BlockRef] ##\
## Slot -> BlockRef mapping for the canonical chain - use getBlockBySlot
## to access, generally
genesis*: BlockRef ##\
## The genesis block of the network

58
beacon_chain/consensus_object_pools/blockchain_dag.nim
View File

@ -137,6 +137,8 @@ func validatorKey*(
## non-head branch)!
validatorKey(epochRef.dag, index)
func epochAncestor*(dag: ChainDAGRef, blck: BlockRef, epoch: Epoch): EpochKey
func init*(
T: type EpochRef, dag: ChainDAGRef, state: StateData,
cache: var StateCache): T =
@ -144,7 +146,7 @@ func init*(
epoch = state.data.get_current_epoch()
epochRef = EpochRef(
dag: dag, # This gives access to the validator pubkeys through an EpochRef
key: state.blck.epochAncestor(epoch),
key: epochAncestor(dag, state.blck, epoch),
eth1_data: getStateField(state.data, eth1_data),
eth1_deposit_index: getStateField(state.data, eth1_deposit_index),
current_justified_checkpoint:
@ -251,7 +253,21 @@ func atEpochStart*(blck: BlockRef, epoch: Epoch): BlockSlot =
## Return the BlockSlot corresponding to the first slot in the given epoch
atSlot(blck, epoch.compute_start_slot_at_epoch)
func epochAncestor*(blck: BlockRef, epoch: Epoch): EpochKey =
func getBlockBySlot*(dag: ChainDAGRef, slot: Slot): BlockSlot =
## Retrieve the canonical block at the given slot, or the last block that
## comes before - similar to atSlot, but without the linear scan
if slot > dag.finalizedHead.slot:
return dag.head.atSlot(slot) # Linear iteration is the fastest we have
var tmp = slot.int
while true:
if dag.finalizedBlocks[tmp] != nil:
return dag.finalizedBlocks[tmp].atSlot(slot)
if tmp == 0:
raiseAssert "At least the genesis block should be available!"
tmp = tmp - 1
func epochAncestor*(dag: ChainDAGRef, blck: BlockRef, epoch: Epoch): EpochKey =
## The state transition works by storing information from blocks in a
## "working" area until the epoch transition, then batching work collected
## during the epoch. Thus, last block in the ancestor epochs is the block
@ -260,15 +276,17 @@ func epochAncestor*(blck: BlockRef, epoch: Epoch): EpochKey =
## This function returns a BlockSlot pointing to that epoch boundary, ie the
## boundary where the last block has been applied to the state and epoch
## processing has been done.
var blck = blck
while blck.slot.epoch >= epoch and not blck.parent.isNil:
blck = blck.parent
let blck =
if epoch == GENESIS_EPOCH:
dag.genesis
else:
dag.getBlockBySlot(compute_start_slot_at_epoch(epoch) - 1).blck
EpochKey(epoch: epoch, blck: blck)
func findEpochRef*(
dag: ChainDAGRef, blck: BlockRef, epoch: Epoch): EpochRef = # may return nil!
let ancestor = blck.epochAncestor(epoch)
let ancestor = epochAncestor(dag, blck, epoch)
doAssert ancestor.blck != nil
for i in 0..<dag.epochRefs.len:
if dag.epochRefs[i] != nil and dag.epochRefs[i].key == ancestor:
@ -540,6 +558,13 @@ proc init*(T: type ChainDAGRef, cfg: RuntimeConfig, db: BeaconChainDB,
tailRef.slot.epoch)
dag.finalizedHead = headRef.atEpochStart(finalizedEpoch)
block:
dag.finalizedBlocks.setLen(dag.finalizedHead.slot.int + 1)
var tmp = dag.finalizedHead.blck
while not isNil(tmp):
dag.finalizedBlocks[tmp.slot.int] = tmp
tmp = tmp.parent
dag.clearanceState = dag.headState
# Pruning metadata
@ -605,7 +630,7 @@ proc getEpochRef*(dag: ChainDAGRef, blck: BlockRef, epoch: Epoch): EpochRef =
beacon_state_data_cache_misses.inc
let
ancestor = blck.epochAncestor(epoch)
ancestor = epochAncestor(dag, blck, epoch)
dag.withState(
dag.epochRefState, ancestor.blck.atEpochStart(ancestor.epoch)):
@ -724,7 +749,7 @@ func getBlockRange*(
endSlot = startSlot + extraBlocks * skipStep
var
b = dag.head.atSlot(endSlot)
b = dag.getBlockBySlot(endSlot)
o = output.len
# Process all blocks that follow the start block (may be zero blocks)
@ -743,11 +768,6 @@ func getBlockRange*(
o # Return the index of the first non-nil item in the output
func getBlockBySlot*(dag: ChainDAGRef, slot: Slot): BlockSlot =
## Retrieves the first block in the current canonical chain
## with slot number less or equal to `slot`.
dag.head.atSlot(slot)
proc getForkedBlock*(dag: ChainDAGRef, blck: BlockRef): ForkedTrustedSignedBeaconBlock =
case dag.cfg.blockForkAtEpoch(blck.slot.epoch)
of BeaconBlockFork.Phase0:
@ -1318,7 +1338,17 @@ proc updateHead*(
finalized = shortLog(getStateField(
dag.headState.data, finalized_checkpoint))
dag.finalizedHead = finalizedHead
block:
# Update `dag.finalizedBlocks` with all newly finalized blocks (those
# newer than the previous finalized head), then update `dag.finalizedHead`
dag.finalizedBlocks.setLen(finalizedHead.slot.int + 1)
var tmp = finalizedHead.blck
while not isNil(tmp) and tmp.slot >= dag.finalizedHead.slot:
dag.finalizedBlocks[tmp.slot.int] = tmp
tmp = tmp.parent
dag.finalizedHead = finalizedHead
beacon_finalized_epoch.set(getStateField(
dag.headState.data, finalized_checkpoint).epoch.toGaugeValue)

28
beacon_chain/rpc/rest_beacon_api.nim
View File

@ -686,20 +686,16 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
$rroot.error())
return RestApiResponse.jsonError(Http500, NoImplementationError)
let bdata =
let blck =
block:
let head =
block:
let res = node.getCurrentHead(qslot)
if res.isErr():
return RestApiResponse.jsonError(Http404, SlotNotFoundError,
$res.error())
res.get()
let blockSlot = head.atSlot(qslot)
if isNil(blockSlot.blck):
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
node.dag.get(blockSlot.blck)
let res = node.getCurrentBlock(qslot)
if res.isErr():
return RestApiResponse.jsonError(Http404, BlockNotFoundError,
$res.error())
res.get()
let bdata = node.dag.get(blck)
return
withBlck(bdata.data):
RestApiResponse.jsonResponse(
@ -871,18 +867,16 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
# https://ethereum.github.io/beacon-APIs/#/Beacon/getBlockRoot
router.api(MethodGet, "/api/eth/v1/beacon/blocks/{block_id}/root") do (
block_id: BlockIdent) -> RestApiResponse:
let bdata =
let blck =
block:
if block_id.isErr():
return RestApiResponse.jsonError(Http400, InvalidBlockIdValueError,
$block_id.error())
let res = node.getBlockDataFromBlockIdent(block_id.get())
let res = node.getBlockRef(block_id.get())
if res.isErr():
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
res.get()
return
withBlck(bdata.data):
RestApiResponse.jsonResponse((root: blck.root))
return RestApiResponse.jsonResponse((root: blck.root))
# https://ethereum.github.io/beacon-APIs/#/Beacon/getBlockAttestations
router.api(MethodGet,

62
beacon_chain/rpc/rest_utils.nim
View File

@ -40,8 +40,22 @@ proc validate(key: string, value: string): int =
else:
1
proc getCurrentHead*(node: BeaconNode,
slot: Slot): Result[BlockRef, cstring] =
func getCurrentSlot*(node: BeaconNode, slot: Slot):
Result[Slot, cstring] =
if slot <= (node.dag.head.slot + (SLOTS_PER_EPOCH * 2)):
ok(slot)
else:
err("Requesting slot too far ahead of the current head")
func getCurrentBlock*(node: BeaconNode, slot: Slot):
Result[BlockRef, cstring] =
let bs = node.dag.getBlockBySlot(? node.getCurrentSlot(slot))
if bs.slot == bs.blck.slot:
ok(bs.blck)
else:
err("Block not found")
proc getCurrentHead*(node: BeaconNode, slot: Slot): Result[BlockRef, cstring] =
let res = node.dag.head
# if not(node.isSynced(res)):
# return err("Cannot fulfill request until node is synced")
@ -62,16 +76,13 @@ proc getBlockSlot*(node: BeaconNode,
stateIdent: StateIdent): Result[BlockSlot, cstring] =
case stateIdent.kind
of StateQueryKind.Slot:
let head = ? getCurrentHead(node, stateIdent.slot)
let bslot = head.atSlot(stateIdent.slot)
if isNil(bslot.blck):
return err("Block not found")
ok(bslot)
ok(node.dag.getBlockBySlot(? node.getCurrentSlot(stateIdent.slot)))
of StateQueryKind.Root:
let blckRef = node.dag.getRef(stateIdent.root)
if isNil(blckRef):
return err("Block not found")
ok(blckRef.toBlockSlot())
if stateIdent.root == getStateRoot(node.dag.headState.data):
ok(node.dag.headState.blck.toBlockSlot())
else:
# We don't have a state root -> BlockSlot mapping
err("State not found")
of StateQueryKind.Named:
case stateIdent.value
of StateIdentType.Head:
@ -84,28 +95,29 @@ proc getBlockSlot*(node: BeaconNode,
ok(node.dag.head.atEpochStart(getStateField(
node.dag.headState.data, current_justified_checkpoint).epoch))
proc getBlockDataFromBlockIdent*(node: BeaconNode,
id: BlockIdent): Result[BlockData, cstring] =
proc getBlockRef*(node: BeaconNode,
id: BlockIdent): Result[BlockRef, cstring] =
case id.kind
of BlockQueryKind.Named:
case id.value
of BlockIdentType.Head:
ok(node.dag.get(node.dag.head))
ok(node.dag.head)
of BlockIdentType.Genesis:
ok(node.dag.getGenesisBlockData())
ok(node.dag.genesis)
of BlockIdentType.Finalized:
ok(node.dag.get(node.dag.finalizedHead.blck))
ok(node.dag.finalizedHead.blck)
of BlockQueryKind.Root:
let res = node.dag.get(id.root)
if res.isNone():
return err("Block not found")
ok(res.get())
let res = node.dag.getRef(id.root)
if isNil(res):
err("Block not found")
else:
ok(res)
of BlockQueryKind.Slot:
let head = ? node.getCurrentHead(id.slot)
let blockSlot = head.atSlot(id.slot)
if isNil(blockSlot.blck):
return err("Block not found")
ok(node.dag.get(blockSlot.blck))
node.getCurrentBlock(id.slot)
proc getBlockDataFromBlockIdent*(node: BeaconNode,
id: BlockIdent): Result[BlockData, cstring] =
ok(node.dag.get(? node.getBlockRef(id)))
template withStateForBlockSlot*(node: BeaconNode,
blockSlot: BlockSlot, body: untyped): untyped =

2
beacon_chain/rpc/rest_validator_api.nim
View File

@ -259,7 +259,7 @@ proc installValidatorApiHandlers*(router: var RestRouter, node: BeaconNode) =
# in order to compute the sync committee for the epoch. See the following
# discussion for more details:
# https://github.com/status-im/nimbus-eth2/pull/3133#pullrequestreview-817184693
node.withStateForBlockSlot(node.dag.head.atSlot(earliestSlotInQSyncPeriod)):
node.withStateForBlockSlot(node.dag.getBlockBySlot(earliestSlotInQSyncPeriod)):
let res = withState(stateData().data):
when stateFork >= BeaconStateFork.Altair:
produceResponse(indexList,

16
tests/test_blockchain_dag.nim
View File

@ -68,22 +68,6 @@ suite "BlockRef and helpers" & preset():
s4.get_ancestor(Slot(3)) == s2
s4.get_ancestor(Slot(4)) == s4
test "epochAncestor sanity" & preset():
let
s0 = BlockRef(slot: Slot(0))
var cur = s0
for i in 1..SLOTS_PER_EPOCH * 2:
cur = BlockRef(slot: Slot(i), parent: cur)
let ancestor = cur.epochAncestor(cur.slot.epoch)
check:
ancestor.epoch == cur.slot.epoch
ancestor.blck != cur # should have selected a parent
ancestor.blck.epochAncestor(cur.slot.epoch) == ancestor
ancestor.blck.epochAncestor(ancestor.blck.slot.epoch) != ancestor
suite "BlockSlot and helpers" & preset():
test "atSlot sanity" & preset():
let