2021-10-06 17:05:06 +00:00
|
|
|
# beacon_chain
|
2022-01-04 09:45:38 +00:00
|
|
|
# Copyright (c) 2018-2022 Status Research & Development GmbH
|
2021-03-17 18:46:45 +00:00
|
|
|
# Licensed and distributed under either of
|
|
|
|
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
|
|
|
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
|
|
|
# at your option. This file may not be copied, modified, or distributed except according to those terms.
|
2021-10-06 17:05:06 +00:00
|
|
|
|
2021-03-17 18:46:45 +00:00
|
|
|
import
|
2022-06-09 08:50:36 +00:00
|
|
|
std/[typetraits, sequtils, sets],
|
2021-03-17 18:46:45 +00:00
|
|
|
stew/[results, base10],
|
|
|
|
chronicles,
|
2021-11-02 18:23:31 +00:00
|
|
|
./rest_utils,
|
2022-12-07 10:24:51 +00:00
|
|
|
./state_ttl_cache,
|
|
|
|
../beacon_node,
|
2021-08-23 10:41:48 +00:00
|
|
|
../consensus_object_pools/[blockchain_dag, exit_pool, spec_cache],
|
2022-12-07 10:24:51 +00:00
|
|
|
../spec/[deposit_snapshots, eth2_merkleization, forks, network, validator],
|
2022-01-05 14:49:10 +00:00
|
|
|
../spec/datatypes/[phase0, altair],
|
2022-12-07 10:24:51 +00:00
|
|
|
../validators/message_router_mev
|
2021-03-17 18:46:45 +00:00
|
|
|
|
2021-10-27 12:01:11 +00:00
|
|
|
export rest_utils
|
|
|
|
|
2021-03-17 18:46:45 +00:00
|
|
|
logScope: topics = "rest_beaconapi"
|
|
|
|
|
2022-08-19 10:30:07 +00:00
|
|
|
proc validateBeaconApiQueries*(key: string, value: string): int =
|
|
|
|
## This is rough validation procedure which should be simple and fast,
|
|
|
|
## because it will be used for query routing.
|
|
|
|
case key
|
|
|
|
of "{epoch}":
|
|
|
|
0
|
|
|
|
of "{slot}":
|
|
|
|
0
|
|
|
|
of "{peer_id}":
|
|
|
|
0
|
|
|
|
of "{state_id}":
|
|
|
|
0
|
|
|
|
of "{block_id}":
|
|
|
|
0
|
|
|
|
of "{validator_id}":
|
|
|
|
0
|
|
|
|
of "{block_root}":
|
|
|
|
0
|
|
|
|
of "{pubkey}":
|
|
|
|
int(value.len != 98)
|
|
|
|
else:
|
|
|
|
1
|
|
|
|
|
2021-03-17 18:46:45 +00:00
|
|
|
proc validateFilter(filters: seq[ValidatorFilter]): Result[ValidatorFilter,
|
|
|
|
cstring] =
|
|
|
|
var res: ValidatorFilter
|
|
|
|
for item in filters:
|
|
|
|
if res * item != {}:
|
|
|
|
return err("Validator status must be unique")
|
|
|
|
res.incl(item)
|
|
|
|
|
|
|
|
if res == {}:
|
|
|
|
res = {ValidatorFilterKind.PendingInitialized,
|
|
|
|
ValidatorFilterKind.PendingQueued,
|
|
|
|
ValidatorFilterKind.ActiveOngoing,
|
|
|
|
ValidatorFilterKind.ActiveExiting,
|
|
|
|
ValidatorFilterKind.ActiveSlashed,
|
|
|
|
ValidatorFilterKind.ExitedUnslashed,
|
|
|
|
ValidatorFilterKind.ExitedSlashed,
|
|
|
|
ValidatorFilterKind.WithdrawalPossible,
|
|
|
|
ValidatorFilterKind.WithdrawalDone}
|
|
|
|
ok(res)
|
|
|
|
|
|
|
|
proc getStatus(validator: Validator,
|
|
|
|
current_epoch: Epoch): Result[ValidatorFilterKind, cstring] =
|
|
|
|
if validator.activation_epoch > current_epoch:
|
|
|
|
# pending
|
|
|
|
if validator.activation_eligibility_epoch == FAR_FUTURE_EPOCH:
|
|
|
|
ok(ValidatorFilterKind.PendingInitialized)
|
|
|
|
else:
|
|
|
|
# validator.activation_eligibility_epoch < FAR_FUTURE_EPOCH:
|
|
|
|
ok(ValidatorFilterKind.PendingQueued)
|
|
|
|
elif (validator.activation_epoch <= current_epoch) and
|
|
|
|
(current_epoch < validator.exit_epoch):
|
|
|
|
# active
|
|
|
|
if validator.exit_epoch == FAR_FUTURE_EPOCH:
|
|
|
|
ok(ValidatorFilterKind.ActiveOngoing)
|
|
|
|
elif not validator.slashed:
|
|
|
|
# validator.exit_epoch < FAR_FUTURE_EPOCH
|
|
|
|
ok(ValidatorFilterKind.ActiveExiting)
|
|
|
|
else:
|
|
|
|
# validator.exit_epoch < FAR_FUTURE_EPOCH and validator.slashed:
|
|
|
|
ok(ValidatorFilterKind.ActiveSlashed)
|
|
|
|
elif (validator.exit_epoch <= current_epoch) and
|
|
|
|
(current_epoch < validator.withdrawable_epoch):
|
|
|
|
# exited
|
|
|
|
if not validator.slashed:
|
|
|
|
ok(ValidatorFilterKind.ExitedUnslashed)
|
|
|
|
else:
|
|
|
|
# validator.slashed
|
|
|
|
ok(ValidatorFilterKind.ExitedSlashed)
|
|
|
|
elif validator.withdrawable_epoch <= current_epoch:
|
|
|
|
# withdrawal
|
|
|
|
if validator.effective_balance != 0:
|
|
|
|
ok(ValidatorFilterKind.WithdrawalPossible)
|
|
|
|
else:
|
|
|
|
# validator.effective_balance == 0
|
|
|
|
ok(ValidatorFilterKind.WithdrawalDone)
|
|
|
|
else:
|
|
|
|
err("Invalid validator status")
|
|
|
|
|
|
|
|
proc toString*(kind: ValidatorFilterKind): string =
|
|
|
|
case kind
|
|
|
|
of ValidatorFilterKind.PendingInitialized:
|
|
|
|
"pending_initialized"
|
|
|
|
of ValidatorFilterKind.PendingQueued:
|
|
|
|
"pending_queued"
|
|
|
|
of ValidatorFilterKind.ActiveOngoing:
|
|
|
|
"active_ongoing"
|
|
|
|
of ValidatorFilterKind.ActiveExiting:
|
|
|
|
"active_exiting"
|
|
|
|
of ValidatorFilterKind.ActiveSlashed:
|
|
|
|
"active_slashed"
|
|
|
|
of ValidatorFilterKind.ExitedUnslashed:
|
|
|
|
"exited_unslashed"
|
|
|
|
of ValidatorFilterKind.ExitedSlashed:
|
|
|
|
"exited_slashed"
|
|
|
|
of ValidatorFilterKind.WithdrawalPossible:
|
|
|
|
"withdrawal_possible"
|
|
|
|
of ValidatorFilterKind.WithdrawalDone:
|
|
|
|
"withdrawal_done"
|
|
|
|
|
|
|
|
proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
|
2022-12-07 10:24:51 +00:00
|
|
|
# https://github.com/ethereum/EIPs/blob/master/EIPS/eip-4881.md
|
|
|
|
router.api(MethodGet, "/eth/v1/beacon/deposit_snapshot") do () -> RestApiResponse:
|
|
|
|
let snapshotOpt = node.db.getDepositTreeSnapshot()
|
|
|
|
if snapshotOpt.isSome():
|
|
|
|
let snapshot = snapshotOpt.get()
|
|
|
|
return RestApiResponse.jsonResponse(
|
|
|
|
RestDepositSnapshot(
|
|
|
|
finalized: snapshot.depositContractState.branch,
|
|
|
|
deposit_root: snapshot.getDepositRoot(),
|
|
|
|
deposit_count: snapshot.getDepositCountU64(),
|
|
|
|
execution_block_hash: snapshot.eth1Block,
|
|
|
|
execution_block_height: snapshot.blockHeight))
|
|
|
|
else:
|
|
|
|
# This can happen in a very short window after the client is started, but the
|
|
|
|
# snapshot record still haven't been upgraded in the database. Returning 404
|
|
|
|
# should be easy to handle for the clients - they just need to retry.
|
|
|
|
return RestApiResponse.jsonError(Http404, NoFinalizedSnapshotAvailableError)
|
|
|
|
|
2021-08-23 10:41:48 +00:00
|
|
|
# https://ethereum.github.io/beacon-APIs/#/Beacon/getGenesis
|
2022-01-06 07:38:40 +00:00
|
|
|
router.api(MethodGet, "/eth/v1/beacon/genesis") do () -> RestApiResponse:
|
2021-03-17 18:46:45 +00:00
|
|
|
return RestApiResponse.jsonResponse(
|
2021-03-23 22:50:18 +00:00
|
|
|
(
|
2022-03-16 07:20:40 +00:00
|
|
|
genesis_time: getStateField(node.dag.headState, genesis_time),
|
2021-03-17 18:46:45 +00:00
|
|
|
genesis_validators_root:
|
2022-03-16 07:20:40 +00:00
|
|
|
getStateField(node.dag.headState, genesis_validators_root),
|
2021-07-13 14:27:10 +00:00
|
|
|
genesis_fork_version: node.dag.cfg.GENESIS_FORK_VERSION
|
2021-03-17 18:46:45 +00:00
|
|
|
)
|
|
|
|
)
|
|
|
|
|
2021-08-23 10:41:48 +00:00
|
|
|
# https://ethereum.github.io/beacon-APIs/#/Beacon/getStateRoot
|
2022-01-06 07:38:40 +00:00
|
|
|
router.api(MethodGet, "/eth/v1/beacon/states/{state_id}/root") do (
|
2021-03-17 18:46:45 +00:00
|
|
|
state_id: StateIdent) -> RestApiResponse:
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
let
|
|
|
|
sid = state_id.valueOr:
|
|
|
|
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
|
|
|
|
$error)
|
Prune `BlockRef` on finalization (#3513)
Up til now, the block dag has been using `BlockRef`, a structure adapted
for a full DAG, to represent all of chain history. This is a correct and
simple design, but does not exploit the linearity of the chain once
parts of it finalize.
By pruning the in-memory `BlockRef` structure at finalization, we save,
at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory
landing us at a steady state of ~750mb normal memory usage for a
validating node.
Above all though, we prevent memory usage from growing proportionally
with the length of the chain, something that would not be sustainable
over time - instead, the steady state memory usage is roughly
determined by the validator set size which grows much more slowly. With
these changes, the core should remain sustainable memory-wise post-merge
all the way to withdrawals (when the validator set is expected to grow).
In-memory indices are still used for the "hot" unfinalized portion of
the chain - this ensure that consensus performance remains unchanged.
What changes is that for historical access, we use a db-based linear
slot index which is cache-and-disk-friendly, keeping the cost for
accessing historical data at a similar level as before, achieving the
savings at no percievable cost to functionality or performance.
A nice collateral benefit is the almost-instant startup since we no
longer load any large indicies at dag init.
The cost of this functionality instead can be found in the complexity of
having to deal with two ways of traversing the chain - by `BlockRef` and
by slot.
* use `BlockId` instead of `BlockRef` where finalized / historical data
may be required
* simplify clearance pre-advancement
* remove dag.finalizedBlocks (~50:ish mb)
* remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead
* `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef`
instance, unlike `BlockRef` traversal
* prune `BlockRef` parents on finality (~200:ish mb)
* speed up ChainDAG init by not loading finalized history index
* mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
|
|
|
bslot = node.getBlockSlotId(sid).valueOr:
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
if sid.kind == StateQueryKind.Root:
|
|
|
|
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
|
|
|
|
# in current version of database.
|
|
|
|
return RestApiResponse.jsonError(Http500, NoImplementationError)
|
|
|
|
return RestApiResponse.jsonError(Http404, StateNotFoundError,
|
2022-09-23 15:51:04 +00:00
|
|
|
$error)
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
|
Prune `BlockRef` on finalization (#3513)
Up til now, the block dag has been using `BlockRef`, a structure adapted
for a full DAG, to represent all of chain history. This is a correct and
simple design, but does not exploit the linearity of the chain once
parts of it finalize.
By pruning the in-memory `BlockRef` structure at finalization, we save,
at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory
landing us at a steady state of ~750mb normal memory usage for a
validating node.
Above all though, we prevent memory usage from growing proportionally
with the length of the chain, something that would not be sustainable
over time - instead, the steady state memory usage is roughly
determined by the validator set size which grows much more slowly. With
these changes, the core should remain sustainable memory-wise post-merge
all the way to withdrawals (when the validator set is expected to grow).
In-memory indices are still used for the "hot" unfinalized portion of
the chain - this ensure that consensus performance remains unchanged.
What changes is that for historical access, we use a db-based linear
slot index which is cache-and-disk-friendly, keeping the cost for
accessing historical data at a similar level as before, achieving the
savings at no percievable cost to functionality or performance.
A nice collateral benefit is the almost-instant startup since we no
longer load any large indicies at dag init.
The cost of this functionality instead can be found in the complexity of
having to deal with two ways of traversing the chain - by `BlockRef` and
by slot.
* use `BlockId` instead of `BlockRef` where finalized / historical data
may be required
* simplify clearance pre-advancement
* remove dag.finalizedBlocks (~50:ish mb)
* remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead
* `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef`
instance, unlike `BlockRef` traversal
* prune `BlockRef` parents on finality (~200:ish mb)
* speed up ChainDAG init by not loading finalized history index
* mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
|
|
|
node.withStateForBlockSlotId(bslot):
|
2022-06-20 05:53:39 +00:00
|
|
|
return RestApiResponse.jsonResponseWOpt(
|
|
|
|
(root: stateRoot),
|
|
|
|
node.getStateOptimistic(state)
|
|
|
|
)
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
|
2021-10-14 10:38:38 +00:00
|
|
|
return RestApiResponse.jsonError(Http404, StateNotFoundError)
|
2021-03-17 18:46:45 +00:00
|
|
|
|
2021-08-23 10:41:48 +00:00
|
|
|
# https://ethereum.github.io/beacon-APIs/#/Beacon/getStateFork
|
2022-01-06 07:38:40 +00:00
|
|
|
router.api(MethodGet, "/eth/v1/beacon/states/{state_id}/fork") do (
|
2021-03-17 18:46:45 +00:00
|
|
|
state_id: StateIdent) -> RestApiResponse:
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
let
|
|
|
|
sid = state_id.valueOr:
|
|
|
|
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
|
|
|
|
$error)
|
Prune `BlockRef` on finalization (#3513)
Up til now, the block dag has been using `BlockRef`, a structure adapted
for a full DAG, to represent all of chain history. This is a correct and
simple design, but does not exploit the linearity of the chain once
parts of it finalize.
By pruning the in-memory `BlockRef` structure at finalization, we save,
at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory
landing us at a steady state of ~750mb normal memory usage for a
validating node.
Above all though, we prevent memory usage from growing proportionally
with the length of the chain, something that would not be sustainable
over time - instead, the steady state memory usage is roughly
determined by the validator set size which grows much more slowly. With
these changes, the core should remain sustainable memory-wise post-merge
all the way to withdrawals (when the validator set is expected to grow).
In-memory indices are still used for the "hot" unfinalized portion of
the chain - this ensure that consensus performance remains unchanged.
What changes is that for historical access, we use a db-based linear
slot index which is cache-and-disk-friendly, keeping the cost for
accessing historical data at a similar level as before, achieving the
savings at no percievable cost to functionality or performance.
A nice collateral benefit is the almost-instant startup since we no
longer load any large indicies at dag init.
The cost of this functionality instead can be found in the complexity of
having to deal with two ways of traversing the chain - by `BlockRef` and
by slot.
* use `BlockId` instead of `BlockRef` where finalized / historical data
may be required
* simplify clearance pre-advancement
* remove dag.finalizedBlocks (~50:ish mb)
* remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead
* `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef`
instance, unlike `BlockRef` traversal
* prune `BlockRef` parents on finality (~200:ish mb)
* speed up ChainDAG init by not loading finalized history index
* mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
|
|
|
bslot = node.getBlockSlotId(sid).valueOr:
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
if sid.kind == StateQueryKind.Root:
|
|
|
|
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
|
|
|
|
# in current version of database.
|
|
|
|
return RestApiResponse.jsonError(Http500, NoImplementationError)
|
|
|
|
return RestApiResponse.jsonError(Http404, StateNotFoundError,
|
|
|
|
$error)
|
|
|
|
|
Prune `BlockRef` on finalization (#3513)
Up til now, the block dag has been using `BlockRef`, a structure adapted
for a full DAG, to represent all of chain history. This is a correct and
simple design, but does not exploit the linearity of the chain once
parts of it finalize.
By pruning the in-memory `BlockRef` structure at finalization, we save,
at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory
landing us at a steady state of ~750mb normal memory usage for a
validating node.
Above all though, we prevent memory usage from growing proportionally
with the length of the chain, something that would not be sustainable
over time - instead, the steady state memory usage is roughly
determined by the validator set size which grows much more slowly. With
these changes, the core should remain sustainable memory-wise post-merge
all the way to withdrawals (when the validator set is expected to grow).
In-memory indices are still used for the "hot" unfinalized portion of
the chain - this ensure that consensus performance remains unchanged.
What changes is that for historical access, we use a db-based linear
slot index which is cache-and-disk-friendly, keeping the cost for
accessing historical data at a similar level as before, achieving the
savings at no percievable cost to functionality or performance.
A nice collateral benefit is the almost-instant startup since we no
longer load any large indicies at dag init.
The cost of this functionality instead can be found in the complexity of
having to deal with two ways of traversing the chain - by `BlockRef` and
by slot.
* use `BlockId` instead of `BlockRef` where finalized / historical data
may be required
* simplify clearance pre-advancement
* remove dag.finalizedBlocks (~50:ish mb)
* remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead
* `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef`
instance, unlike `BlockRef` traversal
* prune `BlockRef` parents on finality (~200:ish mb)
* speed up ChainDAG init by not loading finalized history index
* mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
|
|
|
node.withStateForBlockSlotId(bslot):
|
2022-06-20 05:53:39 +00:00
|
|
|
return RestApiResponse.jsonResponseWOpt(
|
2021-03-23 22:50:18 +00:00
|
|
|
(
|
2022-06-20 05:53:39 +00:00
|
|
|
previous_version:
|
|
|
|
getStateField(state, fork).previous_version,
|
|
|
|
current_version:
|
|
|
|
getStateField(state, fork).current_version,
|
|
|
|
epoch:
|
|
|
|
getStateField(state, fork).epoch
|
|
|
|
),
|
|
|
|
node.getStateOptimistic(state)
|
2021-03-17 18:46:45 +00:00
|
|
|
)
|
2021-10-14 10:38:38 +00:00
|
|
|
return RestApiResponse.jsonError(Http404, StateNotFoundError)
|
2021-03-17 18:46:45 +00:00
|
|
|
|
2021-08-23 10:41:48 +00:00
|
|
|
# https://ethereum.github.io/beacon-APIs/#/Beacon/getStateFinalityCheckpoints
|
2021-03-17 18:46:45 +00:00
|
|
|
router.api(MethodGet,
|
2022-01-06 07:38:40 +00:00
|
|
|
"/eth/v1/beacon/states/{state_id}/finality_checkpoints") do (
|
2021-03-17 18:46:45 +00:00
|
|
|
state_id: StateIdent) -> RestApiResponse:
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
let
|
|
|
|
sid = state_id.valueOr:
|
|
|
|
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
|
|
|
|
$error)
|
Prune `BlockRef` on finalization (#3513)
Up til now, the block dag has been using `BlockRef`, a structure adapted
for a full DAG, to represent all of chain history. This is a correct and
simple design, but does not exploit the linearity of the chain once
parts of it finalize.
By pruning the in-memory `BlockRef` structure at finalization, we save,
at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory
landing us at a steady state of ~750mb normal memory usage for a
validating node.
Above all though, we prevent memory usage from growing proportionally
with the length of the chain, something that would not be sustainable
over time - instead, the steady state memory usage is roughly
determined by the validator set size which grows much more slowly. With
these changes, the core should remain sustainable memory-wise post-merge
all the way to withdrawals (when the validator set is expected to grow).
In-memory indices are still used for the "hot" unfinalized portion of
the chain - this ensure that consensus performance remains unchanged.
What changes is that for historical access, we use a db-based linear
slot index which is cache-and-disk-friendly, keeping the cost for
accessing historical data at a similar level as before, achieving the
savings at no percievable cost to functionality or performance.
A nice collateral benefit is the almost-instant startup since we no
longer load any large indicies at dag init.
The cost of this functionality instead can be found in the complexity of
having to deal with two ways of traversing the chain - by `BlockRef` and
by slot.
* use `BlockId` instead of `BlockRef` where finalized / historical data
may be required
* simplify clearance pre-advancement
* remove dag.finalizedBlocks (~50:ish mb)
* remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead
* `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef`
instance, unlike `BlockRef` traversal
* prune `BlockRef` parents on finality (~200:ish mb)
* speed up ChainDAG init by not loading finalized history index
* mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
|
|
|
bslot = node.getBlockSlotId(sid).valueOr:
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
if sid.kind == StateQueryKind.Root:
|
|
|
|
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
|
|
|
|
# in current version of database.
|
|
|
|
return RestApiResponse.jsonError(Http500, NoImplementationError)
|
|
|
|
return RestApiResponse.jsonError(Http404, StateNotFoundError,
|
2022-06-20 05:53:39 +00:00
|
|
|
$error)
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
|
Prune `BlockRef` on finalization (#3513)
Up til now, the block dag has been using `BlockRef`, a structure adapted
for a full DAG, to represent all of chain history. This is a correct and
simple design, but does not exploit the linearity of the chain once
parts of it finalize.
By pruning the in-memory `BlockRef` structure at finalization, we save,
at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory
landing us at a steady state of ~750mb normal memory usage for a
validating node.
Above all though, we prevent memory usage from growing proportionally
with the length of the chain, something that would not be sustainable
over time - instead, the steady state memory usage is roughly
determined by the validator set size which grows much more slowly. With
these changes, the core should remain sustainable memory-wise post-merge
all the way to withdrawals (when the validator set is expected to grow).
In-memory indices are still used for the "hot" unfinalized portion of
the chain - this ensure that consensus performance remains unchanged.
What changes is that for historical access, we use a db-based linear
slot index which is cache-and-disk-friendly, keeping the cost for
accessing historical data at a similar level as before, achieving the
savings at no percievable cost to functionality or performance.
A nice collateral benefit is the almost-instant startup since we no
longer load any large indicies at dag init.
The cost of this functionality instead can be found in the complexity of
having to deal with two ways of traversing the chain - by `BlockRef` and
by slot.
* use `BlockId` instead of `BlockRef` where finalized / historical data
may be required
* simplify clearance pre-advancement
* remove dag.finalizedBlocks (~50:ish mb)
* remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead
* `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef`
instance, unlike `BlockRef` traversal
* prune `BlockRef` parents on finality (~200:ish mb)
* speed up ChainDAG init by not loading finalized history index
* mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
|
|
|
node.withStateForBlockSlotId(bslot):
|
2022-06-20 05:53:39 +00:00
|
|
|
return RestApiResponse.jsonResponseWOpt(
|
2021-03-23 22:50:18 +00:00
|
|
|
(
|
2022-06-20 05:53:39 +00:00
|
|
|
previous_justified:
|
|
|
|
getStateField(state, previous_justified_checkpoint),
|
|
|
|
current_justified:
|
|
|
|
getStateField(state, current_justified_checkpoint),
|
|
|
|
finalized:
|
|
|
|
getStateField(state, finalized_checkpoint)
|
|
|
|
),
|
|
|
|
node.getStateOptimistic(state)
|
2021-03-17 18:46:45 +00:00
|
|
|
)
|
2021-10-14 10:38:38 +00:00
|
|
|
return RestApiResponse.jsonError(Http404, StateNotFoundError)
|
2021-03-17 18:46:45 +00:00
|
|
|
|
2021-08-23 10:41:48 +00:00
|
|
|
# https://ethereum.github.io/beacon-APIs/#/Beacon/getStateValidators
|
2022-01-06 07:38:40 +00:00
|
|
|
router.api(MethodGet, "/eth/v1/beacon/states/{state_id}/validators") do (
|
2021-03-17 18:46:45 +00:00
|
|
|
state_id: StateIdent, id: seq[ValidatorIdent],
|
|
|
|
status: seq[ValidatorFilter]) -> RestApiResponse:
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
let
|
|
|
|
sid = state_id.valueOr:
|
|
|
|
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
|
|
|
|
$error)
|
Prune `BlockRef` on finalization (#3513)
Up til now, the block dag has been using `BlockRef`, a structure adapted
for a full DAG, to represent all of chain history. This is a correct and
simple design, but does not exploit the linearity of the chain once
parts of it finalize.
By pruning the in-memory `BlockRef` structure at finalization, we save,
at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory
landing us at a steady state of ~750mb normal memory usage for a
validating node.
Above all though, we prevent memory usage from growing proportionally
with the length of the chain, something that would not be sustainable
over time - instead, the steady state memory usage is roughly
determined by the validator set size which grows much more slowly. With
these changes, the core should remain sustainable memory-wise post-merge
all the way to withdrawals (when the validator set is expected to grow).
In-memory indices are still used for the "hot" unfinalized portion of
the chain - this ensure that consensus performance remains unchanged.
What changes is that for historical access, we use a db-based linear
slot index which is cache-and-disk-friendly, keeping the cost for
accessing historical data at a similar level as before, achieving the
savings at no percievable cost to functionality or performance.
A nice collateral benefit is the almost-instant startup since we no
longer load any large indicies at dag init.
The cost of this functionality instead can be found in the complexity of
having to deal with two ways of traversing the chain - by `BlockRef` and
by slot.
* use `BlockId` instead of `BlockRef` where finalized / historical data
may be required
* simplify clearance pre-advancement
* remove dag.finalizedBlocks (~50:ish mb)
* remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead
* `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef`
instance, unlike `BlockRef` traversal
* prune `BlockRef` parents on finality (~200:ish mb)
* speed up ChainDAG init by not loading finalized history index
* mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
|
|
|
bslot = node.getBlockSlotId(sid).valueOr:
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
if sid.kind == StateQueryKind.Root:
|
|
|
|
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
|
|
|
|
# in current version of database.
|
|
|
|
return RestApiResponse.jsonError(Http500, NoImplementationError)
|
|
|
|
return RestApiResponse.jsonError(Http404, StateNotFoundError,
|
|
|
|
$error)
|
2021-03-17 18:46:45 +00:00
|
|
|
let validatorIds =
|
|
|
|
block:
|
|
|
|
if id.isErr():
|
|
|
|
return RestApiResponse.jsonError(Http400,
|
2021-04-08 10:49:28 +00:00
|
|
|
InvalidValidatorIdValueError)
|
2021-03-29 10:59:39 +00:00
|
|
|
let ires = id.get()
|
|
|
|
if len(ires) > MaximumValidatorIds:
|
|
|
|
return RestApiResponse.jsonError(Http400,
|
2021-04-08 10:49:28 +00:00
|
|
|
MaximumNumberOfValidatorIdsError)
|
2021-03-29 10:59:39 +00:00
|
|
|
ires
|
2021-03-17 18:46:45 +00:00
|
|
|
|
|
|
|
let validatorsMask =
|
|
|
|
block:
|
|
|
|
if status.isErr():
|
|
|
|
return RestApiResponse.jsonError(Http400,
|
2021-04-08 10:49:28 +00:00
|
|
|
InvalidValidatorStatusValueError)
|
2021-03-17 18:46:45 +00:00
|
|
|
let res = validateFilter(status.get())
|
|
|
|
if res.isErr():
|
|
|
|
return RestApiResponse.jsonError(Http400,
|
2021-04-08 10:49:28 +00:00
|
|
|
InvalidValidatorStatusValueError,
|
2021-03-17 18:46:45 +00:00
|
|
|
$res.error())
|
|
|
|
res.get()
|
|
|
|
|
Prune `BlockRef` on finalization (#3513)
Up til now, the block dag has been using `BlockRef`, a structure adapted
for a full DAG, to represent all of chain history. This is a correct and
simple design, but does not exploit the linearity of the chain once
parts of it finalize.
By pruning the in-memory `BlockRef` structure at finalization, we save,
at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory
landing us at a steady state of ~750mb normal memory usage for a
validating node.
Above all though, we prevent memory usage from growing proportionally
with the length of the chain, something that would not be sustainable
over time - instead, the steady state memory usage is roughly
determined by the validator set size which grows much more slowly. With
these changes, the core should remain sustainable memory-wise post-merge
all the way to withdrawals (when the validator set is expected to grow).
In-memory indices are still used for the "hot" unfinalized portion of
the chain - this ensure that consensus performance remains unchanged.
What changes is that for historical access, we use a db-based linear
slot index which is cache-and-disk-friendly, keeping the cost for
accessing historical data at a similar level as before, achieving the
savings at no percievable cost to functionality or performance.
A nice collateral benefit is the almost-instant startup since we no
longer load any large indicies at dag init.
The cost of this functionality instead can be found in the complexity of
having to deal with two ways of traversing the chain - by `BlockRef` and
by slot.
* use `BlockId` instead of `BlockRef` where finalized / historical data
may be required
* simplify clearance pre-advancement
* remove dag.finalizedBlocks (~50:ish mb)
* remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead
* `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef`
instance, unlike `BlockRef` traversal
* prune `BlockRef` parents on finality (~200:ish mb)
* speed up ChainDAG init by not loading finalized history index
* mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
|
|
|
node.withStateForBlockSlotId(bslot):
|
2021-10-14 10:38:38 +00:00
|
|
|
let
|
2022-03-16 07:20:40 +00:00
|
|
|
current_epoch = getStateField(state, slot).epoch()
|
|
|
|
validatorsCount = lenu64(getStateField(state, validators))
|
2021-10-14 10:38:38 +00:00
|
|
|
|
|
|
|
let indices =
|
|
|
|
block:
|
|
|
|
var keyset: HashSet[ValidatorPubKey]
|
|
|
|
var indexset: HashSet[ValidatorIndex]
|
|
|
|
for item in validatorIds:
|
|
|
|
case item.kind
|
|
|
|
of ValidatorQueryKind.Key:
|
|
|
|
keyset.incl(item.key)
|
|
|
|
of ValidatorQueryKind.Index:
|
|
|
|
let vindex =
|
|
|
|
block:
|
|
|
|
let vres = item.index.toValidatorIndex()
|
|
|
|
if vres.isErr():
|
|
|
|
case vres.error()
|
|
|
|
of ValidatorIndexError.TooHighValue:
|
|
|
|
return RestApiResponse.jsonError(Http400,
|
2021-04-08 10:49:28 +00:00
|
|
|
TooHighValidatorIndexValueError)
|
2021-10-14 10:38:38 +00:00
|
|
|
of ValidatorIndexError.UnsupportedValue:
|
|
|
|
return RestApiResponse.jsonError(Http500,
|
2021-04-08 10:49:28 +00:00
|
|
|
UnsupportedValidatorIndexValueError)
|
2021-10-14 10:38:38 +00:00
|
|
|
let index = vres.get()
|
|
|
|
index
|
2021-10-18 08:54:20 +00:00
|
|
|
if uint64(vindex) < validatorsCount:
|
|
|
|
# We only adding validator indices which are present in
|
|
|
|
# validators list at this moment.
|
|
|
|
indexset.incl(vindex)
|
2021-10-14 10:38:38 +00:00
|
|
|
|
|
|
|
if len(keyset) > 0:
|
2022-03-16 07:20:40 +00:00
|
|
|
let optIndices = keysToIndices(node.restKeysCache, state,
|
2021-10-14 10:38:38 +00:00
|
|
|
keyset.toSeq())
|
2021-10-18 08:54:20 +00:00
|
|
|
# Remove all the duplicates.
|
2021-10-14 10:38:38 +00:00
|
|
|
for item in optIndices:
|
2021-10-18 08:54:20 +00:00
|
|
|
# We ignore missing keys.
|
|
|
|
if item.isSome():
|
|
|
|
indexset.incl(item.get())
|
2021-10-14 10:38:38 +00:00
|
|
|
indexset.toSeq()
|
2021-03-17 18:46:45 +00:00
|
|
|
|
2021-10-14 10:38:38 +00:00
|
|
|
let response =
|
|
|
|
block:
|
|
|
|
var res: seq[RestValidator]
|
|
|
|
if len(indices) == 0:
|
2021-10-25 13:52:00 +00:00
|
|
|
# Case when `len(indices) == 0 and len(validatorIds) != 0` means
|
|
|
|
# that we can't find validator identifiers in state, so we should
|
|
|
|
# return empty response.
|
|
|
|
if len(validatorIds) == 0:
|
|
|
|
# There is no indices, so we going to filter all the validators.
|
2022-05-10 10:03:40 +00:00
|
|
|
for index, validator in getStateField(state, validators):
|
2021-10-25 13:52:00 +00:00
|
|
|
let
|
2022-05-30 13:30:42 +00:00
|
|
|
balance = getStateField(state, balances).item(index)
|
2021-10-25 13:52:00 +00:00
|
|
|
status =
|
|
|
|
block:
|
|
|
|
let sres = validator.getStatus(current_epoch)
|
|
|
|
if sres.isErr():
|
|
|
|
return RestApiResponse.jsonError(Http400,
|
|
|
|
ValidatorStatusNotFoundError,
|
|
|
|
$sres.get())
|
|
|
|
sres.get()
|
|
|
|
if status in validatorsMask:
|
|
|
|
res.add(RestValidator.init(ValidatorIndex(index), balance,
|
|
|
|
toString(status), validator))
|
2021-10-14 10:38:38 +00:00
|
|
|
else:
|
|
|
|
for index in indices:
|
|
|
|
let
|
2022-05-30 13:30:42 +00:00
|
|
|
validator = getStateField(state, validators).item(index)
|
|
|
|
balance = getStateField(state, balances).item(index)
|
2021-10-14 10:38:38 +00:00
|
|
|
status =
|
|
|
|
block:
|
|
|
|
let sres = validator.getStatus(current_epoch)
|
|
|
|
if sres.isErr():
|
|
|
|
return RestApiResponse.jsonError(Http400,
|
|
|
|
ValidatorStatusNotFoundError,
|
|
|
|
$sres.get())
|
|
|
|
sres.get()
|
|
|
|
if status in validatorsMask:
|
|
|
|
res.add(RestValidator.init(index, balance, toString(status),
|
|
|
|
validator))
|
|
|
|
res
|
2022-06-20 05:53:39 +00:00
|
|
|
return RestApiResponse.jsonResponseWOpt(
|
|
|
|
response,
|
|
|
|
node.getStateOptimistic(state)
|
|
|
|
)
|
2021-10-14 10:38:38 +00:00
|
|
|
return RestApiResponse.jsonError(Http404, StateNotFoundError)
|
2021-03-17 18:46:45 +00:00
|
|
|
|
2021-08-23 10:41:48 +00:00
|
|
|
# https://ethereum.github.io/beacon-APIs/#/Beacon/getStateValidator
|
2021-03-17 18:46:45 +00:00
|
|
|
router.api(MethodGet,
|
2022-01-06 07:38:40 +00:00
|
|
|
"/eth/v1/beacon/states/{state_id}/validators/{validator_id}") do (
|
2021-03-17 18:46:45 +00:00
|
|
|
state_id: StateIdent, validator_id: ValidatorIdent) -> RestApiResponse:
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
let
|
|
|
|
sid = state_id.valueOr:
|
|
|
|
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
|
|
|
|
$error)
|
|
|
|
vid = validator_id.valueOr:
|
|
|
|
return RestApiResponse.jsonError(Http400, InvalidValidatorIdValueError,
|
|
|
|
$error)
|
Prune `BlockRef` on finalization (#3513)
Up til now, the block dag has been using `BlockRef`, a structure adapted
for a full DAG, to represent all of chain history. This is a correct and
simple design, but does not exploit the linearity of the chain once
parts of it finalize.
By pruning the in-memory `BlockRef` structure at finalization, we save,
at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory
landing us at a steady state of ~750mb normal memory usage for a
validating node.
Above all though, we prevent memory usage from growing proportionally
with the length of the chain, something that would not be sustainable
over time - instead, the steady state memory usage is roughly
determined by the validator set size which grows much more slowly. With
these changes, the core should remain sustainable memory-wise post-merge
all the way to withdrawals (when the validator set is expected to grow).
In-memory indices are still used for the "hot" unfinalized portion of
the chain - this ensure that consensus performance remains unchanged.
What changes is that for historical access, we use a db-based linear
slot index which is cache-and-disk-friendly, keeping the cost for
accessing historical data at a similar level as before, achieving the
savings at no percievable cost to functionality or performance.
A nice collateral benefit is the almost-instant startup since we no
longer load any large indicies at dag init.
The cost of this functionality instead can be found in the complexity of
having to deal with two ways of traversing the chain - by `BlockRef` and
by slot.
* use `BlockId` instead of `BlockRef` where finalized / historical data
may be required
* simplify clearance pre-advancement
* remove dag.finalizedBlocks (~50:ish mb)
* remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead
* `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef`
instance, unlike `BlockRef` traversal
* prune `BlockRef` parents on finality (~200:ish mb)
* speed up ChainDAG init by not loading finalized history index
* mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
|
|
|
bslot = node.getBlockSlotId(sid).valueOr:
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
if sid.kind == StateQueryKind.Root:
|
|
|
|
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
|
|
|
|
# in current version of database.
|
|
|
|
return RestApiResponse.jsonError(Http500, NoImplementationError)
|
|
|
|
return RestApiResponse.jsonError(Http404, StateNotFoundError,
|
|
|
|
$error)
|
|
|
|
|
Prune `BlockRef` on finalization (#3513)
Up til now, the block dag has been using `BlockRef`, a structure adapted
for a full DAG, to represent all of chain history. This is a correct and
simple design, but does not exploit the linearity of the chain once
parts of it finalize.
By pruning the in-memory `BlockRef` structure at finalization, we save,
at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory
landing us at a steady state of ~750mb normal memory usage for a
validating node.
Above all though, we prevent memory usage from growing proportionally
with the length of the chain, something that would not be sustainable
over time - instead, the steady state memory usage is roughly
determined by the validator set size which grows much more slowly. With
these changes, the core should remain sustainable memory-wise post-merge
all the way to withdrawals (when the validator set is expected to grow).
In-memory indices are still used for the "hot" unfinalized portion of
the chain - this ensure that consensus performance remains unchanged.
What changes is that for historical access, we use a db-based linear
slot index which is cache-and-disk-friendly, keeping the cost for
accessing historical data at a similar level as before, achieving the
savings at no percievable cost to functionality or performance.
A nice collateral benefit is the almost-instant startup since we no
longer load any large indicies at dag init.
The cost of this functionality instead can be found in the complexity of
having to deal with two ways of traversing the chain - by `BlockRef` and
by slot.
* use `BlockId` instead of `BlockRef` where finalized / historical data
may be required
* simplify clearance pre-advancement
* remove dag.finalizedBlocks (~50:ish mb)
* remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead
* `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef`
instance, unlike `BlockRef` traversal
* prune `BlockRef` parents on finality (~200:ish mb)
* speed up ChainDAG init by not loading finalized history index
* mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
|
|
|
node.withStateForBlockSlotId(bslot):
|
2021-10-14 10:38:38 +00:00
|
|
|
let
|
2022-03-16 07:20:40 +00:00
|
|
|
current_epoch = getStateField(state, slot).epoch()
|
|
|
|
validatorsCount = lenu64(getStateField(state, validators))
|
2021-10-14 10:38:38 +00:00
|
|
|
|
|
|
|
let vindex =
|
|
|
|
block:
|
|
|
|
case vid.kind
|
|
|
|
of ValidatorQueryKind.Key:
|
Prune `BlockRef` on finalization (#3513)
Up til now, the block dag has been using `BlockRef`, a structure adapted
for a full DAG, to represent all of chain history. This is a correct and
simple design, but does not exploit the linearity of the chain once
parts of it finalize.
By pruning the in-memory `BlockRef` structure at finalization, we save,
at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory
landing us at a steady state of ~750mb normal memory usage for a
validating node.
Above all though, we prevent memory usage from growing proportionally
with the length of the chain, something that would not be sustainable
over time - instead, the steady state memory usage is roughly
determined by the validator set size which grows much more slowly. With
these changes, the core should remain sustainable memory-wise post-merge
all the way to withdrawals (when the validator set is expected to grow).
In-memory indices are still used for the "hot" unfinalized portion of
the chain - this ensure that consensus performance remains unchanged.
What changes is that for historical access, we use a db-based linear
slot index which is cache-and-disk-friendly, keeping the cost for
accessing historical data at a similar level as before, achieving the
savings at no percievable cost to functionality or performance.
A nice collateral benefit is the almost-instant startup since we no
longer load any large indicies at dag init.
The cost of this functionality instead can be found in the complexity of
having to deal with two ways of traversing the chain - by `BlockRef` and
by slot.
* use `BlockId` instead of `BlockRef` where finalized / historical data
may be required
* simplify clearance pre-advancement
* remove dag.finalizedBlocks (~50:ish mb)
* remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead
* `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef`
instance, unlike `BlockRef` traversal
* prune `BlockRef` parents on finality (~200:ish mb)
* speed up ChainDAG init by not loading finalized history index
* mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
|
|
|
let optIndices = keysToIndices(node.restKeysCache, state, [vid.key])
|
2021-10-14 10:38:38 +00:00
|
|
|
if optIndices[0].isNone():
|
2021-10-18 08:54:20 +00:00
|
|
|
return RestApiResponse.jsonError(Http404, ValidatorNotFoundError)
|
2021-10-14 10:38:38 +00:00
|
|
|
optIndices[0].get()
|
|
|
|
of ValidatorQueryKind.Index:
|
2021-04-03 00:21:44 +00:00
|
|
|
let vres = vid.index.toValidatorIndex()
|
|
|
|
if vres.isErr():
|
|
|
|
case vres.error()
|
|
|
|
of ValidatorIndexError.TooHighValue:
|
|
|
|
return RestApiResponse.jsonError(Http400,
|
2021-04-08 10:49:28 +00:00
|
|
|
TooHighValidatorIndexValueError)
|
2021-04-03 00:21:44 +00:00
|
|
|
of ValidatorIndexError.UnsupportedValue:
|
|
|
|
return RestApiResponse.jsonError(Http500,
|
2021-04-08 10:49:28 +00:00
|
|
|
UnsupportedValidatorIndexValueError)
|
2021-10-14 10:38:38 +00:00
|
|
|
let index = vres.get()
|
|
|
|
if uint64(index) >= validatorsCount:
|
2021-10-18 08:54:20 +00:00
|
|
|
return RestApiResponse.jsonError(Http404, ValidatorNotFoundError)
|
2021-10-14 10:38:38 +00:00
|
|
|
index
|
|
|
|
|
|
|
|
let
|
2022-05-30 13:30:42 +00:00
|
|
|
validator = getStateField(state, validators).item(vindex)
|
|
|
|
balance = getStateField(state, balances).item(vindex)
|
2021-10-14 10:38:38 +00:00
|
|
|
status =
|
|
|
|
block:
|
|
|
|
let sres = validator.getStatus(current_epoch)
|
|
|
|
if sres.isErr():
|
|
|
|
return RestApiResponse.jsonError(Http400,
|
|
|
|
ValidatorStatusNotFoundError,
|
|
|
|
$sres.get())
|
|
|
|
toString(sres.get())
|
2022-06-20 05:53:39 +00:00
|
|
|
return RestApiResponse.jsonResponseWOpt(
|
|
|
|
RestValidator.init(vindex, balance, status, validator),
|
|
|
|
node.getStateOptimistic(state)
|
2021-10-14 10:38:38 +00:00
|
|
|
)
|
|
|
|
return RestApiResponse.jsonError(Http404, StateNotFoundError)
|
2021-03-17 18:46:45 +00:00
|
|
|
|
2021-08-23 10:41:48 +00:00
|
|
|
# https://ethereum.github.io/beacon-APIs/#/Beacon/getStateValidatorBalances
|
2021-03-17 18:46:45 +00:00
|
|
|
router.api(MethodGet,
|
2022-01-06 07:38:40 +00:00
|
|
|
"/eth/v1/beacon/states/{state_id}/validator_balances") do (
|
2021-03-17 18:46:45 +00:00
|
|
|
state_id: StateIdent, id: seq[ValidatorIdent]) -> RestApiResponse:
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
let
|
|
|
|
sid = state_id.valueOr:
|
|
|
|
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
|
|
|
|
$error)
|
Prune `BlockRef` on finalization (#3513)
Up til now, the block dag has been using `BlockRef`, a structure adapted
for a full DAG, to represent all of chain history. This is a correct and
simple design, but does not exploit the linearity of the chain once
parts of it finalize.
By pruning the in-memory `BlockRef` structure at finalization, we save,
at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory
landing us at a steady state of ~750mb normal memory usage for a
validating node.
Above all though, we prevent memory usage from growing proportionally
with the length of the chain, something that would not be sustainable
over time - instead, the steady state memory usage is roughly
determined by the validator set size which grows much more slowly. With
these changes, the core should remain sustainable memory-wise post-merge
all the way to withdrawals (when the validator set is expected to grow).
In-memory indices are still used for the "hot" unfinalized portion of
the chain - this ensure that consensus performance remains unchanged.
What changes is that for historical access, we use a db-based linear
slot index which is cache-and-disk-friendly, keeping the cost for
accessing historical data at a similar level as before, achieving the
savings at no percievable cost to functionality or performance.
A nice collateral benefit is the almost-instant startup since we no
longer load any large indicies at dag init.
The cost of this functionality instead can be found in the complexity of
having to deal with two ways of traversing the chain - by `BlockRef` and
by slot.
* use `BlockId` instead of `BlockRef` where finalized / historical data
may be required
* simplify clearance pre-advancement
* remove dag.finalizedBlocks (~50:ish mb)
* remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead
* `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef`
instance, unlike `BlockRef` traversal
* prune `BlockRef` parents on finality (~200:ish mb)
* speed up ChainDAG init by not loading finalized history index
* mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
|
|
|
bslot = node.getBlockSlotId(sid).valueOr:
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
if sid.kind == StateQueryKind.Root:
|
|
|
|
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
|
|
|
|
# in current version of database.
|
|
|
|
return RestApiResponse.jsonError(Http500, NoImplementationError)
|
|
|
|
return RestApiResponse.jsonError(Http404, StateNotFoundError,
|
|
|
|
$error)
|
|
|
|
|
2021-03-17 18:46:45 +00:00
|
|
|
let validatorIds =
|
|
|
|
block:
|
|
|
|
if id.isErr():
|
|
|
|
return RestApiResponse.jsonError(Http400,
|
2021-04-08 10:49:28 +00:00
|
|
|
InvalidValidatorIdValueError)
|
2021-03-29 10:59:39 +00:00
|
|
|
let ires = id.get()
|
|
|
|
if len(ires) > MaximumValidatorIds:
|
|
|
|
return RestApiResponse.jsonError(Http400,
|
2021-04-08 10:49:28 +00:00
|
|
|
MaximumNumberOfValidatorIdsError)
|
2021-03-29 10:59:39 +00:00
|
|
|
ires
|
2021-10-14 10:38:38 +00:00
|
|
|
|
Prune `BlockRef` on finalization (#3513)
Up til now, the block dag has been using `BlockRef`, a structure adapted
for a full DAG, to represent all of chain history. This is a correct and
simple design, but does not exploit the linearity of the chain once
parts of it finalize.
By pruning the in-memory `BlockRef` structure at finalization, we save,
at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory
landing us at a steady state of ~750mb normal memory usage for a
validating node.
Above all though, we prevent memory usage from growing proportionally
with the length of the chain, something that would not be sustainable
over time - instead, the steady state memory usage is roughly
determined by the validator set size which grows much more slowly. With
these changes, the core should remain sustainable memory-wise post-merge
all the way to withdrawals (when the validator set is expected to grow).
In-memory indices are still used for the "hot" unfinalized portion of
the chain - this ensure that consensus performance remains unchanged.
What changes is that for historical access, we use a db-based linear
slot index which is cache-and-disk-friendly, keeping the cost for
accessing historical data at a similar level as before, achieving the
savings at no percievable cost to functionality or performance.
A nice collateral benefit is the almost-instant startup since we no
longer load any large indicies at dag init.
The cost of this functionality instead can be found in the complexity of
having to deal with two ways of traversing the chain - by `BlockRef` and
by slot.
* use `BlockId` instead of `BlockRef` where finalized / historical data
may be required
* simplify clearance pre-advancement
* remove dag.finalizedBlocks (~50:ish mb)
* remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead
* `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef`
instance, unlike `BlockRef` traversal
* prune `BlockRef` parents on finality (~200:ish mb)
* speed up ChainDAG init by not loading finalized history index
* mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
|
|
|
node.withStateForBlockSlotId(bslot):
|
2022-03-16 07:20:40 +00:00
|
|
|
let validatorsCount = lenu64(getStateField(state, validators))
|
2021-10-14 10:38:38 +00:00
|
|
|
|
|
|
|
let indices =
|
|
|
|
block:
|
|
|
|
var keyset: HashSet[ValidatorPubKey]
|
|
|
|
var indexset: HashSet[ValidatorIndex]
|
|
|
|
for item in validatorIds:
|
|
|
|
case item.kind
|
|
|
|
of ValidatorQueryKind.Key:
|
|
|
|
keyset.incl(item.key)
|
|
|
|
of ValidatorQueryKind.Index:
|
|
|
|
let vindex =
|
|
|
|
block:
|
|
|
|
let vres = item.index.toValidatorIndex()
|
|
|
|
if vres.isErr():
|
|
|
|
case vres.error()
|
|
|
|
of ValidatorIndexError.TooHighValue:
|
|
|
|
return RestApiResponse.jsonError(Http400,
|
2021-04-08 10:49:28 +00:00
|
|
|
TooHighValidatorIndexValueError)
|
2021-10-14 10:38:38 +00:00
|
|
|
of ValidatorIndexError.UnsupportedValue:
|
|
|
|
return RestApiResponse.jsonError(Http500,
|
2021-04-08 10:49:28 +00:00
|
|
|
UnsupportedValidatorIndexValueError)
|
2021-10-18 08:54:20 +00:00
|
|
|
vres.get()
|
|
|
|
# We only adding validator indices which are present in
|
|
|
|
# validators list at this moment.
|
|
|
|
if uint64(vindex) < validatorsCount:
|
|
|
|
indexset.incl(vindex)
|
2021-10-14 10:38:38 +00:00
|
|
|
|
|
|
|
if len(keyset) > 0:
|
2022-03-16 07:20:40 +00:00
|
|
|
let optIndices = keysToIndices(node.restKeysCache, state,
|
2021-10-14 10:38:38 +00:00
|
|
|
keyset.toSeq())
|
2021-10-18 08:54:20 +00:00
|
|
|
# Remove all the duplicates.
|
2021-10-14 10:38:38 +00:00
|
|
|
for item in optIndices:
|
2021-10-18 08:54:20 +00:00
|
|
|
# We ignore missing keys.
|
|
|
|
if item.isSome():
|
|
|
|
indexset.incl(item.get())
|
2021-10-14 10:38:38 +00:00
|
|
|
indexset.toSeq()
|
|
|
|
|
|
|
|
let response =
|
|
|
|
block:
|
|
|
|
var res: seq[RestValidatorBalance]
|
|
|
|
if len(indices) == 0:
|
2021-10-25 13:52:00 +00:00
|
|
|
# Case when `len(indices) == 0 and len(validatorIds) != 0` means
|
|
|
|
# that we can't find validator identifiers in state, so we should
|
|
|
|
# return empty response.
|
|
|
|
if len(validatorIds) == 0:
|
|
|
|
# There is no indices, so we going to return balances of all
|
|
|
|
# known validators.
|
2022-05-10 10:03:40 +00:00
|
|
|
for index, balance in getStateField(state, balances):
|
2021-10-25 13:52:00 +00:00
|
|
|
res.add(RestValidatorBalance.init(ValidatorIndex(index),
|
|
|
|
balance))
|
2021-10-14 10:38:38 +00:00
|
|
|
else:
|
|
|
|
for index in indices:
|
2022-05-30 13:30:42 +00:00
|
|
|
let balance = getStateField(state, balances).item(index)
|
2021-10-14 10:38:38 +00:00
|
|
|
res.add(RestValidatorBalance.init(index, balance))
|
|
|
|
res
|
2022-06-20 05:53:39 +00:00
|
|
|
return RestApiResponse.jsonResponseWOpt(
|
|
|
|
response,
|
|
|
|
node.getStateOptimistic(state)
|
|
|
|
)
|
2021-03-17 18:46:45 +00:00
|
|
|
|
2021-10-14 10:38:38 +00:00
|
|
|
return RestApiResponse.jsonError(Http404, StateNotFoundError)
|
2021-03-17 18:46:45 +00:00
|
|
|
|
2021-08-23 10:41:48 +00:00
|
|
|
# https://ethereum.github.io/beacon-APIs/#/Beacon/getEpochCommittees
|
2021-03-17 18:46:45 +00:00
|
|
|
router.api(MethodGet,
|
2022-01-06 07:38:40 +00:00
|
|
|
"/eth/v1/beacon/states/{state_id}/committees") do (
|
2021-03-17 18:46:45 +00:00
|
|
|
state_id: StateIdent, epoch: Option[Epoch], index: Option[CommitteeIndex],
|
|
|
|
slot: Option[Slot]) -> RestApiResponse:
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
let
|
|
|
|
sid = state_id.valueOr:
|
|
|
|
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
|
|
|
|
$error)
|
Prune `BlockRef` on finalization (#3513)
Up til now, the block dag has been using `BlockRef`, a structure adapted
for a full DAG, to represent all of chain history. This is a correct and
simple design, but does not exploit the linearity of the chain once
parts of it finalize.
By pruning the in-memory `BlockRef` structure at finalization, we save,
at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory
landing us at a steady state of ~750mb normal memory usage for a
validating node.
Above all though, we prevent memory usage from growing proportionally
with the length of the chain, something that would not be sustainable
over time - instead, the steady state memory usage is roughly
determined by the validator set size which grows much more slowly. With
these changes, the core should remain sustainable memory-wise post-merge
all the way to withdrawals (when the validator set is expected to grow).
In-memory indices are still used for the "hot" unfinalized portion of
the chain - this ensure that consensus performance remains unchanged.
What changes is that for historical access, we use a db-based linear
slot index which is cache-and-disk-friendly, keeping the cost for
accessing historical data at a similar level as before, achieving the
savings at no percievable cost to functionality or performance.
A nice collateral benefit is the almost-instant startup since we no
longer load any large indicies at dag init.
The cost of this functionality instead can be found in the complexity of
having to deal with two ways of traversing the chain - by `BlockRef` and
by slot.
* use `BlockId` instead of `BlockRef` where finalized / historical data
may be required
* simplify clearance pre-advancement
* remove dag.finalizedBlocks (~50:ish mb)
* remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead
* `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef`
instance, unlike `BlockRef` traversal
* prune `BlockRef` parents on finality (~200:ish mb)
* speed up ChainDAG init by not loading finalized history index
* mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
|
|
|
bslot = node.getBlockSlotId(sid).valueOr:
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
if sid.kind == StateQueryKind.Root:
|
|
|
|
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
|
|
|
|
# in current version of database.
|
|
|
|
return RestApiResponse.jsonError(Http500, NoImplementationError)
|
|
|
|
return RestApiResponse.jsonError(Http404, StateNotFoundError,
|
|
|
|
$error)
|
|
|
|
|
2021-03-17 18:46:45 +00:00
|
|
|
let vepoch =
|
|
|
|
if epoch.isSome():
|
|
|
|
let repoch = epoch.get()
|
|
|
|
if repoch.isErr():
|
2021-04-08 10:49:28 +00:00
|
|
|
return RestApiResponse.jsonError(Http400, InvalidEpochValueError,
|
2021-03-17 18:46:45 +00:00
|
|
|
$repoch.error())
|
2021-04-04 09:48:44 +00:00
|
|
|
let res = repoch.get()
|
2022-01-08 20:06:34 +00:00
|
|
|
|
|
|
|
if res > bslot.slot.epoch + MIN_SEED_LOOKAHEAD:
|
|
|
|
return RestApiResponse.jsonError(
|
|
|
|
Http400, InvalidEpochValueError,
|
|
|
|
"Requested epoch more than 1 epoch past state epoch")
|
|
|
|
|
|
|
|
if res + EPOCHS_PER_HISTORICAL_VECTOR <
|
|
|
|
bslot.slot.epoch + MIN_SEED_LOOKAHEAD:
|
|
|
|
return RestApiResponse.jsonError(
|
|
|
|
Http400, InvalidEpochValueError,
|
|
|
|
"Requested epoch earlier than what committees can be computed for")
|
|
|
|
|
2021-04-04 09:48:44 +00:00
|
|
|
some(res)
|
2021-03-17 18:46:45 +00:00
|
|
|
else:
|
|
|
|
none[Epoch]()
|
|
|
|
let vindex =
|
|
|
|
if index.isSome():
|
|
|
|
let rindex = index.get()
|
|
|
|
if rindex.isErr():
|
2021-04-08 10:49:28 +00:00
|
|
|
return RestApiResponse.jsonError(Http400,
|
|
|
|
InvalidCommitteeIndexValueError,
|
2021-03-17 18:46:45 +00:00
|
|
|
$rindex.error())
|
|
|
|
some(rindex.get())
|
|
|
|
else:
|
|
|
|
none[CommitteeIndex]()
|
|
|
|
let vslot =
|
|
|
|
if slot.isSome():
|
|
|
|
let rslot = slot.get()
|
|
|
|
if rslot.isErr():
|
2021-04-08 10:49:28 +00:00
|
|
|
return RestApiResponse.jsonError(Http400, InvalidSlotValueError,
|
2021-03-17 18:46:45 +00:00
|
|
|
$rslot.error())
|
2022-01-08 20:06:34 +00:00
|
|
|
let res = rslot.get()
|
|
|
|
if vepoch.isSome():
|
|
|
|
if res.epoch != vepoch.get():
|
|
|
|
return RestApiResponse.jsonError(Http400, InvalidSlotValueError,
|
|
|
|
"Slot does not match requested epoch")
|
|
|
|
else:
|
|
|
|
if res.epoch > bslot.slot.epoch + 1:
|
|
|
|
return RestApiResponse.jsonError(
|
|
|
|
Http400, InvalidEpochValueError,
|
|
|
|
"Requested slot more than 1 epoch past state epoch")
|
|
|
|
|
|
|
|
if res.epoch + EPOCHS_PER_HISTORICAL_VECTOR <
|
|
|
|
bslot.slot.epoch + MIN_SEED_LOOKAHEAD:
|
|
|
|
return RestApiResponse.jsonError(
|
|
|
|
Http400, InvalidEpochValueError,
|
|
|
|
"Requested slot earlier than what committees can be computed for")
|
|
|
|
|
|
|
|
some(res)
|
2021-03-17 18:46:45 +00:00
|
|
|
else:
|
|
|
|
none[Slot]()
|
Prune `BlockRef` on finalization (#3513)
Up til now, the block dag has been using `BlockRef`, a structure adapted
for a full DAG, to represent all of chain history. This is a correct and
simple design, but does not exploit the linearity of the chain once
parts of it finalize.
By pruning the in-memory `BlockRef` structure at finalization, we save,
at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory
landing us at a steady state of ~750mb normal memory usage for a
validating node.
Above all though, we prevent memory usage from growing proportionally
with the length of the chain, something that would not be sustainable
over time - instead, the steady state memory usage is roughly
determined by the validator set size which grows much more slowly. With
these changes, the core should remain sustainable memory-wise post-merge
all the way to withdrawals (when the validator set is expected to grow).
In-memory indices are still used for the "hot" unfinalized portion of
the chain - this ensure that consensus performance remains unchanged.
What changes is that for historical access, we use a db-based linear
slot index which is cache-and-disk-friendly, keeping the cost for
accessing historical data at a similar level as before, achieving the
savings at no percievable cost to functionality or performance.
A nice collateral benefit is the almost-instant startup since we no
longer load any large indicies at dag init.
The cost of this functionality instead can be found in the complexity of
having to deal with two ways of traversing the chain - by `BlockRef` and
by slot.
* use `BlockId` instead of `BlockRef` where finalized / historical data
may be required
* simplify clearance pre-advancement
* remove dag.finalizedBlocks (~50:ish mb)
* remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead
* `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef`
instance, unlike `BlockRef` traversal
* prune `BlockRef` parents on finality (~200:ish mb)
* speed up ChainDAG init by not loading finalized history index
* mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
|
|
|
node.withStateForBlockSlotId(bslot):
|
2021-03-17 18:46:45 +00:00
|
|
|
proc getCommittee(slot: Slot,
|
2021-09-23 22:13:25 +00:00
|
|
|
index: CommitteeIndex): RestBeaconStatesCommittees =
|
2022-03-16 07:20:40 +00:00
|
|
|
let validators = get_beacon_committee(state, slot, index, cache)
|
2021-05-20 17:56:12 +00:00
|
|
|
RestBeaconStatesCommittees(index: index, slot: slot,
|
|
|
|
validators: validators)
|
2021-03-17 18:46:45 +00:00
|
|
|
|
|
|
|
proc forSlot(slot: Slot, cindex: Option[CommitteeIndex],
|
2021-05-20 17:56:12 +00:00
|
|
|
res: var seq[RestBeaconStatesCommittees]) =
|
2022-01-12 20:42:03 +00:00
|
|
|
let committees_per_slot = get_committee_count_per_slot(
|
2022-03-16 07:20:40 +00:00
|
|
|
state, slot.epoch, cache)
|
2021-03-17 18:46:45 +00:00
|
|
|
|
|
|
|
if cindex.isNone:
|
2022-01-12 20:42:03 +00:00
|
|
|
for committee_index in get_committee_indices(committees_per_slot):
|
2022-01-08 23:28:49 +00:00
|
|
|
res.add(getCommittee(slot, committee_index))
|
2021-03-17 18:46:45 +00:00
|
|
|
else:
|
2022-01-08 23:28:49 +00:00
|
|
|
let
|
|
|
|
idx = cindex.get()
|
|
|
|
if idx < committees_per_slot:
|
2021-06-29 15:09:29 +00:00
|
|
|
res.add(getCommittee(slot, idx))
|
2021-03-17 18:46:45 +00:00
|
|
|
|
2021-05-20 17:56:12 +00:00
|
|
|
var res: seq[RestBeaconStatesCommittees]
|
2021-03-17 18:46:45 +00:00
|
|
|
let qepoch =
|
|
|
|
if vepoch.isNone:
|
2022-03-16 07:20:40 +00:00
|
|
|
epoch(getStateField(state, slot))
|
2021-03-17 18:46:45 +00:00
|
|
|
else:
|
|
|
|
vepoch.get()
|
|
|
|
|
|
|
|
if vslot.isNone():
|
2022-01-11 10:01:54 +00:00
|
|
|
for slot in qepoch.slots():
|
2022-01-08 23:28:49 +00:00
|
|
|
forSlot(slot, vindex, res)
|
2021-03-17 18:46:45 +00:00
|
|
|
else:
|
|
|
|
forSlot(vslot.get(), vindex, res)
|
|
|
|
|
2022-06-20 05:53:39 +00:00
|
|
|
return RestApiResponse.jsonResponseWOpt(
|
|
|
|
res,
|
|
|
|
node.getStateOptimistic(state)
|
|
|
|
)
|
2021-03-17 18:46:45 +00:00
|
|
|
|
2021-10-14 10:38:38 +00:00
|
|
|
return RestApiResponse.jsonError(Http404, StateNotFoundError)
|
2021-03-17 18:46:45 +00:00
|
|
|
|
2021-09-23 22:13:25 +00:00
|
|
|
# https://ethereum.github.io/beacon-APIs/#/Beacon/getEpochSyncCommittees
|
|
|
|
router.api(MethodGet,
|
2022-01-06 07:38:40 +00:00
|
|
|
"/eth/v1/beacon/states/{state_id}/sync_committees") do (
|
2021-09-23 22:13:25 +00:00
|
|
|
state_id: StateIdent, epoch: Option[Epoch]) -> RestApiResponse:
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
let
|
|
|
|
sid = state_id.valueOr:
|
|
|
|
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
|
|
|
|
$error)
|
Prune `BlockRef` on finalization (#3513)
Up til now, the block dag has been using `BlockRef`, a structure adapted
for a full DAG, to represent all of chain history. This is a correct and
simple design, but does not exploit the linearity of the chain once
parts of it finalize.
By pruning the in-memory `BlockRef` structure at finalization, we save,
at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory
landing us at a steady state of ~750mb normal memory usage for a
validating node.
Above all though, we prevent memory usage from growing proportionally
with the length of the chain, something that would not be sustainable
over time - instead, the steady state memory usage is roughly
determined by the validator set size which grows much more slowly. With
these changes, the core should remain sustainable memory-wise post-merge
all the way to withdrawals (when the validator set is expected to grow).
In-memory indices are still used for the "hot" unfinalized portion of
the chain - this ensure that consensus performance remains unchanged.
What changes is that for historical access, we use a db-based linear
slot index which is cache-and-disk-friendly, keeping the cost for
accessing historical data at a similar level as before, achieving the
savings at no percievable cost to functionality or performance.
A nice collateral benefit is the almost-instant startup since we no
longer load any large indicies at dag init.
The cost of this functionality instead can be found in the complexity of
having to deal with two ways of traversing the chain - by `BlockRef` and
by slot.
* use `BlockId` instead of `BlockRef` where finalized / historical data
may be required
* simplify clearance pre-advancement
* remove dag.finalizedBlocks (~50:ish mb)
* remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead
* `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef`
instance, unlike `BlockRef` traversal
* prune `BlockRef` parents on finality (~200:ish mb)
* speed up ChainDAG init by not loading finalized history index
* mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
|
|
|
bslot = node.getBlockSlotId(sid).valueOr:
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
if sid.kind == StateQueryKind.Root:
|
|
|
|
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
|
|
|
|
# in current version of database.
|
|
|
|
return RestApiResponse.jsonError(Http500, NoImplementationError)
|
|
|
|
return RestApiResponse.jsonError(Http404, StateNotFoundError,
|
|
|
|
$error)
|
2021-09-23 22:13:25 +00:00
|
|
|
|
|
|
|
let qepoch =
|
|
|
|
if epoch.isSome():
|
|
|
|
let repoch = epoch.get()
|
|
|
|
if repoch.isErr():
|
|
|
|
return RestApiResponse.jsonError(Http400, InvalidEpochValueError,
|
|
|
|
$repoch.error())
|
|
|
|
let res = repoch.get()
|
|
|
|
if res > MaxEpoch:
|
|
|
|
return RestApiResponse.jsonError(Http400, EpochOverflowValueError)
|
|
|
|
if res < node.dag.cfg.ALTAIR_FORK_EPOCH:
|
|
|
|
return RestApiResponse.jsonError(Http400,
|
|
|
|
EpochFromTheIncorrectForkError)
|
|
|
|
res
|
|
|
|
else:
|
|
|
|
# If ``epoch`` not present then the sync committees for the epoch of
|
|
|
|
# the state will be obtained.
|
|
|
|
bslot.slot.epoch()
|
|
|
|
|
Prune `BlockRef` on finalization (#3513)
Up til now, the block dag has been using `BlockRef`, a structure adapted
for a full DAG, to represent all of chain history. This is a correct and
simple design, but does not exploit the linearity of the chain once
parts of it finalize.
By pruning the in-memory `BlockRef` structure at finalization, we save,
at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory
landing us at a steady state of ~750mb normal memory usage for a
validating node.
Above all though, we prevent memory usage from growing proportionally
with the length of the chain, something that would not be sustainable
over time - instead, the steady state memory usage is roughly
determined by the validator set size which grows much more slowly. With
these changes, the core should remain sustainable memory-wise post-merge
all the way to withdrawals (when the validator set is expected to grow).
In-memory indices are still used for the "hot" unfinalized portion of
the chain - this ensure that consensus performance remains unchanged.
What changes is that for historical access, we use a db-based linear
slot index which is cache-and-disk-friendly, keeping the cost for
accessing historical data at a similar level as before, achieving the
savings at no percievable cost to functionality or performance.
A nice collateral benefit is the almost-instant startup since we no
longer load any large indicies at dag init.
The cost of this functionality instead can be found in the complexity of
having to deal with two ways of traversing the chain - by `BlockRef` and
by slot.
* use `BlockId` instead of `BlockRef` where finalized / historical data
may be required
* simplify clearance pre-advancement
* remove dag.finalizedBlocks (~50:ish mb)
* remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead
* `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef`
instance, unlike `BlockRef` traversal
* prune `BlockRef` parents on finality (~200:ish mb)
* speed up ChainDAG init by not loading finalized history index
* mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
|
|
|
node.withStateForBlockSlotId(bslot):
|
2021-09-23 22:13:25 +00:00
|
|
|
let keys =
|
|
|
|
block:
|
2022-03-16 07:20:40 +00:00
|
|
|
let res = syncCommitteeParticipants(state, qepoch)
|
2021-09-23 22:13:25 +00:00
|
|
|
if res.isErr():
|
|
|
|
return RestApiResponse.jsonError(Http400,
|
|
|
|
$res.error())
|
|
|
|
let kres = res.get()
|
|
|
|
if len(kres) == 0:
|
|
|
|
return RestApiResponse.jsonError(Http500, InternalServerError,
|
|
|
|
"List of sync committee participants is empty")
|
|
|
|
kres
|
|
|
|
|
|
|
|
let indices =
|
|
|
|
block:
|
|
|
|
var res: seq[ValidatorIndex]
|
2022-03-16 07:20:40 +00:00
|
|
|
let optIndices = keysToIndices(node.restKeysCache, state, keys)
|
2021-10-18 08:54:20 +00:00
|
|
|
# Remove all the duplicates.
|
2021-10-14 10:38:38 +00:00
|
|
|
for item in optIndices:
|
|
|
|
if item.isNone():
|
2021-10-18 08:54:20 +00:00
|
|
|
# This should not be happened, because keys are from state.
|
2021-10-14 10:38:38 +00:00
|
|
|
return RestApiResponse.jsonError(Http500, InternalServerError,
|
|
|
|
"Could not get validator indices")
|
|
|
|
res.add(item.get())
|
2021-09-23 22:13:25 +00:00
|
|
|
res
|
|
|
|
|
|
|
|
let aggregates =
|
|
|
|
block:
|
|
|
|
var
|
|
|
|
res: seq[seq[ValidatorIndex]]
|
|
|
|
offset = 0
|
|
|
|
while true:
|
|
|
|
let length = min(SYNC_SUBCOMMITTEE_SIZE, len(indices) - offset)
|
|
|
|
if length == 0:
|
|
|
|
break
|
|
|
|
res.add(@(indices.toOpenArray(offset, offset + length - 1)))
|
|
|
|
offset.inc(length)
|
|
|
|
res
|
|
|
|
|
2022-06-20 05:53:39 +00:00
|
|
|
return RestApiResponse.jsonResponseWOpt(
|
|
|
|
RestEpochSyncCommittee(validators: indices,
|
|
|
|
validator_aggregates: aggregates),
|
|
|
|
node.getStateOptimistic(state)
|
2021-09-28 18:08:23 +00:00
|
|
|
)
|
2021-09-23 22:13:25 +00:00
|
|
|
|
2021-10-14 10:38:38 +00:00
|
|
|
return RestApiResponse.jsonError(Http404, StateNotFoundError)
|
2021-09-23 22:13:25 +00:00
|
|
|
|
2021-08-23 10:41:48 +00:00
|
|
|
# https://ethereum.github.io/beacon-APIs/#/Beacon/getBlockHeaders
|
2022-01-06 07:38:40 +00:00
|
|
|
router.api(MethodGet, "/eth/v1/beacon/headers") do (
|
2021-03-17 18:46:45 +00:00
|
|
|
slot: Option[Slot], parent_root: Option[Eth2Digest]) -> RestApiResponse:
|
2021-08-09 06:08:18 +00:00
|
|
|
# TODO (cheatfate): This call is incomplete, because structure
|
2021-03-29 10:59:39 +00:00
|
|
|
# of database do not allow to query blocks by `parent_root`.
|
2021-04-03 01:19:16 +00:00
|
|
|
let qslot =
|
|
|
|
if slot.isSome():
|
|
|
|
let rslot = slot.get()
|
|
|
|
if rslot.isErr():
|
2021-04-08 10:49:28 +00:00
|
|
|
return RestApiResponse.jsonError(Http400, InvalidSlotValueError,
|
2021-04-03 01:19:16 +00:00
|
|
|
$rslot.error())
|
|
|
|
rslot.get()
|
|
|
|
else:
|
2021-06-01 11:13:40 +00:00
|
|
|
node.dag.head.slot
|
2021-04-03 01:19:16 +00:00
|
|
|
|
|
|
|
if parent_root.isSome():
|
|
|
|
let rroot = parent_root.get()
|
|
|
|
if rroot.isErr():
|
2021-04-08 10:49:28 +00:00
|
|
|
return RestApiResponse.jsonError(Http400, InvalidParentRootValueError,
|
2021-04-03 01:19:16 +00:00
|
|
|
$rroot.error())
|
2021-04-08 10:49:28 +00:00
|
|
|
return RestApiResponse.jsonError(Http500, NoImplementationError)
|
2021-04-03 01:19:16 +00:00
|
|
|
|
2022-03-11 12:08:17 +00:00
|
|
|
let bdata = node.getForkedBlock(BlockIdent.init(qslot)).valueOr:
|
|
|
|
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
|
2021-04-03 01:19:16 +00:00
|
|
|
|
2021-08-09 06:08:18 +00:00
|
|
|
return
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
withBlck(bdata):
|
2022-06-20 05:53:39 +00:00
|
|
|
RestApiResponse.jsonResponseWOpt(
|
2021-08-09 06:08:18 +00:00
|
|
|
[
|
|
|
|
(
|
|
|
|
root: blck.root,
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
canonical: node.dag.isCanonical(
|
|
|
|
BlockId(root: blck.root, slot: blck.message.slot)),
|
2021-08-09 06:08:18 +00:00
|
|
|
header: (
|
2022-02-18 20:35:52 +00:00
|
|
|
message: blck.toBeaconBlockHeader,
|
2021-08-09 06:08:18 +00:00
|
|
|
signature: blck.signature
|
|
|
|
)
|
|
|
|
)
|
2022-06-20 05:53:39 +00:00
|
|
|
],
|
|
|
|
node.getBlockOptimistic(bdata)
|
2021-04-03 01:19:16 +00:00
|
|
|
)
|
2021-03-17 18:46:45 +00:00
|
|
|
|
2021-08-23 10:41:48 +00:00
|
|
|
# https://ethereum.github.io/beacon-APIs/#/Beacon/getBlockHeader
|
2022-01-06 07:38:40 +00:00
|
|
|
router.api(MethodGet, "/eth/v1/beacon/headers/{block_id}") do (
|
2021-03-17 18:46:45 +00:00
|
|
|
block_id: BlockIdent) -> RestApiResponse:
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
let
|
|
|
|
bid = block_id.valueOr:
|
|
|
|
return RestApiResponse.jsonError(Http400, InvalidBlockIdValueError,
|
|
|
|
$error)
|
|
|
|
|
|
|
|
bdata = node.getForkedBlock(bid).valueOr:
|
|
|
|
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
|
2021-03-23 22:50:18 +00:00
|
|
|
|
2021-08-09 06:08:18 +00:00
|
|
|
return
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
withBlck(bdata):
|
2022-06-20 05:53:39 +00:00
|
|
|
RestApiResponse.jsonResponseWOpt(
|
2021-08-09 06:08:18 +00:00
|
|
|
(
|
|
|
|
root: blck.root,
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
canonical: node.dag.isCanonical(
|
|
|
|
BlockId(root: blck.root, slot: blck.message.slot)),
|
2021-08-09 06:08:18 +00:00
|
|
|
header: (
|
2022-02-18 20:35:52 +00:00
|
|
|
message: blck.toBeaconBlockHeader,
|
2021-08-09 06:08:18 +00:00
|
|
|
signature: blck.signature
|
|
|
|
)
|
2022-06-20 05:53:39 +00:00
|
|
|
),
|
|
|
|
node.getBlockOptimistic(bdata)
|
2021-03-17 18:46:45 +00:00
|
|
|
)
|
|
|
|
|
2021-08-23 10:41:48 +00:00
|
|
|
# https://ethereum.github.io/beacon-APIs/#/Beacon/publishBlock
|
2022-01-06 07:38:40 +00:00
|
|
|
router.api(MethodPost, "/eth/v1/beacon/blocks") do (
|
2021-03-17 18:46:45 +00:00
|
|
|
contentBody: Option[ContentBody]) -> RestApiResponse:
|
2022-07-06 16:11:44 +00:00
|
|
|
let res =
|
2021-03-23 22:50:18 +00:00
|
|
|
block:
|
|
|
|
if contentBody.isNone():
|
2021-04-08 10:49:28 +00:00
|
|
|
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
|
2022-09-29 21:00:53 +00:00
|
|
|
let
|
|
|
|
body = contentBody.get()
|
|
|
|
version = request.headers.getString("eth-consensus-version")
|
|
|
|
var
|
|
|
|
restBlock = decodeBody(RestPublishedSignedBeaconBlock, body,
|
|
|
|
version).valueOr:
|
|
|
|
return RestApiResponse.jsonError(Http400, InvalidBlockObjectError,
|
|
|
|
$error)
|
|
|
|
forked = ForkedSignedBeaconBlock(restBlock)
|
|
|
|
|
2022-02-13 15:21:55 +00:00
|
|
|
if forked.kind != node.dag.cfg.blockForkAtEpoch(
|
2022-09-29 21:00:53 +00:00
|
|
|
getForkedBlockField(forked, slot).epoch):
|
2022-02-13 15:21:55 +00:00
|
|
|
return RestApiResponse.jsonError(Http400, InvalidBlockObjectError)
|
|
|
|
|
|
|
|
withBlck(forked):
|
|
|
|
blck.root = hash_tree_root(blck.message)
|
2022-07-06 16:11:44 +00:00
|
|
|
await node.router.routeSignedBeaconBlock(blck)
|
2021-05-27 11:53:53 +00:00
|
|
|
|
2021-08-23 10:41:48 +00:00
|
|
|
if res.isErr():
|
2022-07-06 16:11:44 +00:00
|
|
|
return RestApiResponse.jsonError(
|
|
|
|
Http503, BeaconNodeInSyncError, $res.error())
|
|
|
|
if res.get().isNone():
|
2021-04-08 10:49:28 +00:00
|
|
|
return RestApiResponse.jsonError(Http202, BlockValidationError)
|
2021-12-03 13:58:12 +00:00
|
|
|
|
2021-08-23 10:41:48 +00:00
|
|
|
return RestApiResponse.jsonMsgResponse(BlockValidationSuccess)
|
2021-03-17 18:46:45 +00:00
|
|
|
|
2022-11-08 18:08:43 +00:00
|
|
|
# https://ethereum.github.io/beacon-APIs/#/Beacon/publishBlindedBlock
|
|
|
|
# https://github.com/ethereum/beacon-APIs/blob/v2.3.0/apis/beacon/blocks/blinded_blocks.yaml
|
|
|
|
router.api(MethodPost, "/eth/v1/beacon/blinded_blocks") do (
|
|
|
|
contentBody: Option[ContentBody]) -> RestApiResponse:
|
|
|
|
## Instructs the beacon node to use the components of the
|
|
|
|
## `SignedBlindedBeaconBlock` to construct and publish a
|
|
|
|
## `SignedBeaconBlock` by swapping out the transactions_root for the
|
|
|
|
## corresponding full list of transactions. The beacon node should
|
|
|
|
## broadcast a newly constructed `SignedBeaconBlock` to the beacon network,
|
|
|
|
## to be included in the beacon chain. The beacon node is not required to
|
|
|
|
## validate the signed `BeaconBlock`, and a successful response (20X) only
|
|
|
|
## indicates that the broadcast has been successful.
|
|
|
|
if contentBody.isNone():
|
|
|
|
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
|
|
|
|
|
|
|
|
let
|
|
|
|
currentEpochFork =
|
|
|
|
node.dag.cfg.stateForkAtEpoch(node.currentSlot().epoch())
|
|
|
|
version = request.headers.getString("eth-consensus-version")
|
|
|
|
body = contentBody.get()
|
|
|
|
|
|
|
|
if body.contentType == OctetStreamMediaType and
|
|
|
|
currentEpochFork.toString != version:
|
|
|
|
return RestApiResponse.jsonError(Http400, BlockIncorrectFork)
|
|
|
|
|
|
|
|
case currentEpochFork
|
2022-12-07 16:47:23 +00:00
|
|
|
of BeaconStateFork.EIP4844:
|
|
|
|
return RestApiResponse.jsonError(Http500, $eip4844ImplementationMissing)
|
2022-11-08 18:08:43 +00:00
|
|
|
of BeaconStateFork.Capella:
|
|
|
|
return RestApiResponse.jsonError(Http500, $capellaImplementationMissing)
|
|
|
|
of BeaconStateFork.Bellatrix:
|
|
|
|
let res =
|
|
|
|
block:
|
|
|
|
var restBlock = decodeBodyJsonOrSsz(SignedBlindedBeaconBlock, body).valueOr:
|
|
|
|
return RestApiResponse.jsonError(Http400, InvalidBlockObjectError,
|
|
|
|
$error)
|
|
|
|
await node.unblindAndRouteBlockMEV(restBlock)
|
|
|
|
|
2022-11-27 00:49:30 +00:00
|
|
|
if res.isErr():
|
2022-11-08 18:08:43 +00:00
|
|
|
return RestApiResponse.jsonError(
|
|
|
|
Http503, BeaconNodeInSyncError, $res.error())
|
|
|
|
if res.get().isNone():
|
|
|
|
return RestApiResponse.jsonError(Http202, BlockValidationError)
|
|
|
|
|
|
|
|
return RestApiResponse.jsonMsgResponse(BlockValidationSuccess)
|
|
|
|
of BeaconStateFork.Altair, BeaconStateFork.Phase0:
|
|
|
|
# Pre-Bellatrix, this endpoint will accept a `SignedBeaconBlock`.
|
|
|
|
#
|
|
|
|
# This is mostly the same as /eth/v1/beacon/blocks for phase 0 and
|
|
|
|
# altair.
|
|
|
|
var
|
|
|
|
restBlock = decodeBody(RestPublishedSignedBeaconBlock, body,
|
|
|
|
version).valueOr:
|
|
|
|
return RestApiResponse.jsonError(Http400, InvalidBlockObjectError,
|
|
|
|
$error)
|
|
|
|
forked = ForkedSignedBeaconBlock(restBlock)
|
|
|
|
|
|
|
|
if forked.kind != node.dag.cfg.blockForkAtEpoch(
|
|
|
|
getForkedBlockField(forked, slot).epoch):
|
|
|
|
return RestApiResponse.jsonError(Http400, InvalidBlockObjectError)
|
|
|
|
|
|
|
|
let res = withBlck(forked):
|
|
|
|
blck.root = hash_tree_root(blck.message)
|
|
|
|
await node.router.routeSignedBeaconBlock(blck)
|
|
|
|
|
|
|
|
if res.isErr():
|
|
|
|
return RestApiResponse.jsonError(
|
|
|
|
Http503, BeaconNodeInSyncError, $res.error())
|
|
|
|
elif res.get().isNone():
|
|
|
|
return RestApiResponse.jsonError(Http202, BlockValidationError)
|
|
|
|
|
|
|
|
return RestApiResponse.jsonMsgResponse(BlockValidationSuccess)
|
|
|
|
|
2021-08-23 10:41:48 +00:00
|
|
|
# https://ethereum.github.io/beacon-APIs/#/Beacon/getBlock
|
2022-01-06 07:38:40 +00:00
|
|
|
router.api(MethodGet, "/eth/v1/beacon/blocks/{block_id}") do (
|
2021-03-17 18:46:45 +00:00
|
|
|
block_id: BlockIdent) -> RestApiResponse:
|
2022-11-02 10:56:55 +00:00
|
|
|
return RestApiResponse.jsonError(
|
|
|
|
Http410, DeprecatedRemovalBeaconBlocksDebugStateV1)
|
2021-08-09 06:08:18 +00:00
|
|
|
|
2021-08-23 10:41:48 +00:00
|
|
|
# https://ethereum.github.io/beacon-APIs/#/Beacon/getBlockV2
|
2022-01-06 07:38:40 +00:00
|
|
|
router.api(MethodGet, "/eth/v2/beacon/blocks/{block_id}") do (
|
2021-08-09 06:08:18 +00:00
|
|
|
block_id: BlockIdent) -> RestApiResponse:
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
let
|
2022-03-11 12:08:17 +00:00
|
|
|
blockIdent = block_id.valueOr:
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
return RestApiResponse.jsonError(Http400, InvalidBlockIdValueError,
|
|
|
|
$error)
|
2022-03-11 12:08:17 +00:00
|
|
|
bid = node.getBlockId(blockIdent).valueOr:
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
|
2022-03-11 12:08:17 +00:00
|
|
|
|
2021-09-16 13:32:32 +00:00
|
|
|
let contentType =
|
|
|
|
block:
|
2022-01-21 16:52:34 +00:00
|
|
|
let res = preferredContentType(jsonMediaType,
|
|
|
|
sszMediaType)
|
2021-09-16 13:32:32 +00:00
|
|
|
if res.isErr():
|
|
|
|
return RestApiResponse.jsonError(Http406, ContentNotAcceptableError)
|
|
|
|
res.get()
|
|
|
|
return
|
2022-01-21 16:52:34 +00:00
|
|
|
if contentType == sszMediaType:
|
2022-03-11 12:08:17 +00:00
|
|
|
var data: seq[byte]
|
|
|
|
if not node.dag.getBlockSSZ(bid, data):
|
|
|
|
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
|
|
|
|
|
2022-06-20 05:53:39 +00:00
|
|
|
let
|
|
|
|
fork = node.dag.cfg.blockForkAtEpoch(bid.slot.epoch)
|
|
|
|
headers = [("eth-consensus-version", fork.toString())]
|
|
|
|
|
2022-09-23 15:51:04 +00:00
|
|
|
RestApiResponse.sszResponsePlain(data, headers)
|
2022-01-21 16:52:34 +00:00
|
|
|
elif contentType == jsonMediaType:
|
2022-03-11 12:08:17 +00:00
|
|
|
let bdata = node.dag.getForkedBlock(bid).valueOr:
|
|
|
|
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
|
|
|
|
|
2022-06-20 05:53:39 +00:00
|
|
|
RestApiResponse.jsonResponseBlock(
|
|
|
|
bdata.asSigned(),
|
2022-10-03 22:05:52 +00:00
|
|
|
node.getBlockOptimistic(bdata)
|
2022-06-20 05:53:39 +00:00
|
|
|
)
|
2021-09-16 13:32:32 +00:00
|
|
|
else:
|
|
|
|
RestApiResponse.jsonError(Http500, InvalidAcceptError)
|
2021-03-17 18:46:45 +00:00
|
|
|
|
2021-08-23 10:41:48 +00:00
|
|
|
# https://ethereum.github.io/beacon-APIs/#/Beacon/getBlockRoot
|
2022-01-06 07:38:40 +00:00
|
|
|
router.api(MethodGet, "/eth/v1/beacon/blocks/{block_id}/root") do (
|
2021-03-17 18:46:45 +00:00
|
|
|
block_id: BlockIdent) -> RestApiResponse:
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
let
|
2022-03-11 12:08:17 +00:00
|
|
|
blockIdent = block_id.valueOr:
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
return RestApiResponse.jsonError(Http400, InvalidBlockIdValueError,
|
|
|
|
$error)
|
|
|
|
|
2022-03-11 12:08:17 +00:00
|
|
|
bid = node.getBlockId(blockIdent).valueOr:
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
|
|
|
|
|
2022-06-20 05:53:39 +00:00
|
|
|
bdata = node.dag.getForkedBlock(bid).valueOr:
|
|
|
|
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
|
|
|
|
|
|
|
|
return RestApiResponse.jsonResponseWOpt(
|
|
|
|
(root: bid.root),
|
|
|
|
node.getBlockOptimistic(bdata)
|
|
|
|
)
|
2021-03-17 18:46:45 +00:00
|
|
|
|
2021-08-23 10:41:48 +00:00
|
|
|
# https://ethereum.github.io/beacon-APIs/#/Beacon/getBlockAttestations
|
2021-03-17 18:46:45 +00:00
|
|
|
router.api(MethodGet,
|
2022-01-06 07:38:40 +00:00
|
|
|
"/eth/v1/beacon/blocks/{block_id}/attestations") do (
|
2021-03-17 18:46:45 +00:00
|
|
|
block_id: BlockIdent) -> RestApiResponse:
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
let
|
|
|
|
bid = block_id.valueOr:
|
|
|
|
return RestApiResponse.jsonError(Http400, InvalidBlockIdValueError,
|
|
|
|
$error)
|
|
|
|
|
|
|
|
bdata = node.getForkedBlock(bid).valueOr:
|
|
|
|
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
|
|
|
|
|
2021-08-09 06:08:18 +00:00
|
|
|
return
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
withBlck(bdata):
|
2022-06-20 05:53:39 +00:00
|
|
|
RestApiResponse.jsonResponseWOpt(
|
|
|
|
blck.message.body.attestations.asSeq(),
|
|
|
|
node.getBlockOptimistic(bdata)
|
|
|
|
)
|
2021-03-17 18:46:45 +00:00
|
|
|
|
2021-08-23 10:41:48 +00:00
|
|
|
# https://ethereum.github.io/beacon-APIs/#/Beacon/getPoolAttestations
|
2022-01-06 07:38:40 +00:00
|
|
|
router.api(MethodGet, "/eth/v1/beacon/pool/attestations") do (
|
2021-03-17 18:46:45 +00:00
|
|
|
slot: Option[Slot],
|
|
|
|
committee_index: Option[CommitteeIndex]) -> RestApiResponse:
|
|
|
|
let vindex =
|
|
|
|
if committee_index.isSome():
|
|
|
|
let rindex = committee_index.get()
|
|
|
|
if rindex.isErr():
|
|
|
|
return RestApiResponse.jsonError(Http400,
|
2021-04-08 10:49:28 +00:00
|
|
|
InvalidCommitteeIndexValueError,
|
2021-03-17 18:46:45 +00:00
|
|
|
$rindex.error())
|
2022-07-06 16:11:44 +00:00
|
|
|
Opt.some(rindex.get())
|
2021-03-17 18:46:45 +00:00
|
|
|
else:
|
2022-07-06 16:11:44 +00:00
|
|
|
Opt.none(CommitteeIndex)
|
2021-03-17 18:46:45 +00:00
|
|
|
let vslot =
|
|
|
|
if slot.isSome():
|
|
|
|
let rslot = slot.get()
|
|
|
|
if rslot.isErr():
|
2021-04-08 10:49:28 +00:00
|
|
|
return RestApiResponse.jsonError(Http400, InvalidSlotValueError,
|
2021-03-17 18:46:45 +00:00
|
|
|
$rslot.error())
|
2022-07-06 16:11:44 +00:00
|
|
|
Opt.some(rslot.get())
|
2021-03-17 18:46:45 +00:00
|
|
|
else:
|
2022-07-06 16:11:44 +00:00
|
|
|
Opt.none(Slot)
|
2021-03-17 18:46:45 +00:00
|
|
|
var res: seq[Attestation]
|
|
|
|
for item in node.attestationPool[].attestations(vslot, vindex):
|
|
|
|
res.add(item)
|
2021-03-23 22:50:18 +00:00
|
|
|
return RestApiResponse.jsonResponse(res)
|
2021-03-17 18:46:45 +00:00
|
|
|
|
2021-08-23 10:41:48 +00:00
|
|
|
# https://ethereum.github.io/beacon-APIs/#/Beacon/submitPoolAttestations
|
2022-01-06 07:38:40 +00:00
|
|
|
router.api(MethodPost, "/eth/v1/beacon/pool/attestations") do (
|
2021-03-17 18:46:45 +00:00
|
|
|
contentBody: Option[ContentBody]) -> RestApiResponse:
|
2021-03-23 22:50:18 +00:00
|
|
|
let attestations =
|
|
|
|
block:
|
|
|
|
if contentBody.isNone():
|
2021-04-08 10:49:28 +00:00
|
|
|
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
|
2021-03-23 22:50:18 +00:00
|
|
|
let dres = decodeBody(seq[Attestation], contentBody.get())
|
|
|
|
if dres.isErr():
|
2021-04-08 10:49:28 +00:00
|
|
|
return RestApiResponse.jsonError(Http400,
|
|
|
|
InvalidAttestationObjectError,
|
2021-03-23 22:50:18 +00:00
|
|
|
$dres.error())
|
|
|
|
dres.get()
|
|
|
|
|
2021-08-23 10:41:48 +00:00
|
|
|
# Since our validation logic supports batch processing, we will submit all
|
|
|
|
# attestations for validation.
|
|
|
|
let pending =
|
|
|
|
block:
|
|
|
|
var res: seq[Future[SendResult]]
|
|
|
|
for attestation in attestations:
|
2022-07-06 16:11:44 +00:00
|
|
|
res.add(node.router.routeAttestation(attestation))
|
2021-08-23 10:41:48 +00:00
|
|
|
res
|
|
|
|
let failures =
|
|
|
|
block:
|
2022-09-29 20:55:18 +00:00
|
|
|
var res: seq[RestIndexedErrorMessageItem]
|
2021-08-23 10:41:48 +00:00
|
|
|
await allFutures(pending)
|
2022-05-10 10:03:40 +00:00
|
|
|
for index, future in pending:
|
2021-08-23 10:41:48 +00:00
|
|
|
if future.done():
|
|
|
|
let fres = future.read()
|
|
|
|
if fres.isErr():
|
2022-09-29 20:55:18 +00:00
|
|
|
let failure = RestIndexedErrorMessageItem(index: index,
|
|
|
|
message: $fres.error())
|
2021-08-23 10:41:48 +00:00
|
|
|
res.add(failure)
|
|
|
|
elif future.failed() or future.cancelled():
|
|
|
|
# This is unexpected failure, so we log the error message.
|
|
|
|
let exc = future.readError()
|
2022-09-29 20:55:18 +00:00
|
|
|
let failure = RestIndexedErrorMessageItem(index: index,
|
|
|
|
message: $exc.msg)
|
2021-08-23 10:41:48 +00:00
|
|
|
res.add(failure)
|
|
|
|
res
|
2021-03-23 22:50:18 +00:00
|
|
|
|
|
|
|
if len(failures) > 0:
|
2021-04-08 10:49:28 +00:00
|
|
|
return RestApiResponse.jsonErrorList(Http400, AttestationValidationError,
|
2021-03-23 22:50:18 +00:00
|
|
|
failures)
|
|
|
|
else:
|
2021-07-13 11:15:07 +00:00
|
|
|
return RestApiResponse.jsonMsgResponse(AttestationValidationSuccess)
|
2021-03-17 18:46:45 +00:00
|
|
|
|
2021-08-23 10:41:48 +00:00
|
|
|
# https://ethereum.github.io/beacon-APIs/#/Beacon/getPoolAttesterSlashings
|
2022-01-06 07:38:40 +00:00
|
|
|
router.api(MethodGet, "/eth/v1/beacon/pool/attester_slashings") do (
|
2021-03-17 18:46:45 +00:00
|
|
|
) -> RestApiResponse:
|
|
|
|
var res: seq[AttesterSlashing]
|
|
|
|
if isNil(node.exitPool):
|
2021-03-23 22:50:18 +00:00
|
|
|
return RestApiResponse.jsonResponse(res)
|
2021-03-17 18:46:45 +00:00
|
|
|
let length = len(node.exitPool.attester_slashings)
|
|
|
|
res = newSeqOfCap[AttesterSlashing](length)
|
|
|
|
for item in node.exitPool.attester_slashings.items():
|
|
|
|
res.add(item)
|
2021-03-23 22:50:18 +00:00
|
|
|
return RestApiResponse.jsonResponse(res)
|
2021-03-17 18:46:45 +00:00
|
|
|
|
2021-08-23 10:41:48 +00:00
|
|
|
# https://ethereum.github.io/beacon-APIs/#/Beacon/submitPoolAttesterSlashings
|
2022-01-06 07:38:40 +00:00
|
|
|
router.api(MethodPost, "/eth/v1/beacon/pool/attester_slashings") do (
|
2021-03-17 18:46:45 +00:00
|
|
|
contentBody: Option[ContentBody]) -> RestApiResponse:
|
2021-03-23 22:50:18 +00:00
|
|
|
let slashing =
|
|
|
|
block:
|
|
|
|
if contentBody.isNone():
|
2021-04-08 10:49:28 +00:00
|
|
|
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
|
2021-03-23 22:50:18 +00:00
|
|
|
let dres = decodeBody(AttesterSlashing, contentBody.get())
|
|
|
|
if dres.isErr():
|
2021-04-08 10:49:28 +00:00
|
|
|
return RestApiResponse.jsonError(Http400,
|
|
|
|
InvalidAttesterSlashingObjectError,
|
|
|
|
$dres.error())
|
2021-08-23 10:41:48 +00:00
|
|
|
dres.get()
|
2022-07-06 16:11:44 +00:00
|
|
|
let res = await node.router.routeAttesterSlashing(slashing)
|
2021-08-23 10:41:48 +00:00
|
|
|
if res.isErr():
|
|
|
|
return RestApiResponse.jsonError(Http400,
|
|
|
|
AttesterSlashingValidationError,
|
|
|
|
$res.error())
|
2021-07-13 11:15:07 +00:00
|
|
|
return RestApiResponse.jsonMsgResponse(AttesterSlashingValidationSuccess)
|
2021-03-17 18:46:45 +00:00
|
|
|
|
2021-08-23 10:41:48 +00:00
|
|
|
# https://ethereum.github.io/beacon-APIs/#/Beacon/getPoolProposerSlashings
|
2022-01-06 07:38:40 +00:00
|
|
|
router.api(MethodGet, "/eth/v1/beacon/pool/proposer_slashings") do (
|
2021-03-17 18:46:45 +00:00
|
|
|
) -> RestApiResponse:
|
|
|
|
var res: seq[ProposerSlashing]
|
|
|
|
if isNil(node.exitPool):
|
2021-03-23 22:50:18 +00:00
|
|
|
return RestApiResponse.jsonResponse(res)
|
2021-03-17 18:46:45 +00:00
|
|
|
let length = len(node.exitPool.proposer_slashings)
|
|
|
|
res = newSeqOfCap[ProposerSlashing](length)
|
|
|
|
for item in node.exitPool.proposer_slashings.items():
|
|
|
|
res.add(item)
|
2021-03-23 22:50:18 +00:00
|
|
|
return RestApiResponse.jsonResponse(res)
|
2021-03-17 18:46:45 +00:00
|
|
|
|
2021-08-23 10:41:48 +00:00
|
|
|
# https://ethereum.github.io/beacon-APIs/#/Beacon/submitPoolProposerSlashings
|
2022-01-06 07:38:40 +00:00
|
|
|
router.api(MethodPost, "/eth/v1/beacon/pool/proposer_slashings") do (
|
2021-03-17 18:46:45 +00:00
|
|
|
contentBody: Option[ContentBody]) -> RestApiResponse:
|
2021-03-23 22:50:18 +00:00
|
|
|
let slashing =
|
|
|
|
block:
|
|
|
|
if contentBody.isNone():
|
2021-04-08 10:49:28 +00:00
|
|
|
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
|
2021-03-23 22:50:18 +00:00
|
|
|
let dres = decodeBody(ProposerSlashing, contentBody.get())
|
|
|
|
if dres.isErr():
|
2021-04-08 10:49:28 +00:00
|
|
|
return RestApiResponse.jsonError(Http400,
|
|
|
|
InvalidProposerSlashingObjectError,
|
|
|
|
$dres.error())
|
2021-08-23 10:41:48 +00:00
|
|
|
dres.get()
|
2022-07-06 16:11:44 +00:00
|
|
|
let res = await node.router.routeProposerSlashing(slashing)
|
2021-08-23 10:41:48 +00:00
|
|
|
if res.isErr():
|
|
|
|
return RestApiResponse.jsonError(Http400,
|
|
|
|
ProposerSlashingValidationError,
|
|
|
|
$res.error())
|
2021-07-13 11:15:07 +00:00
|
|
|
return RestApiResponse.jsonMsgResponse(ProposerSlashingValidationSuccess)
|
2021-03-17 18:46:45 +00:00
|
|
|
|
2021-09-23 22:13:25 +00:00
|
|
|
# https://ethereum.github.io/beacon-APIs/#/Beacon/submitPoolSyncCommitteeSignatures
|
2022-01-06 07:38:40 +00:00
|
|
|
router.api(MethodPost, "/eth/v1/beacon/pool/sync_committees") do (
|
2021-09-23 22:13:25 +00:00
|
|
|
contentBody: Option[ContentBody]) -> RestApiResponse:
|
|
|
|
let messages =
|
|
|
|
block:
|
|
|
|
if contentBody.isNone():
|
|
|
|
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
|
|
|
|
let dres = decodeBody(seq[SyncCommitteeMessage], contentBody.get())
|
|
|
|
if dres.isErr():
|
|
|
|
return RestApiResponse.jsonError(Http400,
|
|
|
|
InvalidSyncCommitteeSignatureMessageError)
|
|
|
|
dres.get()
|
|
|
|
|
2022-07-06 16:11:44 +00:00
|
|
|
let results = await node.router.routeSyncCommitteeMessages(messages)
|
2021-09-23 22:13:25 +00:00
|
|
|
|
|
|
|
let failures =
|
|
|
|
block:
|
2022-09-29 20:55:18 +00:00
|
|
|
var res: seq[RestIndexedErrorMessageItem]
|
2022-05-10 10:03:40 +00:00
|
|
|
for index, item in results:
|
2021-09-23 22:13:25 +00:00
|
|
|
if item.isErr():
|
2022-09-29 20:55:18 +00:00
|
|
|
res.add(RestIndexedErrorMessageItem(index: index,
|
|
|
|
message: $item.error()))
|
2021-09-23 22:13:25 +00:00
|
|
|
res
|
|
|
|
if len(failures) > 0:
|
|
|
|
return RestApiResponse.jsonErrorList(Http400,
|
|
|
|
SyncCommitteeMessageValidationError,
|
|
|
|
failures)
|
|
|
|
else:
|
|
|
|
return RestApiResponse.jsonMsgResponse(
|
|
|
|
SyncCommitteeMessageValidationSuccess)
|
|
|
|
|
2021-08-23 10:41:48 +00:00
|
|
|
# https://ethereum.github.io/beacon-APIs/#/Beacon/getPoolVoluntaryExits
|
2022-01-06 07:38:40 +00:00
|
|
|
router.api(MethodGet, "/eth/v1/beacon/pool/voluntary_exits") do (
|
2021-03-17 18:46:45 +00:00
|
|
|
) -> RestApiResponse:
|
|
|
|
var res: seq[SignedVoluntaryExit]
|
|
|
|
if isNil(node.exitPool):
|
2021-03-23 22:50:18 +00:00
|
|
|
return RestApiResponse.jsonResponse(res)
|
2021-03-17 18:46:45 +00:00
|
|
|
let length = len(node.exitPool.voluntary_exits)
|
|
|
|
res = newSeqOfCap[SignedVoluntaryExit](length)
|
|
|
|
for item in node.exitPool.voluntary_exits.items():
|
|
|
|
res.add(item)
|
2021-03-23 22:50:18 +00:00
|
|
|
return RestApiResponse.jsonResponse(res)
|
2021-03-17 18:46:45 +00:00
|
|
|
|
2021-08-23 10:41:48 +00:00
|
|
|
# https://ethereum.github.io/beacon-APIs/#/Beacon/submitPoolVoluntaryExit
|
2022-01-06 07:38:40 +00:00
|
|
|
router.api(MethodPost, "/eth/v1/beacon/pool/voluntary_exits") do (
|
2021-03-17 18:46:45 +00:00
|
|
|
contentBody: Option[ContentBody]) -> RestApiResponse:
|
2021-03-23 22:50:18 +00:00
|
|
|
let exit =
|
|
|
|
block:
|
|
|
|
if contentBody.isNone():
|
2021-04-08 10:49:28 +00:00
|
|
|
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
|
2021-03-23 22:50:18 +00:00
|
|
|
let dres = decodeBody(SignedVoluntaryExit, contentBody.get())
|
|
|
|
if dres.isErr():
|
2021-04-08 10:49:28 +00:00
|
|
|
return RestApiResponse.jsonError(Http400,
|
|
|
|
InvalidVoluntaryExitObjectError,
|
|
|
|
$dres.error())
|
2021-08-23 10:41:48 +00:00
|
|
|
dres.get()
|
2022-07-06 16:11:44 +00:00
|
|
|
let res = await node.router.routeSignedVoluntaryExit(exit)
|
2021-08-23 10:41:48 +00:00
|
|
|
if res.isErr():
|
|
|
|
return RestApiResponse.jsonError(Http400,
|
|
|
|
VoluntaryExitValidationError,
|
|
|
|
$res.error())
|
2021-07-13 11:15:07 +00:00
|
|
|
return RestApiResponse.jsonMsgResponse(VoluntaryExitValidationSuccess)
|