nimbus-eth2/beacon_chain/rpc/rest_beacon_api.nim

1730 lines
70 KiB
Nim
Raw Normal View History

# beacon_chain
# Copyright (c) 2018-2024 Status Research & Development GmbH
2021-03-17 18:46:45 +00:00
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [].}
2021-03-17 18:46:45 +00:00
import
2022-06-09 08:50:36 +00:00
std/[typetraits, sequtils, sets],
stew/base10,
chronicles, metrics,
./rest_utils,
./state_ttl_cache,
../beacon_node,
../consensus_object_pools/[blockchain_dag, spec_cache, validator_change_pool],
../spec/[deposit_snapshots, eth2_merkleization, forks, network, validator],
../spec/mev/[bellatrix_mev, capella_mev],
../validators/message_router_mev
2021-03-17 18:46:45 +00:00
export rest_utils
2021-03-17 18:46:45 +00:00
logScope: topics = "rest_beaconapi"
proc validateBeaconApiQueries*(key: string, value: string): int =
## This is rough validation procedure which should be simple and fast,
## because it will be used for query routing.
case key
of "{epoch}":
0
of "{slot}":
0
of "{peer_id}":
0
of "{state_id}":
0
of "{block_id}":
0
of "{validator_id}":
0
of "{block_root}":
0
of "{pubkey}":
int(value.len != 98)
else:
1
2024-01-19 23:34:11 +00:00
const
AllValidatorFilterKinds = {
ValidatorFilterKind.PendingInitialized,
ValidatorFilterKind.PendingQueued,
ValidatorFilterKind.ActiveOngoing,
ValidatorFilterKind.ActiveExiting,
ValidatorFilterKind.ActiveSlashed,
ValidatorFilterKind.ExitedUnslashed,
ValidatorFilterKind.ExitedSlashed,
ValidatorFilterKind.WithdrawalPossible,
ValidatorFilterKind.WithdrawalDone
}
2021-03-17 18:46:45 +00:00
proc validateFilter(filters: seq[ValidatorFilter]): Result[ValidatorFilter,
cstring] =
var res: ValidatorFilter
for item in filters:
if res * item != {}:
return err("Validator status must be unique")
res.incl(item)
if res == {}:
2024-01-19 23:34:11 +00:00
return ok(AllValidatorFilterKinds)
2021-03-17 18:46:45 +00:00
ok(res)
proc getStatus(validator: Validator,
current_epoch: Epoch): Result[ValidatorFilterKind, cstring] =
if validator.activation_epoch > current_epoch:
# pending
if validator.activation_eligibility_epoch == FAR_FUTURE_EPOCH:
ok(ValidatorFilterKind.PendingInitialized)
else:
# validator.activation_eligibility_epoch < FAR_FUTURE_EPOCH:
ok(ValidatorFilterKind.PendingQueued)
elif (validator.activation_epoch <= current_epoch) and
(current_epoch < validator.exit_epoch):
# active
if validator.exit_epoch == FAR_FUTURE_EPOCH:
ok(ValidatorFilterKind.ActiveOngoing)
elif not validator.slashed:
# validator.exit_epoch < FAR_FUTURE_EPOCH
ok(ValidatorFilterKind.ActiveExiting)
else:
# validator.exit_epoch < FAR_FUTURE_EPOCH and validator.slashed:
ok(ValidatorFilterKind.ActiveSlashed)
elif (validator.exit_epoch <= current_epoch) and
(current_epoch < validator.withdrawable_epoch):
# exited
if not validator.slashed:
ok(ValidatorFilterKind.ExitedUnslashed)
else:
# validator.slashed
ok(ValidatorFilterKind.ExitedSlashed)
elif validator.withdrawable_epoch <= current_epoch:
# withdrawal
if validator.effective_balance != 0.Gwei:
2021-03-17 18:46:45 +00:00
ok(ValidatorFilterKind.WithdrawalPossible)
else:
# validator.effective_balance == 0.Gwei
2021-03-17 18:46:45 +00:00
ok(ValidatorFilterKind.WithdrawalDone)
else:
err("Invalid validator status")
proc toString*(kind: ValidatorFilterKind): string =
case kind
of ValidatorFilterKind.PendingInitialized:
"pending_initialized"
of ValidatorFilterKind.PendingQueued:
"pending_queued"
of ValidatorFilterKind.ActiveOngoing:
"active_ongoing"
of ValidatorFilterKind.ActiveExiting:
"active_exiting"
of ValidatorFilterKind.ActiveSlashed:
"active_slashed"
of ValidatorFilterKind.ExitedUnslashed:
"exited_unslashed"
of ValidatorFilterKind.ExitedSlashed:
"exited_slashed"
of ValidatorFilterKind.WithdrawalPossible:
"withdrawal_possible"
of ValidatorFilterKind.WithdrawalDone:
"withdrawal_done"
proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
# https://github.com/ethereum/EIPs/blob/master/EIPS/eip-4881.md
router.api2(MethodGet, "/eth/v1/beacon/deposit_snapshot") do (
) -> RestApiResponse:
let snapshot = node.db.getDepositContractSnapshot().valueOr:
# This can happen in a very short window after the client is started,
# but the snapshot record still haven't been upgraded in the database.
# Returning 404 should be easy to handle for the clients - they just need
# to retry.
return RestApiResponse.jsonError(Http404,
NoFinalizedSnapshotAvailableError)
RestApiResponse.jsonResponse(snapshot.getTreeSnapshot())
2023-01-11 12:29:21 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getGenesis
router.api2(MethodGet, "/eth/v1/beacon/genesis") do () -> RestApiResponse:
RestApiResponse.jsonResponse(
(
genesis_time: getStateField(node.dag.headState, genesis_time),
2021-03-17 18:46:45 +00:00
genesis_validators_root:
getStateField(node.dag.headState, genesis_validators_root),
genesis_fork_version: node.dag.cfg.GENESIS_FORK_VERSION
2021-03-17 18:46:45 +00:00
)
)
# https://ethereum.github.io/beacon-APIs/#/Beacon/getStateRoot
router.api2(MethodGet, "/eth/v1/beacon/states/{state_id}/root") do (
2021-03-17 18:46:45 +00:00
state_id: StateIdent) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
bslot = node.getBlockSlotId(sid).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(Http404, StateNotFoundError,
$error)
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
node.withStateForBlockSlotId(bslot):
return RestApiResponse.jsonResponseFinalized(
(root: stateRoot),
node.getStateOptimistic(state),
node.dag.isFinalized(bslot.bid)
)
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
RestApiResponse.jsonError(Http404, StateNotFoundError)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getStateFork
router.metricsApi2(
MethodGet, "/eth/v1/beacon/states/{state_id}/fork",
{RestServerMetricsType.Status, Response}) do (
2021-03-17 18:46:45 +00:00
state_id: StateIdent) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
bslot = node.getBlockSlotId(sid).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(Http404, StateNotFoundError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
node.withStateForBlockSlotId(bslot):
return RestApiResponse.jsonResponseFinalized(
(
previous_version:
getStateField(state, fork).previous_version,
current_version:
getStateField(state, fork).current_version,
epoch:
getStateField(state, fork).epoch
),
node.getStateOptimistic(state),
node.dag.isFinalized(bslot.bid)
2021-03-17 18:46:45 +00:00
)
RestApiResponse.jsonError(Http404, StateNotFoundError)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getStateFinalityCheckpoints
router.metricsApi2(
MethodGet, "/eth/v1/beacon/states/{state_id}/finality_checkpoints",
{RestServerMetricsType.Status, Response}) do (
2021-03-17 18:46:45 +00:00
state_id: StateIdent) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
bslot = node.getBlockSlotId(sid).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(Http404, StateNotFoundError,
$error)
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
node.withStateForBlockSlotId(bslot):
return RestApiResponse.jsonResponseFinalized(
(
previous_justified:
getStateField(state, previous_justified_checkpoint),
current_justified:
getStateField(state, current_justified_checkpoint),
finalized:
getStateField(state, finalized_checkpoint)
),
node.getStateOptimistic(state),
node.dag.isFinalized(bslot.bid)
2021-03-17 18:46:45 +00:00
)
RestApiResponse.jsonError(Http404, StateNotFoundError)
2021-03-17 18:46:45 +00:00
proc getIndices(
node: BeaconNode,
validatorIds: openArray[ValidatorIdent],
state: ForkedHashedBeaconState
): Result[seq[ValidatorIndex], RestErrorMessage] =
var
keyset: HashSet[ValidatorPubKey]
indexset: HashSet[ValidatorIndex]
let validatorsCount = lenu64(getStateField(state, validators))
for item in validatorIds:
case item.kind
of ValidatorQueryKind.Key:
# Test for uniqueness of value.
if keyset.containsOrIncl(item.key):
return err(RestErrorMessage.init(
Http400, NonUniqueValidatorIdError, $item.key))
of ValidatorQueryKind.Index:
let vindex = item.index.toValidatorIndex().valueOr:
case error
of ValidatorIndexError.TooHighValue:
return err(RestErrorMessage.init(
Http400, TooHighValidatorIndexValueError))
of ValidatorIndexError.UnsupportedValue:
return err(RestErrorMessage.init(
Http500, UnsupportedValidatorIndexValueError))
if uint64(vindex) < validatorsCount:
# We're only adding validator indices which are present in
# validators list at this moment.
if indexset.containsOrIncl(vindex):
return err(RestErrorMessage.init(
Http400, NonUniqueValidatorIdError,
Base10.toString(uint64(vindex))))
if len(keyset) > 0:
let optIndices = keysToIndices(node.restKeysCache, state, keyset.toSeq())
# Remove all the duplicates.
for item in optIndices:
# We ignore missing keys.
if item.isSome():
indexset.incl(item.get())
ok(indexset.toSeq())
proc getValidators(
node: BeaconNode,
bslot: BlockSlotId,
validatorsMask: ValidatorFilter,
validatorIds: openArray[ValidatorIdent]
): RestApiResponse =
node.withStateForBlockSlotId(bslot):
let
stateEpoch = getStateField(state, slot).epoch()
indices = node.getIndices(validatorIds, state).valueOr:
return RestApiResponse.jsonError(error)
response =
block:
var res: seq[RestValidator]
if len(indices) == 0:
# Case when `len(indices) == 0 and len(validatorIds) != 0` means
# that we can't find validator identifiers in state, so we should
# return empty response.
if len(validatorIds) == 0:
# There are no indices, so we're going to filter all the
# validators.
for index, validator in getStateField(state, validators):
let
balance = getStateField(state, balances).item(index)
status = validator.getStatus(stateEpoch).valueOr:
return RestApiResponse.jsonError(
Http400, ValidatorStatusNotFoundError, $error)
if status in validatorsMask:
res.add(RestValidator.init(ValidatorIndex(index), balance,
toString(status), validator))
else:
for index in indices:
let
validator = getStateField(state, validators).item(index)
balance = getStateField(state, balances).item(index)
status = validator.getStatus(stateEpoch).valueOr:
return RestApiResponse.jsonError(
Http400, ValidatorStatusNotFoundError, $error)
if status in validatorsMask:
res.add(RestValidator.init(index, balance, toString(status),
validator))
res
return RestApiResponse.jsonResponseFinalized(
response,
node.getStateOptimistic(state),
node.dag.isFinalized(bslot.bid)
)
RestApiResponse.jsonError(Http404, StateNotFoundError)
proc getBalances(
node: BeaconNode,
bslot: BlockSlotId,
validatorIds: openArray[ValidatorIdent]
): RestApiResponse =
node.withStateForBlockSlotId(bslot):
let
indices = node.getIndices(validatorIds, state).valueOr:
return RestApiResponse.jsonError(error)
response =
block:
var res: seq[RestValidatorBalance]
if len(indices) == 0:
# Case when `len(indices) == 0 and len(validatorIds) != 0` means
# that we can't find validator identifiers in state, so we should
# return empty response.
if len(validatorIds) == 0:
# There are no indices, so we're going to return balances of all
# known validators.
for index, balance in getStateField(state, balances):
res.add(RestValidatorBalance.init(ValidatorIndex(index),
balance))
else:
for index in indices:
let balance = getStateField(state, balances).item(index)
res.add(RestValidatorBalance.init(index, balance))
res
return RestApiResponse.jsonResponseFinalized(
response,
node.getStateOptimistic(state),
node.dag.isFinalized(bslot.bid)
)
RestApiResponse.jsonError(Http404, StateNotFoundError)
# https://ethereum.github.io/beacon-APIs/#/Beacon/getStateValidators
router.metricsApi2(
MethodGet, "/eth/v1/beacon/states/{state_id}/validators",
{RestServerMetricsType.Status, Response}) do (
2021-03-17 18:46:45 +00:00
state_id: StateIdent, id: seq[ValidatorIdent],
status: seq[ValidatorFilter]) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
bslot = node.getBlockSlotId(sid).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(
Http404, StateNotFoundError, $error)
validatorIds =
block:
if id.isErr():
return RestApiResponse.jsonError(
Http400, InvalidValidatorIdValueError)
let ires = id.get()
if len(ires) > ServerMaximumValidatorIds:
return RestApiResponse.jsonError(
Http414, MaximumNumberOfValidatorIdsError)
ires
validatorsMask =
block:
if status.isErr():
return RestApiResponse.jsonError(Http400,
InvalidValidatorStatusValueError)
validateFilter(status.get()).valueOr:
return RestApiResponse.jsonError(
Http400, InvalidValidatorStatusValueError, $error)
getValidators(node, bslot, validatorsMask, validatorIds)
# https://ethereum.github.io/beacon-APIs/#/Beacon/postStateValidators
router.metricsApi2(
MethodPost, "/eth/v1/beacon/states/{state_id}/validators",
{RestServerMetricsType.Status, Response}) do (
state_id: StateIdent, contentBody: Option[ContentBody]) -> RestApiResponse:
let
(validatorIds, validatorsMask) =
block:
if contentBody.isNone():
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
let request =
decodeBody(RestValidatorRequest, contentBody.get()).valueOr:
return RestApiResponse.jsonError(
Http400, InvalidRequestBodyError, $error)
let
ids = request.ids.valueOr: @[]
filter =
if request.status.isNone() or len(request.status.get) == 0:
AllValidatorFilterKinds
else:
request.status.get
(ids, filter)
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
bslot = node.getBlockSlotId(sid).valueOr:
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(Http404, StateNotFoundError, $error)
getValidators(node, bslot, validatorsMask, validatorIds)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getStateValidator
router.metricsApi2(
MethodGet, "/eth/v1/beacon/states/{state_id}/validators/{validator_id}",
{RestServerMetricsType.Status, Response}) do (
2021-03-17 18:46:45 +00:00
state_id: StateIdent, validator_id: ValidatorIdent) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
vid = validator_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidValidatorIdValueError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
bslot = node.getBlockSlotId(sid).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(Http404, StateNotFoundError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
node.withStateForBlockSlotId(bslot):
let
current_epoch = getStateField(state, slot).epoch()
validatorsCount = lenu64(getStateField(state, validators))
let vindex =
block:
case vid.kind
of ValidatorQueryKind.Key:
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
let optIndices = keysToIndices(node.restKeysCache, state, [vid.key])
if optIndices[0].isNone():
return RestApiResponse.jsonError(Http404, ValidatorNotFoundError)
optIndices[0].get()
of ValidatorQueryKind.Index:
let vres = vid.index.toValidatorIndex()
if vres.isErr():
case vres.error
of ValidatorIndexError.TooHighValue:
return RestApiResponse.jsonError(Http400,
2021-04-08 10:49:28 +00:00
TooHighValidatorIndexValueError)
of ValidatorIndexError.UnsupportedValue:
return RestApiResponse.jsonError(Http500,
2021-04-08 10:49:28 +00:00
UnsupportedValidatorIndexValueError)
let index = vres.get()
if uint64(index) >= validatorsCount:
return RestApiResponse.jsonError(Http404, ValidatorNotFoundError)
index
let
validator = getStateField(state, validators).item(vindex)
balance = getStateField(state, balances).item(vindex)
status =
block:
let sres = validator.getStatus(current_epoch)
if sres.isErr():
return RestApiResponse.jsonError(Http400,
ValidatorStatusNotFoundError,
$sres.get())
toString(sres.get())
return RestApiResponse.jsonResponseFinalized(
RestValidator.init(vindex, balance, status, validator),
node.getStateOptimistic(state),
node.dag.isFinalized(bslot.bid)
)
RestApiResponse.jsonError(Http404, StateNotFoundError)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getStateValidatorBalances
router.metricsApi2(
MethodGet, "/eth/v1/beacon/states/{state_id}/validator_balances",
{RestServerMetricsType.Status, Response}) do (
2021-03-17 18:46:45 +00:00
state_id: StateIdent, id: seq[ValidatorIdent]) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
bslot = node.getBlockSlotId(sid).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(Http404, StateNotFoundError, $error)
validatorIds =
block:
if id.isErr():
return RestApiResponse.jsonError(
Http400, InvalidValidatorIdValueError)
let ires = id.get()
if len(ires) > ServerMaximumValidatorIds:
return RestApiResponse.jsonError(
Http400, MaximumNumberOfValidatorIdsError)
ires
getBalances(node, bslot, validatorIds)
# https://ethereum.github.io/beacon-APIs/#/Beacon/postStateValidatorBalances
router.metricsApi2(
MethodPost, "/eth/v1/beacon/states/{state_id}/validator_balances",
{RestServerMetricsType.Status, Response}) do (
state_id: StateIdent, contentBody: Option[ContentBody]) -> RestApiResponse:
let
validatorIds =
block:
if contentBody.isNone():
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
let body = contentBody.get()
decodeBody(seq[ValidatorIdent], body).valueOr:
return RestApiResponse.jsonError(
Http400, InvalidValidatorIdValueError, $error)
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
bslot = node.getBlockSlotId(sid).valueOr:
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(Http404, StateNotFoundError, $error)
getBalances(node, bslot, validatorIds)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getEpochCommittees
router.metricsApi2(
MethodGet, "/eth/v1/beacon/states/{state_id}/committees",
{RestServerMetricsType.Status, Response}) do (
2021-03-17 18:46:45 +00:00
state_id: StateIdent, epoch: Option[Epoch], index: Option[CommitteeIndex],
slot: Option[Slot]) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
bslot = node.getBlockSlotId(sid).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(Http404, StateNotFoundError,
$error)
2021-03-17 18:46:45 +00:00
let vepoch =
if epoch.isSome():
let repoch = epoch.get()
if repoch.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, InvalidEpochValueError,
$repoch.error)
let res = repoch.get()
if res > bslot.slot.epoch + MIN_SEED_LOOKAHEAD:
return RestApiResponse.jsonError(
Http400, InvalidEpochValueError,
"Requested epoch more than 1 epoch past state epoch")
if res + EPOCHS_PER_HISTORICAL_VECTOR <
bslot.slot.epoch + MIN_SEED_LOOKAHEAD:
return RestApiResponse.jsonError(
Http400, InvalidEpochValueError,
"Requested epoch earlier than what committees can be computed for")
some(res)
2021-03-17 18:46:45 +00:00
else:
none[Epoch]()
let vindex =
if index.isSome():
let rindex = index.get()
if rindex.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400,
InvalidCommitteeIndexValueError,
$rindex.error)
2021-03-17 18:46:45 +00:00
some(rindex.get())
else:
none[CommitteeIndex]()
let vslot =
if slot.isSome():
let rslot = slot.get()
if rslot.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, InvalidSlotValueError,
$rslot.error)
let res = rslot.get()
if vepoch.isSome():
if res.epoch != vepoch.get():
return RestApiResponse.jsonError(
Http400, InvalidSlotValueError,
"Slot does not match requested epoch")
else:
if res.epoch > bslot.slot.epoch + 1:
return RestApiResponse.jsonError(
Http400, InvalidEpochValueError,
"Requested slot more than 1 epoch past state epoch")
if res.epoch + EPOCHS_PER_HISTORICAL_VECTOR <
bslot.slot.epoch + MIN_SEED_LOOKAHEAD:
return RestApiResponse.jsonError(
Http400, InvalidEpochValueError,
"Requested slot earlier than what committees can be computed for")
some(res)
2021-03-17 18:46:45 +00:00
else:
none[Slot]()
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
node.withStateForBlockSlotId(bslot):
2021-03-17 18:46:45 +00:00
proc getCommittee(slot: Slot,
index: CommitteeIndex): RestBeaconStatesCommittees =
let validators = get_beacon_committee(state, slot, index, cache)
RestBeaconStatesCommittees(index: index, slot: slot,
validators: validators)
2021-03-17 18:46:45 +00:00
proc forSlot(slot: Slot, cindex: Option[CommitteeIndex],
res: var seq[RestBeaconStatesCommittees]) =
let committees_per_slot = get_committee_count_per_slot(
state, slot.epoch, cache)
2021-03-17 18:46:45 +00:00
if cindex.isNone:
for committee_index in get_committee_indices(committees_per_slot):
res.add(getCommittee(slot, committee_index))
2021-03-17 18:46:45 +00:00
else:
let
idx = cindex.get()
if idx < committees_per_slot:
res.add(getCommittee(slot, idx))
2021-03-17 18:46:45 +00:00
var res: seq[RestBeaconStatesCommittees]
2021-03-17 18:46:45 +00:00
let qepoch =
if vepoch.isNone:
epoch(getStateField(state, slot))
2021-03-17 18:46:45 +00:00
else:
vepoch.get()
if vslot.isNone():
for slot in qepoch.slots():
forSlot(slot, vindex, res)
2021-03-17 18:46:45 +00:00
else:
forSlot(vslot.get(), vindex, res)
return RestApiResponse.jsonResponseFinalized(
res,
node.getStateOptimistic(state),
node.dag.isFinalized(bslot.bid)
)
2021-03-17 18:46:45 +00:00
RestApiResponse.jsonError(Http404, StateNotFoundError)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getEpochSyncCommittees
router.metricsApi2(
MethodGet, "/eth/v1/beacon/states/{state_id}/sync_committees",
{RestServerMetricsType.Status, Response}) do (
state_id: StateIdent, epoch: Option[Epoch]) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
bslot = node.getBlockSlotId(sid).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(Http404, StateNotFoundError,
$error)
let qepoch =
if epoch.isSome():
let repoch = epoch.get()
if repoch.isErr():
return RestApiResponse.jsonError(Http400, InvalidEpochValueError,
$repoch.error)
let res = repoch.get()
if res > MaxEpoch:
return RestApiResponse.jsonError(Http400, EpochOverflowValueError)
if res < node.dag.cfg.ALTAIR_FORK_EPOCH:
return RestApiResponse.jsonError(Http400,
EpochFromTheIncorrectForkError)
res
else:
# If ``epoch`` not present then the sync committees for the epoch of
# the state will be obtained.
bslot.slot.epoch()
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
node.withStateForBlockSlotId(bslot):
let keys =
block:
let res = syncCommitteeParticipants(state, qepoch)
if res.isErr():
return RestApiResponse.jsonError(Http400,
$res.error)
let kres = res.get()
if len(kres) == 0:
return RestApiResponse.jsonError(Http500, InternalServerError,
"List of sync committee participants is empty")
kres
let indices =
block:
var res: seq[ValidatorIndex]
let optIndices = keysToIndices(node.restKeysCache, state, keys)
# Remove all the duplicates.
for item in optIndices:
if item.isNone():
# This should not be happened, because keys are from state.
return RestApiResponse.jsonError(Http500, InternalServerError,
"Could not get validator indices")
res.add(item.get())
res
let aggregates =
block:
var
res: seq[seq[ValidatorIndex]]
offset = 0
while true:
let length = min(SYNC_SUBCOMMITTEE_SIZE, len(indices) - offset)
if length == 0:
break
res.add(@(indices.toOpenArray(offset, offset + length - 1)))
offset.inc(length)
res
return RestApiResponse.jsonResponseFinalized(
RestEpochSyncCommittee(validators: indices,
validator_aggregates: aggregates),
node.getStateOptimistic(state),
node.dag.isFinalized(bslot.bid)
)
RestApiResponse.jsonError(Http404, StateNotFoundError)
# https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/getStateRandao
# https://github.com/ethereum/beacon-APIs/blob/b3c4defa238aaa74bf22aa602aa1b24b68a4c78e/apis/beacon/states/randao.yaml
router.metricsApi2(
MethodGet, "/eth/v1/beacon/states/{state_id}/randao",
{RestServerMetricsType.Status, Response}) do (
state_id: StateIdent, epoch: Option[Epoch]) -> RestApiResponse:
let
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
bslot = node.getBlockSlotId(sid).valueOr:
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(Http404, StateNotFoundError,
$error)
let qepoch =
if epoch.isSome():
let repoch = epoch.get()
if repoch.isErr():
return RestApiResponse.jsonError(Http400, InvalidEpochValueError,
$repoch.error)
let res = repoch.get()
if res > MaxEpoch:
return RestApiResponse.jsonError(Http400, EpochOverflowValueError)
if res < node.dag.cfg.ALTAIR_FORK_EPOCH:
return RestApiResponse.jsonError(Http400,
EpochFromTheIncorrectForkError)
if res > bslot.slot.epoch() + 1:
return RestApiResponse.jsonError(Http400,
EpochFromFutureError)
res
else:
# If ``epoch`` not present then the RANDAO mix for the epoch of
# the state will be obtained.
bslot.slot.epoch()
# Try to obtain RANDAO in an accelerated way
let bsi = node.dag.atSlot(bslot.bid, (qepoch + 1).start_slot - 1)
if bsi.isSome:
let mix = node.dag.computeRandaoMix(bsi.get.bid)
if mix.isSome:
return RestApiResponse.jsonResponseWOpt(
RestEpochRandao(randao: mix.get),
node.getBidOptimistic(bsi.get.bid)
)
# Fall back to full state computation
node.withStateForBlockSlotId(bslot):
withState(state):
return RestApiResponse.jsonResponseFinalized(
RestEpochRandao(randao: get_randao_mix(forkyState.data, qepoch)),
node.getStateOptimistic(state),
node.dag.isFinalized(bslot.bid)
)
RestApiResponse.jsonError(Http404, StateNotFoundError)
# https://ethereum.github.io/beacon-APIs/#/Beacon/getBlockHeaders
router.api2(MethodGet, "/eth/v1/beacon/headers") do (
2021-03-17 18:46:45 +00:00
slot: Option[Slot], parent_root: Option[Eth2Digest]) -> RestApiResponse:
# TODO (cheatfate): This call is incomplete, because structure
# of database do not allow to query blocks by `parent_root`.
let qslot =
if slot.isSome():
let rslot = slot.get()
if rslot.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, InvalidSlotValueError,
$rslot.error)
rslot.get()
else:
node.dag.head.slot
if parent_root.isSome():
let rroot = parent_root.get()
if rroot.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, InvalidParentRootValueError,
$rroot.error)
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http500, NoImplementationError)
let bdata = node.getForkedBlock(BlockIdent.init(qslot)).valueOr:
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
withBlck(bdata):
let bid = BlockId(root: forkyBlck.root, slot: forkyBlck.message.slot)
RestApiResponse.jsonResponseFinalized(
[
(
root: forkyBlck.root,
canonical: node.dag.isCanonical(bid),
header: (
message: forkyBlck.toBeaconBlockHeader,
signature: forkyBlck.signature
)
)
],
node.getBlockOptimistic(bdata),
node.dag.isFinalized(bid)
)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getBlockHeader
router.api2(MethodGet, "/eth/v1/beacon/headers/{block_id}") do (
2021-03-17 18:46:45 +00:00
block_id: BlockIdent) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
bid = block_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidBlockIdValueError,
$error)
bdata = node.getForkedBlock(bid).valueOr:
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
withBlck(bdata):
let bid = BlockId(root: forkyBlck.root, slot: forkyBlck.message.slot)
RestApiResponse.jsonResponseFinalized(
(
root: forkyBlck.root,
canonical: node.dag.isCanonical(bid),
header: (
message: forkyBlck.toBeaconBlockHeader,
signature: forkyBlck.signature
)
),
node.getBlockOptimistic(bdata),
node.dag.isFinalized(bid)
)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/publishBlock
router.api(MethodPost, "/eth/v1/beacon/blocks") do (
2021-03-17 18:46:45 +00:00
contentBody: Option[ContentBody]) -> RestApiResponse:
let res =
block:
if contentBody.isNone():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
let
body = contentBody.get()
currentEpochFork =
node.dag.cfg.consensusForkAtEpoch(node.currentSlot().epoch())
rawVersion = request.headers.getString("eth-consensus-version")
# The V1 endpoint doesn't require the version to be specified but the
# only fork which works is the current gossip fork. Either it can use
# and broadcast a block in that fork or that broadcast will not prove
# useful anyway, so allow it to fail at the decoding stage.
version =
if rawVersion == "":
currentEpochFork.toString
else:
rawVersion
let restBlock = decodeBody(
RestPublishedSignedBlockContents, body, version).valueOr:
return RestApiResponse.jsonError(error)
withForkyBlck(restBlock):
if restBlock.kind != node.dag.cfg.consensusForkAtEpoch(
forkyBlck.message.slot.epoch):
doAssert strictVerification notin node.dag.updateFlags
return RestApiResponse.jsonError(Http400, InvalidBlockObjectError)
when consensusFork >= ConsensusFork.Deneb:
await node.router.routeSignedBeaconBlock(
forkyBlck, Opt.some(
forkyBlck.create_blob_sidecars(kzg_proofs, blobs)),
checkValidator = true)
else:
await node.router.routeSignedBeaconBlock(
forkyBlck, Opt.none(seq[BlobSidecar]),
checkValidator = true)
if res.isErr():
return RestApiResponse.jsonError(
Http503, BeaconNodeInSyncError, $res.error)
if res.get().isNone():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http202, BlockValidationError)
RestApiResponse.jsonMsgResponse(BlockValidationSuccess)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/publishBlockV2
router.api(MethodPost, "/eth/v2/beacon/blocks") do (
broadcast_validation: Option[BroadcastValidationType],
contentBody: Option[ContentBody]) -> RestApiResponse:
let res =
block:
let
version = request.headers.getString("eth-consensus-version")
validation =
block:
let res =
if broadcast_validation.isNone():
BroadcastValidationType.Gossip
else:
broadcast_validation.get().valueOr:
return RestApiResponse.jsonError(Http400,
InvalidBroadcastValidationType)
# TODO (henridf): support 'consensus' and
# 'consensus_and_equivocation' broadcast_validation types.
if res != BroadcastValidationType.Gossip:
return RestApiResponse.jsonError(Http500,
"Only `gossip` broadcast_validation option supported")
res
body =
block:
if contentBody.isNone():
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
contentBody.get()
restBlock = decodeBody(
RestPublishedSignedBlockContents, body, version).valueOr:
return RestApiResponse.jsonError(error)
withForkyBlck(restBlock):
# TODO (henridf): handle broadcast_validation flag
if restBlock.kind != node.dag.cfg.consensusForkAtEpoch(
forkyBlck.message.slot.epoch):
doAssert strictVerification notin node.dag.updateFlags
return RestApiResponse.jsonError(Http400, InvalidBlockObjectError)
when consensusFork >= ConsensusFork.Deneb:
await node.router.routeSignedBeaconBlock(
forkyBlck, Opt.some(
forkyBlck.create_blob_sidecars(kzg_proofs, blobs)),
checkValidator = true)
else:
await node.router.routeSignedBeaconBlock(
forkyBlck, Opt.none(seq[BlobSidecar]),
checkValidator = true)
if res.isErr():
return RestApiResponse.jsonError(
Http503, BeaconNodeInSyncError, $res.error)
if res.get().isNone():
return RestApiResponse.jsonError(Http202, BlockValidationError)
RestApiResponse.jsonMsgResponse(BlockValidationSuccess)
# https://ethereum.github.io/beacon-APIs/?urls.primaryName=v2.4.2#/Beacon/getBlindedBlock
# https://github.com/ethereum/beacon-APIs/blob/v2.4.2/apis/beacon/blocks/blinded_block.yaml
router.api2(MethodGet, "/eth/v1/beacon/blinded_blocks/{block_id}") do (
block_id: BlockIdent) -> RestApiResponse:
let
blockIdent = block_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidBlockIdValueError,
$error)
bid = node.getBlockId(blockIdent).valueOr:
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
contentType =
block:
let res = preferredContentType(jsonMediaType,
sszMediaType)
if res.isErr():
return RestApiResponse.jsonError(Http406, ContentNotAcceptableError)
res.get()
bdata = node.dag.getForkedBlock(bid).valueOr:
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
template respondSszOrJson(
signedMaybeBlindedBlck: auto, consensusFork: ConsensusFork): untyped =
if contentType == sszMediaType:
RestApiResponse.sszResponse(
signedMaybeBlindedBlck,
[("eth-consensus-version", consensusFork.toString())])
elif contentType == jsonMediaType:
RestApiResponse.jsonResponseBlock(
signedMaybeBlindedBlck,
consensusFork,
node.getBlockOptimistic(bdata),
node.dag.isFinalized(bid)
)
else:
RestApiResponse.jsonError(Http500, InvalidAcceptError)
withBlck(bdata.asSigned()):
when consensusFork <= ConsensusFork.Altair:
respondSszOrJson(forkyBlck, consensusFork)
else:
respondSszOrJson(toSignedBlindedBeaconBlock(forkyBlck), consensusFork)
# https://ethereum.github.io/beacon-APIs/#/Beacon/publishBlindedBlock
# https://github.com/ethereum/beacon-APIs/blob/v2.4.0/apis/beacon/blocks/blinded_blocks.yaml
router.api(MethodPost, "/eth/v1/beacon/blinded_blocks") do (
contentBody: Option[ContentBody]) -> RestApiResponse:
## Instructs the beacon node to use the components of the
## `SignedBlindedBeaconBlock` to construct and publish a
## `SignedBeaconBlock` by swapping out the transactions_root for the
## corresponding full list of transactions. The beacon node should
## broadcast a newly constructed `SignedBeaconBlock` to the beacon network,
## to be included in the beacon chain. The beacon node is not required to
## validate the signed `BeaconBlock`, and a successful response (20X) only
## indicates that the broadcast has been successful.
if contentBody.isNone():
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
let
currentEpochFork =
node.dag.cfg.consensusForkAtEpoch(node.currentSlot().epoch())
rawVersion = request.headers.getString("eth-consensus-version")
body = contentBody.get()
# The V1 endpoint doesn't require the version to be specified but the
# only fork which works is the current gossip fork. Either it can use
# and broadcast a block in that fork or that broadcast will not prove
# useful anyway, so allow it to fail at the decoding stage.
version =
if rawVersion == "":
currentEpochFork.toString
else:
rawVersion
if (body.contentType == OctetStreamMediaType) and
(currentEpochFork.toString != version):
return RestApiResponse.jsonError(Http400, BlockIncorrectFork)
withConsensusFork(currentEpochFork):
when consensusFork >= ConsensusFork.Deneb:
let
restBlock = decodeBodyJsonOrSsz(
consensusFork.SignedBlindedBeaconBlock, body).valueOr:
return RestApiResponse.jsonError(error)
payloadBuilderClient = node.getPayloadBuilderClient(
restBlock.message.proposer_index).valueOr:
return RestApiResponse.jsonError(
Http400, "Unable to initialize payload builder client: " & $error)
res = await node.unblindAndRouteBlockMEV(
payloadBuilderClient, restBlock)
if res.isErr():
return RestApiResponse.jsonError(
Http500, InternalServerError, $res.error)
if res.get().isNone():
return RestApiResponse.jsonError(Http202, BlockValidationError)
return RestApiResponse.jsonMsgResponse(BlockValidationSuccess)
elif consensusFork >= ConsensusFork.Bellatrix:
return RestApiResponse.jsonError(
Http400, $consensusFork & " builder API unsupported")
else:
# Pre-Bellatrix, this endpoint will accept a `SignedBeaconBlock`.
#
# This is mostly the same as /eth/v1/beacon/blocks for phase 0 and
# altair.
var
restBlock = decodeBody(
RestPublishedSignedBeaconBlock, body, version).valueOr:
return RestApiResponse.jsonError(error)
forked = ForkedSignedBeaconBlock(restBlock)
if forked.kind != node.dag.cfg.consensusForkAtEpoch(
getForkedBlockField(forked, slot).epoch):
return RestApiResponse.jsonError(Http400, InvalidBlockObjectError)
let res = withBlck(forked):
forkyBlck.root = hash_tree_root(forkyBlck.message)
await node.router.routeSignedBeaconBlock(
forkyBlck, Opt.none(seq[BlobSidecar]),
checkValidator = true)
if res.isErr():
return RestApiResponse.jsonError(
Http503, BeaconNodeInSyncError, $res.error)
elif res.get().isNone():
return RestApiResponse.jsonError(Http202, BlockValidationError)
RestApiResponse.jsonMsgResponse(BlockValidationSuccess)
# https://ethereum.github.io/beacon-APIs/#/Beacon/publishBlindedBlockV2
router.api(MethodPost, "/eth/v2/beacon/blinded_blocks") do (
broadcast_validation: Option[BroadcastValidationType],
contentBody: Option[ContentBody]) -> RestApiResponse:
if contentBody.isNone():
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
let
currentEpochFork =
node.dag.cfg.consensusForkAtEpoch(node.currentSlot().epoch())
version = request.headers.getString("eth-consensus-version")
validation =
if broadcast_validation.isNone():
BroadcastValidationType.Gossip
else:
let res = broadcast_validation.get().valueOr:
return RestApiResponse.jsonError(Http400,
InvalidBroadcastValidationType)
# TODO (cheatfate): support 'consensus' and
# 'consensus_and_equivocation' broadcast_validation types.
if res != BroadcastValidationType.Gossip:
return RestApiResponse.jsonError(Http500,
"Only `gossip` broadcast_validation option supported")
res
body = contentBody.get()
if (body.contentType == OctetStreamMediaType) and
(currentEpochFork.toString != version):
return RestApiResponse.jsonError(Http400, BlockIncorrectFork)
withConsensusFork(currentEpochFork):
# TODO (cheatfate): handle broadcast_validation flag
when consensusFork >= ConsensusFork.Deneb:
let
restBlock = decodeBodyJsonOrSsz(
consensusFork.SignedBlindedBeaconBlock, body).valueOr:
return RestApiResponse.jsonError(error)
payloadBuilderClient = node.getPayloadBuilderClient(
restBlock.message.proposer_index).valueOr:
return RestApiResponse.jsonError(
Http400, "Unable to initialize payload builder client: " & $error)
res = await node.unblindAndRouteBlockMEV(
payloadBuilderClient, restBlock)
if res.isErr():
return RestApiResponse.jsonError(
Http500, InternalServerError, $res.error)
if res.get().isNone():
return RestApiResponse.jsonError(Http202, BlockValidationError)
return RestApiResponse.jsonMsgResponse(BlockValidationSuccess)
elif consensusFork >= ConsensusFork.Bellatrix:
return RestApiResponse.jsonError(
Http400, $consensusFork & " builder API unsupported")
else:
# Pre-Bellatrix, this endpoint will accept a `SignedBeaconBlock`.
#
# This is mostly the same as /eth/v1/beacon/blocks for phase 0 and
# altair.
var
restBlock = decodeBody(
RestPublishedSignedBeaconBlock, body, version).valueOr:
return RestApiResponse.jsonError(error)
forked = ForkedSignedBeaconBlock(restBlock)
if forked.kind != node.dag.cfg.consensusForkAtEpoch(
getForkedBlockField(forked, slot).epoch):
return RestApiResponse.jsonError(Http400, InvalidBlockObjectError)
let res = withBlck(forked):
forkyBlck.root = hash_tree_root(forkyBlck.message)
await node.router.routeSignedBeaconBlock(
forkyBlck, Opt.none(seq[BlobSidecar]),
checkValidator = true)
if res.isErr():
return RestApiResponse.jsonError(
Http503, BeaconNodeInSyncError, $res.error)
elif res.get().isNone():
return RestApiResponse.jsonError(Http202, BlockValidationError)
RestApiResponse.jsonMsgResponse(BlockValidationSuccess)
# https://ethereum.github.io/beacon-APIs/#/Beacon/getBlock
router.api2(MethodGet, "/eth/v1/beacon/blocks/{block_id}") do (
2021-03-17 18:46:45 +00:00
block_id: BlockIdent) -> RestApiResponse:
RestApiResponse.jsonError(
Http410, DeprecatedRemovalBeaconBlocksDebugStateV1)
# https://ethereum.github.io/beacon-APIs/#/Beacon/getBlockV2
router.api2(MethodGet, "/eth/v2/beacon/blocks/{block_id}") do (
block_id: BlockIdent) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
blockIdent = block_id.valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
return RestApiResponse.jsonError(Http400, InvalidBlockIdValueError,
$error)
bid = node.getBlockId(blockIdent).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
let contentType =
block:
let res = preferredContentType(jsonMediaType,
sszMediaType)
if res.isErr():
return RestApiResponse.jsonError(Http406, ContentNotAcceptableError)
res.get()
if contentType == sszMediaType:
var data: seq[byte]
if not node.dag.getBlockSSZ(bid, data):
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
let
fork = node.dag.cfg.consensusForkAtEpoch(bid.slot.epoch)
headers = [("eth-consensus-version", fork.toString())]
RestApiResponse.sszResponsePlain(data, headers)
elif contentType == jsonMediaType:
let bdata = node.dag.getForkedBlock(bid).valueOr:
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
RestApiResponse.jsonResponseBlock(
bdata.asSigned(),
node.getBlockOptimistic(bdata),
node.dag.isFinalized(bid)
)
else:
RestApiResponse.jsonError(Http500, InvalidAcceptError)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getBlockRoot
router.api2(MethodGet, "/eth/v1/beacon/blocks/{block_id}/root") do (
2021-03-17 18:46:45 +00:00
block_id: BlockIdent) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
blockIdent = block_id.valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
return RestApiResponse.jsonError(Http400, InvalidBlockIdValueError,
$error)
bid = node.getBlockId(blockIdent).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
bdata = node.dag.getForkedBlock(bid).valueOr:
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
RestApiResponse.jsonResponseFinalized(
(root: bid.root),
node.getBlockOptimistic(bdata),
node.dag.isFinalized(bid)
)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getBlockAttestations
router.api2(MethodGet,
"/eth/v1/beacon/blocks/{block_id}/attestations") do (
2021-03-17 18:46:45 +00:00
block_id: BlockIdent) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
blockIdent = block_id.valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
return RestApiResponse.jsonError(Http400, InvalidBlockIdValueError,
$error)
bdata = node.getForkedBlock(blockIdent).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
withBlck(bdata):
let bid = BlockId(root: forkyBlck.root, slot: forkyBlck.message.slot)
RestApiResponse.jsonResponseFinalized(
forkyBlck.message.body.attestations.asSeq(),
node.getBlockOptimistic(bdata),
node.dag.isFinalized(bid)
)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/getBlockAttestationsV2
router.api2(MethodGet,
"/eth/v2/beacon/blocks/{block_id}/attestations") do (
block_id: BlockIdent) -> RestApiResponse:
let
blockIdent = block_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidBlockIdValueError,
$error)
bdata = node.getForkedBlock(blockIdent).valueOr:
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
withBlck(bdata):
let bid = BlockId(root: forkyBlck.root, slot: forkyBlck.message.slot)
RestApiResponse.jsonResponseFinalizedWVersion(
forkyBlck.message.body.attestations.asSeq(),
node.getBlockOptimistic(bdata),
node.dag.isFinalized(bid),
consensusFork
)
# https://ethereum.github.io/beacon-APIs/#/Beacon/getPoolAttestations
router.api2(MethodGet, "/eth/v1/beacon/pool/attestations") do (
2021-03-17 18:46:45 +00:00
slot: Option[Slot],
committee_index: Option[CommitteeIndex]) -> RestApiResponse:
let vindex =
if committee_index.isSome():
let rindex = committee_index.get()
if rindex.isErr():
return RestApiResponse.jsonError(Http400,
2021-04-08 10:49:28 +00:00
InvalidCommitteeIndexValueError,
$rindex.error)
Opt.some(rindex.get())
2021-03-17 18:46:45 +00:00
else:
Opt.none(CommitteeIndex)
2021-03-17 18:46:45 +00:00
let vslot =
if slot.isSome():
let rslot = slot.get()
if rslot.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, InvalidSlotValueError,
$rslot.error)
Opt.some(rslot.get())
2021-03-17 18:46:45 +00:00
else:
Opt.none(Slot)
var res: seq[phase0.Attestation]
2021-03-17 18:46:45 +00:00
for item in node.attestationPool[].attestations(vslot, vindex):
res.add(item)
RestApiResponse.jsonResponse(res)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/getPoolAttestationsV2
router.api2(MethodGet, "/eth/v2/beacon/pool/attestations") do (
slot: Option[Slot],
committee_index: Option[CommitteeIndex]) -> RestApiResponse:
let vindex =
if committee_index.isSome():
let rindex = committee_index.get()
if rindex.isErr():
return RestApiResponse.jsonError(Http400,
InvalidCommitteeIndexValueError,
$rindex.error)
Opt.some(rindex.get())
else:
Opt.none(CommitteeIndex)
let vslot =
if slot.isSome():
let rslot = slot.get()
if rslot.isErr():
return RestApiResponse.jsonError(Http400, InvalidSlotValueError,
$rslot.error)
Opt.some(rslot.get())
else:
Opt.none(Slot)
let consensusFork =
if vslot.isNone():
node.dag.cfg.consensusForkAtEpoch(node.currentSlot().epoch())
else:
node.dag.cfg.consensusForkAtEpoch(vslot.get().epoch)
if consensusFork < ConsensusFork.Electra:
return RestApiResponse.jsonResponseWVersion(
toSeq(node.attestationPool[].attestations(vslot, vindex)),
consensusFork)
else:
return RestApiResponse.jsonResponseWVersion(
toSeq(node.attestationPool[].electraAttestations(vslot, vindex)),
consensusFork)
# https://ethereum.github.io/beacon-APIs/#/Beacon/submitPoolAttestations
router.api2(MethodPost, "/eth/v1/beacon/pool/attestations") do (
2021-03-17 18:46:45 +00:00
contentBody: Option[ContentBody]) -> RestApiResponse:
let attestations =
block:
if contentBody.isNone():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
let dres = decodeBody(seq[phase0.Attestation], contentBody.get())
if dres.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400,
InvalidAttestationObjectError,
$dres.error)
dres.get()
# Since our validation logic supports batch processing, we will submit all
# attestations for validation.
let pending =
mapIt(attestations, node.router.routeAttestation(it))
let failures =
block:
var res: seq[RestIndexedErrorMessageItem]
await allFutures(pending)
for index, future in pending:
if future.completed():
let fres = future.value()
if fres.isErr():
let failure = RestIndexedErrorMessageItem(index: index,
message: $fres.error)
res.add(failure)
elif future.failed() or future.cancelled():
# This is unexpected failure, so we log the error message.
let exc = future.error()
let failure = RestIndexedErrorMessageItem(index: index,
message: $exc.msg)
res.add(failure)
res
if len(failures) > 0:
RestApiResponse.jsonErrorList(Http400, AttestationValidationError,
failures)
else:
RestApiResponse.jsonMsgResponse(AttestationValidationSuccess)
# https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/submitPoolAttestationsV2
router.api2(MethodPost, "/eth/v2/beacon/pool/attestations") do (
contentBody: Option[ContentBody]) -> RestApiResponse:
let
headerVersion = request.headers.getString("Eth-Consensus-Version")
consensusVersion = ConsensusFork.init(headerVersion)
if consensusVersion.isNone():
return RestApiResponse.jsonError(Http400, FailedToObtainConsensusForkError)
if contentBody.isNone():
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
var pendingAttestations: seq[Future[SendResult]]
template decodeAttestations(AttestationType: untyped) =
let dres = decodeBody(seq[AttestationType], contentBody.get())
if dres.isErr():
return RestApiResponse.jsonError(Http400,
InvalidAttestationObjectError,
$dres.error)
# Since our validation logic supports batch processing, we will submit all
# attestations for validation.
for attestation in dres.get():
pendingAttestations.add(node.router.routeAttestation(attestation))
case consensusVersion.get():
of ConsensusFork.Phase0 .. ConsensusFork.Deneb:
decodeAttestations(phase0.Attestation)
of ConsensusFork.Electra:
decodeAttestations(electra.Attestation)
let failures =
block:
var res: seq[RestIndexedErrorMessageItem]
await allFutures(pendingAttestations)
for index, future in pendingAttestations:
if future.completed():
let fres = future.value()
if fres.isErr():
let failure = RestIndexedErrorMessageItem(index: index,
message: $fres.error)
res.add(failure)
elif future.failed() or future.cancelled():
# This is unexpected failure, so we log the error message.
let exc = future.error()
let failure = RestIndexedErrorMessageItem(index: index,
message: $exc.msg)
res.add(failure)
res
if len(failures) > 0:
RestApiResponse.jsonErrorList(Http400, AttestationValidationError,
failures)
else:
RestApiResponse.jsonMsgResponse(AttestationValidationSuccess)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getPoolAttesterSlashings
router.api2(MethodGet, "/eth/v1/beacon/pool/attester_slashings") do (
2021-03-17 18:46:45 +00:00
) -> RestApiResponse:
RestApiResponse.jsonResponse(
toSeq(node.validatorChangePool.phase0_attester_slashings))
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/submitPoolAttesterSlashings
router.api(MethodPost, "/eth/v1/beacon/pool/attester_slashings") do (
2021-03-17 18:46:45 +00:00
contentBody: Option[ContentBody]) -> RestApiResponse:
let slashing =
block:
if contentBody.isNone():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
let dres = decodeBody(phase0.AttesterSlashing, contentBody.get())
if dres.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400,
InvalidAttesterSlashingObjectError,
$dres.error)
dres.get()
let res = await node.router.routeAttesterSlashing(slashing)
if res.isErr():
return RestApiResponse.jsonError(Http400,
AttesterSlashingValidationError,
$res.error)
RestApiResponse.jsonMsgResponse(AttesterSlashingValidationSuccess)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/getPoolAttesterSlashingsV2
router.api2(MethodGet, "/eth/v2/beacon/pool/attester_slashings") do (
) -> RestApiResponse:
let contextFork =
node.dag.cfg.consensusForkAtEpoch(node.currentSlot.epoch)
withConsensusFork(contextFork):
when consensusFork < ConsensusFork.Electra:
RestApiResponse.jsonResponseWVersion(
toSeq(node.validatorChangePool.phase0_attester_slashings),
contextFork)
else:
RestApiResponse.jsonResponseWVersion(
toSeq(node.validatorChangePool.electra_attester_slashings),
contextFork)
# https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/submitPoolAttesterSlashingsV2
router.api(MethodPost, "/eth/v2/beacon/pool/attester_slashings") do (
contentBody: Option[ContentBody]) -> RestApiResponse:
let
headerVersion = request.headers.getString("Eth-Consensus-Version")
consensusVersion = ConsensusFork.init(headerVersion)
if consensusVersion.isNone():
return RestApiResponse.jsonError(Http400, FailedToObtainConsensusForkError)
if contentBody.isNone():
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
template decodeAttesterSlashing(AttesterSlashingType: untyped) =
let dres = decodeBody(AttesterSlashingType, contentBody.get())
if dres.isErr():
return RestApiResponse.jsonError(Http400,
InvalidAttesterSlashingObjectError,
$dres.error)
let res = await node.router.routeAttesterSlashing(dres.get())
if res.isErr():
return RestApiResponse.jsonError(Http400,
AttesterSlashingValidationError,
$res.error)
return RestApiResponse.jsonMsgResponse(AttesterSlashingValidationSuccess)
case consensusVersion.get():
of ConsensusFork.Phase0 .. ConsensusFork.Deneb:
decodeAttesterSlashing(phase0.AttesterSlashing)
of ConsensusFork.Electra:
decodeAttesterSlashing(electra.AttesterSlashing)
# https://ethereum.github.io/beacon-APIs/#/Beacon/getPoolProposerSlashings
router.api2(MethodGet, "/eth/v1/beacon/pool/proposer_slashings") do (
2021-03-17 18:46:45 +00:00
) -> RestApiResponse:
RestApiResponse.jsonResponse(
toSeq(node.validatorChangePool.proposer_slashings))
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/submitPoolProposerSlashings
router.api(MethodPost, "/eth/v1/beacon/pool/proposer_slashings") do (
2021-03-17 18:46:45 +00:00
contentBody: Option[ContentBody]) -> RestApiResponse:
let slashing =
block:
if contentBody.isNone():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
let dres = decodeBody(ProposerSlashing, contentBody.get())
if dres.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400,
InvalidProposerSlashingObjectError,
$dres.error)
dres.get()
let res = await node.router.routeProposerSlashing(slashing)
if res.isErr():
return RestApiResponse.jsonError(Http400,
ProposerSlashingValidationError,
$res.error)
RestApiResponse.jsonMsgResponse(ProposerSlashingValidationSuccess)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/getPoolBLSToExecutionChanges
# https://github.com/ethereum/beacon-APIs/blob/86850001845df9163da5ae9605dbf15cd318d5d0/apis/beacon/pool/bls_to_execution_changes.yaml
router.api2(MethodGet, "/eth/v1/beacon/pool/bls_to_execution_changes") do (
) -> RestApiResponse:
RestApiResponse.jsonResponse(
toSeq(node.validatorChangePool.bls_to_execution_changes_gossip) &
toSeq(node.validatorChangePool.bls_to_execution_changes_api))
# https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/submitPoolBLSToExecutionChange
# https://github.com/ethereum/beacon-APIs/blob/86850001845df9163da5ae9605dbf15cd318d5d0/apis/beacon/pool/bls_to_execution_changes.yaml
router.api2(MethodPost, "/eth/v1/beacon/pool/bls_to_execution_changes") do (
contentBody: Option[ContentBody]) -> RestApiResponse:
if node.currentSlot().epoch() < node.dag.cfg.CAPELLA_FORK_EPOCH:
return RestApiResponse.jsonError(Http400,
InvalidBlsToExecutionChangeObjectError,
"Attempt to add to BLS to execution change pool pre-Capella")
let
bls_to_execution_changes =
block:
if contentBody.isNone():
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
let dres =
decodeBody(seq[SignedBLSToExecutionChange], contentBody.get())
if dres.isErr():
return RestApiResponse.jsonError(
Http400, InvalidBlsToExecutionChangeObjectError, $dres.error)
dres.get()
pending = mapIt(bls_to_execution_changes,
node.router.routeBlsToExecutionChange(it))
await allFutures(pending)
for future in pending:
if future.failed() or future.cancelled():
return RestApiResponse.jsonError(Http400,
BlsToExecutionChangeValidationError,
$future.error().msg)
let res = future.value()
if res.isErr():
return RestApiResponse.jsonError(Http400,
BlsToExecutionChangeValidationError,
$res.error)
RestApiResponse.jsonMsgResponse(BlsToExecutionChangeValidationSuccess)
# https://ethereum.github.io/beacon-APIs/#/Beacon/submitPoolSyncCommitteeSignatures
router.api(MethodPost, "/eth/v1/beacon/pool/sync_committees") do (
contentBody: Option[ContentBody]) -> RestApiResponse:
let messages =
block:
if contentBody.isNone():
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
let dres = decodeBody(seq[SyncCommitteeMessage], contentBody.get())
if dres.isErr():
return RestApiResponse.jsonError(Http400,
InvalidSyncCommitteeSignatureMessageError)
dres.get()
let results = await node.router.routeSyncCommitteeMessages(messages)
let failures =
block:
var res: seq[RestIndexedErrorMessageItem]
for index, item in results:
if item.isErr():
res.add(RestIndexedErrorMessageItem(index: index,
message: $item.error))
res
if len(failures) > 0:
RestApiResponse.jsonErrorList(
Http400, SyncCommitteeMessageValidationError, failures)
else:
RestApiResponse.jsonMsgResponse(
SyncCommitteeMessageValidationSuccess)
# https://ethereum.github.io/beacon-APIs/#/Beacon/getPoolVoluntaryExits
router.api2(MethodGet, "/eth/v1/beacon/pool/voluntary_exits") do (
2021-03-17 18:46:45 +00:00
) -> RestApiResponse:
return RestApiResponse.jsonResponse(
toSeq(node.validatorChangePool.voluntary_exits))
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/submitPoolVoluntaryExit
router.api(MethodPost, "/eth/v1/beacon/pool/voluntary_exits") do (
2021-03-17 18:46:45 +00:00
contentBody: Option[ContentBody]) -> RestApiResponse:
let exit =
block:
if contentBody.isNone():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
let dres = decodeBody(SignedVoluntaryExit, contentBody.get())
if dres.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400,
InvalidVoluntaryExitObjectError,
$dres.error)
dres.get()
let res = await node.router.routeSignedVoluntaryExit(exit)
if res.isErr():
return RestApiResponse.jsonError(
Http400, VoluntaryExitValidationError, $res.error)
New validator client using REST API. (#2651) * Initial commit. * Exporting getConfig(). * Add beacon node checking procedures. * Post rebase fixes. * Use runSlotLoop() from nimbus_beacon_node. Fallback implementation. Fixes for ETH2 REST serialization. * Add beacon_clock.durationToNextSlot(). Move type declarations from beacon_rest_api to json_rest_serialization. Fix seq[ValidatorIndex] serialization. Refactor ValidatorPool and add some utility procedures. Create separate version of validator_client. * Post-rebase fixes. Remove CookedPubKey from validator_pool.nim. * Now we should be able to produce attestations and aggregate and proofs. But its not working yet. * Debugging attestation sending. * Add durationToNextAttestation. Optimize some debug logs. Fix aggregation_bits encoding. Bump chronos/presto. * Its alive. * Fixes for launch_local_testnet script. Bump chronos. * Switch client API to not use `/api` prefix. * Post-rebase adjustments. * Fix endpoint for publishBlock(). * Add CONFIG_NAME. Add more checks to ensure that beacon_node is compatible. * Add beacon committee subscription support to validator_client. * Fix stacktrace should be an array of strings. Fix committee subscriptions should not be `data` keyed. * Log duration to next block proposal. * Fix beacon_node_status import. * Use jsonMsgResponse() instead of jsonError(). * Fix graffityBytes usage. Remove unnecessary `await`. Adjust creation of SignedBlock instance. Remove legacy files. * Rework durationToNextSlot() and durationToNextEpoch() to use `fromNow`. * Fix race condition for block proposal and attestations for same slot. Fix local_testnet script to properly kill tasks on Windows. Bump chronos and nim-http-tools, to allow connections to infura.io (basic auth). * Catch services errors. Improve performance of local_testnet.sh script on Windows. Fix race condition when attestation producing. * Post-rebase fixes. * Bump chronos and presto. * Calculate block publishing delay. Fix pkill in one more place. * Add error handling and timeouts to firstSuccess() template. Add onceToAll() template. Add checkNodes() procedure. Refactor firstSuccess() template. Add error checking to api.nim calls. * Deprecated usage onceToAll() for better stability. Address comment and send attestations asap. * Avoid unnecessary loop when calculating minimal duration.
2021-07-13 11:15:07 +00:00
return RestApiResponse.jsonMsgResponse(VoluntaryExitValidationSuccess)
# https://ethereum.github.io/beacon-APIs/?urls.primaryName=v2.4.2#/Beacon/getBlobSidecars
# https://github.com/ethereum/beacon-APIs/blob/v2.4.2/apis/beacon/blob_sidecars/blob_sidecars.yaml
router.api2(MethodGet, "/eth/v1/beacon/blob_sidecars/{block_id}") do (
block_id: BlockIdent, indices: seq[uint64]) -> RestApiResponse:
let
blockIdent = block_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidBlockIdValueError,
$error)
bid = node.getBlockId(blockIdent).valueOr:
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
contentType = block:
let res = preferredContentType(jsonMediaType,
sszMediaType)
if res.isErr():
return RestApiResponse.jsonError(Http406, ContentNotAcceptableError)
res.get()
# https://github.com/ethereum/beacon-APIs/blob/v2.4.2/types/deneb/blob_sidecar.yaml#L2-L28
let data = newClone(default(List[BlobSidecar, Limit MAX_BLOBS_PER_BLOCK]))
if indices.isErr:
return RestApiResponse.jsonError(Http400,
InvalidSidecarIndexValueError)
let indexFilter = indices.get.toHashSet
for blobIndex in 0'u64 ..< MAX_BLOBS_PER_BLOCK:
if indexFilter.len > 0 and blobIndex notin indexFilter:
continue
var blobSidecar = new BlobSidecar
if node.dag.db.getBlobSidecar(bid.root, blobIndex, blobSidecar[]):
discard data[].add blobSidecar[]
if contentType == sszMediaType:
RestApiResponse.sszResponse(
data[], headers = [("eth-consensus-version",
node.dag.cfg.consensusForkAtEpoch(bid.slot.epoch).toString())])
elif contentType == jsonMediaType:
RestApiResponse.jsonResponse(data)
else:
RestApiResponse.jsonError(Http500, InvalidAcceptError)