nimbus-eth2/beacon_chain/rpc/rest_beacon_api.nim

1346 lines
56 KiB
Nim
Raw Normal View History

# beacon_chain
# Copyright (c) 2018-2023 Status Research & Development GmbH
2021-03-17 18:46:45 +00:00
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
2021-03-17 18:46:45 +00:00
import
2022-06-09 08:50:36 +00:00
std/[typetraits, sequtils, sets],
2021-03-17 18:46:45 +00:00
stew/[results, base10],
chronicles,
./rest_utils,
./state_ttl_cache,
../beacon_node,
../consensus_object_pools/[blockchain_dag, exit_pool, spec_cache],
../spec/[deposit_snapshots, eth2_merkleization, forks, network, validator],
../spec/datatypes/[phase0, altair, deneb],
../validators/message_router_mev
2021-03-17 18:46:45 +00:00
export rest_utils
2021-03-17 18:46:45 +00:00
logScope: topics = "rest_beaconapi"
proc validateBeaconApiQueries*(key: string, value: string): int =
## This is rough validation procedure which should be simple and fast,
## because it will be used for query routing.
case key
of "{epoch}":
0
of "{slot}":
0
of "{peer_id}":
0
of "{state_id}":
0
of "{block_id}":
0
of "{validator_id}":
0
of "{block_root}":
0
of "{pubkey}":
int(value.len != 98)
else:
1
2021-03-17 18:46:45 +00:00
proc validateFilter(filters: seq[ValidatorFilter]): Result[ValidatorFilter,
cstring] =
var res: ValidatorFilter
for item in filters:
if res * item != {}:
return err("Validator status must be unique")
res.incl(item)
if res == {}:
res = {ValidatorFilterKind.PendingInitialized,
ValidatorFilterKind.PendingQueued,
ValidatorFilterKind.ActiveOngoing,
ValidatorFilterKind.ActiveExiting,
ValidatorFilterKind.ActiveSlashed,
ValidatorFilterKind.ExitedUnslashed,
ValidatorFilterKind.ExitedSlashed,
ValidatorFilterKind.WithdrawalPossible,
ValidatorFilterKind.WithdrawalDone}
ok(res)
proc getStatus(validator: Validator,
current_epoch: Epoch): Result[ValidatorFilterKind, cstring] =
if validator.activation_epoch > current_epoch:
# pending
if validator.activation_eligibility_epoch == FAR_FUTURE_EPOCH:
ok(ValidatorFilterKind.PendingInitialized)
else:
# validator.activation_eligibility_epoch < FAR_FUTURE_EPOCH:
ok(ValidatorFilterKind.PendingQueued)
elif (validator.activation_epoch <= current_epoch) and
(current_epoch < validator.exit_epoch):
# active
if validator.exit_epoch == FAR_FUTURE_EPOCH:
ok(ValidatorFilterKind.ActiveOngoing)
elif not validator.slashed:
# validator.exit_epoch < FAR_FUTURE_EPOCH
ok(ValidatorFilterKind.ActiveExiting)
else:
# validator.exit_epoch < FAR_FUTURE_EPOCH and validator.slashed:
ok(ValidatorFilterKind.ActiveSlashed)
elif (validator.exit_epoch <= current_epoch) and
(current_epoch < validator.withdrawable_epoch):
# exited
if not validator.slashed:
ok(ValidatorFilterKind.ExitedUnslashed)
else:
# validator.slashed
ok(ValidatorFilterKind.ExitedSlashed)
elif validator.withdrawable_epoch <= current_epoch:
# withdrawal
if validator.effective_balance != 0:
ok(ValidatorFilterKind.WithdrawalPossible)
else:
# validator.effective_balance == 0
ok(ValidatorFilterKind.WithdrawalDone)
else:
err("Invalid validator status")
proc toString*(kind: ValidatorFilterKind): string =
case kind
of ValidatorFilterKind.PendingInitialized:
"pending_initialized"
of ValidatorFilterKind.PendingQueued:
"pending_queued"
of ValidatorFilterKind.ActiveOngoing:
"active_ongoing"
of ValidatorFilterKind.ActiveExiting:
"active_exiting"
of ValidatorFilterKind.ActiveSlashed:
"active_slashed"
of ValidatorFilterKind.ExitedUnslashed:
"exited_unslashed"
of ValidatorFilterKind.ExitedSlashed:
"exited_slashed"
of ValidatorFilterKind.WithdrawalPossible:
"withdrawal_possible"
of ValidatorFilterKind.WithdrawalDone:
"withdrawal_done"
proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
# https://github.com/ethereum/EIPs/blob/master/EIPS/eip-4881.md
router.api(MethodGet, "/eth/v1/beacon/deposit_snapshot") do () -> RestApiResponse:
2023-01-11 12:29:21 +00:00
let snapshot = node.db.getDepositTreeSnapshot().valueOr:
# This can happen in a very short window after the client is started, but the
# snapshot record still haven't been upgraded in the database. Returning 404
# should be easy to handle for the clients - they just need to retry.
return RestApiResponse.jsonError(Http404, NoFinalizedSnapshotAvailableError)
2023-01-11 12:29:21 +00:00
return RestApiResponse.jsonResponse(
RestDepositSnapshot(
finalized: snapshot.depositContractState.branch,
deposit_root: snapshot.getDepositRoot(),
deposit_count: snapshot.getDepositCountU64(),
execution_block_hash: snapshot.eth1Block,
execution_block_height: snapshot.blockHeight))
# https://ethereum.github.io/beacon-APIs/#/Beacon/getGenesis
router.api(MethodGet, "/eth/v1/beacon/genesis") do () -> RestApiResponse:
2021-03-17 18:46:45 +00:00
return RestApiResponse.jsonResponse(
(
genesis_time: getStateField(node.dag.headState, genesis_time),
2021-03-17 18:46:45 +00:00
genesis_validators_root:
getStateField(node.dag.headState, genesis_validators_root),
genesis_fork_version: node.dag.cfg.GENESIS_FORK_VERSION
2021-03-17 18:46:45 +00:00
)
)
# https://ethereum.github.io/beacon-APIs/#/Beacon/getStateRoot
router.api(MethodGet, "/eth/v1/beacon/states/{state_id}/root") do (
2021-03-17 18:46:45 +00:00
state_id: StateIdent) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
bslot = node.getBlockSlotId(sid).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(Http404, StateNotFoundError,
$error)
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
node.withStateForBlockSlotId(bslot):
return RestApiResponse.jsonResponseWOpt(
(root: stateRoot),
node.getStateOptimistic(state)
)
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
return RestApiResponse.jsonError(Http404, StateNotFoundError)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getStateFork
router.api(MethodGet, "/eth/v1/beacon/states/{state_id}/fork") do (
2021-03-17 18:46:45 +00:00
state_id: StateIdent) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
bslot = node.getBlockSlotId(sid).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(Http404, StateNotFoundError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
node.withStateForBlockSlotId(bslot):
return RestApiResponse.jsonResponseWOpt(
(
previous_version:
getStateField(state, fork).previous_version,
current_version:
getStateField(state, fork).current_version,
epoch:
getStateField(state, fork).epoch
),
node.getStateOptimistic(state)
2021-03-17 18:46:45 +00:00
)
return RestApiResponse.jsonError(Http404, StateNotFoundError)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getStateFinalityCheckpoints
2021-03-17 18:46:45 +00:00
router.api(MethodGet,
"/eth/v1/beacon/states/{state_id}/finality_checkpoints") do (
2021-03-17 18:46:45 +00:00
state_id: StateIdent) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
bslot = node.getBlockSlotId(sid).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(Http404, StateNotFoundError,
$error)
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
node.withStateForBlockSlotId(bslot):
return RestApiResponse.jsonResponseWOpt(
(
previous_justified:
getStateField(state, previous_justified_checkpoint),
current_justified:
getStateField(state, current_justified_checkpoint),
finalized:
getStateField(state, finalized_checkpoint)
),
node.getStateOptimistic(state)
2021-03-17 18:46:45 +00:00
)
return RestApiResponse.jsonError(Http404, StateNotFoundError)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getStateValidators
router.api(MethodGet, "/eth/v1/beacon/states/{state_id}/validators") do (
2021-03-17 18:46:45 +00:00
state_id: StateIdent, id: seq[ValidatorIdent],
status: seq[ValidatorFilter]) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
bslot = node.getBlockSlotId(sid).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(Http404, StateNotFoundError,
$error)
2021-03-17 18:46:45 +00:00
let validatorIds =
block:
if id.isErr():
return RestApiResponse.jsonError(Http400,
2021-04-08 10:49:28 +00:00
InvalidValidatorIdValueError)
let ires = id.get()
if len(ires) > ServerMaximumValidatorIds:
return RestApiResponse.jsonError(Http400,
2021-04-08 10:49:28 +00:00
MaximumNumberOfValidatorIdsError)
ires
2021-03-17 18:46:45 +00:00
let validatorsMask =
block:
if status.isErr():
return RestApiResponse.jsonError(Http400,
2021-04-08 10:49:28 +00:00
InvalidValidatorStatusValueError)
2021-03-17 18:46:45 +00:00
let res = validateFilter(status.get())
if res.isErr():
return RestApiResponse.jsonError(Http400,
2021-04-08 10:49:28 +00:00
InvalidValidatorStatusValueError,
2021-03-17 18:46:45 +00:00
$res.error())
res.get()
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
node.withStateForBlockSlotId(bslot):
let
current_epoch = getStateField(state, slot).epoch()
validatorsCount = lenu64(getStateField(state, validators))
let indices =
block:
var keyset: HashSet[ValidatorPubKey]
var indexset: HashSet[ValidatorIndex]
for item in validatorIds:
case item.kind
of ValidatorQueryKind.Key:
keyset.incl(item.key)
of ValidatorQueryKind.Index:
let vindex =
block:
let vres = item.index.toValidatorIndex()
if vres.isErr():
case vres.error()
of ValidatorIndexError.TooHighValue:
return RestApiResponse.jsonError(Http400,
2021-04-08 10:49:28 +00:00
TooHighValidatorIndexValueError)
of ValidatorIndexError.UnsupportedValue:
return RestApiResponse.jsonError(Http500,
2021-04-08 10:49:28 +00:00
UnsupportedValidatorIndexValueError)
let index = vres.get()
index
if uint64(vindex) < validatorsCount:
# We only adding validator indices which are present in
# validators list at this moment.
indexset.incl(vindex)
if len(keyset) > 0:
let optIndices = keysToIndices(node.restKeysCache, state,
keyset.toSeq())
# Remove all the duplicates.
for item in optIndices:
# We ignore missing keys.
if item.isSome():
indexset.incl(item.get())
indexset.toSeq()
2021-03-17 18:46:45 +00:00
let response =
block:
var res: seq[RestValidator]
if len(indices) == 0:
# Case when `len(indices) == 0 and len(validatorIds) != 0` means
# that we can't find validator identifiers in state, so we should
# return empty response.
if len(validatorIds) == 0:
# There is no indices, so we going to filter all the validators.
for index, validator in getStateField(state, validators):
let
balance = getStateField(state, balances).item(index)
status =
block:
let sres = validator.getStatus(current_epoch)
if sres.isErr():
return RestApiResponse.jsonError(Http400,
ValidatorStatusNotFoundError,
$sres.get())
sres.get()
if status in validatorsMask:
res.add(RestValidator.init(ValidatorIndex(index), balance,
toString(status), validator))
else:
for index in indices:
let
validator = getStateField(state, validators).item(index)
balance = getStateField(state, balances).item(index)
status =
block:
let sres = validator.getStatus(current_epoch)
if sres.isErr():
return RestApiResponse.jsonError(Http400,
ValidatorStatusNotFoundError,
$sres.get())
sres.get()
if status in validatorsMask:
res.add(RestValidator.init(index, balance, toString(status),
validator))
res
return RestApiResponse.jsonResponseWOpt(
response,
node.getStateOptimistic(state)
)
return RestApiResponse.jsonError(Http404, StateNotFoundError)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getStateValidator
2021-03-17 18:46:45 +00:00
router.api(MethodGet,
"/eth/v1/beacon/states/{state_id}/validators/{validator_id}") do (
2021-03-17 18:46:45 +00:00
state_id: StateIdent, validator_id: ValidatorIdent) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
vid = validator_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidValidatorIdValueError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
bslot = node.getBlockSlotId(sid).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(Http404, StateNotFoundError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
node.withStateForBlockSlotId(bslot):
let
current_epoch = getStateField(state, slot).epoch()
validatorsCount = lenu64(getStateField(state, validators))
let vindex =
block:
case vid.kind
of ValidatorQueryKind.Key:
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
let optIndices = keysToIndices(node.restKeysCache, state, [vid.key])
if optIndices[0].isNone():
return RestApiResponse.jsonError(Http404, ValidatorNotFoundError)
optIndices[0].get()
of ValidatorQueryKind.Index:
let vres = vid.index.toValidatorIndex()
if vres.isErr():
case vres.error()
of ValidatorIndexError.TooHighValue:
return RestApiResponse.jsonError(Http400,
2021-04-08 10:49:28 +00:00
TooHighValidatorIndexValueError)
of ValidatorIndexError.UnsupportedValue:
return RestApiResponse.jsonError(Http500,
2021-04-08 10:49:28 +00:00
UnsupportedValidatorIndexValueError)
let index = vres.get()
if uint64(index) >= validatorsCount:
return RestApiResponse.jsonError(Http404, ValidatorNotFoundError)
index
let
validator = getStateField(state, validators).item(vindex)
balance = getStateField(state, balances).item(vindex)
status =
block:
let sres = validator.getStatus(current_epoch)
if sres.isErr():
return RestApiResponse.jsonError(Http400,
ValidatorStatusNotFoundError,
$sres.get())
toString(sres.get())
return RestApiResponse.jsonResponseWOpt(
RestValidator.init(vindex, balance, status, validator),
node.getStateOptimistic(state)
)
return RestApiResponse.jsonError(Http404, StateNotFoundError)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getStateValidatorBalances
2021-03-17 18:46:45 +00:00
router.api(MethodGet,
"/eth/v1/beacon/states/{state_id}/validator_balances") do (
2021-03-17 18:46:45 +00:00
state_id: StateIdent, id: seq[ValidatorIdent]) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
bslot = node.getBlockSlotId(sid).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(Http404, StateNotFoundError,
$error)
2021-03-17 18:46:45 +00:00
let validatorIds =
block:
if id.isErr():
return RestApiResponse.jsonError(Http400,
2021-04-08 10:49:28 +00:00
InvalidValidatorIdValueError)
let ires = id.get()
if len(ires) > ServerMaximumValidatorIds:
return RestApiResponse.jsonError(Http400,
2021-04-08 10:49:28 +00:00
MaximumNumberOfValidatorIdsError)
ires
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
node.withStateForBlockSlotId(bslot):
let validatorsCount = lenu64(getStateField(state, validators))
let indices =
block:
var keyset: HashSet[ValidatorPubKey]
var indexset: HashSet[ValidatorIndex]
for item in validatorIds:
case item.kind
of ValidatorQueryKind.Key:
keyset.incl(item.key)
of ValidatorQueryKind.Index:
let vindex =
block:
let vres = item.index.toValidatorIndex()
if vres.isErr():
case vres.error()
of ValidatorIndexError.TooHighValue:
return RestApiResponse.jsonError(Http400,
2021-04-08 10:49:28 +00:00
TooHighValidatorIndexValueError)
of ValidatorIndexError.UnsupportedValue:
return RestApiResponse.jsonError(Http500,
2021-04-08 10:49:28 +00:00
UnsupportedValidatorIndexValueError)
vres.get()
# We only adding validator indices which are present in
# validators list at this moment.
if uint64(vindex) < validatorsCount:
indexset.incl(vindex)
if len(keyset) > 0:
let optIndices = keysToIndices(node.restKeysCache, state,
keyset.toSeq())
# Remove all the duplicates.
for item in optIndices:
# We ignore missing keys.
if item.isSome():
indexset.incl(item.get())
indexset.toSeq()
let response =
block:
var res: seq[RestValidatorBalance]
if len(indices) == 0:
# Case when `len(indices) == 0 and len(validatorIds) != 0` means
# that we can't find validator identifiers in state, so we should
# return empty response.
if len(validatorIds) == 0:
# There is no indices, so we going to return balances of all
# known validators.
for index, balance in getStateField(state, balances):
res.add(RestValidatorBalance.init(ValidatorIndex(index),
balance))
else:
for index in indices:
let balance = getStateField(state, balances).item(index)
res.add(RestValidatorBalance.init(index, balance))
res
return RestApiResponse.jsonResponseWOpt(
response,
node.getStateOptimistic(state)
)
2021-03-17 18:46:45 +00:00
return RestApiResponse.jsonError(Http404, StateNotFoundError)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getEpochCommittees
2021-03-17 18:46:45 +00:00
router.api(MethodGet,
"/eth/v1/beacon/states/{state_id}/committees") do (
2021-03-17 18:46:45 +00:00
state_id: StateIdent, epoch: Option[Epoch], index: Option[CommitteeIndex],
slot: Option[Slot]) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
bslot = node.getBlockSlotId(sid).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(Http404, StateNotFoundError,
$error)
2021-03-17 18:46:45 +00:00
let vepoch =
if epoch.isSome():
let repoch = epoch.get()
if repoch.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, InvalidEpochValueError,
2021-03-17 18:46:45 +00:00
$repoch.error())
let res = repoch.get()
if res > bslot.slot.epoch + MIN_SEED_LOOKAHEAD:
return RestApiResponse.jsonError(
Http400, InvalidEpochValueError,
"Requested epoch more than 1 epoch past state epoch")
if res + EPOCHS_PER_HISTORICAL_VECTOR <
bslot.slot.epoch + MIN_SEED_LOOKAHEAD:
return RestApiResponse.jsonError(
Http400, InvalidEpochValueError,
"Requested epoch earlier than what committees can be computed for")
some(res)
2021-03-17 18:46:45 +00:00
else:
none[Epoch]()
let vindex =
if index.isSome():
let rindex = index.get()
if rindex.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400,
InvalidCommitteeIndexValueError,
2021-03-17 18:46:45 +00:00
$rindex.error())
some(rindex.get())
else:
none[CommitteeIndex]()
let vslot =
if slot.isSome():
let rslot = slot.get()
if rslot.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, InvalidSlotValueError,
2021-03-17 18:46:45 +00:00
$rslot.error())
let res = rslot.get()
if vepoch.isSome():
if res.epoch != vepoch.get():
return RestApiResponse.jsonError(Http400, InvalidSlotValueError,
"Slot does not match requested epoch")
else:
if res.epoch > bslot.slot.epoch + 1:
return RestApiResponse.jsonError(
Http400, InvalidEpochValueError,
"Requested slot more than 1 epoch past state epoch")
if res.epoch + EPOCHS_PER_HISTORICAL_VECTOR <
bslot.slot.epoch + MIN_SEED_LOOKAHEAD:
return RestApiResponse.jsonError(
Http400, InvalidEpochValueError,
"Requested slot earlier than what committees can be computed for")
some(res)
2021-03-17 18:46:45 +00:00
else:
none[Slot]()
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
node.withStateForBlockSlotId(bslot):
2021-03-17 18:46:45 +00:00
proc getCommittee(slot: Slot,
index: CommitteeIndex): RestBeaconStatesCommittees =
let validators = get_beacon_committee(state, slot, index, cache)
RestBeaconStatesCommittees(index: index, slot: slot,
validators: validators)
2021-03-17 18:46:45 +00:00
proc forSlot(slot: Slot, cindex: Option[CommitteeIndex],
res: var seq[RestBeaconStatesCommittees]) =
let committees_per_slot = get_committee_count_per_slot(
state, slot.epoch, cache)
2021-03-17 18:46:45 +00:00
if cindex.isNone:
for committee_index in get_committee_indices(committees_per_slot):
res.add(getCommittee(slot, committee_index))
2021-03-17 18:46:45 +00:00
else:
let
idx = cindex.get()
if idx < committees_per_slot:
res.add(getCommittee(slot, idx))
2021-03-17 18:46:45 +00:00
var res: seq[RestBeaconStatesCommittees]
2021-03-17 18:46:45 +00:00
let qepoch =
if vepoch.isNone:
epoch(getStateField(state, slot))
2021-03-17 18:46:45 +00:00
else:
vepoch.get()
if vslot.isNone():
for slot in qepoch.slots():
forSlot(slot, vindex, res)
2021-03-17 18:46:45 +00:00
else:
forSlot(vslot.get(), vindex, res)
return RestApiResponse.jsonResponseWOpt(
res,
node.getStateOptimistic(state)
)
2021-03-17 18:46:45 +00:00
return RestApiResponse.jsonError(Http404, StateNotFoundError)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getEpochSyncCommittees
router.api(MethodGet,
"/eth/v1/beacon/states/{state_id}/sync_committees") do (
state_id: StateIdent, epoch: Option[Epoch]) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
bslot = node.getBlockSlotId(sid).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(Http404, StateNotFoundError,
$error)
let qepoch =
if epoch.isSome():
let repoch = epoch.get()
if repoch.isErr():
return RestApiResponse.jsonError(Http400, InvalidEpochValueError,
$repoch.error())
let res = repoch.get()
if res > MaxEpoch:
return RestApiResponse.jsonError(Http400, EpochOverflowValueError)
if res < node.dag.cfg.ALTAIR_FORK_EPOCH:
return RestApiResponse.jsonError(Http400,
EpochFromTheIncorrectForkError)
res
else:
# If ``epoch`` not present then the sync committees for the epoch of
# the state will be obtained.
bslot.slot.epoch()
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
node.withStateForBlockSlotId(bslot):
let keys =
block:
let res = syncCommitteeParticipants(state, qepoch)
if res.isErr():
return RestApiResponse.jsonError(Http400,
$res.error())
let kres = res.get()
if len(kres) == 0:
return RestApiResponse.jsonError(Http500, InternalServerError,
"List of sync committee participants is empty")
kres
let indices =
block:
var res: seq[ValidatorIndex]
let optIndices = keysToIndices(node.restKeysCache, state, keys)
# Remove all the duplicates.
for item in optIndices:
if item.isNone():
# This should not be happened, because keys are from state.
return RestApiResponse.jsonError(Http500, InternalServerError,
"Could not get validator indices")
res.add(item.get())
res
let aggregates =
block:
var
res: seq[seq[ValidatorIndex]]
offset = 0
while true:
let length = min(SYNC_SUBCOMMITTEE_SIZE, len(indices) - offset)
if length == 0:
break
res.add(@(indices.toOpenArray(offset, offset + length - 1)))
offset.inc(length)
res
return RestApiResponse.jsonResponseWOpt(
RestEpochSyncCommittee(validators: indices,
validator_aggregates: aggregates),
node.getStateOptimistic(state)
)
return RestApiResponse.jsonError(Http404, StateNotFoundError)
# https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/getStateRandao
# https://github.com/ethereum/beacon-APIs/blob/b3c4defa238aaa74bf22aa602aa1b24b68a4c78e/apis/beacon/states/randao.yaml
router.api(MethodGet,
"/eth/v1/beacon/states/{state_id}/randao") do (
state_id: StateIdent, epoch: Option[Epoch]) -> RestApiResponse:
let
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
bslot = node.getBlockSlotId(sid).valueOr:
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(Http404, StateNotFoundError,
$error)
let qepoch =
if epoch.isSome():
let repoch = epoch.get()
if repoch.isErr():
return RestApiResponse.jsonError(Http400, InvalidEpochValueError,
$repoch.error())
let res = repoch.get()
if res > MaxEpoch:
return RestApiResponse.jsonError(Http400, EpochOverflowValueError)
if res < node.dag.cfg.ALTAIR_FORK_EPOCH:
return RestApiResponse.jsonError(Http400,
EpochFromTheIncorrectForkError)
if res > bslot.slot.epoch() + 1:
return RestApiResponse.jsonError(Http400,
EpochFromFutureError)
res
else:
# If ``epoch`` not present then the RANDAO mix for the epoch of
# the state will be obtained.
bslot.slot.epoch()
# Try to obtain RANDAO in an accelerated way
let bsi = node.dag.atSlot(bslot.bid, (qepoch + 1).start_slot - 1)
if bsi.isSome:
let mix = node.dag.computeRandaoMix(bsi.get.bid)
if mix.isSome:
return RestApiResponse.jsonResponseWOpt(
RestEpochRandao(randao: mix.get),
node.getBidOptimistic(bsi.get.bid)
)
# Fall back to full state computation
node.withStateForBlockSlotId(bslot):
withState(state):
return RestApiResponse.jsonResponseWOpt(
RestEpochRandao(randao: get_randao_mix(forkyState.data, qepoch)),
node.getStateOptimistic(state)
)
return RestApiResponse.jsonError(Http404, StateNotFoundError)
# https://ethereum.github.io/beacon-APIs/#/Beacon/getBlockHeaders
router.api(MethodGet, "/eth/v1/beacon/headers") do (
2021-03-17 18:46:45 +00:00
slot: Option[Slot], parent_root: Option[Eth2Digest]) -> RestApiResponse:
# TODO (cheatfate): This call is incomplete, because structure
# of database do not allow to query blocks by `parent_root`.
let qslot =
if slot.isSome():
let rslot = slot.get()
if rslot.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, InvalidSlotValueError,
$rslot.error())
rslot.get()
else:
node.dag.head.slot
if parent_root.isSome():
let rroot = parent_root.get()
if rroot.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, InvalidParentRootValueError,
$rroot.error())
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http500, NoImplementationError)
let bdata = node.getForkedBlock(BlockIdent.init(qslot)).valueOr:
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
return
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
withBlck(bdata):
RestApiResponse.jsonResponseWOpt(
[
(
root: forkyBlck.root,
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
canonical: node.dag.isCanonical(
BlockId(root: forkyBlck.root, slot: forkyBlck.message.slot)),
header: (
message: forkyBlck.toBeaconBlockHeader,
signature: forkyBlck.signature
)
)
],
node.getBlockOptimistic(bdata)
)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getBlockHeader
router.api(MethodGet, "/eth/v1/beacon/headers/{block_id}") do (
2021-03-17 18:46:45 +00:00
block_id: BlockIdent) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
bid = block_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidBlockIdValueError,
$error)
bdata = node.getForkedBlock(bid).valueOr:
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
return
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
withBlck(bdata):
RestApiResponse.jsonResponseWOpt(
(
root: forkyBlck.root,
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
canonical: node.dag.isCanonical(
BlockId(root: forkyBlck.root, slot: forkyBlck.message.slot)),
header: (
message: forkyBlck.toBeaconBlockHeader,
signature: forkyBlck.signature
)
),
node.getBlockOptimistic(bdata)
2021-03-17 18:46:45 +00:00
)
# https://ethereum.github.io/beacon-APIs/#/Beacon/publishBlock
router.api(MethodPost, "/eth/v1/beacon/blocks") do (
2021-03-17 18:46:45 +00:00
contentBody: Option[ContentBody]) -> RestApiResponse:
let res =
block:
if contentBody.isNone():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
let
body = contentBody.get()
version = request.headers.getString("eth-consensus-version")
var
restBlock = decodeBody(RestPublishedSignedBlockContents, body,
version).valueOr:
return RestApiResponse.jsonError(Http400, InvalidBlockObjectError,
$error)
forked = ForkedSignedBeaconBlock.init(restBlock)
if restBlock.kind != node.dag.cfg.consensusForkAtEpoch(
getForkedBlockField(forked, slot).epoch):
2023-02-03 15:12:11 +00:00
doAssert strictVerification notin node.dag.updateFlags
return RestApiResponse.jsonError(Http400, InvalidBlockObjectError)
case restBlock.kind
of ConsensusFork.Phase0:
var blck = restBlock.phase0Data
blck.root = hash_tree_root(blck.message)
await node.router.routeSignedBeaconBlock(blck,
Opt.none(SignedBlobSidecars))
of ConsensusFork.Altair:
var blck = restBlock.altairData
blck.root = hash_tree_root(blck.message)
await node.router.routeSignedBeaconBlock(blck,
Opt.none(SignedBlobSidecars))
of ConsensusFork.Bellatrix:
var blck = restBlock.bellatrixData
blck.root = hash_tree_root(blck.message)
await node.router.routeSignedBeaconBlock(blck,
Opt.none(SignedBlobSidecars))
of ConsensusFork.Capella:
var blck = restBlock.capellaData
blck.root = hash_tree_root(blck.message)
await node.router.routeSignedBeaconBlock(blck,
Opt.none(SignedBlobSidecars))
of ConsensusFork.Deneb:
var blck = restBlock.denebData.signed_block
blck.root = hash_tree_root(blck.message)
await node.router.routeSignedBeaconBlock(
blck, Opt.some(asSeq restBlock.denebData.signed_blob_sidecars))
if res.isErr():
return RestApiResponse.jsonError(
Http503, BeaconNodeInSyncError, $res.error())
if res.get().isNone():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http202, BlockValidationError)
return RestApiResponse.jsonMsgResponse(BlockValidationSuccess)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/publishBlockV2
router.api(MethodPost, "/eth/v2/beacon/blocks") do (
contentBody: Option[ContentBody]) -> RestApiResponse:
let res =
block:
if contentBody.isNone():
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
if request.headers.getString("broadcast_validation") != "gossip":
# TODO (henridf): support 'consensus' and 'consensus_and_equivocation'
# broadcast_validation
return RestApiResponse.jsonError(
Http500, "gossip broadcast_validation only supported")
let
body = contentBody.get()
version = request.headers.getString("eth-consensus-version")
var
restBlock = decodeBodyJsonOrSsz(RestPublishedSignedBlockContents,
body, version).valueOr:
return RestApiResponse.jsonError(Http400, InvalidBlockObjectError,
$error)
forked = ForkedSignedBeaconBlock.init(restBlock)
# TODO (henridf): handle broadcast_validation flag
if restBlock.kind != node.dag.cfg.consensusForkAtEpoch(
getForkedBlockField(forked, slot).epoch):
doAssert strictVerification notin node.dag.updateFlags
return RestApiResponse.jsonError(Http400, InvalidBlockObjectError)
case restBlock.kind
of ConsensusFork.Phase0:
var blck = restBlock.phase0Data
blck.root = hash_tree_root(blck.message)
await node.router.routeSignedBeaconBlock(blck,
Opt.none(SignedBlobSidecars))
of ConsensusFork.Altair:
var blck = restBlock.altairData
blck.root = hash_tree_root(blck.message)
await node.router.routeSignedBeaconBlock(blck,
Opt.none(SignedBlobSidecars))
of ConsensusFork.Bellatrix:
var blck = restBlock.bellatrixData
blck.root = hash_tree_root(blck.message)
await node.router.routeSignedBeaconBlock(blck,
Opt.none(SignedBlobSidecars))
of ConsensusFork.Capella:
var blck = restBlock.capellaData
blck.root = hash_tree_root(blck.message)
await node.router.routeSignedBeaconBlock(blck,
Opt.none(SignedBlobSidecars))
of ConsensusFork.Deneb:
var blck = restBlock.denebData.signed_block
blck.root = hash_tree_root(blck.message)
await node.router.routeSignedBeaconBlock(
blck, Opt.some(asSeq restBlock.denebData.signed_blob_sidecars))
if res.isErr():
return RestApiResponse.jsonError(
Http503, BeaconNodeInSyncError, $res.error())
if res.get().isNone():
return RestApiResponse.jsonError(Http202, BlockValidationError)
return RestApiResponse.jsonMsgResponse(BlockValidationSuccess)
# https://ethereum.github.io/beacon-APIs/#/Beacon/publishBlindedBlock
# https://github.com/ethereum/beacon-APIs/blob/v2.4.0/apis/beacon/blocks/blinded_blocks.yaml
router.api(MethodPost, "/eth/v1/beacon/blinded_blocks") do (
contentBody: Option[ContentBody]) -> RestApiResponse:
## Instructs the beacon node to use the components of the
## `SignedBlindedBeaconBlock` to construct and publish a
## `SignedBeaconBlock` by swapping out the transactions_root for the
## corresponding full list of transactions. The beacon node should
## broadcast a newly constructed `SignedBeaconBlock` to the beacon network,
## to be included in the beacon chain. The beacon node is not required to
## validate the signed `BeaconBlock`, and a successful response (20X) only
## indicates that the broadcast has been successful.
if contentBody.isNone():
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
let
currentEpochFork =
node.dag.cfg.consensusForkAtEpoch(node.currentSlot().epoch())
version = request.headers.getString("eth-consensus-version")
body = contentBody.get()
if body.contentType == OctetStreamMediaType and
currentEpochFork.toString != version:
return RestApiResponse.jsonError(Http400, BlockIncorrectFork)
case currentEpochFork
of ConsensusFork.Deneb:
return RestApiResponse.jsonError(Http500, $denebImplementationMissing)
of ConsensusFork.Capella:
let
restBlock = decodeBodyJsonOrSsz(
capella_mev.SignedBlindedBeaconBlock, body).valueOr:
return RestApiResponse.jsonError(Http400, InvalidBlockObjectError,
$error)
payloadBuilderClient = node.getPayloadBuilderClient(
restBlock.message.proposer_index).valueOr:
return RestApiResponse.jsonError(
Http400, "Unable to initialize payload builder client: " & $error)
res = await node.unblindAndRouteBlockMEV(
payloadBuilderClient, restBlock)
if res.isErr():
return RestApiResponse.jsonError(
Http503, BeaconNodeInSyncError, $res.error())
if res.get().isNone():
return RestApiResponse.jsonError(Http202, BlockValidationError)
return RestApiResponse.jsonMsgResponse(BlockValidationSuccess)
of ConsensusFork.Bellatrix:
return RestApiResponse.jsonError(Http400, "FOO")
of ConsensusFork.Altair, ConsensusFork.Phase0:
# Pre-Bellatrix, this endpoint will accept a `SignedBeaconBlock`.
#
# This is mostly the same as /eth/v1/beacon/blocks for phase 0 and
# altair.
var
restBlock = decodeBody(RestPublishedSignedBeaconBlock, body,
version).valueOr:
return RestApiResponse.jsonError(Http400, InvalidBlockObjectError,
$error)
forked = ForkedSignedBeaconBlock(restBlock)
if forked.kind != node.dag.cfg.consensusForkAtEpoch(
getForkedBlockField(forked, slot).epoch):
return RestApiResponse.jsonError(Http400, InvalidBlockObjectError)
let res = withBlck(forked):
forkyBlck.root = hash_tree_root(forkyBlck.message)
await node.router.routeSignedBeaconBlock(forkyBlck,
Opt.none(SignedBlobSidecars))
if res.isErr():
return RestApiResponse.jsonError(
Http503, BeaconNodeInSyncError, $res.error())
elif res.get().isNone():
return RestApiResponse.jsonError(Http202, BlockValidationError)
return RestApiResponse.jsonMsgResponse(BlockValidationSuccess)
# https://ethereum.github.io/beacon-APIs/#/Beacon/getBlock
router.api(MethodGet, "/eth/v1/beacon/blocks/{block_id}") do (
2021-03-17 18:46:45 +00:00
block_id: BlockIdent) -> RestApiResponse:
return RestApiResponse.jsonError(
Http410, DeprecatedRemovalBeaconBlocksDebugStateV1)
# https://ethereum.github.io/beacon-APIs/#/Beacon/getBlockV2
router.api(MethodGet, "/eth/v2/beacon/blocks/{block_id}") do (
block_id: BlockIdent) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
blockIdent = block_id.valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
return RestApiResponse.jsonError(Http400, InvalidBlockIdValueError,
$error)
bid = node.getBlockId(blockIdent).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
let contentType =
block:
let res = preferredContentType(jsonMediaType,
sszMediaType)
if res.isErr():
return RestApiResponse.jsonError(Http406, ContentNotAcceptableError)
res.get()
return
if contentType == sszMediaType:
var data: seq[byte]
if not node.dag.getBlockSSZ(bid, data):
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
let
fork = node.dag.cfg.consensusForkAtEpoch(bid.slot.epoch)
headers = [("eth-consensus-version", fork.toString())]
RestApiResponse.sszResponsePlain(data, headers)
elif contentType == jsonMediaType:
let bdata = node.dag.getForkedBlock(bid).valueOr:
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
RestApiResponse.jsonResponseBlock(
bdata.asSigned(),
node.getBlockOptimistic(bdata)
)
else:
RestApiResponse.jsonError(Http500, InvalidAcceptError)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getBlockRoot
router.api(MethodGet, "/eth/v1/beacon/blocks/{block_id}/root") do (
2021-03-17 18:46:45 +00:00
block_id: BlockIdent) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
blockIdent = block_id.valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
return RestApiResponse.jsonError(Http400, InvalidBlockIdValueError,
$error)
bid = node.getBlockId(blockIdent).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
bdata = node.dag.getForkedBlock(bid).valueOr:
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
return RestApiResponse.jsonResponseWOpt(
(root: bid.root),
node.getBlockOptimistic(bdata)
)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getBlockAttestations
2021-03-17 18:46:45 +00:00
router.api(MethodGet,
"/eth/v1/beacon/blocks/{block_id}/attestations") do (
2021-03-17 18:46:45 +00:00
block_id: BlockIdent) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
bid = block_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidBlockIdValueError,
$error)
bdata = node.getForkedBlock(bid).valueOr:
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
return
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
withBlck(bdata):
RestApiResponse.jsonResponseWOpt(
forkyBlck.message.body.attestations.asSeq(),
node.getBlockOptimistic(bdata)
)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getPoolAttestations
router.api(MethodGet, "/eth/v1/beacon/pool/attestations") do (
2021-03-17 18:46:45 +00:00
slot: Option[Slot],
committee_index: Option[CommitteeIndex]) -> RestApiResponse:
let vindex =
if committee_index.isSome():
let rindex = committee_index.get()
if rindex.isErr():
return RestApiResponse.jsonError(Http400,
2021-04-08 10:49:28 +00:00
InvalidCommitteeIndexValueError,
2021-03-17 18:46:45 +00:00
$rindex.error())
Opt.some(rindex.get())
2021-03-17 18:46:45 +00:00
else:
Opt.none(CommitteeIndex)
2021-03-17 18:46:45 +00:00
let vslot =
if slot.isSome():
let rslot = slot.get()
if rslot.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, InvalidSlotValueError,
2021-03-17 18:46:45 +00:00
$rslot.error())
Opt.some(rslot.get())
2021-03-17 18:46:45 +00:00
else:
Opt.none(Slot)
2021-03-17 18:46:45 +00:00
var res: seq[Attestation]
for item in node.attestationPool[].attestations(vslot, vindex):
res.add(item)
return RestApiResponse.jsonResponse(res)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/submitPoolAttestations
router.api(MethodPost, "/eth/v1/beacon/pool/attestations") do (
2021-03-17 18:46:45 +00:00
contentBody: Option[ContentBody]) -> RestApiResponse:
let attestations =
block:
if contentBody.isNone():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
let dres = decodeBody(seq[Attestation], contentBody.get())
if dres.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400,
InvalidAttestationObjectError,
$dres.error())
dres.get()
# Since our validation logic supports batch processing, we will submit all
# attestations for validation.
let pending =
block:
var res: seq[Future[SendResult]]
for attestation in attestations:
res.add(node.router.routeAttestation(attestation))
res
let failures =
block:
var res: seq[RestIndexedErrorMessageItem]
await allFutures(pending)
for index, future in pending:
if future.completed():
let fres = future.read()
if fres.isErr():
let failure = RestIndexedErrorMessageItem(index: index,
message: $fres.error())
res.add(failure)
elif future.failed() or future.cancelled():
# This is unexpected failure, so we log the error message.
let exc = future.readError()
let failure = RestIndexedErrorMessageItem(index: index,
message: $exc.msg)
res.add(failure)
res
if len(failures) > 0:
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonErrorList(Http400, AttestationValidationError,
failures)
else:
New validator client using REST API. (#2651) * Initial commit. * Exporting getConfig(). * Add beacon node checking procedures. * Post rebase fixes. * Use runSlotLoop() from nimbus_beacon_node. Fallback implementation. Fixes for ETH2 REST serialization. * Add beacon_clock.durationToNextSlot(). Move type declarations from beacon_rest_api to json_rest_serialization. Fix seq[ValidatorIndex] serialization. Refactor ValidatorPool and add some utility procedures. Create separate version of validator_client. * Post-rebase fixes. Remove CookedPubKey from validator_pool.nim. * Now we should be able to produce attestations and aggregate and proofs. But its not working yet. * Debugging attestation sending. * Add durationToNextAttestation. Optimize some debug logs. Fix aggregation_bits encoding. Bump chronos/presto. * Its alive. * Fixes for launch_local_testnet script. Bump chronos. * Switch client API to not use `/api` prefix. * Post-rebase adjustments. * Fix endpoint for publishBlock(). * Add CONFIG_NAME. Add more checks to ensure that beacon_node is compatible. * Add beacon committee subscription support to validator_client. * Fix stacktrace should be an array of strings. Fix committee subscriptions should not be `data` keyed. * Log duration to next block proposal. * Fix beacon_node_status import. * Use jsonMsgResponse() instead of jsonError(). * Fix graffityBytes usage. Remove unnecessary `await`. Adjust creation of SignedBlock instance. Remove legacy files. * Rework durationToNextSlot() and durationToNextEpoch() to use `fromNow`. * Fix race condition for block proposal and attestations for same slot. Fix local_testnet script to properly kill tasks on Windows. Bump chronos and nim-http-tools, to allow connections to infura.io (basic auth). * Catch services errors. Improve performance of local_testnet.sh script on Windows. Fix race condition when attestation producing. * Post-rebase fixes. * Bump chronos and presto. * Calculate block publishing delay. Fix pkill in one more place. * Add error handling and timeouts to firstSuccess() template. Add onceToAll() template. Add checkNodes() procedure. Refactor firstSuccess() template. Add error checking to api.nim calls. * Deprecated usage onceToAll() for better stability. Address comment and send attestations asap. * Avoid unnecessary loop when calculating minimal duration.
2021-07-13 11:15:07 +00:00
return RestApiResponse.jsonMsgResponse(AttestationValidationSuccess)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getPoolAttesterSlashings
router.api(MethodGet, "/eth/v1/beacon/pool/attester_slashings") do (
2021-03-17 18:46:45 +00:00
) -> RestApiResponse:
return RestApiResponse.jsonResponse(
toSeq(node.validatorChangePool.attester_slashings))
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/submitPoolAttesterSlashings
router.api(MethodPost, "/eth/v1/beacon/pool/attester_slashings") do (
2021-03-17 18:46:45 +00:00
contentBody: Option[ContentBody]) -> RestApiResponse:
let slashing =
block:
if contentBody.isNone():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
let dres = decodeBody(AttesterSlashing, contentBody.get())
if dres.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400,
InvalidAttesterSlashingObjectError,
$dres.error())
dres.get()
let res = await node.router.routeAttesterSlashing(slashing)
if res.isErr():
return RestApiResponse.jsonError(Http400,
AttesterSlashingValidationError,
$res.error())
New validator client using REST API. (#2651) * Initial commit. * Exporting getConfig(). * Add beacon node checking procedures. * Post rebase fixes. * Use runSlotLoop() from nimbus_beacon_node. Fallback implementation. Fixes for ETH2 REST serialization. * Add beacon_clock.durationToNextSlot(). Move type declarations from beacon_rest_api to json_rest_serialization. Fix seq[ValidatorIndex] serialization. Refactor ValidatorPool and add some utility procedures. Create separate version of validator_client. * Post-rebase fixes. Remove CookedPubKey from validator_pool.nim. * Now we should be able to produce attestations and aggregate and proofs. But its not working yet. * Debugging attestation sending. * Add durationToNextAttestation. Optimize some debug logs. Fix aggregation_bits encoding. Bump chronos/presto. * Its alive. * Fixes for launch_local_testnet script. Bump chronos. * Switch client API to not use `/api` prefix. * Post-rebase adjustments. * Fix endpoint for publishBlock(). * Add CONFIG_NAME. Add more checks to ensure that beacon_node is compatible. * Add beacon committee subscription support to validator_client. * Fix stacktrace should be an array of strings. Fix committee subscriptions should not be `data` keyed. * Log duration to next block proposal. * Fix beacon_node_status import. * Use jsonMsgResponse() instead of jsonError(). * Fix graffityBytes usage. Remove unnecessary `await`. Adjust creation of SignedBlock instance. Remove legacy files. * Rework durationToNextSlot() and durationToNextEpoch() to use `fromNow`. * Fix race condition for block proposal and attestations for same slot. Fix local_testnet script to properly kill tasks on Windows. Bump chronos and nim-http-tools, to allow connections to infura.io (basic auth). * Catch services errors. Improve performance of local_testnet.sh script on Windows. Fix race condition when attestation producing. * Post-rebase fixes. * Bump chronos and presto. * Calculate block publishing delay. Fix pkill in one more place. * Add error handling and timeouts to firstSuccess() template. Add onceToAll() template. Add checkNodes() procedure. Refactor firstSuccess() template. Add error checking to api.nim calls. * Deprecated usage onceToAll() for better stability. Address comment and send attestations asap. * Avoid unnecessary loop when calculating minimal duration.
2021-07-13 11:15:07 +00:00
return RestApiResponse.jsonMsgResponse(AttesterSlashingValidationSuccess)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getPoolProposerSlashings
router.api(MethodGet, "/eth/v1/beacon/pool/proposer_slashings") do (
2021-03-17 18:46:45 +00:00
) -> RestApiResponse:
return RestApiResponse.jsonResponse(
toSeq(node.validatorChangePool.proposer_slashings))
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/submitPoolProposerSlashings
router.api(MethodPost, "/eth/v1/beacon/pool/proposer_slashings") do (
2021-03-17 18:46:45 +00:00
contentBody: Option[ContentBody]) -> RestApiResponse:
let slashing =
block:
if contentBody.isNone():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
let dres = decodeBody(ProposerSlashing, contentBody.get())
if dres.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400,
InvalidProposerSlashingObjectError,
$dres.error())
dres.get()
let res = await node.router.routeProposerSlashing(slashing)
if res.isErr():
return RestApiResponse.jsonError(Http400,
ProposerSlashingValidationError,
$res.error())
New validator client using REST API. (#2651) * Initial commit. * Exporting getConfig(). * Add beacon node checking procedures. * Post rebase fixes. * Use runSlotLoop() from nimbus_beacon_node. Fallback implementation. Fixes for ETH2 REST serialization. * Add beacon_clock.durationToNextSlot(). Move type declarations from beacon_rest_api to json_rest_serialization. Fix seq[ValidatorIndex] serialization. Refactor ValidatorPool and add some utility procedures. Create separate version of validator_client. * Post-rebase fixes. Remove CookedPubKey from validator_pool.nim. * Now we should be able to produce attestations and aggregate and proofs. But its not working yet. * Debugging attestation sending. * Add durationToNextAttestation. Optimize some debug logs. Fix aggregation_bits encoding. Bump chronos/presto. * Its alive. * Fixes for launch_local_testnet script. Bump chronos. * Switch client API to not use `/api` prefix. * Post-rebase adjustments. * Fix endpoint for publishBlock(). * Add CONFIG_NAME. Add more checks to ensure that beacon_node is compatible. * Add beacon committee subscription support to validator_client. * Fix stacktrace should be an array of strings. Fix committee subscriptions should not be `data` keyed. * Log duration to next block proposal. * Fix beacon_node_status import. * Use jsonMsgResponse() instead of jsonError(). * Fix graffityBytes usage. Remove unnecessary `await`. Adjust creation of SignedBlock instance. Remove legacy files. * Rework durationToNextSlot() and durationToNextEpoch() to use `fromNow`. * Fix race condition for block proposal and attestations for same slot. Fix local_testnet script to properly kill tasks on Windows. Bump chronos and nim-http-tools, to allow connections to infura.io (basic auth). * Catch services errors. Improve performance of local_testnet.sh script on Windows. Fix race condition when attestation producing. * Post-rebase fixes. * Bump chronos and presto. * Calculate block publishing delay. Fix pkill in one more place. * Add error handling and timeouts to firstSuccess() template. Add onceToAll() template. Add checkNodes() procedure. Refactor firstSuccess() template. Add error checking to api.nim calls. * Deprecated usage onceToAll() for better stability. Address comment and send attestations asap. * Avoid unnecessary loop when calculating minimal duration.
2021-07-13 11:15:07 +00:00
return RestApiResponse.jsonMsgResponse(ProposerSlashingValidationSuccess)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/getPoolBLSToExecutionChanges
# https://github.com/ethereum/beacon-APIs/blob/86850001845df9163da5ae9605dbf15cd318d5d0/apis/beacon/pool/bls_to_execution_changes.yaml
router.api(MethodGet, "/eth/v1/beacon/pool/bls_to_execution_changes") do (
) -> RestApiResponse:
return RestApiResponse.jsonResponse(
toSeq(node.validatorChangePool.bls_to_execution_changes_gossip) &
toSeq(node.validatorChangePool.bls_to_execution_changes_api))
# https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/submitPoolBLSToExecutionChange
# https://github.com/ethereum/beacon-APIs/blob/86850001845df9163da5ae9605dbf15cd318d5d0/apis/beacon/pool/bls_to_execution_changes.yaml
router.api(MethodPost, "/eth/v1/beacon/pool/bls_to_execution_changes") do (
contentBody: Option[ContentBody]) -> RestApiResponse:
if node.currentSlot().epoch() < node.dag.cfg.CAPELLA_FORK_EPOCH:
return RestApiResponse.jsonError(Http400,
InvalidBlsToExecutionChangeObjectError,
"Attempt to add to BLS to execution change pool pre-Capella")
let bls_to_execution_changes =
block:
if contentBody.isNone():
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
let dres = decodeBody(seq[SignedBLSToExecutionChange], contentBody.get())
if dres.isErr():
return RestApiResponse.jsonError(Http400,
InvalidBlsToExecutionChangeObjectError,
$dres.error())
dres.get()
let res = await allFinished(mapIt(
bls_to_execution_changes, node.router.routeBlsToExecutionChange(it)))
for individual_res in res:
doAssert individual_res.finished()
if individual_res.failed():
return RestApiResponse.jsonError(Http400,
BlsToExecutionChangeValidationError,
$individual_res.error[].msg)
let fut_result = individual_res.read()
if fut_result.isErr():
return RestApiResponse.jsonError(Http400,
BlsToExecutionChangeValidationError,
$fut_result.error())
return RestApiResponse.jsonMsgResponse(BlsToExecutionChangeValidationSuccess)
# https://ethereum.github.io/beacon-APIs/#/Beacon/submitPoolSyncCommitteeSignatures
router.api(MethodPost, "/eth/v1/beacon/pool/sync_committees") do (
contentBody: Option[ContentBody]) -> RestApiResponse:
let messages =
block:
if contentBody.isNone():
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
let dres = decodeBody(seq[SyncCommitteeMessage], contentBody.get())
if dres.isErr():
return RestApiResponse.jsonError(Http400,
InvalidSyncCommitteeSignatureMessageError)
dres.get()
let results = await node.router.routeSyncCommitteeMessages(messages)
let failures =
block:
var res: seq[RestIndexedErrorMessageItem]
for index, item in results:
if item.isErr():
res.add(RestIndexedErrorMessageItem(index: index,
message: $item.error()))
res
if len(failures) > 0:
return RestApiResponse.jsonErrorList(Http400,
SyncCommitteeMessageValidationError,
failures)
else:
return RestApiResponse.jsonMsgResponse(
SyncCommitteeMessageValidationSuccess)
# https://ethereum.github.io/beacon-APIs/#/Beacon/getPoolVoluntaryExits
router.api(MethodGet, "/eth/v1/beacon/pool/voluntary_exits") do (
2021-03-17 18:46:45 +00:00
) -> RestApiResponse:
return RestApiResponse.jsonResponse(
toSeq(node.validatorChangePool.voluntary_exits))
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/submitPoolVoluntaryExit
router.api(MethodPost, "/eth/v1/beacon/pool/voluntary_exits") do (
2021-03-17 18:46:45 +00:00
contentBody: Option[ContentBody]) -> RestApiResponse:
let exit =
block:
if contentBody.isNone():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
let dres = decodeBody(SignedVoluntaryExit, contentBody.get())
if dres.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400,
InvalidVoluntaryExitObjectError,
$dres.error())
dres.get()
let res = await node.router.routeSignedVoluntaryExit(exit)
if res.isErr():
return RestApiResponse.jsonError(Http400,
VoluntaryExitValidationError,
$res.error())
New validator client using REST API. (#2651) * Initial commit. * Exporting getConfig(). * Add beacon node checking procedures. * Post rebase fixes. * Use runSlotLoop() from nimbus_beacon_node. Fallback implementation. Fixes for ETH2 REST serialization. * Add beacon_clock.durationToNextSlot(). Move type declarations from beacon_rest_api to json_rest_serialization. Fix seq[ValidatorIndex] serialization. Refactor ValidatorPool and add some utility procedures. Create separate version of validator_client. * Post-rebase fixes. Remove CookedPubKey from validator_pool.nim. * Now we should be able to produce attestations and aggregate and proofs. But its not working yet. * Debugging attestation sending. * Add durationToNextAttestation. Optimize some debug logs. Fix aggregation_bits encoding. Bump chronos/presto. * Its alive. * Fixes for launch_local_testnet script. Bump chronos. * Switch client API to not use `/api` prefix. * Post-rebase adjustments. * Fix endpoint for publishBlock(). * Add CONFIG_NAME. Add more checks to ensure that beacon_node is compatible. * Add beacon committee subscription support to validator_client. * Fix stacktrace should be an array of strings. Fix committee subscriptions should not be `data` keyed. * Log duration to next block proposal. * Fix beacon_node_status import. * Use jsonMsgResponse() instead of jsonError(). * Fix graffityBytes usage. Remove unnecessary `await`. Adjust creation of SignedBlock instance. Remove legacy files. * Rework durationToNextSlot() and durationToNextEpoch() to use `fromNow`. * Fix race condition for block proposal and attestations for same slot. Fix local_testnet script to properly kill tasks on Windows. Bump chronos and nim-http-tools, to allow connections to infura.io (basic auth). * Catch services errors. Improve performance of local_testnet.sh script on Windows. Fix race condition when attestation producing. * Post-rebase fixes. * Bump chronos and presto. * Calculate block publishing delay. Fix pkill in one more place. * Add error handling and timeouts to firstSuccess() template. Add onceToAll() template. Add checkNodes() procedure. Refactor firstSuccess() template. Add error checking to api.nim calls. * Deprecated usage onceToAll() for better stability. Address comment and send attestations asap. * Avoid unnecessary loop when calculating minimal duration.
2021-07-13 11:15:07 +00:00
return RestApiResponse.jsonMsgResponse(VoluntaryExitValidationSuccess)