nimbus-eth2/beacon_chain/rpc/rest_beacon_api.nim

1229 lines
47 KiB
Nim
Raw Normal View History

# beacon_chain
# Copyright (c) 2018-2022 Status Research & Development GmbH
2021-03-17 18:46:45 +00:00
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
2021-03-17 18:46:45 +00:00
import
2022-06-09 08:50:36 +00:00
std/[typetraits, sequtils, sets],
2021-03-17 18:46:45 +00:00
stew/[results, base10],
chronicles,
./rest_utils,
../beacon_node, ../networking/eth2_network,
../consensus_object_pools/[blockchain_dag, exit_pool, spec_cache],
../validators/validator_duties,
../spec/[eth2_merkleization, forks, network, validator],
../spec/datatypes/[phase0, altair],
./state_ttl_cache
2021-03-17 18:46:45 +00:00
export rest_utils
2021-03-17 18:46:45 +00:00
logScope: topics = "rest_beaconapi"
proc validateFilter(filters: seq[ValidatorFilter]): Result[ValidatorFilter,
cstring] =
var res: ValidatorFilter
for item in filters:
if res * item != {}:
return err("Validator status must be unique")
res.incl(item)
if res == {}:
res = {ValidatorFilterKind.PendingInitialized,
ValidatorFilterKind.PendingQueued,
ValidatorFilterKind.ActiveOngoing,
ValidatorFilterKind.ActiveExiting,
ValidatorFilterKind.ActiveSlashed,
ValidatorFilterKind.ExitedUnslashed,
ValidatorFilterKind.ExitedSlashed,
ValidatorFilterKind.WithdrawalPossible,
ValidatorFilterKind.WithdrawalDone}
ok(res)
proc getStatus(validator: Validator,
current_epoch: Epoch): Result[ValidatorFilterKind, cstring] =
if validator.activation_epoch > current_epoch:
# pending
if validator.activation_eligibility_epoch == FAR_FUTURE_EPOCH:
ok(ValidatorFilterKind.PendingInitialized)
else:
# validator.activation_eligibility_epoch < FAR_FUTURE_EPOCH:
ok(ValidatorFilterKind.PendingQueued)
elif (validator.activation_epoch <= current_epoch) and
(current_epoch < validator.exit_epoch):
# active
if validator.exit_epoch == FAR_FUTURE_EPOCH:
ok(ValidatorFilterKind.ActiveOngoing)
elif not validator.slashed:
# validator.exit_epoch < FAR_FUTURE_EPOCH
ok(ValidatorFilterKind.ActiveExiting)
else:
# validator.exit_epoch < FAR_FUTURE_EPOCH and validator.slashed:
ok(ValidatorFilterKind.ActiveSlashed)
elif (validator.exit_epoch <= current_epoch) and
(current_epoch < validator.withdrawable_epoch):
# exited
if not validator.slashed:
ok(ValidatorFilterKind.ExitedUnslashed)
else:
# validator.slashed
ok(ValidatorFilterKind.ExitedSlashed)
elif validator.withdrawable_epoch <= current_epoch:
# withdrawal
if validator.effective_balance != 0:
ok(ValidatorFilterKind.WithdrawalPossible)
else:
# validator.effective_balance == 0
ok(ValidatorFilterKind.WithdrawalDone)
else:
err("Invalid validator status")
proc toString*(kind: ValidatorFilterKind): string =
case kind
of ValidatorFilterKind.PendingInitialized:
"pending_initialized"
of ValidatorFilterKind.PendingQueued:
"pending_queued"
of ValidatorFilterKind.ActiveOngoing:
"active_ongoing"
of ValidatorFilterKind.ActiveExiting:
"active_exiting"
of ValidatorFilterKind.ActiveSlashed:
"active_slashed"
of ValidatorFilterKind.ExitedUnslashed:
"exited_unslashed"
of ValidatorFilterKind.ExitedSlashed:
"exited_slashed"
of ValidatorFilterKind.WithdrawalPossible:
"withdrawal_possible"
of ValidatorFilterKind.WithdrawalDone:
"withdrawal_done"
proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
# https://ethereum.github.io/beacon-APIs/#/Beacon/getGenesis
router.api(MethodGet, "/eth/v1/beacon/genesis") do () -> RestApiResponse:
2021-03-17 18:46:45 +00:00
return RestApiResponse.jsonResponse(
(
genesis_time: getStateField(node.dag.headState, genesis_time),
2021-03-17 18:46:45 +00:00
genesis_validators_root:
getStateField(node.dag.headState, genesis_validators_root),
genesis_fork_version: node.dag.cfg.GENESIS_FORK_VERSION
2021-03-17 18:46:45 +00:00
)
)
# https://ethereum.github.io/beacon-APIs/#/Beacon/getStateRoot
router.api(MethodGet, "/eth/v1/beacon/states/{state_id}/root") do (
2021-03-17 18:46:45 +00:00
state_id: StateIdent) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
bslot = node.getBlockSlotId(sid).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(Http404, StateNotFoundError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
node.withStateForBlockSlotId(bslot):
return RestApiResponse.jsonResponseWOpt(
(root: stateRoot),
node.getStateOptimistic(state)
)
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
return RestApiResponse.jsonError(Http404, StateNotFoundError)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getStateFork
router.api(MethodGet, "/eth/v1/beacon/states/{state_id}/fork") do (
2021-03-17 18:46:45 +00:00
state_id: StateIdent) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
bslot = node.getBlockSlotId(sid).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(Http404, StateNotFoundError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
node.withStateForBlockSlotId(bslot):
return RestApiResponse.jsonResponseWOpt(
(
previous_version:
getStateField(state, fork).previous_version,
current_version:
getStateField(state, fork).current_version,
epoch:
getStateField(state, fork).epoch
),
node.getStateOptimistic(state)
2021-03-17 18:46:45 +00:00
)
return RestApiResponse.jsonError(Http404, StateNotFoundError)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getStateFinalityCheckpoints
2021-03-17 18:46:45 +00:00
router.api(MethodGet,
"/eth/v1/beacon/states/{state_id}/finality_checkpoints") do (
2021-03-17 18:46:45 +00:00
state_id: StateIdent) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
bslot = node.getBlockSlotId(sid).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(Http404, StateNotFoundError,
$error)
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
node.withStateForBlockSlotId(bslot):
return RestApiResponse.jsonResponseWOpt(
(
previous_justified:
getStateField(state, previous_justified_checkpoint),
current_justified:
getStateField(state, current_justified_checkpoint),
finalized:
getStateField(state, finalized_checkpoint)
),
node.getStateOptimistic(state)
2021-03-17 18:46:45 +00:00
)
return RestApiResponse.jsonError(Http404, StateNotFoundError)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getStateValidators
router.api(MethodGet, "/eth/v1/beacon/states/{state_id}/validators") do (
2021-03-17 18:46:45 +00:00
state_id: StateIdent, id: seq[ValidatorIdent],
status: seq[ValidatorFilter]) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
bslot = node.getBlockSlotId(sid).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(Http404, StateNotFoundError,
$error)
2021-03-17 18:46:45 +00:00
let validatorIds =
block:
if id.isErr():
return RestApiResponse.jsonError(Http400,
2021-04-08 10:49:28 +00:00
InvalidValidatorIdValueError)
let ires = id.get()
if len(ires) > MaximumValidatorIds:
return RestApiResponse.jsonError(Http400,
2021-04-08 10:49:28 +00:00
MaximumNumberOfValidatorIdsError)
ires
2021-03-17 18:46:45 +00:00
let validatorsMask =
block:
if status.isErr():
return RestApiResponse.jsonError(Http400,
2021-04-08 10:49:28 +00:00
InvalidValidatorStatusValueError)
2021-03-17 18:46:45 +00:00
let res = validateFilter(status.get())
if res.isErr():
return RestApiResponse.jsonError(Http400,
2021-04-08 10:49:28 +00:00
InvalidValidatorStatusValueError,
2021-03-17 18:46:45 +00:00
$res.error())
res.get()
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
node.withStateForBlockSlotId(bslot):
let
current_epoch = getStateField(state, slot).epoch()
validatorsCount = lenu64(getStateField(state, validators))
let indices =
block:
var keyset: HashSet[ValidatorPubKey]
var indexset: HashSet[ValidatorIndex]
for item in validatorIds:
case item.kind
of ValidatorQueryKind.Key:
keyset.incl(item.key)
of ValidatorQueryKind.Index:
let vindex =
block:
let vres = item.index.toValidatorIndex()
if vres.isErr():
case vres.error()
of ValidatorIndexError.TooHighValue:
return RestApiResponse.jsonError(Http400,
2021-04-08 10:49:28 +00:00
TooHighValidatorIndexValueError)
of ValidatorIndexError.UnsupportedValue:
return RestApiResponse.jsonError(Http500,
2021-04-08 10:49:28 +00:00
UnsupportedValidatorIndexValueError)
let index = vres.get()
index
if uint64(vindex) < validatorsCount:
# We only adding validator indices which are present in
# validators list at this moment.
indexset.incl(vindex)
if len(keyset) > 0:
let optIndices = keysToIndices(node.restKeysCache, state,
keyset.toSeq())
# Remove all the duplicates.
for item in optIndices:
# We ignore missing keys.
if item.isSome():
indexset.incl(item.get())
indexset.toSeq()
2021-03-17 18:46:45 +00:00
let response =
block:
var res: seq[RestValidator]
if len(indices) == 0:
# Case when `len(indices) == 0 and len(validatorIds) != 0` means
# that we can't find validator identifiers in state, so we should
# return empty response.
if len(validatorIds) == 0:
# There is no indices, so we going to filter all the validators.
for index, validator in getStateField(state, validators):
let
balance = getStateField(state, balances).item(index)
status =
block:
let sres = validator.getStatus(current_epoch)
if sres.isErr():
return RestApiResponse.jsonError(Http400,
ValidatorStatusNotFoundError,
$sres.get())
sres.get()
if status in validatorsMask:
res.add(RestValidator.init(ValidatorIndex(index), balance,
toString(status), validator))
else:
for index in indices:
let
validator = getStateField(state, validators).item(index)
balance = getStateField(state, balances).item(index)
status =
block:
let sres = validator.getStatus(current_epoch)
if sres.isErr():
return RestApiResponse.jsonError(Http400,
ValidatorStatusNotFoundError,
$sres.get())
sres.get()
if status in validatorsMask:
res.add(RestValidator.init(index, balance, toString(status),
validator))
res
return RestApiResponse.jsonResponseWOpt(
response,
node.getStateOptimistic(state)
)
return RestApiResponse.jsonError(Http404, StateNotFoundError)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getStateValidator
2021-03-17 18:46:45 +00:00
router.api(MethodGet,
"/eth/v1/beacon/states/{state_id}/validators/{validator_id}") do (
2021-03-17 18:46:45 +00:00
state_id: StateIdent, validator_id: ValidatorIdent) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
vid = validator_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidValidatorIdValueError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
bslot = node.getBlockSlotId(sid).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(Http404, StateNotFoundError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
node.withStateForBlockSlotId(bslot):
let
current_epoch = getStateField(state, slot).epoch()
validatorsCount = lenu64(getStateField(state, validators))
let vindex =
block:
let vid = validator_id.get()
case vid.kind
of ValidatorQueryKind.Key:
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
let optIndices = keysToIndices(node.restKeysCache, state, [vid.key])
if optIndices[0].isNone():
return RestApiResponse.jsonError(Http404, ValidatorNotFoundError)
optIndices[0].get()
of ValidatorQueryKind.Index:
let vres = vid.index.toValidatorIndex()
if vres.isErr():
case vres.error()
of ValidatorIndexError.TooHighValue:
return RestApiResponse.jsonError(Http400,
2021-04-08 10:49:28 +00:00
TooHighValidatorIndexValueError)
of ValidatorIndexError.UnsupportedValue:
return RestApiResponse.jsonError(Http500,
2021-04-08 10:49:28 +00:00
UnsupportedValidatorIndexValueError)
let index = vres.get()
if uint64(index) >= validatorsCount:
return RestApiResponse.jsonError(Http404, ValidatorNotFoundError)
index
let
validator = getStateField(state, validators).item(vindex)
balance = getStateField(state, balances).item(vindex)
status =
block:
let sres = validator.getStatus(current_epoch)
if sres.isErr():
return RestApiResponse.jsonError(Http400,
ValidatorStatusNotFoundError,
$sres.get())
toString(sres.get())
return RestApiResponse.jsonResponseWOpt(
RestValidator.init(vindex, balance, status, validator),
node.getStateOptimistic(state)
)
return RestApiResponse.jsonError(Http404, StateNotFoundError)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getStateValidatorBalances
2021-03-17 18:46:45 +00:00
router.api(MethodGet,
"/eth/v1/beacon/states/{state_id}/validator_balances") do (
2021-03-17 18:46:45 +00:00
state_id: StateIdent, id: seq[ValidatorIdent]) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
bslot = node.getBlockSlotId(sid).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(Http404, StateNotFoundError,
$error)
2021-03-17 18:46:45 +00:00
let validatorIds =
block:
if id.isErr():
return RestApiResponse.jsonError(Http400,
2021-04-08 10:49:28 +00:00
InvalidValidatorIdValueError)
let ires = id.get()
if len(ires) > MaximumValidatorIds:
return RestApiResponse.jsonError(Http400,
2021-04-08 10:49:28 +00:00
MaximumNumberOfValidatorIdsError)
ires
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
node.withStateForBlockSlotId(bslot):
let validatorsCount = lenu64(getStateField(state, validators))
let indices =
block:
var keyset: HashSet[ValidatorPubKey]
var indexset: HashSet[ValidatorIndex]
for item in validatorIds:
case item.kind
of ValidatorQueryKind.Key:
keyset.incl(item.key)
of ValidatorQueryKind.Index:
let vindex =
block:
let vres = item.index.toValidatorIndex()
if vres.isErr():
case vres.error()
of ValidatorIndexError.TooHighValue:
return RestApiResponse.jsonError(Http400,
2021-04-08 10:49:28 +00:00
TooHighValidatorIndexValueError)
of ValidatorIndexError.UnsupportedValue:
return RestApiResponse.jsonError(Http500,
2021-04-08 10:49:28 +00:00
UnsupportedValidatorIndexValueError)
vres.get()
# We only adding validator indices which are present in
# validators list at this moment.
if uint64(vindex) < validatorsCount:
indexset.incl(vindex)
if len(keyset) > 0:
let optIndices = keysToIndices(node.restKeysCache, state,
keyset.toSeq())
# Remove all the duplicates.
for item in optIndices:
# We ignore missing keys.
if item.isSome():
indexset.incl(item.get())
indexset.toSeq()
let response =
block:
var res: seq[RestValidatorBalance]
if len(indices) == 0:
# Case when `len(indices) == 0 and len(validatorIds) != 0` means
# that we can't find validator identifiers in state, so we should
# return empty response.
if len(validatorIds) == 0:
# There is no indices, so we going to return balances of all
# known validators.
for index, balance in getStateField(state, balances):
res.add(RestValidatorBalance.init(ValidatorIndex(index),
balance))
else:
for index in indices:
let balance = getStateField(state, balances).item(index)
res.add(RestValidatorBalance.init(index, balance))
res
return RestApiResponse.jsonResponseWOpt(
response,
node.getStateOptimistic(state)
)
2021-03-17 18:46:45 +00:00
return RestApiResponse.jsonError(Http404, StateNotFoundError)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getEpochCommittees
2021-03-17 18:46:45 +00:00
router.api(MethodGet,
"/eth/v1/beacon/states/{state_id}/committees") do (
2021-03-17 18:46:45 +00:00
state_id: StateIdent, epoch: Option[Epoch], index: Option[CommitteeIndex],
slot: Option[Slot]) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
bslot = node.getBlockSlotId(sid).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(Http404, StateNotFoundError,
$error)
2021-03-17 18:46:45 +00:00
let vepoch =
if epoch.isSome():
let repoch = epoch.get()
if repoch.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, InvalidEpochValueError,
2021-03-17 18:46:45 +00:00
$repoch.error())
let res = repoch.get()
if res > bslot.slot.epoch + MIN_SEED_LOOKAHEAD:
return RestApiResponse.jsonError(
Http400, InvalidEpochValueError,
"Requested epoch more than 1 epoch past state epoch")
if res + EPOCHS_PER_HISTORICAL_VECTOR <
bslot.slot.epoch + MIN_SEED_LOOKAHEAD:
return RestApiResponse.jsonError(
Http400, InvalidEpochValueError,
"Requested epoch earlier than what committees can be computed for")
some(res)
2021-03-17 18:46:45 +00:00
else:
none[Epoch]()
let vindex =
if index.isSome():
let rindex = index.get()
if rindex.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400,
InvalidCommitteeIndexValueError,
2021-03-17 18:46:45 +00:00
$rindex.error())
some(rindex.get())
else:
none[CommitteeIndex]()
let vslot =
if slot.isSome():
let rslot = slot.get()
if rslot.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, InvalidSlotValueError,
2021-03-17 18:46:45 +00:00
$rslot.error())
let res = rslot.get()
if vepoch.isSome():
if res.epoch != vepoch.get():
return RestApiResponse.jsonError(Http400, InvalidSlotValueError,
"Slot does not match requested epoch")
else:
if res.epoch > bslot.slot.epoch + 1:
return RestApiResponse.jsonError(
Http400, InvalidEpochValueError,
"Requested slot more than 1 epoch past state epoch")
if res.epoch + EPOCHS_PER_HISTORICAL_VECTOR <
bslot.slot.epoch + MIN_SEED_LOOKAHEAD:
return RestApiResponse.jsonError(
Http400, InvalidEpochValueError,
"Requested slot earlier than what committees can be computed for")
some(res)
2021-03-17 18:46:45 +00:00
else:
none[Slot]()
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
node.withStateForBlockSlotId(bslot):
2021-03-17 18:46:45 +00:00
proc getCommittee(slot: Slot,
index: CommitteeIndex): RestBeaconStatesCommittees =
let validators = get_beacon_committee(state, slot, index, cache)
RestBeaconStatesCommittees(index: index, slot: slot,
validators: validators)
2021-03-17 18:46:45 +00:00
proc forSlot(slot: Slot, cindex: Option[CommitteeIndex],
res: var seq[RestBeaconStatesCommittees]) =
let committees_per_slot = get_committee_count_per_slot(
state, slot.epoch, cache)
2021-03-17 18:46:45 +00:00
if cindex.isNone:
for committee_index in get_committee_indices(committees_per_slot):
res.add(getCommittee(slot, committee_index))
2021-03-17 18:46:45 +00:00
else:
let
idx = cindex.get()
if idx < committees_per_slot:
res.add(getCommittee(slot, idx))
2021-03-17 18:46:45 +00:00
var res: seq[RestBeaconStatesCommittees]
2021-03-17 18:46:45 +00:00
let qepoch =
if vepoch.isNone:
epoch(getStateField(state, slot))
2021-03-17 18:46:45 +00:00
else:
vepoch.get()
if vslot.isNone():
for slot in qepoch.slots():
forSlot(slot, vindex, res)
2021-03-17 18:46:45 +00:00
else:
forSlot(vslot.get(), vindex, res)
return RestApiResponse.jsonResponseWOpt(
res,
node.getStateOptimistic(state)
)
2021-03-17 18:46:45 +00:00
return RestApiResponse.jsonError(Http404, StateNotFoundError)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getEpochSyncCommittees
router.api(MethodGet,
"/eth/v1/beacon/states/{state_id}/sync_committees") do (
state_id: StateIdent, epoch: Option[Epoch]) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
sid = state_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
$error)
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
bslot = node.getBlockSlotId(sid).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
if sid.kind == StateQueryKind.Root:
# TODO (cheatfate): Its impossible to retrieve state by `state_root`
# in current version of database.
return RestApiResponse.jsonError(Http500, NoImplementationError)
return RestApiResponse.jsonError(Http404, StateNotFoundError,
$error)
let qepoch =
if epoch.isSome():
let repoch = epoch.get()
if repoch.isErr():
return RestApiResponse.jsonError(Http400, InvalidEpochValueError,
$repoch.error())
let res = repoch.get()
if res > MaxEpoch:
return RestApiResponse.jsonError(Http400, EpochOverflowValueError)
if res < node.dag.cfg.ALTAIR_FORK_EPOCH:
return RestApiResponse.jsonError(Http400,
EpochFromTheIncorrectForkError)
res
else:
# If ``epoch`` not present then the sync committees for the epoch of
# the state will be obtained.
bslot.slot.epoch()
Prune `BlockRef` on finalization (#3513) Up til now, the block dag has been using `BlockRef`, a structure adapted for a full DAG, to represent all of chain history. This is a correct and simple design, but does not exploit the linearity of the chain once parts of it finalize. By pruning the in-memory `BlockRef` structure at finalization, we save, at the time of writing, a cool ~250mb (or 25%:ish) chunk of memory landing us at a steady state of ~750mb normal memory usage for a validating node. Above all though, we prevent memory usage from growing proportionally with the length of the chain, something that would not be sustainable over time - instead, the steady state memory usage is roughly determined by the validator set size which grows much more slowly. With these changes, the core should remain sustainable memory-wise post-merge all the way to withdrawals (when the validator set is expected to grow). In-memory indices are still used for the "hot" unfinalized portion of the chain - this ensure that consensus performance remains unchanged. What changes is that for historical access, we use a db-based linear slot index which is cache-and-disk-friendly, keeping the cost for accessing historical data at a similar level as before, achieving the savings at no percievable cost to functionality or performance. A nice collateral benefit is the almost-instant startup since we no longer load any large indicies at dag init. The cost of this functionality instead can be found in the complexity of having to deal with two ways of traversing the chain - by `BlockRef` and by slot. * use `BlockId` instead of `BlockRef` where finalized / historical data may be required * simplify clearance pre-advancement * remove dag.finalizedBlocks (~50:ish mb) * remove `getBlockAtSlot` - use `getBlockIdAtSlot` instead * `parent` and `atSlot` for `BlockId` now require a `ChainDAGRef` instance, unlike `BlockRef` traversal * prune `BlockRef` parents on finality (~200:ish mb) * speed up ChainDAG init by not loading finalized history index * mess up light client server error handling - this need revisiting :)
2022-03-17 17:42:56 +00:00
node.withStateForBlockSlotId(bslot):
let keys =
block:
let res = syncCommitteeParticipants(state, qepoch)
if res.isErr():
return RestApiResponse.jsonError(Http400,
$res.error())
let kres = res.get()
if len(kres) == 0:
return RestApiResponse.jsonError(Http500, InternalServerError,
"List of sync committee participants is empty")
kres
let indices =
block:
var res: seq[ValidatorIndex]
let optIndices = keysToIndices(node.restKeysCache, state, keys)
# Remove all the duplicates.
for item in optIndices:
if item.isNone():
# This should not be happened, because keys are from state.
return RestApiResponse.jsonError(Http500, InternalServerError,
"Could not get validator indices")
res.add(item.get())
res
let aggregates =
block:
var
res: seq[seq[ValidatorIndex]]
offset = 0
while true:
let length = min(SYNC_SUBCOMMITTEE_SIZE, len(indices) - offset)
if length == 0:
break
res.add(@(indices.toOpenArray(offset, offset + length - 1)))
offset.inc(length)
res
return RestApiResponse.jsonResponseWOpt(
RestEpochSyncCommittee(validators: indices,
validator_aggregates: aggregates),
node.getStateOptimistic(state)
)
return RestApiResponse.jsonError(Http404, StateNotFoundError)
# https://ethereum.github.io/beacon-APIs/#/Beacon/getBlockHeaders
router.api(MethodGet, "/eth/v1/beacon/headers") do (
2021-03-17 18:46:45 +00:00
slot: Option[Slot], parent_root: Option[Eth2Digest]) -> RestApiResponse:
# TODO (cheatfate): This call is incomplete, because structure
# of database do not allow to query blocks by `parent_root`.
let qslot =
if slot.isSome():
let rslot = slot.get()
if rslot.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, InvalidSlotValueError,
$rslot.error())
rslot.get()
else:
node.dag.head.slot
if parent_root.isSome():
let rroot = parent_root.get()
if rroot.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, InvalidParentRootValueError,
$rroot.error())
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http500, NoImplementationError)
let bdata = node.getForkedBlock(BlockIdent.init(qslot)).valueOr:
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
return
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
withBlck(bdata):
RestApiResponse.jsonResponseWOpt(
[
(
root: blck.root,
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
canonical: node.dag.isCanonical(
BlockId(root: blck.root, slot: blck.message.slot)),
header: (
message: blck.toBeaconBlockHeader,
signature: blck.signature
)
)
],
node.getBlockOptimistic(bdata)
)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getBlockHeader
router.api(MethodGet, "/eth/v1/beacon/headers/{block_id}") do (
2021-03-17 18:46:45 +00:00
block_id: BlockIdent) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
bid = block_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidBlockIdValueError,
$error)
bdata = node.getForkedBlock(bid).valueOr:
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
return
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
withBlck(bdata):
RestApiResponse.jsonResponseWOpt(
(
root: blck.root,
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
canonical: node.dag.isCanonical(
BlockId(root: blck.root, slot: blck.message.slot)),
header: (
message: blck.toBeaconBlockHeader,
signature: blck.signature
)
),
node.getBlockOptimistic(bdata)
2021-03-17 18:46:45 +00:00
)
# https://ethereum.github.io/beacon-APIs/#/Beacon/publishBlock
router.api(MethodPost, "/eth/v1/beacon/blocks") do (
2021-03-17 18:46:45 +00:00
contentBody: Option[ContentBody]) -> RestApiResponse:
let forkedBlock =
block:
if contentBody.isNone():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
let body = contentBody.get()
let res = decodeBody(RestPublishedSignedBeaconBlock, body)
if res.isErr():
return RestApiResponse.jsonError(Http400, InvalidBlockObjectError,
$res.error())
var forked = ForkedSignedBeaconBlock(res.get())
if forked.kind != node.dag.cfg.blockForkAtEpoch(
getForkedBlockField(forked, slot).epoch):
return RestApiResponse.jsonError(Http400, InvalidBlockObjectError)
withBlck(forked):
blck.root = hash_tree_root(blck.message)
forked
let res = await node.sendBeaconBlock(forkedBlock)
if res.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http503, BeaconNodeInSyncError)
if not(res.get()):
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http202, BlockValidationError)
return RestApiResponse.jsonMsgResponse(BlockValidationSuccess)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getBlock
router.api(MethodGet, "/eth/v1/beacon/blocks/{block_id}") do (
2021-03-17 18:46:45 +00:00
block_id: BlockIdent) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
blockIdent = block_id.valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
return RestApiResponse.jsonError(Http400, InvalidBlockIdValueError,
$error)
bid = node.getBlockId(blockIdent).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
if node.dag.cfg.blockForkAtEpoch(bid.slot.epoch) !=
BeaconBlockFork.Phase0:
return RestApiResponse.jsonError(
Http404, BlockNotFoundError, "v1 API supports only phase 0 blocks")
let contentType =
block:
let res = preferredContentType(jsonMediaType,
sszMediaType)
if res.isErr():
return RestApiResponse.jsonError(Http406, ContentNotAcceptableError)
res.get()
return
if contentType == sszMediaType:
var data: seq[byte]
if not node.dag.getBlockSSZ(bid, data):
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
RestApiResponse.response(data, Http200, $sszMediaType)
elif contentType == jsonMediaType:
let bdata = node.dag.getForkedBlock(bid).valueOr:
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
if bdata.kind == BeaconBlockFork.Phase0:
RestApiResponse.jsonResponse(bdata.phase0Data.asSigned())
else:
# Shouldn't happen, but in case there's some weird block database
# issue..
RestApiResponse.jsonError(
Http404, BlockNotFoundError, "v1 API supports only phase 0 blocks")
else:
RestApiResponse.jsonError(Http500, InvalidAcceptError)
# https://ethereum.github.io/beacon-APIs/#/Beacon/getBlockV2
router.api(MethodGet, "/eth/v2/beacon/blocks/{block_id}") do (
block_id: BlockIdent) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
blockIdent = block_id.valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
return RestApiResponse.jsonError(Http400, InvalidBlockIdValueError,
$error)
bid = node.getBlockId(blockIdent).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
let contentType =
block:
let res = preferredContentType(jsonMediaType,
sszMediaType)
if res.isErr():
return RestApiResponse.jsonError(Http406, ContentNotAcceptableError)
res.get()
return
if contentType == sszMediaType:
var data: seq[byte]
if not node.dag.getBlockSSZ(bid, data):
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
let
fork = node.dag.cfg.blockForkAtEpoch(bid.slot.epoch)
headers = [("eth-consensus-version", fork.toString())]
RestApiResponse.response(data, Http200, $sszMediaType,
headers = headers)
elif contentType == jsonMediaType:
let bdata = node.dag.getForkedBlock(bid).valueOr:
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
let
fork = node.dag.cfg.blockForkAtEpoch(bid.slot.epoch)
headers = [("eth-consensus-version", fork.toString())]
RestApiResponse.jsonResponseBlock(
bdata.asSigned(),
node.getBlockOptimistic(bdata),
headers
)
else:
RestApiResponse.jsonError(Http500, InvalidAcceptError)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getBlockRoot
router.api(MethodGet, "/eth/v1/beacon/blocks/{block_id}/root") do (
2021-03-17 18:46:45 +00:00
block_id: BlockIdent) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
blockIdent = block_id.valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
return RestApiResponse.jsonError(Http400, InvalidBlockIdValueError,
$error)
bid = node.getBlockId(blockIdent).valueOr:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
bdata = node.dag.getForkedBlock(bid).valueOr:
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
return RestApiResponse.jsonResponseWOpt(
(root: bid.root),
node.getBlockOptimistic(bdata)
)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getBlockAttestations
2021-03-17 18:46:45 +00:00
router.api(MethodGet,
"/eth/v1/beacon/blocks/{block_id}/attestations") do (
2021-03-17 18:46:45 +00:00
block_id: BlockIdent) -> RestApiResponse:
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
let
bid = block_id.valueOr:
return RestApiResponse.jsonError(Http400, InvalidBlockIdValueError,
$error)
bdata = node.getForkedBlock(bid).valueOr:
return RestApiResponse.jsonError(Http404, BlockNotFoundError)
return
limit by-root requests to non-finalized blocks (#3293) * limit by-root requests to non-finalized blocks Presently, we keep a mapping from block root to `BlockRef` in memory - this has simplified reasoning about the dag, but is not sustainable with the chain growing. We can distinguish between two cases where by-root access is useful: * unfinalized blocks - this is where the beacon chain is operating generally, by validating incoming data as interesting for future fork choice decisions - bounded by the length of the unfinalized period * finalized blocks - historical access in the REST API etc - no bounds, really In this PR, we limit the by-root block index to the first use case: finalized chain data can more efficiently be addressed by slot number. Future work includes: * limiting the `BlockRef` horizon in general - each instance is 40 bytes+overhead which adds up - this needs further refactoring to deal with the tail vs state problem * persisting the finalized slot-to-hash index - this one also keeps growing unbounded (albeit slowly) Anyway, this PR easily shaves ~128mb of memory usage at the time of writing. * No longer honor `BeaconBlocksByRoot` requests outside of the non-finalized period - previously, Nimbus would generously return any block through this libp2p request - per the spec, finalized blocks should be fetched via `BeaconBlocksByRange` instead. * return `Opt[BlockRef]` instead of `nil` when blocks can't be found - this becomes a lot more common now and thus deserves more attention * `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized blocks from now - `finalizedBlocks` covers the other `BlockRef` instances * in backfill, verify that the last backfilled block leads back to genesis, or panic * add backfill timings to log * fix missing check that `BlockRef` block can be fetched with `getForkedBlock` reliably * shortcut doppelganger check when feature is not enabled * in REST/JSON-RPC, fetch blocks without involving `BlockRef` * fix dag.blocks ref
2022-01-21 11:33:16 +00:00
withBlck(bdata):
RestApiResponse.jsonResponseWOpt(
blck.message.body.attestations.asSeq(),
node.getBlockOptimistic(bdata)
)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getPoolAttestations
router.api(MethodGet, "/eth/v1/beacon/pool/attestations") do (
2021-03-17 18:46:45 +00:00
slot: Option[Slot],
committee_index: Option[CommitteeIndex]) -> RestApiResponse:
let vindex =
if committee_index.isSome():
let rindex = committee_index.get()
if rindex.isErr():
return RestApiResponse.jsonError(Http400,
2021-04-08 10:49:28 +00:00
InvalidCommitteeIndexValueError,
2021-03-17 18:46:45 +00:00
$rindex.error())
some(rindex.get())
else:
none[CommitteeIndex]()
let vslot =
if slot.isSome():
let rslot = slot.get()
if rslot.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, InvalidSlotValueError,
2021-03-17 18:46:45 +00:00
$rslot.error())
some(rslot.get())
else:
none[Slot]()
var res: seq[Attestation]
for item in node.attestationPool[].attestations(vslot, vindex):
res.add(item)
return RestApiResponse.jsonResponse(res)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/submitPoolAttestations
router.api(MethodPost, "/eth/v1/beacon/pool/attestations") do (
2021-03-17 18:46:45 +00:00
contentBody: Option[ContentBody]) -> RestApiResponse:
let attestations =
block:
if contentBody.isNone():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
let dres = decodeBody(seq[Attestation], contentBody.get())
if dres.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400,
InvalidAttestationObjectError,
$dres.error())
dres.get()
# Since our validation logic supports batch processing, we will submit all
# attestations for validation.
let pending =
block:
var res: seq[Future[SendResult]]
for attestation in attestations:
res.add(node.sendAttestation(attestation))
res
let failures =
block:
var res: seq[RestAttestationsFailure]
await allFutures(pending)
for index, future in pending:
if future.done():
let fres = future.read()
if fres.isErr():
let failure = RestAttestationsFailure(index: uint64(index),
message: $fres.error())
res.add(failure)
elif future.failed() or future.cancelled():
# This is unexpected failure, so we log the error message.
let exc = future.readError()
let failure = RestAttestationsFailure(index: uint64(index),
message: $exc.msg)
res.add(failure)
res
if len(failures) > 0:
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonErrorList(Http400, AttestationValidationError,
failures)
else:
New validator client using REST API. (#2651) * Initial commit. * Exporting getConfig(). * Add beacon node checking procedures. * Post rebase fixes. * Use runSlotLoop() from nimbus_beacon_node. Fallback implementation. Fixes for ETH2 REST serialization. * Add beacon_clock.durationToNextSlot(). Move type declarations from beacon_rest_api to json_rest_serialization. Fix seq[ValidatorIndex] serialization. Refactor ValidatorPool and add some utility procedures. Create separate version of validator_client. * Post-rebase fixes. Remove CookedPubKey from validator_pool.nim. * Now we should be able to produce attestations and aggregate and proofs. But its not working yet. * Debugging attestation sending. * Add durationToNextAttestation. Optimize some debug logs. Fix aggregation_bits encoding. Bump chronos/presto. * Its alive. * Fixes for launch_local_testnet script. Bump chronos. * Switch client API to not use `/api` prefix. * Post-rebase adjustments. * Fix endpoint for publishBlock(). * Add CONFIG_NAME. Add more checks to ensure that beacon_node is compatible. * Add beacon committee subscription support to validator_client. * Fix stacktrace should be an array of strings. Fix committee subscriptions should not be `data` keyed. * Log duration to next block proposal. * Fix beacon_node_status import. * Use jsonMsgResponse() instead of jsonError(). * Fix graffityBytes usage. Remove unnecessary `await`. Adjust creation of SignedBlock instance. Remove legacy files. * Rework durationToNextSlot() and durationToNextEpoch() to use `fromNow`. * Fix race condition for block proposal and attestations for same slot. Fix local_testnet script to properly kill tasks on Windows. Bump chronos and nim-http-tools, to allow connections to infura.io (basic auth). * Catch services errors. Improve performance of local_testnet.sh script on Windows. Fix race condition when attestation producing. * Post-rebase fixes. * Bump chronos and presto. * Calculate block publishing delay. Fix pkill in one more place. * Add error handling and timeouts to firstSuccess() template. Add onceToAll() template. Add checkNodes() procedure. Refactor firstSuccess() template. Add error checking to api.nim calls. * Deprecated usage onceToAll() for better stability. Address comment and send attestations asap. * Avoid unnecessary loop when calculating minimal duration.
2021-07-13 11:15:07 +00:00
return RestApiResponse.jsonMsgResponse(AttestationValidationSuccess)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getPoolAttesterSlashings
router.api(MethodGet, "/eth/v1/beacon/pool/attester_slashings") do (
2021-03-17 18:46:45 +00:00
) -> RestApiResponse:
var res: seq[AttesterSlashing]
if isNil(node.exitPool):
return RestApiResponse.jsonResponse(res)
2021-03-17 18:46:45 +00:00
let length = len(node.exitPool.attester_slashings)
res = newSeqOfCap[AttesterSlashing](length)
for item in node.exitPool.attester_slashings.items():
res.add(item)
return RestApiResponse.jsonResponse(res)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/submitPoolAttesterSlashings
router.api(MethodPost, "/eth/v1/beacon/pool/attester_slashings") do (
2021-03-17 18:46:45 +00:00
contentBody: Option[ContentBody]) -> RestApiResponse:
let slashing =
block:
if contentBody.isNone():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
let dres = decodeBody(AttesterSlashing, contentBody.get())
if dres.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400,
InvalidAttesterSlashingObjectError,
$dres.error())
dres.get()
let res = await node.sendAttesterSlashing(slashing)
if res.isErr():
return RestApiResponse.jsonError(Http400,
AttesterSlashingValidationError,
$res.error())
New validator client using REST API. (#2651) * Initial commit. * Exporting getConfig(). * Add beacon node checking procedures. * Post rebase fixes. * Use runSlotLoop() from nimbus_beacon_node. Fallback implementation. Fixes for ETH2 REST serialization. * Add beacon_clock.durationToNextSlot(). Move type declarations from beacon_rest_api to json_rest_serialization. Fix seq[ValidatorIndex] serialization. Refactor ValidatorPool and add some utility procedures. Create separate version of validator_client. * Post-rebase fixes. Remove CookedPubKey from validator_pool.nim. * Now we should be able to produce attestations and aggregate and proofs. But its not working yet. * Debugging attestation sending. * Add durationToNextAttestation. Optimize some debug logs. Fix aggregation_bits encoding. Bump chronos/presto. * Its alive. * Fixes for launch_local_testnet script. Bump chronos. * Switch client API to not use `/api` prefix. * Post-rebase adjustments. * Fix endpoint for publishBlock(). * Add CONFIG_NAME. Add more checks to ensure that beacon_node is compatible. * Add beacon committee subscription support to validator_client. * Fix stacktrace should be an array of strings. Fix committee subscriptions should not be `data` keyed. * Log duration to next block proposal. * Fix beacon_node_status import. * Use jsonMsgResponse() instead of jsonError(). * Fix graffityBytes usage. Remove unnecessary `await`. Adjust creation of SignedBlock instance. Remove legacy files. * Rework durationToNextSlot() and durationToNextEpoch() to use `fromNow`. * Fix race condition for block proposal and attestations for same slot. Fix local_testnet script to properly kill tasks on Windows. Bump chronos and nim-http-tools, to allow connections to infura.io (basic auth). * Catch services errors. Improve performance of local_testnet.sh script on Windows. Fix race condition when attestation producing. * Post-rebase fixes. * Bump chronos and presto. * Calculate block publishing delay. Fix pkill in one more place. * Add error handling and timeouts to firstSuccess() template. Add onceToAll() template. Add checkNodes() procedure. Refactor firstSuccess() template. Add error checking to api.nim calls. * Deprecated usage onceToAll() for better stability. Address comment and send attestations asap. * Avoid unnecessary loop when calculating minimal duration.
2021-07-13 11:15:07 +00:00
return RestApiResponse.jsonMsgResponse(AttesterSlashingValidationSuccess)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/getPoolProposerSlashings
router.api(MethodGet, "/eth/v1/beacon/pool/proposer_slashings") do (
2021-03-17 18:46:45 +00:00
) -> RestApiResponse:
var res: seq[ProposerSlashing]
if isNil(node.exitPool):
return RestApiResponse.jsonResponse(res)
2021-03-17 18:46:45 +00:00
let length = len(node.exitPool.proposer_slashings)
res = newSeqOfCap[ProposerSlashing](length)
for item in node.exitPool.proposer_slashings.items():
res.add(item)
return RestApiResponse.jsonResponse(res)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/submitPoolProposerSlashings
router.api(MethodPost, "/eth/v1/beacon/pool/proposer_slashings") do (
2021-03-17 18:46:45 +00:00
contentBody: Option[ContentBody]) -> RestApiResponse:
let slashing =
block:
if contentBody.isNone():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
let dres = decodeBody(ProposerSlashing, contentBody.get())
if dres.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400,
InvalidProposerSlashingObjectError,
$dres.error())
dres.get()
let res = await node.sendProposerSlashing(slashing)
if res.isErr():
return RestApiResponse.jsonError(Http400,
ProposerSlashingValidationError,
$res.error())
New validator client using REST API. (#2651) * Initial commit. * Exporting getConfig(). * Add beacon node checking procedures. * Post rebase fixes. * Use runSlotLoop() from nimbus_beacon_node. Fallback implementation. Fixes for ETH2 REST serialization. * Add beacon_clock.durationToNextSlot(). Move type declarations from beacon_rest_api to json_rest_serialization. Fix seq[ValidatorIndex] serialization. Refactor ValidatorPool and add some utility procedures. Create separate version of validator_client. * Post-rebase fixes. Remove CookedPubKey from validator_pool.nim. * Now we should be able to produce attestations and aggregate and proofs. But its not working yet. * Debugging attestation sending. * Add durationToNextAttestation. Optimize some debug logs. Fix aggregation_bits encoding. Bump chronos/presto. * Its alive. * Fixes for launch_local_testnet script. Bump chronos. * Switch client API to not use `/api` prefix. * Post-rebase adjustments. * Fix endpoint for publishBlock(). * Add CONFIG_NAME. Add more checks to ensure that beacon_node is compatible. * Add beacon committee subscription support to validator_client. * Fix stacktrace should be an array of strings. Fix committee subscriptions should not be `data` keyed. * Log duration to next block proposal. * Fix beacon_node_status import. * Use jsonMsgResponse() instead of jsonError(). * Fix graffityBytes usage. Remove unnecessary `await`. Adjust creation of SignedBlock instance. Remove legacy files. * Rework durationToNextSlot() and durationToNextEpoch() to use `fromNow`. * Fix race condition for block proposal and attestations for same slot. Fix local_testnet script to properly kill tasks on Windows. Bump chronos and nim-http-tools, to allow connections to infura.io (basic auth). * Catch services errors. Improve performance of local_testnet.sh script on Windows. Fix race condition when attestation producing. * Post-rebase fixes. * Bump chronos and presto. * Calculate block publishing delay. Fix pkill in one more place. * Add error handling and timeouts to firstSuccess() template. Add onceToAll() template. Add checkNodes() procedure. Refactor firstSuccess() template. Add error checking to api.nim calls. * Deprecated usage onceToAll() for better stability. Address comment and send attestations asap. * Avoid unnecessary loop when calculating minimal duration.
2021-07-13 11:15:07 +00:00
return RestApiResponse.jsonMsgResponse(ProposerSlashingValidationSuccess)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/submitPoolSyncCommitteeSignatures
router.api(MethodPost, "/eth/v1/beacon/pool/sync_committees") do (
contentBody: Option[ContentBody]) -> RestApiResponse:
let messages =
block:
if contentBody.isNone():
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
let dres = decodeBody(seq[SyncCommitteeMessage], contentBody.get())
if dres.isErr():
return RestApiResponse.jsonError(Http400,
InvalidSyncCommitteeSignatureMessageError)
dres.get()
let results = await node.sendSyncCommitteeMessages(messages)
let failures =
block:
var res: seq[RestAttestationsFailure]
for index, item in results:
if item.isErr():
res.add(RestAttestationsFailure(index: uint64(index),
message: $item.error()))
res
if len(failures) > 0:
return RestApiResponse.jsonErrorList(Http400,
SyncCommitteeMessageValidationError,
failures)
else:
return RestApiResponse.jsonMsgResponse(
SyncCommitteeMessageValidationSuccess)
# https://ethereum.github.io/beacon-APIs/#/Beacon/getPoolVoluntaryExits
router.api(MethodGet, "/eth/v1/beacon/pool/voluntary_exits") do (
2021-03-17 18:46:45 +00:00
) -> RestApiResponse:
var res: seq[SignedVoluntaryExit]
if isNil(node.exitPool):
return RestApiResponse.jsonResponse(res)
2021-03-17 18:46:45 +00:00
let length = len(node.exitPool.voluntary_exits)
res = newSeqOfCap[SignedVoluntaryExit](length)
for item in node.exitPool.voluntary_exits.items():
res.add(item)
return RestApiResponse.jsonResponse(res)
2021-03-17 18:46:45 +00:00
# https://ethereum.github.io/beacon-APIs/#/Beacon/submitPoolVoluntaryExit
router.api(MethodPost, "/eth/v1/beacon/pool/voluntary_exits") do (
2021-03-17 18:46:45 +00:00
contentBody: Option[ContentBody]) -> RestApiResponse:
let exit =
block:
if contentBody.isNone():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
let dres = decodeBody(SignedVoluntaryExit, contentBody.get())
if dres.isErr():
2021-04-08 10:49:28 +00:00
return RestApiResponse.jsonError(Http400,
InvalidVoluntaryExitObjectError,
$dres.error())
dres.get()
let res = await node.sendVoluntaryExit(exit)
if res.isErr():
return RestApiResponse.jsonError(Http400,
VoluntaryExitValidationError,
$res.error())
New validator client using REST API. (#2651) * Initial commit. * Exporting getConfig(). * Add beacon node checking procedures. * Post rebase fixes. * Use runSlotLoop() from nimbus_beacon_node. Fallback implementation. Fixes for ETH2 REST serialization. * Add beacon_clock.durationToNextSlot(). Move type declarations from beacon_rest_api to json_rest_serialization. Fix seq[ValidatorIndex] serialization. Refactor ValidatorPool and add some utility procedures. Create separate version of validator_client. * Post-rebase fixes. Remove CookedPubKey from validator_pool.nim. * Now we should be able to produce attestations and aggregate and proofs. But its not working yet. * Debugging attestation sending. * Add durationToNextAttestation. Optimize some debug logs. Fix aggregation_bits encoding. Bump chronos/presto. * Its alive. * Fixes for launch_local_testnet script. Bump chronos. * Switch client API to not use `/api` prefix. * Post-rebase adjustments. * Fix endpoint for publishBlock(). * Add CONFIG_NAME. Add more checks to ensure that beacon_node is compatible. * Add beacon committee subscription support to validator_client. * Fix stacktrace should be an array of strings. Fix committee subscriptions should not be `data` keyed. * Log duration to next block proposal. * Fix beacon_node_status import. * Use jsonMsgResponse() instead of jsonError(). * Fix graffityBytes usage. Remove unnecessary `await`. Adjust creation of SignedBlock instance. Remove legacy files. * Rework durationToNextSlot() and durationToNextEpoch() to use `fromNow`. * Fix race condition for block proposal and attestations for same slot. Fix local_testnet script to properly kill tasks on Windows. Bump chronos and nim-http-tools, to allow connections to infura.io (basic auth). * Catch services errors. Improve performance of local_testnet.sh script on Windows. Fix race condition when attestation producing. * Post-rebase fixes. * Bump chronos and presto. * Calculate block publishing delay. Fix pkill in one more place. * Add error handling and timeouts to firstSuccess() template. Add onceToAll() template. Add checkNodes() procedure. Refactor firstSuccess() template. Add error checking to api.nim calls. * Deprecated usage onceToAll() for better stability. Address comment and send attestations asap. * Avoid unnecessary loop when calculating minimal duration.
2021-07-13 11:15:07 +00:00
return RestApiResponse.jsonMsgResponse(VoluntaryExitValidationSuccess)
2021-04-13 10:19:31 +00:00
# Legacy URLS - Nimbus <= 1.5.5 used to expose the REST API with an additional
# `/api` path component
2021-04-13 10:19:31 +00:00
router.redirect(
MethodGet,
"/api/eth/v1/beacon/genesis",
2021-04-13 10:19:31 +00:00
"/eth/v1/beacon/genesis",
)
router.redirect(
MethodGet,
"/api/eth/v1/beacon/states/{state_id}/root",
2021-04-13 10:19:31 +00:00
"/eth/v1/beacon/states/{state_id}/root",
)
router.redirect(
MethodGet,
"/api/eth/v1/beacon/states/{state_id}/fork",
2021-04-13 10:19:31 +00:00
"/eth/v1/beacon/states/{state_id}/fork",
)
router.redirect(
MethodGet,
"/api/eth/v1/beacon/states/{state_id}/finality_checkpoints",
"/eth/v1/beacon/states/{state_id}/finality_checkpoints"
2021-04-13 10:19:31 +00:00
)
router.redirect(
MethodGet,
"/api/eth/v1/beacon/states/{state_id}/validators",
"/eth/v1/beacon/states/{state_id}/validators"
2021-04-13 10:19:31 +00:00
)
router.redirect(
MethodGet,
"/api/eth/v1/beacon/states/{state_id}/validators/{validator_id}",
"/eth/v1/beacon/states/{state_id}/validators/{validator_id}"
2021-04-13 10:19:31 +00:00
)
router.redirect(
MethodGet,
"/api/eth/v1/beacon/states/{state_id}/validator_balances",
"/eth/v1/beacon/states/{state_id}/validator_balances"
2021-04-13 10:19:31 +00:00
)
router.redirect(
MethodGet,
"/api/eth/v1/beacon/states/{state_id}/committees",
"/eth/v1/beacon/states/{state_id}/committees"
2021-04-13 10:19:31 +00:00
)
router.redirect(
MethodGet,
"/api/eth/v1/beacon/states/{state_id}/sync_committees",
"/eth/v1/beacon/states/{state_id}/sync_committees"
)
2021-04-13 10:19:31 +00:00
router.redirect(
MethodGet,
"/api/eth/v1/beacon/headers",
"/eth/v1/beacon/headers"
2021-04-13 10:19:31 +00:00
)
router.redirect(
MethodGet,
"/api/eth/v1/beacon/headers/{block_id}",
"/eth/v1/beacon/headers/{block_id}"
2021-04-13 10:19:31 +00:00
)
router.redirect(
MethodPost,
"/api/eth/v1/beacon/blocks",
"/eth/v1/beacon/blocks"
2021-04-13 10:19:31 +00:00
)
router.redirect(
MethodGet,
"/api/eth/v1/beacon/blocks/{block_id}",
"/eth/v1/beacon/blocks/{block_id}"
2021-04-13 10:19:31 +00:00
)
router.redirect(
MethodGet,
"/api/eth/v2/beacon/blocks/{block_id}",
"/eth/v2/beacon/blocks/{block_id}"
)
2021-04-13 10:19:31 +00:00
router.redirect(
MethodGet,
"/api/eth/v1/beacon/blocks/{block_id}/root",
"/eth/v1/beacon/blocks/{block_id}/root"
2021-04-13 10:19:31 +00:00
)
router.redirect(
MethodGet,
"/api/eth/v1/beacon/blocks/{block_id}/attestations",
"/eth/v1/beacon/blocks/{block_id}/attestations"
2021-04-13 10:19:31 +00:00
)
router.redirect(
MethodGet,
"/api/eth/v1/beacon/pool/attestations",
"/eth/v1/beacon/pool/attestations"
2021-04-13 10:19:31 +00:00
)
router.redirect(
MethodPost,
"/api/eth/v1/beacon/pool/attestations",
"/eth/v1/beacon/pool/attestations"
)
2021-04-13 10:19:31 +00:00
router.redirect(
MethodPost,
"/api/eth/v1/beacon/pool/attester_slashings",
"/eth/v1/beacon/pool/attester_slashings"
2021-04-13 10:19:31 +00:00
)
router.redirect(
MethodGet,
"/api/eth/v1/beacon/pool/attester_slashings",
"/eth/v1/beacon/pool/attester_slashings"
2021-04-13 10:19:31 +00:00
)
router.redirect(
MethodPost,
"/api/eth/v1/beacon/pool/proposer_slashings",
"/eth/v1/beacon/pool/proposer_slashings"
2021-04-13 10:19:31 +00:00
)
router.redirect(
MethodGet,
"/api/eth/v1/beacon/pool/proposer_slashings",
"/eth/v1/beacon/pool/proposer_slashings"
2021-04-13 10:19:31 +00:00
)
router.redirect(
MethodPost,
"/api/eth/v1/beacon/pool/sync_committees",
"/eth/v1/beacon/pool/sync_committees"
)
2021-04-13 10:19:31 +00:00
router.redirect(
MethodPost,
"/api/eth/v1/beacon/pool/voluntary_exits",
"/eth/v1/beacon/pool/voluntary_exits"
2021-04-13 10:19:31 +00:00
)
router.redirect(
MethodGet,
"/api/eth/v1/beacon/pool/voluntary_exits",
"/eth/v1/beacon/pool/voluntary_exits"
2021-04-13 10:19:31 +00:00
)