era file verification (#3605)
* era file verification Implement and document era file verification * era file states now come with block applied for easier verification * clarify conflicting version handling * document verification requirements * remove count from name, use start-era, end-root to discover range * remove obsolete todo * abstract out block root loading
This commit is contained in:
Jacek Sieka
GitHub
parent
fc75c3ce36
commit
011e0ca02f
|
|
@ -873,60 +873,43 @@ proc init*(T: type ChainDAGRef, cfg: RuntimeConfig, db: BeaconChainDB,
|
|||
historical_roots = getStateField(dag.headState, historical_roots).asSeq()
|
||||
|
||||
var
|
||||
files = 0
|
||||
blocks = 0
|
||||
parent: Eth2Digest
|
||||
|
||||
# Here, we'll build up the slot->root mapping in memory for the range of
|
||||
# blocks from genesis to backfill, if possible.
|
||||
for i in 0'u64..<historical_roots.lenu64():
|
||||
var
|
||||
found = false
|
||||
done = false
|
||||
for summary in dag.era.getBlockIds(historical_roots, Slot(0)):
|
||||
if summary.slot >= dag.backfill.slot:
|
||||
# If we end up in here, we failed the root comparison just below in
|
||||
# an earlier iteration
|
||||
fatal "Era summaries don't lead up to backfill, database or era files corrupt?",
|
||||
slot = summary.slot
|
||||
quit 1
|
||||
|
||||
for summary in dag.era.getBlockIds(historical_roots, Era(i)):
|
||||
if summary.slot >= dag.backfill.slot:
|
||||
# If we end up in here, we failed the root comparison just below in
|
||||
# an earlier iteration
|
||||
fatal "Era summaries don't lead up to backfill, database or era files corrupt?",
|
||||
slot = summary.slot
|
||||
quit 1
|
||||
# In BeaconState.block_roots, empty slots are filled with the root of
|
||||
# the previous block - in our data structure, we use a zero hash instead
|
||||
if summary.root != parent:
|
||||
dag.frontfillBlocks.setLen(summary.slot.int + 1)
|
||||
dag.frontfillBlocks[summary.slot.int] = summary.root
|
||||
|
||||
# In BeaconState.block_roots, empty slots are filled with the root of
|
||||
# the previous block - in our data structure, we use a zero hash instead
|
||||
if summary.root != parent:
|
||||
dag.frontfillBlocks.setLen(summary.slot.int + 1)
|
||||
dag.frontfillBlocks[summary.slot.int] = summary.root
|
||||
if summary.root == dag.backfill.parent_root:
|
||||
# We've reached the backfill point, meaning blocks are available
|
||||
# in the sqlite database from here onwards - remember this point in
|
||||
# time so that we can write summaries to the database - it's a lot
|
||||
# faster to load from database than to iterate over era files with
|
||||
# the current naive era file reader.
|
||||
reset(dag.backfill)
|
||||
|
||||
if summary.root == dag.backfill.parent_root:
|
||||
# We've reached the backfill point, meaning blocks are available
|
||||
# in the sqlite database from here onwards - remember this point in
|
||||
# time so that we can write summaries to the database - it's a lot
|
||||
# faster to load from database than to iterate over era files with
|
||||
# the current naive era file reader.
|
||||
done = true
|
||||
reset(dag.backfill)
|
||||
dag.updateFrontfillBlocks()
|
||||
|
||||
dag.updateFrontfillBlocks()
|
||||
break
|
||||
|
||||
break
|
||||
parent = summary.root
|
||||
|
||||
parent = summary.root
|
||||
blocks += 1
|
||||
|
||||
found = true
|
||||
blocks += 1
|
||||
|
||||
if found:
|
||||
files += 1
|
||||
|
||||
# Try to load as many era files as possible, but stop when there's a
|
||||
# gap - the current logic for loading finalized blocks from the
|
||||
# database is unable to deal with gaps correctly
|
||||
if not found or done: break
|
||||
|
||||
if files > 0:
|
||||
info "Front-filled blocks from era files",
|
||||
files, blocks
|
||||
if blocks > 0:
|
||||
info "Front-filled blocks from era files", blocks
|
||||
|
||||
let frontfillTick = Moment.now()
|
||||
|
||||
|
|
|
|||
|
|
@ -6,17 +6,17 @@
|
|||
|
||||
import
|
||||
std/os,
|
||||
stew/results, snappy,
|
||||
../ncli/e2store,
|
||||
stew/results, snappy, taskpools,
|
||||
../ncli/e2store, eth/keys,
|
||||
./spec/datatypes/[altair, bellatrix, phase0],
|
||||
./spec/forks,
|
||||
./spec/[beaconstate, forks, signatures_batch],
|
||||
./consensus_object_pools/block_dag # TODO move to somewhere else to avoid circular deps
|
||||
|
||||
export results, forks, e2store
|
||||
|
||||
type
|
||||
EraFile = ref object
|
||||
handle: IoHandle
|
||||
EraFile* = ref object
|
||||
handle: Opt[IoHandle]
|
||||
stateIdx: Index
|
||||
blockIdx: Index
|
||||
|
||||
|
|
@ -29,25 +29,9 @@ type
|
|||
|
||||
files: seq[EraFile]
|
||||
|
||||
proc getEraFile(
|
||||
db: EraDB, historical_roots: openArray[Eth2Digest], era: Era):
|
||||
Result[EraFile, string] =
|
||||
for f in db.files:
|
||||
if f.stateIdx.startSlot.era == era:
|
||||
return ok(f)
|
||||
|
||||
if db.files.len > 16:
|
||||
discard closeFile(db.files[0].handle)
|
||||
db.files.delete(0)
|
||||
|
||||
if era.uint64 > historical_roots.lenu64():
|
||||
return err("Era outside of known history")
|
||||
|
||||
let
|
||||
name = eraFileName(db.cfg, db.genesis_validators_root, historical_roots, era)
|
||||
|
||||
proc open*(_: type EraFile, name: string): Result[EraFile, string] =
|
||||
var
|
||||
f = Opt[IoHandle].ok(? openFile(db.path / name, {OpenFlags.Read}).mapErr(ioErrorMsg))
|
||||
f = Opt[IoHandle].ok(? openFile(name, {OpenFlags.Read}).mapErr(ioErrorMsg))
|
||||
|
||||
defer:
|
||||
if f.isSome(): discard closeFile(f[])
|
||||
|
|
@ -81,40 +65,46 @@ proc getEraFile(
|
|||
else:
|
||||
Index()
|
||||
|
||||
let res = EraFile(handle: f[], stateIdx: stateIdx, blockIdx: blockIdx)
|
||||
let res = EraFile(handle: f, stateIdx: stateIdx, blockIdx: blockIdx)
|
||||
reset(f)
|
||||
ok res
|
||||
|
||||
db.files.add(res)
|
||||
ok(res)
|
||||
proc close(f: EraFile) =
|
||||
if f.handle.isSome():
|
||||
discard closeFile(f.handle.get())
|
||||
reset(f.handle)
|
||||
|
||||
proc getBlockSZ*(
|
||||
db: EraDB, historical_roots: openArray[Eth2Digest], slot: Slot, bytes: var seq[byte]):
|
||||
Result[void, string] =
|
||||
f: EraFile, slot: Slot, bytes: var seq[byte]): Result[void, string] =
|
||||
## Get a snappy-frame-compressed version of the block data - may overwrite
|
||||
## `bytes` on error
|
||||
##
|
||||
## Sets `bytes` to an empty seq and returns success if there is no block at
|
||||
## the given slot, according to the index
|
||||
|
||||
# Block content for the blocks of an era is found in the file for the _next_
|
||||
# era
|
||||
doAssert not isNil(f) and f[].handle.isSome
|
||||
|
||||
let
|
||||
f = ? db.getEraFile(historical_roots, slot.era + 1)
|
||||
pos = f[].blockIdx.offsets[slot - f[].blockIdx.startSlot]
|
||||
|
||||
if pos == 0:
|
||||
return err("No block at given slot")
|
||||
bytes = @[]
|
||||
return ok()
|
||||
|
||||
? f.handle.setFilePos(pos, SeekPosition.SeekBegin).mapErr(ioErrorMsg)
|
||||
? f[].handle.get().setFilePos(pos, SeekPosition.SeekBegin).mapErr(ioErrorMsg)
|
||||
|
||||
let header = ? f.handle.readRecord(bytes)
|
||||
let header = ? f[].handle.get().readRecord(bytes)
|
||||
if header.typ != SnappyBeaconBlock:
|
||||
return err("Invalid era file: didn't find block at index position")
|
||||
|
||||
ok()
|
||||
|
||||
proc getBlockSSZ*(
|
||||
db: EraDB, historical_roots: openArray[Eth2Digest], slot: Slot,
|
||||
bytes: var seq[byte]): Result[void, string] =
|
||||
f: EraFile, slot: Slot, bytes: var seq[byte]): Result[void, string] =
|
||||
var tmp: seq[byte]
|
||||
? db.getBlockSZ(historical_roots, slot, tmp)
|
||||
? f.getBlockSZ(slot, tmp)
|
||||
|
||||
try:
|
||||
bytes = decodeFramed(tmp)
|
||||
|
|
@ -122,6 +112,168 @@ proc getBlockSSZ*(
|
|||
except CatchableError as exc:
|
||||
err(exc.msg)
|
||||
|
||||
proc getStateSZ*(
|
||||
f: EraFile, slot: Slot, bytes: var seq[byte]): Result[void, string] =
|
||||
## Get a snappy-frame-compressed version of the state data - may overwrite
|
||||
## `bytes` on error
|
||||
## https://github.com/google/snappy/blob/8dd58a519f79f0742d4c68fbccb2aed2ddb651e8/framing_format.txt#L34
|
||||
doAssert not isNil(f) and f[].handle.isSome
|
||||
|
||||
# TODO consider multi-era files
|
||||
if f[].stateIdx.startSlot != slot:
|
||||
return err("State not found in era file")
|
||||
|
||||
let pos = f[].stateIdx.offsets[0]
|
||||
if pos == 0:
|
||||
return err("No state at given slot")
|
||||
|
||||
? f[].handle.get().setFilePos(pos, SeekPosition.SeekBegin).mapErr(ioErrorMsg)
|
||||
|
||||
let header = ? f[].handle.get().readRecord(bytes)
|
||||
if header.typ != SnappyBeaconState:
|
||||
return err("Invalid era file: didn't find state at index position")
|
||||
|
||||
ok()
|
||||
|
||||
proc getStateSSZ*(
|
||||
f: EraFile, slot: Slot, bytes: var seq[byte],
|
||||
partial: Opt[int] = default(Opt[int])): Result[void, string] =
|
||||
var tmp: seq[byte]
|
||||
? f.getStateSZ(slot, tmp)
|
||||
|
||||
let
|
||||
len = uncompressedLenFramed(tmp).valueOr:
|
||||
return err("Cannot read uncompressed length, era file corrupt?")
|
||||
wanted =
|
||||
if partial.isSome():
|
||||
min(len, partial.get().uint64 + maxUncompressedFrameDataLen - 1)
|
||||
else: len
|
||||
|
||||
bytes = newSeqUninitialized[byte](wanted)
|
||||
let (_, written) = uncompressFramed(tmp, bytes).valueOr:
|
||||
return err("State failed to decompress, era file corrupt?")
|
||||
|
||||
ok()
|
||||
|
||||
proc verify*(f: EraFile, cfg: RuntimeConfig): Result[Eth2Digest, string] =
|
||||
## Verify that an era file is internally consistent, returning the state root
|
||||
## Verification is dominated by block signature checks - about 4-10s on
|
||||
## decent hardware.
|
||||
|
||||
# We'll load the full state and compute its root - then we'll load the blocks
|
||||
# and make sure that they match the state and that their signatures check out
|
||||
let
|
||||
startSlot = f.stateIdx.startSlot
|
||||
era = startSlot.era
|
||||
|
||||
var
|
||||
taskpool = Taskpool.new()
|
||||
verifier = BatchVerifier(rng: keys.newRng(), taskpool: taskpool)
|
||||
|
||||
var tmp: seq[byte]
|
||||
? f.getStateSSZ(startSlot, tmp)
|
||||
|
||||
let
|
||||
state =
|
||||
try: newClone(readSszForkedHashedBeaconState(cfg, tmp))
|
||||
except CatchableError as exc:
|
||||
return err("Unable to read state: " & exc.msg)
|
||||
|
||||
if era > 0:
|
||||
var sigs: seq[SignatureSet]
|
||||
|
||||
for slot in (era - 1).start_slot()..<era.start_slot():
|
||||
? f.getBlockSSZ(slot, tmp)
|
||||
|
||||
# TODO verify that missing blocks correspond to "repeated" block roots
|
||||
# in state.block_roots - how to do this for "initial" empty slots in
|
||||
# the era?
|
||||
if tmp.len > 0:
|
||||
let
|
||||
blck =
|
||||
try: newClone(readSszForkedSignedBeaconBlock(cfg, tmp))
|
||||
except CatchableError as exc:
|
||||
return err("Unable to read block: " & exc.msg)
|
||||
if getForkedBlockField(blck[], slot) != slot:
|
||||
return err("Block slot does not match era index")
|
||||
if blck[].root !=
|
||||
state[].get_block_root_at_slot(getForkedBlockField(blck[], slot)):
|
||||
return err("Block does not match state")
|
||||
|
||||
let
|
||||
proposer = getForkedBlockField(blck[], proposer_index)
|
||||
key = withState(state[]):
|
||||
if proposer >= state.data.validators.asSeq().lenu64:
|
||||
return err("Invalid proposer in block")
|
||||
state.data.validators.asSeq()[proposer].pubkey
|
||||
cooked = key.load()
|
||||
sig = blck[].signature.load()
|
||||
|
||||
if cooked.isNone():
|
||||
return err("Cannot load proposer key")
|
||||
if sig.isNone():
|
||||
return err("Cannot load block signature")
|
||||
|
||||
if slot == GENESIS_SLOT:
|
||||
if blck[].signature != default(type(blck[].signature)):
|
||||
return err("Genesis slot signature not empty")
|
||||
else:
|
||||
# Batch-verification more than doubles total verification speed
|
||||
sigs.add block_signature_set(
|
||||
getStateField(state[], fork),
|
||||
getStateField(state[], genesis_validators_root), slot, blck[].root,
|
||||
cooked.get(), sig.get())
|
||||
|
||||
if not batchVerify(verifier, sigs):
|
||||
return err("Invalid block signature")
|
||||
|
||||
ok(getStateRoot(state[]))
|
||||
|
||||
proc getEraFile(
|
||||
db: EraDB, historical_roots: openArray[Eth2Digest], era: Era):
|
||||
Result[EraFile, string] =
|
||||
for f in db.files:
|
||||
if f.stateIdx.startSlot.era == era:
|
||||
return ok(f)
|
||||
|
||||
let
|
||||
eraRoot = eraRoot(
|
||||
db.genesis_validators_root, historical_roots, era).valueOr:
|
||||
return err("Era outside of known history")
|
||||
name = eraFileName(db.cfg, era, eraRoot)
|
||||
f = ? EraFile.open(db.path / name)
|
||||
|
||||
if db.files.len > 16: # TODO LRU
|
||||
close(db.files[0])
|
||||
db.files.delete(0)
|
||||
|
||||
db.files.add(f)
|
||||
ok(f)
|
||||
|
||||
proc getBlockSZ*(
|
||||
db: EraDB, historical_roots: openArray[Eth2Digest], slot: Slot,
|
||||
bytes: var seq[byte]): Result[void, string] =
|
||||
## Get a snappy-frame-compressed version of the block data - may overwrite
|
||||
## `bytes` on error
|
||||
##
|
||||
## Sets `bytes` to an empty seq and returns success if there is no block at
|
||||
## the given slot, according to the index
|
||||
|
||||
# Block content for the blocks of an era is found in the file for the _next_
|
||||
# era
|
||||
let
|
||||
f = ? db.getEraFile(historical_roots, slot.era + 1)
|
||||
|
||||
f.getBlockSZ(slot, bytes)
|
||||
|
||||
proc getBlockSSZ*(
|
||||
db: EraDB, historical_roots: openArray[Eth2Digest], slot: Slot,
|
||||
bytes: var seq[byte]): Result[void, string] =
|
||||
let
|
||||
f = ? db.getEraFile(historical_roots, slot.era + 1)
|
||||
|
||||
f.getBlockSSZ(slot, bytes)
|
||||
|
||||
proc getBlock*(
|
||||
db: EraDB, historical_roots: openArray[Eth2Digest], slot: Slot,
|
||||
root: Opt[Eth2Digest], T: type ForkyTrustedSignedBeaconBlock): Opt[T] =
|
||||
|
|
@ -149,32 +301,15 @@ proc getStateSZ*(
|
|||
let
|
||||
f = ? db.getEraFile(historical_roots, slot.era)
|
||||
|
||||
if f.stateIdx.startSlot != slot:
|
||||
return err("State not found in era file")
|
||||
|
||||
let pos = f.stateIdx.offsets[0]
|
||||
if pos == 0:
|
||||
return err("No state at given slot")
|
||||
|
||||
? f.handle.setFilePos(pos, SeekPosition.SeekBegin).mapErr(ioErrorMsg)
|
||||
|
||||
let header = ? f.handle.readRecord(bytes)
|
||||
if header.typ != SnappyBeaconState:
|
||||
return err("Invalid era file: didn't find state at index position")
|
||||
|
||||
ok()
|
||||
f.getStateSZ(slot, bytes)
|
||||
|
||||
proc getStateSSZ*(
|
||||
db: EraDB, historical_roots: openArray[Eth2Digest], slot: Slot,
|
||||
bytes: var seq[byte]): Result[void, string] =
|
||||
var tmp: seq[byte]
|
||||
? db.getStateSZ(historical_roots, slot, tmp)
|
||||
bytes: var seq[byte], partial = Opt[int].err()): Result[void, string] =
|
||||
let
|
||||
f = ? db.getEraFile(historical_roots, slot.era)
|
||||
|
||||
try:
|
||||
bytes = decodeFramed(tmp)
|
||||
ok()
|
||||
except CatchableError as exc:
|
||||
err(exc.msg)
|
||||
f.getStateSSZ(slot, bytes)
|
||||
|
||||
type
|
||||
PartialBeaconState = object
|
||||
|
|
@ -197,16 +332,18 @@ type
|
|||
proc getPartialState(
|
||||
db: EraDB, historical_roots: openArray[Eth2Digest], slot: Slot,
|
||||
output: var PartialBeaconState): bool =
|
||||
# TODO don't read all bytes: we only need a few, and shouldn't decompress the
|
||||
# rest - our snappy impl is very slow, in part to the crc32 check it
|
||||
# performs
|
||||
var tmp: seq[byte]
|
||||
if (let e = db.getStateSSZ(historical_roots, slot, tmp); e.isErr):
|
||||
return false
|
||||
|
||||
static: doAssert isFixedSize(PartialBeaconState)
|
||||
const partialBytes = fixedPortionSize(PartialBeaconState)
|
||||
|
||||
# TODO we don't need to read all bytes: ideally we could use something like
|
||||
# faststreams to read uncompressed bytes up to a limit and it would take care
|
||||
# of reading the minimal number of bytes from disk
|
||||
var tmp: seq[byte]
|
||||
if (let e = db.getStateSSZ(
|
||||
historical_roots, slot, tmp, Opt[int].ok(partialBytes));
|
||||
e.isErr):
|
||||
return false
|
||||
|
||||
try:
|
||||
readSszBytes(tmp.toOpenArray(0, partialBytes - 1), output)
|
||||
true
|
||||
|
|
@ -215,26 +352,29 @@ proc getPartialState(
|
|||
false
|
||||
|
||||
iterator getBlockIds*(
|
||||
db: EraDB, historical_roots: openArray[Eth2Digest], era: Era): BlockId =
|
||||
# The state from which we load block roots is stored in the file corresponding
|
||||
# to the "next" era
|
||||
let fileEra = era + 1
|
||||
|
||||
db: EraDB, historical_roots: openArray[Eth2Digest], startSlot: Slot): BlockId =
|
||||
var
|
||||
state = (ref PartialBeaconState)() # avoid stack overflow
|
||||
slot = startSlot
|
||||
|
||||
# `case` ensures we're on a fork for which the `PartialBeaconState`
|
||||
# definition is consistent
|
||||
case db.cfg.stateForkAtEpoch(fileEra.start_slot().epoch)
|
||||
of BeaconStateFork.Phase0, BeaconStateFork.Altair, BeaconStateFork.Bellatrix:
|
||||
if not getPartialState(db, historical_roots, fileEra.start_slot(), state[]):
|
||||
state = nil # No `return` in iterators
|
||||
while true:
|
||||
# `case` ensures we're on a fork for which the `PartialBeaconState`
|
||||
# definition is consistent
|
||||
case db.cfg.stateForkAtEpoch(slot.epoch)
|
||||
of BeaconStateFork.Phase0, BeaconStateFork.Altair, BeaconStateFork.Bellatrix:
|
||||
let stateSlot = (slot.era() + 1).start_slot()
|
||||
if not getPartialState(db, historical_roots, stateSlot, state[]):
|
||||
state = nil # No `return` in iterators
|
||||
|
||||
if state != nil:
|
||||
var
|
||||
slot = era.start_slot()
|
||||
for root in state[].block_roots:
|
||||
yield BlockId(root: root, slot: slot)
|
||||
if state == nil:
|
||||
break
|
||||
|
||||
let
|
||||
x = slot.int mod state[].block_roots.len
|
||||
for i in x..<state[].block_roots.len():
|
||||
# TODO these are not actually valid BlockId instances in the case where
|
||||
# the slot is missing a block - use index to filter..
|
||||
yield BlockId(root: state[].block_roots[i], slot: slot)
|
||||
slot += 1
|
||||
|
||||
proc new*(
|
||||
|
|
|
|||
|
|
@ -949,6 +949,14 @@ func process_randao_mixes_reset*(state: var ForkyBeaconState) =
|
|||
state.randao_mixes[next_epoch mod EPOCHS_PER_HISTORICAL_VECTOR] =
|
||||
get_randao_mix(state, current_epoch)
|
||||
|
||||
func compute_historical_root*(state: var ForkyBeaconState): Eth2Digest =
|
||||
# Equivalent to hash_tree_root(foo: HistoricalBatch), but without using
|
||||
# significant additional stack or heap.
|
||||
# https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/beacon-chain.md#historicalbatch
|
||||
# In response to https://github.com/status-im/nimbus-eth2/issues/921
|
||||
hash_tree_root([
|
||||
hash_tree_root(state.block_roots), hash_tree_root(state.state_roots)])
|
||||
|
||||
# https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/beacon-chain.md#historical-roots-updates
|
||||
func process_historical_roots_update*(state: var ForkyBeaconState) =
|
||||
## Set historical root accumulator
|
||||
|
|
@ -959,8 +967,7 @@ func process_historical_roots_update*(state: var ForkyBeaconState) =
|
|||
# significant additional stack or heap.
|
||||
# https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/beacon-chain.md#historicalbatch
|
||||
# In response to https://github.com/status-im/nimbus-eth2/issues/921
|
||||
if not state.historical_roots.add hash_tree_root(
|
||||
[hash_tree_root(state.block_roots), hash_tree_root(state.state_roots)]):
|
||||
if not state.historical_roots.add state.compute_historical_root():
|
||||
raiseAssert "no more room for historical roots, so long and thanks for the fish!"
|
||||
|
||||
# https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/beacon-chain.md#participation-records-rotation
|
||||
|
|
|
|||
118
docs/e2store.md
118
docs/e2store.md
|
|
@ -1,6 +1,6 @@
|
|||
# Introduction
|
||||
|
||||
The `e2store` (extension: `.e2s`) is a simple linear [TLV](https://en.wikipedia.org/wiki/Type-length-value) file for storing arbitrary items typically encoded using serialization techniques used in ethereum 2 in general: SSZ, varint, snappy.
|
||||
The `e2store` (extension: `.e2s`) is a simple linear [Type-Length-Value](https://en.wikipedia.org/wiki/Type-length-value) file for long-term cold storage of arbitrary items typically found in Ethereum. Entries encoded using serialization techniques used in ethereum 2 in general: SSZ, varint, snappy.
|
||||
|
||||
# General structure
|
||||
|
||||
|
|
@ -23,11 +23,13 @@ The `length` is the first 6 bytes of a little-endian encoded `uint64`, not inclu
|
|||
|
||||
`.e2s` files may freely be concatenated, and may contain out-of-order records.
|
||||
|
||||
Types that have the high bit in the first byte set (those in the range `[0x80-0xff]`) are application and/or vendor specific.
|
||||
Types that have the high bit in the first byte set (those in the range `[0x80-0xff]`) are application and/or vendor specific - other types are reserved for future versions of this specification.
|
||||
|
||||
Records may be traversed without any further out-of-band knowledge, but in order to interpret contents the decode must know the preset and runtime configuration of the network.
|
||||
|
||||
## Reading
|
||||
|
||||
The following python code can be used to read an e2 file:
|
||||
The following python code can be used to read an `e2` file:
|
||||
|
||||
```python
|
||||
import sys, struct
|
||||
|
|
@ -67,7 +69,11 @@ def print_stats(name):
|
|||
|
||||
## Writing
|
||||
|
||||
`e2s` files are by design intended to be append-only, making them suitable for cold storage of finalized chain data.
|
||||
`e2s` files are written record-by-record starting with a version record. Files may be concatenated freely, meaning that the version record may appear multiple times in the file and a single file may have multiple versions.
|
||||
|
||||
The version record is used to introduce backwards-incompatible changes to the file format - readers should not attempt to read unknown versions.
|
||||
|
||||
When splitting a multi-record file, a version record must appear first in each of the new files.
|
||||
|
||||
# Known types
|
||||
|
||||
|
|
@ -77,7 +83,9 @@ def print_stats(name):
|
|||
type: [0x65, 0x32]
|
||||
```
|
||||
|
||||
The `version` type must be the first record in the file. Its type is `[0x65, 0x32]` (`e2` in ascii) and the length of its data field is always 0, thus the first 8 bytes of an `e2s` file are always `[0x65, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]`. When a new version record is encountered, it applies to all records following the version entry - this can happen when two e2s files are concatenated.
|
||||
The `version` type must be the first record in the file. Its type is `[0x65, 0x32]` (`e2` in ascii) and the length of its data field is always 0, thus the first 8 bytes of an `e2s` file are always `[0x65, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]`.
|
||||
|
||||
When a new version record is encountered, it applies to all records following the version entry - this can happen when two e2s files are concatenated.
|
||||
|
||||
## CompressedSignedBeaconBlock
|
||||
|
||||
|
|
@ -86,7 +94,11 @@ type: [0x01, 0x00]
|
|||
data: snappyFramed(ssz(SignedBeaconBlock))
|
||||
```
|
||||
|
||||
`CompressedSignedBeaconBlock` entries are entries whose data field matches the payload of `BeaconBlocksByRange` and `BeaconBlocksByRoot` chunks in the phase0 p2p specification. In particular, the SignedBeaconBlock is serialized using SSZ, then compressed using the snappy [framing format](https://github.com/google/snappy/blob/master/framing_format.txt).
|
||||
`CompressedSignedBeackBlock` contain `SignedBeaconBlock` objects encoded using `SSZ` then compressed using the snappy [framing format](https://github.com/google/snappy/blob/master/framing_format.txt).
|
||||
|
||||
The encoding matches that of the `BeaconBlocksByRoot` and `BeaconBlocksByRange` requests from the p2p specification.
|
||||
|
||||
The fork and thus the exact format of the `SignedBeaconBlock` should be derived from the `slot`.
|
||||
|
||||
## CompressedBeaconState
|
||||
|
||||
|
|
@ -95,7 +107,9 @@ type: [0x02, 0x00]
|
|||
data: snappyFramed(ssz(BeaconState))
|
||||
```
|
||||
|
||||
`CompressedBeaconState` entries are entries whose data field match that of `CompressedSignedBeaconBlock` but carry a `BeaconState` instead.
|
||||
`CompressedBeaconState` entries contain a `BeaconState`, and are encoded the same way as `CompressedSignedBeaconBlock`.
|
||||
|
||||
The fork and thus the exact format of the `BeaconState` should be derived from the `slot`.
|
||||
|
||||
## Empty
|
||||
|
||||
|
|
@ -103,7 +117,7 @@ data: snappyFramed(ssz(BeaconState))
|
|||
type: [0x00, 0x00]
|
||||
```
|
||||
|
||||
The `Empty` type contains no data, but may have a length. The corresponding amount of data should be skiped while reading the file.
|
||||
The `Empty` type contains no data, but may have a length. The corresponding amount of data should be skipped while reading the file.
|
||||
|
||||
## SlotIndex
|
||||
|
||||
|
|
@ -114,7 +128,7 @@ data: starting-slot | index | index | index ... | count
|
|||
|
||||
`SlotIndex` records store offsets, in bytes, from the beginning of the index record to the beginning of the corresponding data at that slot. An offset of `0` indicates that no data is present for the given slot.
|
||||
|
||||
Each entry in the slot index is a fixed-length 8-byte signed integer, meaning that the entry for slot `N` can be found at index `(N * 8) + 16` in the index. The length of a `SlotIndex` record can be computed as `count * 8 + 24` - one entry for every slot and 8 bytes each for type header, starting slot and count. In particular, knowing where the slot index ends allows finding its beginning as well.
|
||||
Each entry in the slot index is a fixed-length 8-byte two's complement signed integer in little-endian, meaning that the entry for slot `N` can be found at index `(N * 8) + 16` in the index. The length of a `SlotIndex` record can be computed as `count * 8 + 24` - one entry for every slot and 8 bytes each for type header, starting slot and count. In particular, knowing where the slot index ends allows finding its beginning as well.
|
||||
|
||||
Only one entry per slot is supported, meaning that only one canonical history can be indexed this way.
|
||||
|
||||
|
|
@ -152,22 +166,26 @@ def read_slot_index(f):
|
|||
|
||||
# Era files
|
||||
|
||||
`.era` files are special instances of `.e2s` files that follow a more strict content format optimised for reading and long-term storage and distribution. Era files contain groups consisting of a state and the blocks that led up to it, limited to `SLOTS_PER_HISTORICAL_ROOT` slots each, allowing quick verification of the data contained in the file.
|
||||
`.era` files are special instances of `.e2s` files that follow a more strict content format optimised for reading and long-term storage and distribution.
|
||||
|
||||
Each era is identified by when it ends. Thus, the genesis era is era 0, followed by era 1 which ends when slot 8192 has been processed, but the block that potentially exists at slot 8192 has not yet been applied.
|
||||
Era files contain groups consisting of a state and the blocks that led up to it, limited to `SLOTS_PER_HISTORICAL_ROOT` slots each.
|
||||
|
||||
In examples, we assume the mainnet configuration: `SLOTS_PER_HISTORICAL_ROOT == 8192`.
|
||||
|
||||
Each era is identified by when it ends. Thus, the genesis era is era `0`, followed by era `1` which ends when slot `8192` has been processed.
|
||||
|
||||
## File name
|
||||
|
||||
`.era` file names follow a simple convention: `<config-name>-<era-number>-<era-count>-<short-historical-root>.era`:
|
||||
|
||||
* `config-name` is the `CONFIG_NAME` field of the runtime configation (`mainnet`, `prater`, etc)
|
||||
* `era-number` is the number of the _last_ era stored in the file - for example, the genesis era file has number 0 - as a 5-digit 0-filled decimal integer
|
||||
* `era-count` is the number of eras stored in the file, as a 5-digit 0-filled decimal integer
|
||||
* `short-historical-root` is the first 4 bytes of the last historical root in the last state in the era file, lower-case hex-encoded (8 characters), except the genesis era which instead uses the `genesis_validators_root` field from the genesis state.
|
||||
* `era-number` is the number of the _first_ era stored in the file - for example, the genesis era file has number 0 - as a 5-digit 0-filled decimal integer
|
||||
* `short-era-root` is the first 4 bytes of the last historical root in the _last_ state in the era file, lower-case hex-encoded (8 characters), except the genesis era which instead uses the `genesis_validators_root` field from the genesis state.
|
||||
* The root is available as `state.historical_roots[era - 1]` except for genesis, which is `state.genesis_validators_root`
|
||||
* Era files with multiple eras use the root of the highest era - this determines the earlier eras as well
|
||||
|
||||
An era file containing the mainnet genesis is thus named `mainnet-00000-00001-4b363db9.era`, and the era after that `mainnet-00001-00001-40cf2f3c.era`.
|
||||
Era files with multiple eras use the era number of the lowest era stored in the file, and the root of the highest era.
|
||||
|
||||
An era file containing the mainnet genesis is thus named `mainnet-00000-4b363db9.era`, and the era after that `mainnet-00001-40cf2f3c.era`.
|
||||
|
||||
## Structure
|
||||
|
||||
|
|
@ -180,22 +198,22 @@ block := CompressedSignedBeaconBlock
|
|||
era-state := CompressedBeaconState
|
||||
```
|
||||
|
||||
The `block` entries of a group include all blocks pertaining to an era. For example, the group representing era one will have all blocks from slot 0 up to and including block 8191.
|
||||
The `block` entries of a group include all blocks leading up to the era transition in slot order. For example, the group representing era `1` contains blocks from slot `0` up to and including block `8191`. Empty slots are skipped.
|
||||
|
||||
The `era-state` is the state of the slot that immediately follows the end of the era without applying blocks from the next era. For example, era 1 that covers the first 8192 slots will have all blocks applied up to slot 8191 and will `process_slots` up to 8192. The genesis group contains only the genesis state but no blocks.
|
||||
|
||||
`slot-index(state)` is a `SlotIndex` entry with `count = 1` for the `CompressedBeaconState` entry of that era, pointing out the offset where the state entry begins.
|
||||
The `era-state` is the state in the era transition slot. The genesis group contains only the genesis state but no blocks. For example, the group representing era `1` contains the canonical state of slot `8192`.
|
||||
|
||||
`slot-index(block)` is a `SlotIndex` entry with `count = SLOTS_PER_HISTORICAL_ROOT` for the `CompressedSignedBeaconBlock` entries in that era, pointing out the offsets of each block in the era. It is omitted for the genesis era.
|
||||
|
||||
`other-entries` is the extension point for future record types in the era file. The positioning of these allows the indices to continue to be looked up from the back.
|
||||
`slot-index(state)` is a `SlotIndex` entry with `count = 1` for the `CompressedBeaconState` entry of that era, pointing out the offset where the state entry begins.
|
||||
|
||||
`other-entries` is an extension point for future record types in the era file. The positioning of these allows the indices to continue to be looked up from the back of the group.
|
||||
|
||||
The structure of the era file gives it the following properties:
|
||||
|
||||
* the indices at the end are fixed-length: they can be used to discover the beginning of an era if the end of it is known
|
||||
* the start slot field of the state slot index idenfifies which era the group pertains to
|
||||
* the state in the era file is the end state after having applied all the blocks in the era - the `block_roots` entries in the state can be used to discover the digest of the blocks - either to verify the intergrity of the era file or to quickly load block roots without computing them
|
||||
* each group in the era file is full, indendent era file - eras can freely be split and combined
|
||||
* the state in the era file is the end state after having applied all the blocks in the era and, if applicable, the block at the first slot - the `block_roots` entries in the state can be used to discover the digest of the blocks - either to verify the intergrity of the era file or to quickly load block roots without computing them.
|
||||
* each group in the era file is full, indendent era file - groups can freely be split and combined
|
||||
|
||||
## Reading era files
|
||||
|
||||
|
|
@ -245,24 +263,47 @@ def read_era_file(name):
|
|||
print("Groups in file:", groups)
|
||||
```
|
||||
|
||||
## Verifying era files
|
||||
|
||||
To verify the internal consistency of an era file, the following checks should be made to verify that an era file is valid for a given network:
|
||||
|
||||
* each group follows the given structure of era files with regards to blocks, states and their indices
|
||||
* offsets within indices must point to entries of the correct kind that can be decompressed and deserialized
|
||||
* era file readers must be prepared to handle malicious inputs, including out-of-range offsets, invalid length prefixes and other trivial errors
|
||||
* unknown record types should be ignored, but it is recommended that verifiers report their size and tag
|
||||
* the state is loadable and consistent with the given runtime configuration
|
||||
* the root of each block in the era file matches that of `state.block_roots` - if a slot is empty according to the block index, this should be confirmed by verifying that
|
||||
`state.get_block_root_at_slot(empty_slot - 1) == state.get_block_root_at_slot(empty_slot)` except for the first slot of the era which, if possible, should be verified against `era - 1`
|
||||
* the genesis era file does not have any blocks
|
||||
* the signature of each block can be verified by the keys in the given state (or any newer state).
|
||||
|
||||
Extended verification consists of verifying a list of era files against a particular history anchored in a checkpoint or a head block. Verification starts from a well-known finalized checkpoint for a slot within the era, using `anchor_state_root = checkpoint_state.state_roots[0]` as anchor and walking the era files as a linked list.
|
||||
|
||||
For each era file:
|
||||
|
||||
* verify that `hash_tree_root(state) == anchor_state_root`
|
||||
* this anchors the era in a particular history, starting from the given state root - the state root is available from any state within the anchor era.
|
||||
* verify the internal consistency of the era, as above
|
||||
* set `anchor_state_root == state.state_roots[0]`
|
||||
|
||||
# FAQ
|
||||
|
||||
## Why snappy framed compression?
|
||||
## Why snappy (sz) framed compression?
|
||||
|
||||
* The networking protocol uses snappy framed compression, avoiding the need to re-compress data to serve blocks
|
||||
* Each entry can be decompressed separately
|
||||
* Each entry in the file can be decompressed independently (and partially!)
|
||||
* It's fast and compresses decently - some compression stats for the first 100 eras:
|
||||
* Uncompressed: 8.4gb
|
||||
* Snappy compression: 4.7gb
|
||||
* `xz` of uncompressed: 3.8gb
|
||||
* `sz`-compressed: 4.7gb
|
||||
* `xz`-compressed: 3.8gb
|
||||
|
||||
## Why SLOTS_PER_HISTORICAL_ROOT blocks per state?
|
||||
## Why `SLOTS_PER_HISTORICAL_ROOT` blocks per state?
|
||||
|
||||
The state stores the block root of the latest `SLOTS_PER_HISTORICAL_ROOT` blocks - storing one state per that many blocks allows verifying the integrity of the blocks easily against the given state, and ensures that all block and state root information remains available, for example to validate states and blocks against `historical_roots`.
|
||||
|
||||
## Why include the state at all?
|
||||
|
||||
This is a tradeoff between being able to access state data such as validator keys and balances directly vs and recreating it by applying each block one by one from from genesis. Given an era file, you can always start processing the chain from there onwards.
|
||||
This is a tradeoff between being able to access state data such as validator keys and balances directly vs and recreating it by applying each block one by one from from genesis. Given an era file, it is possible to start processing the chain from there onwards.
|
||||
|
||||
## Why the weird file name?
|
||||
|
||||
|
|
@ -270,22 +311,27 @@ Historical roots for the entire beacon chain history are stored in the state - t
|
|||
|
||||
The genesis era file uses the genesis validators root for two reasons: it allows disambiguating otherwise similar chains and the genesis state does not yet have a historical root to use.
|
||||
|
||||
The era numbers are zero-filled so that they trivially can be sorted - 5 digits is enough for 99999 eras or ~312 years
|
||||
The era numbers are zero-filled so that they trivially can be sorted - 5 digits is enough for 99999 eras or ~312 years.
|
||||
|
||||
Using the first era number and the last root allows a reading application to quickly determine the range of data in the era file.
|
||||
|
||||
## How long is an era?
|
||||
|
||||
An era is typically 8192 slots, or roughly 27.3 hours - a bit more than a day.
|
||||
An era is typically `8192` slots (in the mainnet configuration), or roughly 27.3 hours.
|
||||
|
||||
## What happens after the merge?
|
||||
|
||||
Era files will store execution block contents, but not execution states (these are too large) - a full era history thus gives the full ethereum history from the merge onwards, for convenient cold storage.
|
||||
Era files will store execution block contents, but not execution states (these are too large) - a full era history thus gives the full ethereum history from the merge onwards for convenient cold storage. Work is underway to similarily cover the rest of history.
|
||||
|
||||
## What is a "era state" and why use it?
|
||||
## Which state should be stored in the era file?
|
||||
|
||||
The state transition function in ethereum does 3 things: slot processing, epoch processing and block processing, in that order. In particular, the slot and epoch processing is done for every slot and epoch, but the block processing may be skipped. When epoch processing is done, all the epoch-related fields in the state have been written, and a new epoch can begin - it's thus reasonable to say that the epoch processing is the last thing that happens in an epoch and the block processing happens in the context of the new epoch.
|
||||
|
||||
Storing the "era state" without the block applied means that any block from the new epoch can be applied to it - if two histories exist, one that skips the first block in the epoch and one that includes it, one can use the same era state in both cases.
|
||||
The protocol favours the state root with the block applied, as both `BeaconState.state_roots` and `BeaconBlock.state_root`, thus era files follow suit.
|
||||
|
||||
One downside is that future blocks will store the state root of the "era state" with the block applied, making it slightly harder to verify that the state in a given era file is part of a particular history.
|
||||
The alternative that was considered is to store the state without the block applied - this has several advantages:
|
||||
|
||||
TODO: consider workarounds for the above point - one can state-transition to find the right state root, but that increases verification requirements significantly.
|
||||
* the era file to be used both for future histories with and without a block at the beginning
|
||||
* no special case is needed when replaying blocks from era files - all are applied in the order they appear in the era file
|
||||
|
||||
In the end though, the applied block state is used throughout in the protocol - given a block, the state root in the block is computed with the data from the block applied and this later gets stored in `state_roots` which forms the basis for `historical_roots`. In API:s such as the beacon API, the canonical state root of a slot is the state with the block of that slot applied, if it is part of the canonical history given by the head.
|
||||
|
|
|
|||
|
|
@ -26,17 +26,12 @@ const
|
|||
FAR_FUTURE_ERA* = Era(not 0'u64)
|
||||
|
||||
type
|
||||
|
||||
Type* = array[2, byte]
|
||||
|
||||
Header* = object
|
||||
typ*: Type
|
||||
len*: int
|
||||
|
||||
EraFile* = object
|
||||
handle: IoHandle
|
||||
start: Slot
|
||||
|
||||
Index* = object
|
||||
startSlot*: Slot
|
||||
offsets*: seq[int64] # Absolute positions in file
|
||||
|
|
@ -56,17 +51,17 @@ proc toString(v: IoErrorCode): string =
|
|||
try: ioErrorMsg(v)
|
||||
except Exception as e: raiseAssert e.msg
|
||||
|
||||
func eraFileName*(
|
||||
cfg: RuntimeConfig, genesis_validators_root: Eth2Digest,
|
||||
historical_roots: openArray[Eth2Digest], era: Era): string =
|
||||
try:
|
||||
let
|
||||
historicalRoot =
|
||||
if era == Era(0): genesis_validators_root
|
||||
elif era > historical_roots.lenu64(): Eth2Digest()
|
||||
else: historical_roots[int(uint64(era)) - 1]
|
||||
func eraRoot*(
|
||||
genesis_validators_root: Eth2Digest,
|
||||
historical_roots: openArray[Eth2Digest], era: Era): Opt[Eth2Digest] =
|
||||
if era == Era(0): ok(genesis_validators_root)
|
||||
elif era <= historical_roots.lenu64(): ok(historical_roots[int(uint64(era) - 1)])
|
||||
else: err()
|
||||
|
||||
&"{cfg.name()}-{era.uint64:05}-{1:05}-{shortLog(historicalRoot)}.era"
|
||||
func eraFileName*(
|
||||
cfg: RuntimeConfig, era: Era, eraRoot: Eth2Digest): string =
|
||||
try:
|
||||
&"{cfg.name()}-{era.uint64:05}-{shortLog(eraRoot)}.era"
|
||||
except ValueError as exc:
|
||||
raiseAssert exc.msg
|
||||
|
||||
|
|
@ -83,7 +78,8 @@ proc appendHeader(f: IoHandle, typ: Type, dataLen: int): Result[int64, string] =
|
|||
|
||||
ok(start)
|
||||
|
||||
proc appendRecord*(f: IoHandle, typ: Type, data: openArray[byte]): Result[int64, string] =
|
||||
proc appendRecord*(
|
||||
f: IoHandle, typ: Type, data: openArray[byte]): Result[int64, string] =
|
||||
let start = ? appendHeader(f, typ, data.len())
|
||||
? append(f, data)
|
||||
ok(start)
|
||||
|
|
@ -91,13 +87,16 @@ proc appendRecord*(f: IoHandle, typ: Type, data: openArray[byte]): Result[int64,
|
|||
proc toCompressedBytes(item: auto): seq[byte] =
|
||||
snappy.encodeFramed(SSZ.encode(item))
|
||||
|
||||
proc appendRecord*(f: IoHandle, v: ForkyTrustedSignedBeaconBlock): Result[int64, string] =
|
||||
proc appendRecord*(
|
||||
f: IoHandle, v: ForkyTrustedSignedBeaconBlock): Result[int64, string] =
|
||||
f.appendRecord(SnappyBeaconBlock, toCompressedBytes(v))
|
||||
|
||||
proc appendRecord*(f: IoHandle, v: ForkyBeaconState): Result[int64, string] =
|
||||
f.appendRecord(SnappyBeaconState, toCompressedBytes(v))
|
||||
|
||||
proc appendIndex*(f: IoHandle, startSlot: Slot, offsets: openArray[int64]): Result[int64, string] =
|
||||
proc appendIndex*(
|
||||
f: IoHandle, startSlot: Slot, offsets: openArray[int64]):
|
||||
Result[int64, string] =
|
||||
let
|
||||
len = offsets.len() * sizeof(int64) + 16
|
||||
pos = ? f.appendHeader(E2Index, len)
|
||||
|
|
@ -225,14 +224,13 @@ proc readIndex*(f: IoHandle): Result[Index, string] =
|
|||
|
||||
type
|
||||
EraGroup* = object
|
||||
eraStart: int64
|
||||
slotIndex*: Index
|
||||
|
||||
proc init*(T: type EraGroup, f: IoHandle, startSlot: Option[Slot]): Result[T, string] =
|
||||
let eraStart = ? f.appendHeader(E2Version, 0)
|
||||
proc init*(
|
||||
T: type EraGroup, f: IoHandle, startSlot: Option[Slot]): Result[T, string] =
|
||||
discard ? f.appendHeader(E2Version, 0)
|
||||
|
||||
ok(EraGroup(
|
||||
eraStart: eraStart,
|
||||
slotIndex: Index(
|
||||
startSlot: startSlot.get(Slot(0)),
|
||||
offsets: newSeq[int64](
|
||||
|
|
@ -240,16 +238,20 @@ proc init*(T: type EraGroup, f: IoHandle, startSlot: Option[Slot]): Result[T, st
|
|||
else: 0
|
||||
))))
|
||||
|
||||
proc update*(g: var EraGroup, f: IoHandle, slot: Slot, szBytes: openArray[byte]): Result[void, string] =
|
||||
proc update*(
|
||||
g: var EraGroup, f: IoHandle, slot: Slot, szBytes: openArray[byte]):
|
||||
Result[void, string] =
|
||||
doAssert slot >= g.slotIndex.startSlot
|
||||
# doAssert slot < g.slotIndex.startSlot + g.slotIndex.offsets.len
|
||||
|
||||
g.slotIndex.offsets[int(slot - g.slotIndex.startSlot)] =
|
||||
try:
|
||||
? f.appendRecord(SnappyBeaconBlock, szBytes)
|
||||
except CatchableError as e: raiseAssert e.msg # TODO fix snappy
|
||||
? f.appendRecord(SnappyBeaconBlock, szBytes)
|
||||
|
||||
ok()
|
||||
|
||||
proc finish*(g: var EraGroup, f: IoHandle, state: ForkyBeaconState): Result[void, string] =
|
||||
proc finish*(
|
||||
g: var EraGroup, f: IoHandle, state: ForkyBeaconState):
|
||||
Result[void, string] =
|
||||
let
|
||||
statePos = ? f.appendRecord(state)
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import
|
|||
snappy,
|
||||
chronicles, confutils, stew/[byteutils, io2], eth/db/kvstore_sqlite3,
|
||||
../beacon_chain/networking/network_metadata,
|
||||
../beacon_chain/[beacon_chain_db],
|
||||
../beacon_chain/[beacon_chain_db, era_db],
|
||||
../beacon_chain/consensus_object_pools/[blockchain_dag],
|
||||
../beacon_chain/spec/datatypes/[phase0, altair, bellatrix],
|
||||
../beacon_chain/spec/[
|
||||
|
|
@ -41,6 +41,7 @@ type
|
|||
dumpBlock = "Extract a (trusted) SignedBeaconBlock from the database"
|
||||
putBlock = "Store a given SignedBeaconBlock in the database, potentially updating some of the pointers"
|
||||
rewindState = "Extract any state from the database based on a given block and slot, replaying if needed"
|
||||
verifyEra = "Verify a single era file"
|
||||
exportEra = "Write an experimental era file"
|
||||
importEra = "Import era files to the database"
|
||||
validatorPerf
|
||||
|
|
@ -134,6 +135,10 @@ type
|
|||
argument
|
||||
desc: "Slot".}: uint64
|
||||
|
||||
of DbCmd.verifyEra:
|
||||
eraFile* {.
|
||||
desc: "Era file name".}: string
|
||||
|
||||
of DbCmd.exportEra:
|
||||
era* {.
|
||||
defaultValue: 0
|
||||
|
|
@ -436,6 +441,16 @@ func atCanonicalSlot(dag: ChainDAGRef, bid: BlockId, slot: Slot): Opt[BlockSlotI
|
|||
else:
|
||||
ok BlockSlotId.init((? dag.atSlot(bid, slot - 1)).bid, slot)
|
||||
|
||||
proc cmdVerifyEra(conf: DbConf, cfg: RuntimeConfig) =
|
||||
let
|
||||
f = EraFile.open(conf.eraFile).valueOr:
|
||||
echo error
|
||||
quit 1
|
||||
root = f.verify(cfg).valueOr:
|
||||
echo error
|
||||
quit 1
|
||||
echo root
|
||||
|
||||
proc cmdExportEra(conf: DbConf, cfg: RuntimeConfig) =
|
||||
let db = BeaconChainDB.new(conf.databaseDir.string, readOnly = true)
|
||||
defer: db.close()
|
||||
|
|
@ -468,7 +483,7 @@ proc cmdExportEra(conf: DbConf, cfg: RuntimeConfig) =
|
|||
if era == 0: none(Slot)
|
||||
else: some((era - 1).start_slot)
|
||||
endSlot = era.start_slot
|
||||
canonical = dag.atCanonicalSlot(dag.head.bid, endSlot).valueOr:
|
||||
eraBid = dag.atSlot(dag.head.bid, endSlot).valueOr:
|
||||
echo "Skipping ", era, ", blocks not available"
|
||||
continue
|
||||
|
||||
|
|
@ -476,10 +491,12 @@ proc cmdExportEra(conf: DbConf, cfg: RuntimeConfig) =
|
|||
echo "Written all complete eras"
|
||||
break
|
||||
|
||||
let name = withState(dag.headState):
|
||||
eraFileName(
|
||||
cfg, state.data.genesis_validators_root,
|
||||
state.data.historical_roots.asSeq, era)
|
||||
let
|
||||
eraRoot = withState(dag.headState):
|
||||
eraRoot(
|
||||
state.data.genesis_validators_root, state.data.historical_roots.asSeq,
|
||||
era).expect("have era root since we checked slot")
|
||||
name = eraFileName(cfg, era, eraRoot)
|
||||
|
||||
if isFile(name):
|
||||
echo "Skipping ", name, " (already exists)"
|
||||
|
|
@ -498,7 +515,7 @@ proc cmdExportEra(conf: DbConf, cfg: RuntimeConfig) =
|
|||
group.update(e2, blocks[i].slot, tmp).get()
|
||||
|
||||
withTimer(timers[tState]):
|
||||
dag.withUpdatedState(tmpState[], canonical) do:
|
||||
dag.withUpdatedState(tmpState[], eraBid) do:
|
||||
withState(state):
|
||||
group.finish(e2, state.data).get()
|
||||
do: raiseAssert "withUpdatedState failed"
|
||||
|
|
@ -546,15 +563,7 @@ proc cmdImportEra(conf: DbConf, cfg: RuntimeConfig) =
|
|||
db.putBlock(blck)
|
||||
blocks += 1
|
||||
elif header.typ == SnappyBeaconState:
|
||||
withTimer(timers[tState]):
|
||||
let uncompressed = decodeFramed(data)
|
||||
let state = try: newClone(
|
||||
readSszForkedHashedBeaconState(cfg, uncompressed))
|
||||
except CatchableError as exc:
|
||||
error "Invalid snappy state", msg = exc.msg, file
|
||||
continue
|
||||
withState(state[]):
|
||||
db.putState(state)
|
||||
info "Skipping beacon state (use reindexing to recreate state snapshots)"
|
||||
states += 1
|
||||
else:
|
||||
info "Skipping record", typ = toHex(header.typ)
|
||||
|
|
@ -1036,6 +1045,8 @@ when isMainModule:
|
|||
cmdPutBlock(conf, cfg)
|
||||
of DbCmd.rewindState:
|
||||
cmdRewindState(conf, cfg)
|
||||
of DbCmd.verifyEra:
|
||||
cmdVerifyEra(conf, cfg)
|
||||
of DbCmd.exportEra:
|
||||
cmdExportEra(conf, cfg)
|
||||
of DbCmd.importEra:
|
||||
|
|
|
|||
Loading…
Reference in New Issue