From ba99c8fe4f3649b8997133b2f92c5365c18e4ed3 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Fri, 7 Jan 2022 11:13:19 +0100 Subject: [PATCH] update era file documentation / impl (#3226) Overhaul of era files, including documentation and reference implementations * store blocks, then state, then slot indices for easy lookup at low cost * document era file rationale * altair+ support in era writer --- beacon_chain/beacon_chain_db.nim | 34 +++ .../consensus_object_pools/blockchain_dag.nim | 13 + beacon_chain/rpc/rest_config_api.nim | 3 +- beacon_chain/spec/presets.nim | 10 +- docs/e2store.md | 225 +++++++++++--- ncli/e2store.nim | 278 +++++++++++++----- ncli/e2store.py | 138 ++++++--- ncli/ncli_db.nim | 47 +-- tests/test_beacon_chain_db.nim | 12 + 9 files changed, 584 insertions(+), 176 deletions(-) diff --git a/beacon_chain/beacon_chain_db.nim b/beacon_chain/beacon_chain_db.nim index 57fc83ae6..1859bc6ca 100644 --- a/beacon_chain/beacon_chain_db.nim +++ b/beacon_chain/beacon_chain_db.nim @@ -633,6 +633,40 @@ proc getMergeBlock*(db: BeaconChainDB, key: Eth2Digest): else: result.err() +proc getPhase0BlockSSZ(db: BeaconChainDBV0, key: Eth2Digest, data: var seq[byte]): bool = + let dataPtr = unsafeAddr data # Short-lived + var success = true + proc decode(data: openArray[byte]) = + try: dataPtr[] = snappy.decode(data, maxDecompressedDbRecordSize) + except CatchableError: success = false + db.backend.get(subkey(phase0.SignedBeaconBlock, key), decode).expectDb() and success + +proc getPhase0BlockSSZ*(db: BeaconChainDB, key: Eth2Digest, data: var seq[byte]): bool = + let dataPtr = unsafeAddr data # Short-lived + var success = true + proc decode(data: openArray[byte]) = + try: dataPtr[] = snappy.decode(data, maxDecompressedDbRecordSize) + except CatchableError: success = false + db.blocks.get(key.data, decode).expectDb() and success or + db.v0.getPhase0BlockSSZ(key, data) + +proc getAltairBlockSSZ*(db: BeaconChainDB, key: Eth2Digest, data: var seq[byte]): bool = + let dataPtr = unsafeAddr data # Short-lived + var success = true + proc decode(data: openArray[byte]) = + try: dataPtr[] = snappy.decode(data, maxDecompressedDbRecordSize) + except CatchableError: success = false + db.altairBlocks.get(key.data, decode).expectDb() and success + +proc getMergeBlockSSZ*(db: BeaconChainDB, key: Eth2Digest, data: var seq[byte]): bool = + let dataPtr = unsafeAddr data # Short-lived + var success = true + proc decode(data: openArray[byte]) = + try: dataPtr[] = snappy.decode(data, maxDecompressedDbRecordSize) + except CatchableError: success = false + + db.mergeBlocks.get(key.data, decode).expectDb() and success + proc getStateOnlyMutableValidators( immutableValidators: openArray[ImmutableValidatorData2], store: KvStoreRef, key: openArray[byte], output: var ForkyBeaconState, diff --git a/beacon_chain/consensus_object_pools/blockchain_dag.nim b/beacon_chain/consensus_object_pools/blockchain_dag.nim index 8e05f05bc..39f189c8c 100644 --- a/beacon_chain/consensus_object_pools/blockchain_dag.nim +++ b/beacon_chain/consensus_object_pools/blockchain_dag.nim @@ -1661,3 +1661,16 @@ proc aggregateAll*( err("aggregate: no attesting keys") else: ok(finish(aggregateKey)) + +proc getBlockSSZ*(dag: ChainDAGRef, id: BlockId, bytes: var seq[byte]): bool = + # Load the SSZ-encoded data of a block into `bytes`, overwriting the existing + # content + # careful: there are two snappy encodings in use, with and without framing! + # Returns true if the block is found, false if not + case dag.cfg.blockForkAtEpoch(id.slot.epoch) + of BeaconBlockFork.Phase0: + dag.db.getPhase0BlockSSZ(id.root, bytes) + of BeaconBlockFork.Altair: + dag.db.getAltairBlockSSZ(id.root, bytes) + of BeaconBlockFork.Bellatrix: + dag.db.getMergeBlockSSZ(id.root, bytes) diff --git a/beacon_chain/rpc/rest_config_api.nim b/beacon_chain/rpc/rest_config_api.nim index 019d986d0..2715b9645 100644 --- a/beacon_chain/rpc/rest_config_api.nim +++ b/beacon_chain/rpc/rest_config_api.nim @@ -23,8 +23,7 @@ proc installConfigApiHandlers*(router: var RestRouter, node: BeaconNode) = RestApiResponse.prepareJsonResponse( ( # https://github.com/ethereum/consensus-specs/blob/v1.0.1/configs/mainnet/phase0.yaml - CONFIG_NAME: - const_preset, + CONFIG_NAME: node.dag.cfg.name(), # https://github.com/ethereum/consensus-specs/blob/v1.1.3/presets/mainnet/phase0.yaml MAX_COMMITTEES_PER_SLOT: diff --git a/beacon_chain/spec/presets.nim b/beacon_chain/spec/presets.nim index 0a9b3789c..d52950c6f 100644 --- a/beacon_chain/spec/presets.nim +++ b/beacon_chain/spec/presets.nim @@ -36,6 +36,8 @@ type PRESET_BASE*: string + CONFIG_NAME*: string + # Transition TERMINAL_TOTAL_DIFFICULTY*: UInt256 TERMINAL_BLOCK_HASH*: BlockHash @@ -148,8 +150,6 @@ const "DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF", "DOMAIN_CONTRIBUTION_AND_PROOF", - "CONFIG_NAME", - "TRANSITION_TOTAL_DIFFICULTY", # Name that appears in some altair alphas, obsolete, remove when no more testnets ] @@ -444,3 +444,9 @@ proc readRuntimeConfig*( msg: "Config not compatible with binary, compile with -d:const_preset=" & cfg.PRESET_BASE) (cfg, unknowns) + +template name*(cfg: RuntimeConfig): string = + if cfg.CONFIG_NAME.len() > 0: + cfg.CONFIG_NAME + else: + const_preset diff --git a/docs/e2store.md b/docs/e2store.md index 2225dd9be..ce6787cc1 100644 --- a/docs/e2store.md +++ b/docs/e2store.md @@ -32,34 +32,39 @@ The following python code can be used to read an e2 file: ```python import sys, struct -with open(sys.argv[1], "rb") as f: +def read_entry(f): header = f.read(8) - typ = header[0:2] # First 2 bytes for type + if not header: return None - if typ != b"e2": - raise RuntimeError("this is not an e2store file") + typ = header[0:2] # 2 bytes of type + dlen = struct.unpack("--.era` with era and count hex-encoded to 8 digits. +## File name + +`.era` file names follow a simple convention: `---.era`: + +* `config-name` is the `CONFIG_NAME` field of the runtime configation (`mainnet`, `prater`, etc) +* `era-number` is the number of the _last_ era stored in the file - for example, the genesis era file has number 0 - as a 5-digit 0-filled decimal integer +* `era-count` is the number of eras stored in the file, as a 5-digit 0-filled decimal integer +* `short-historical-root` is the first 4 bytes of the last historical root in the last state in the era file, lower-case hex-encoded (8 characters), except the genesis era which instead uses the `genesis_validators_root` field from the genesis state. + * The root is available as `state.historical_roots[era - 1]` except genesis, whose historical root is all `0` + +An era file containing the mainnet genesis is thus named `mainnet-00000000-0000-0001.era`, and the era after that `mainnet-40cf2f3c-0001-0001.era`. + +## Structure An `.era` file is structured in the following way: ``` era := group+ -group := canonical-state | blocks* +group := Version | block* | canonical-state | other-entries* | slot-index(block)? | slot-index(state) +block := CompressedSignedBeaconBlock +canonical-state := CompressedBeaconState ``` -The `canonical-state` is the state of the slot that immediately follows the end of the era without applying blocks from the next era. For example, for the era that covers the first 8192 slots will have all blocks applied up to slot 8191 and will `process_slots` up to 8192. The genesis group contains only the genesis state but no blocks. +The `block` entries of a group include all blocks pertaining to an era. For example, the group representing era one will have all blocks from slot 0 up to and including block 8191. + +The `canonical-state` is the state of the slot that immediately follows the end of the era without applying blocks from the next era. For example, era 1 that covers the first 8192 slots will have all blocks applied up to slot 8191 and will `process_slots` up to 8192. The genesis group contains only the genesis state but no blocks. + +`slot-index(state)` is a `SlotIndex` entry with `count = 1` for the `CompressedBeaconState` entry of that era, pointing out the offset where the state entry begins. (TODO: consider count > 1 for files that cover multiple eras - breaks trivial composability of each era snippet but allows instant lookup in multi-era files) + +`slot-index(block)` is a `SlotIndex` entry with `count = SLOTS_PER_HISTORICAL_ROOT` for the `CompressedSignedBeaconBlock` entries in that era, pointing out the offsets of each block in the era. It is omitted for the genesis era. + +`other-entries` is the extension point for future record types in the era file. The positioning of these allows the indices to continue to be looked up from the back. + +The structure of the era file gives it the following properties: + +* the indices at the end are fixed-length: they can be used to discover the beginning of an era if the end of it is known +* the start slot field of the state slot index idenfifies which era the group pertains to +* the state in the era file is the end state after having applied all the blocks in the era - the `block_roots` entries in the state can be used to discover the digest of the blocks - either to verify the intergrity of the era file or to quickly load block roots without computing them +* each group in the era file is full, indendent era file - eras can freely be split and combined + +## Reading era files + +```python +def read_era_file(name): + # Print contents of an era file, backwards + with open(name, "rb") as f: + + # Seek to end of file to figure out the indices of the state and blocks + f.seek(0, 2) + + groups = 0 + while True: + if f.tell() < 8: + break + + (start_slot, state_index_start, state_slot_offsets) = read_slot_index(f) + + print( + "State slot:", start_slot, + "state index start:", state_index_start, + "offsets", state_slot_offsets) + + # The start of the state index record is the end of the block index record, if any + f.seek(state_index_start) + + # This can underflow! Python should complain when seeking - ymmv + prev_group = state_index_start + state_slot_offsets[0] - 8 + if start_slot > 0: + (block_slot, block_index_start, block_slot_offsets) = read_slot_index(f) + + print( + "Block start slot:", block_slot, + "block index start:", block_index_start, + "offsets", len(block_slot_offsets)) + + if any((x for x in block_slot_offsets if x != 0)): + # This can underflow! Python should complain when seeking - ymmv + prev_group = block_index_start + [x for x in block_slot_offsets if x != 0][0] - 8 + + print("Previous group starts at:", prev_group) + # The beginning of the first block (or the state, if there are no blocks) + # is the end of the previous group + f.seek(prev_group) # Skip header + + groups += 1 + print("Groups in file:", groups) +``` + +# FAQ + +## Why snappy framed compression? + +* The networking protocol uses snappy framed compression, avoiding the need to re-compress data to serve blocks +* Each entry can be decompressed separately +* It's fast and compresses decently - some compression stats for the first 100 eras: + * Uncompressed: 8.4gb + * Snappy compression: 4.7gb + * `xz` of uncompressed: 3.8gb + +## Why SLOTS_PER_HISTORICAL_ROOT blocks per state? + +The state stores the block root of the latest `SLOTS_PER_HISTORICAL_ROOT` blocks - storing one state per that many blocks allows verifying the integrity of the blocks easily against the given state, and ensures that all block and state root information remains available, for example to validate states and blocks against `historical_roots`. + +## Why include the state at all? + +This is a tradeoff between being able to access state data such as validator keys and balances directly vs and recreating it by applying each block one by one from from genesis. Given an era file, you can always start processing the chain from there onwards. + +## Why the weird file name? + +Historical roots for the entire beacon chain history are stored in the state - thus, with a recent state one can quickly judge if an era file is part of the same history - this is useful for example when performing checkpoint sync. + +The genesis era file uses the genesis validators root for two reasons: it allows disambiguating otherwise similar chains and the genesis state does not yet have a historical root to use. + +The era numbers are zero-filled so that they trivially can be sorted - 5 digits is enough for 99999 eras or ~312 years + +## How long is an era? + +An era is typically 8192 slots, or roughly 27.3 hours - a bit more than a day. + +## What happens after the merge? + +Era files will store execution block contents, but not execution states (these are too large) - a full era history thus gives the full ethereum history from the merge onwards, for convenient cold storage. + +## What is a "canonical state" and why use it? + +The state transition function in ethereum does 3 things: slot processing, epoch processing and block processing, in that order. In particular, the slot and epoch processing is done for every slot and epoch, but the block processing may be skipped. When epoch processing is done, all the epoch-related fields in the state have been written, and a new epoch can begin - it's thus reasonable to say that the epoch processing is the last thing that happens in an epoch and the block processing happens in the context of the new epoch. + +Storing the "canonical state" without the block applied means that any block from the new epoch can be applied to it - if two histories exist, one that skips the first block in the epoch and one that includes it, one can use the same canonical state in both cases. -Era files place the state first for a number of reasons: the state is then guaranteed to contain all public keys and block roots needed to verify the blocks in the file. A special case is the genesis era file - this file contains only the genesis state. diff --git a/ncli/e2store.nim b/ncli/e2store.nim index df823966b..49d20eed4 100644 --- a/ncli/e2store.nim +++ b/ncli/e2store.nim @@ -1,96 +1,240 @@ {.push raises: [Defect].} import - stew/[endians2, results], + std/strformat, + stew/[arrayops, endians2, io2, results], snappy, snappy/framing, - ../beacon_chain/spec/datatypes/phase0, + ../beacon_chain/spec/forks, ../beacon_chain/spec/eth2_ssz_serialization const - E2Version = [byte 0x65, 0x32] - E2Index = [byte 0x69, 0x32] - SnappyBeaconBlock = [byte 0x01, 0x00] - SnappyBeaconState = [byte 0x02, 0x00] + E2Version* = [byte 0x65, 0x32] + E2Index* = [byte 0x69, 0x32] + + SnappyBeaconBlock* = [byte 0x01, 0x00] + SnappyBeaconState* = [byte 0x02, 0x00] + + TypeFieldLen = 2 + LengthFieldLen = 6 + HeaderFieldLen = TypeFieldLen + LengthFieldLen type - E2Store* = object - data: File - index: File - slot: Slot + Type* = array[2, byte] Header* = object - typ*: array[2, byte] - len*: uint64 + typ*: Type + len*: int -proc append(f: File, data: openArray[byte]): Result[void, string] = + EraFile* = object + handle: IoHandle + start: Slot + + Index* = object + startSlot*: Slot + offsets*: seq[int64] # Absolute positions in file + +proc toString(v: IoErrorCode): string = + try: ioErrorMsg(v) + except Exception as e: raiseAssert e.msg + +func eraFileName*(cfg: RuntimeConfig, state: ForkyBeaconState, era: uint64): string = try: - if writeBytes(f, data, 0, data.len()) != data.len: - err("Cannot write to file") - else: - ok() - except CatchableError as exc: - err(exc.msg) + let + historicalRoot = + if era == 0: state.genesis_validators_root + elif era > state.historical_roots.lenu64(): Eth2Digest() + else: state.historical_roots.asSeq()[era - 1] -proc readHeader(f: File): Result[Header, string] = - try: - var buf: array[8, byte] - if system.readBuffer(f, addr buf[0], 8) != 8: - return err("Not enough bytes for header") - except CatchableError as e: - return err("Cannot read header") + &"{cfg.name()}-{era.int:05}-{1:05}-{shortLog(historicalRoot)}.era" + except ValueError as exc: + raiseAssert exc.msg -proc appendRecord(f: File, typ: array[2, byte], data: openArray[byte]): Result[int64, string] = - try: - let start = getFilePos(f) - let dlen = toBytesLE(data.len().uint64) +proc append(f: IoHandle, data: openArray[byte]): Result[void, string] = + if (? writeFile(f, data).mapErr(toString)) != data.len.uint: + return err("could not write data") + ok() - ? append(f, typ) - ? append(f, dlen.toOpenArray(0, 5)) - ? append(f, data) - ok(start) - except CatchableError as e: - err(e.msg) +proc appendHeader(f: IoHandle, typ: Type, dataLen: int): Result[int64, string] = + let start = ? getFilePos(f).mapErr(toString) -proc open*(T: type E2Store, path: string, name: string, firstSlot: Slot): Result[E2Store, string] = - let - data = - try: open(path / name & ".e2s", fmWrite) - except CatchableError as e: return err(e.msg) - index = - try: system.open(path / name & ".e2i", fmWrite) - except CatchableError as e: - close(data) - return err(e.msg) - discard ? appendRecord(data, E2Version, []) - discard ? appendRecord(index, E2Index, []) - ? append(index, toBytesLE(firstSlot.uint64)) + ? append(f, typ) + ? append(f, toBytesLE(dataLen.uint64).toOpenArray(0, 5)) - ok(E2Store(data: data, index: index, slot: firstSlot)) + ok(start) -func close*(store: var E2Store) = - store.data.close() - store.index.close() +proc appendRecord*(f: IoHandle, typ: Type, data: openArray[byte]): Result[int64, string] = + let start = ? appendHeader(f, typ, data.len()) + ? append(f, data) + ok(start) proc toCompressedBytes(item: auto): seq[byte] = try: - let - payload = SSZ.encode(item) - framingFormatCompress(payload) + framingFormatCompress(SSZ.encode(item)) except CatchableError as exc: raiseAssert exc.msg # shouldn't happen -proc appendRecord*(store: var E2Store, v: phase0.TrustedSignedBeaconBlock): Result[void, string] = - if v.message.slot < store.slot: - return err("Blocks must be written in order") - let start = store.data.appendRecord(SnappyBeaconBlock, toCompressedBytes(v)).get() - while store.slot < v.message.slot: - ? append(store.index, toBytesLE(0'u64)) - store.slot += 1 - ? append(store.index, toBytesLE(start.uint64)) - store.slot += 1 +proc appendRecord*(f: IoHandle, v: ForkyTrustedSignedBeaconBlock): Result[int64, string] = + f.appendRecord(SnappyBeaconBlock, toCompressedBytes(v)) + +proc appendRecord*(f: IoHandle, v: ForkyBeaconState): Result[int64, string] = + f.appendRecord(SnappyBeaconState, toCompressedBytes(v)) + +proc appendIndex*(f: IoHandle, startSlot: Slot, offsets: openArray[int64]): Result[int64, string] = + let + len = offsets.len() * sizeof(int64) + 16 + pos = ? f.appendHeader(E2Index, len) + + ? f.append(startSlot.uint64.toBytesLE()) + + for v in offsets: + ? f.append(cast[uint64](v - pos).toBytesLE()) + + ? f.append(offsets.lenu64().toBytesLE()) + + ok(pos) + +proc appendRecord(f: IoHandle, index: Index): Result[int64, string] = + f.appendIndex(index.startSlot, index.offsets) + +proc checkBytesLeft(f: IoHandle, expected: int64): Result[void, string] = + let size = ? getFileSize(f).mapErr(toString) + if expected > size: + return err("Record extends past end of file") + + let pos = ? getFilePos(f).mapErr(toString) + if expected > size - pos: + return err("Record extends past end of file") ok() -proc appendRecord*(store: var E2Store, v: phase0.BeaconState): Result[void, string] = - discard ? store.data.appendRecord(SnappyBeaconState, toCompressedBytes(v)) +proc readFileExact(f: IoHandle, buf: var openArray[byte]): Result[void, string] = + if (? f.readFile(buf).mapErr(toString)) != buf.len().uint: + return err("missing data") + ok() + +proc readHeader(f: IoHandle): Result[Header, string] = + var buf: array[10, byte] + ? readFileExact(f, buf.toOpenArray(0, 7)) + + var + typ: Type + discard typ.copyFrom(buf) + + # Cast safe because we had only 6 bytes of length data + let + len = cast[int64](uint64.fromBytesLE(buf.toOpenArray(2, 9))) + + # No point reading these.. + if len > int.high(): return err("header length exceeds int.high") + + # Must have at least that much data, or header is invalid + ? f.checkBytesLeft(len) + + ok(Header(typ: typ, len: int(len))) + +proc readRecord(f: IoHandle, data: var seq[byte]): Result[Header, string] = + let header = ? readHeader(f) + if header.len > 0: + ? f.checkBytesLeft(header.len) + + data.setLen(header.len) + + ? readFileExact(f, data) + + ok(header) + +proc readIndexCount*(f: IoHandle): Result[int, string] = + var bytes: array[8, byte] + ? f.readFileExact(bytes) + + let count = uint64.fromBytesLE(bytes) + if count > (int.high() div 8) - 3: return err("count: too large") + + let size = uint64(? f.getFileSize().mapErr(toString)) + # Need to have at least this much data in the file to read an index with + # this count + if count > (size div 8 + 3): return err("count: too large") + + ok(int(count)) # Sizes checked against int above + +proc findIndexStartOffset*(f: IoHandle): Result[int64, string] = + ? f.setFilePos(-8, SeekPosition.SeekCurrent).mapErr(toString) + + let + count = ? f.readIndexCount() # Now we're back at the end of the index + bytes = count.int64 * 8 + 24 + + ok(-bytes) + +proc readIndex*(f: IoHandle): Result[Index, string] = + let + startPos = ? f.getFilePos().mapErr(toString) + fileSize = ? f.getFileSize().mapErr(toString) + header = ? f.readHeader() + + if header.typ != E2Index: return err("not an index") + if header.len < 16: return err("index entry too small") + if header.len mod 8 != 0: return err("index length invalid") + + var buf: array[8, byte] + ? f.readFileExact(buf) + let + slot = uint64.fromBytesLE(buf) + count = header.len div 8 - 2 + + var offsets = newSeqUninitialized[int64](count) + for i in 0.. fileSize: return err("Invalid offset") + offsets[i] = absolute + + ? f.readFileExact(buf) + if uint64(count) != uint64.fromBytesLE(buf): return err("invalid count") + + # technically not an error, but we'll throw this sanity check in here.. + if slot > int32.high().uint64: return err("fishy slot") + + ok(Index(startSlot: Slot(slot), offsets: offsets)) + +type + EraGroup* = object + eraStart: int64 + slotIndex*: Index + +proc init*(T: type EraGroup, f: IoHandle, startSlot: Option[Slot]): Result[T, string] = + let eraStart = ? f.appendHeader(E2Version, 0) + + ok(EraGroup( + eraStart: eraStart, + slotIndex: Index( + startSlot: startSlot.get(Slot(0)), + offsets: newSeq[int64]( + if startSlot.isSome(): SLOTS_PER_HISTORICAL_ROOT.int + else: 0 + )))) + +proc update*(g: var EraGroup, f: IoHandle, slot: Slot, sszBytes: openArray[byte]): Result[void, string] = + doAssert slot >= g.slotIndex.startSlot + g.slotIndex.offsets[int(slot - g.slotIndex.startSlot)] = + try: + ? f.appendRecord(SnappyBeaconBlock, framingFormatCompress(sszBytes)) + except CatchableError as e: raiseAssert e.msg # TODO fix snappy + + ok() + +proc finish*(g: var EraGroup, f: IoHandle, state: ForkyBeaconState): Result[void, string] = + let + statePos = ? f.appendRecord(state) + + if state.slot > Slot(0): + discard ? f.appendRecord(g.slotIndex) + + discard ? f.appendIndex(state.slot, [statePos]) + ok() diff --git a/ncli/e2store.py b/ncli/e2store.py index b858dda65..f7825aa09 100644 --- a/ncli/e2store.py +++ b/ncli/e2store.py @@ -1,49 +1,113 @@ import sys, struct -def read_e2store(name): - with open(name, "rb") as f: - header = f.read(8) - typ = header[0:2] # First 2 bytes for type +def read_entry(f): + header = f.read(8) + if not header: return (None, None) - if typ != b"e2": - raise RuntimeError("this is not an e2store file") + typ = header[0:2] # 2 bytes of type + dlen = struct.unpack(" 0: + (block_slot, block_index_start, block_slot_offsets) = read_slot_index(f) + + print( + "Block start slot:", block_slot, + "block index start:", block_index_start, + "offsets", len(block_slot_offsets)) + + if any((x for x in block_slot_offsets if x != 0)): + # This can underflow! Python should complain when seeking - ymmv + prev_group = block_index_start + [x for x in block_slot_offsets if x != 0][0] - 8 + + print("Previous group starts at:", prev_group) + # The beginning of the first block (or the state, if there are no blocks) + # is the end of the previous group + f.seek(prev_group) # Skip header + + groups += 1 + print("Groups in file:", groups) + +def print_stats(name): + with open(name, "rb") as f: + sizes = {} + entries = 0 while True: - header = f.read(8) # Header is 8 bytes - if not header: break + (typ, data) = read_entry(f) - typ = header[0:2] # First 2 bytes for type - dlen = struct.unpack(" dag.head.slot: echo "Written all complete eras" break - var e2s = E2Store.open(".", name, firstSlot).get() - defer: e2s.close() + let name = withState(dag.headState.data): eraFileName(cfg, state.data, era) + echo "Writing ", name - dag.withUpdatedState(tmpState[], canonical) do: - e2s.appendRecord(stateData.data.phase0Data.data).get() - do: raiseAssert "withUpdatedState failed" + let e2 = openFile(name, {OpenFlags.Write, OpenFlags.Create}).get() + defer: discard closeFile(e2) - var - ancestors: seq[BlockRef] - cur = canonical.blck - if era != 0: - while cur != nil and cur.slot >= firstSlot: - ancestors.add(cur) - cur = cur.parent + var group = EraGroup.init(e2, firstSlot).get() + if firstSlot.isSome(): + withTimer(timers[tBlocks]): + var blocks: array[SLOTS_PER_HISTORICAL_ROOT.int, BlockId] + for i in dag.getBlockRange(firstSlot.get(), 1, blocks)..