speed up state/block loading (#5207)
* speed up state/block loading When loading blocks and states from db/era, we currently redundantly check their CRC32 - for a state, this costs 50ms of loading time presently (110mb uncompressed size) on a decent laptop. * remove `maxDecompressedDbRecordSize` - not actually used on recent data since we store the framed format - also, we're in luck: we blew past the limit quite some time ago * fix obsolete exception-based error checking * avoid `zeroMem` when reading from era store see https://github.com/status-im/nim-snappy/pull/22 for benchmarks * bump snappy
This commit is contained in:
parent
df80ae68fa
commit
e8379389e7
|
@ -203,13 +203,6 @@ type
|
||||||
slot*: Slot
|
slot*: Slot
|
||||||
parent_root*: Eth2Digest
|
parent_root*: Eth2Digest
|
||||||
|
|
||||||
const
|
|
||||||
# The largest object we're saving is the BeaconState, and by far, the largest
|
|
||||||
# part of it is the validator - each validator takes up at least 129 bytes
|
|
||||||
# in phase0, which means 100k validators is >12mb - in addition to this,
|
|
||||||
# there are several MB of hashes.
|
|
||||||
maxDecompressedDbRecordSize = 64*1024*1024
|
|
||||||
|
|
||||||
# Subkeys essentially create "tables" within the key-value store by prefixing
|
# Subkeys essentially create "tables" within the key-value store by prefixing
|
||||||
# each entry with a table id
|
# each entry with a table id
|
||||||
|
|
||||||
|
@ -628,7 +621,7 @@ proc decodeSSZ*[T](data: openArray[byte], output: var T): bool =
|
||||||
|
|
||||||
proc decodeSnappySSZ[T](data: openArray[byte], output: var T): bool =
|
proc decodeSnappySSZ[T](data: openArray[byte], output: var T): bool =
|
||||||
try:
|
try:
|
||||||
let decompressed = snappy.decode(data, maxDecompressedDbRecordSize)
|
let decompressed = snappy.decode(data)
|
||||||
readSszBytes(decompressed, output, updateRoot = false)
|
readSszBytes(decompressed, output, updateRoot = false)
|
||||||
true
|
true
|
||||||
except SerializationError as e:
|
except SerializationError as e:
|
||||||
|
@ -640,7 +633,7 @@ proc decodeSnappySSZ[T](data: openArray[byte], output: var T): bool =
|
||||||
|
|
||||||
proc decodeSZSSZ[T](data: openArray[byte], output: var T): bool =
|
proc decodeSZSSZ[T](data: openArray[byte], output: var T): bool =
|
||||||
try:
|
try:
|
||||||
let decompressed = decodeFramed(data)
|
let decompressed = decodeFramed(data, checkIntegrity = false)
|
||||||
readSszBytes(decompressed, output, updateRoot = false)
|
readSszBytes(decompressed, output, updateRoot = false)
|
||||||
true
|
true
|
||||||
except CatchableError as e:
|
except CatchableError as e:
|
||||||
|
@ -989,8 +982,8 @@ proc getPhase0BlockSSZ(
|
||||||
let dataPtr = addr data # Short-lived
|
let dataPtr = addr data # Short-lived
|
||||||
var success = true
|
var success = true
|
||||||
func decode(data: openArray[byte]) =
|
func decode(data: openArray[byte]) =
|
||||||
try: dataPtr[] = snappy.decode(data, maxDecompressedDbRecordSize)
|
dataPtr[] = snappy.decode(data)
|
||||||
except CatchableError: success = false
|
success = dataPtr[].len > 0
|
||||||
db.backend.get(subkey(phase0.SignedBeaconBlock, key), decode).expectDb() and
|
db.backend.get(subkey(phase0.SignedBeaconBlock, key), decode).expectDb() and
|
||||||
success
|
success
|
||||||
|
|
||||||
|
@ -999,9 +992,8 @@ proc getPhase0BlockSZ(
|
||||||
let dataPtr = addr data # Short-lived
|
let dataPtr = addr data # Short-lived
|
||||||
var success = true
|
var success = true
|
||||||
func decode(data: openArray[byte]) =
|
func decode(data: openArray[byte]) =
|
||||||
try: dataPtr[] = snappy.encodeFramed(
|
dataPtr[] = snappy.encodeFramed(snappy.decode(data))
|
||||||
snappy.decode(data, maxDecompressedDbRecordSize))
|
success = dataPtr[].len > 0
|
||||||
except CatchableError: success = false
|
|
||||||
db.backend.get(subkey(phase0.SignedBeaconBlock, key), decode).expectDb() and
|
db.backend.get(subkey(phase0.SignedBeaconBlock, key), decode).expectDb() and
|
||||||
success
|
success
|
||||||
|
|
||||||
|
@ -1012,8 +1004,8 @@ proc getBlockSSZ*(
|
||||||
let dataPtr = addr data # Short-lived
|
let dataPtr = addr data # Short-lived
|
||||||
var success = true
|
var success = true
|
||||||
func decode(data: openArray[byte]) =
|
func decode(data: openArray[byte]) =
|
||||||
try: dataPtr[] = snappy.decode(data, maxDecompressedDbRecordSize)
|
dataPtr[] = snappy.decode(data)
|
||||||
except CatchableError: success = false
|
success = dataPtr[].len > 0
|
||||||
db.blocks[ConsensusFork.Phase0].get(key.data, decode).expectDb() and success or
|
db.blocks[ConsensusFork.Phase0].get(key.data, decode).expectDb() and success or
|
||||||
db.v0.getPhase0BlockSSZ(key, data)
|
db.v0.getPhase0BlockSSZ(key, data)
|
||||||
|
|
||||||
|
@ -1023,8 +1015,8 @@ proc getBlockSSZ*(
|
||||||
let dataPtr = addr data # Short-lived
|
let dataPtr = addr data # Short-lived
|
||||||
var success = true
|
var success = true
|
||||||
func decode(data: openArray[byte]) =
|
func decode(data: openArray[byte]) =
|
||||||
try: dataPtr[] = snappy.decode(data, maxDecompressedDbRecordSize)
|
dataPtr[] = snappy.decode(data)
|
||||||
except CatchableError: success = false
|
success = dataPtr[].len > 0
|
||||||
db.blocks[T.toFork].get(key.data, decode).expectDb() and success
|
db.blocks[T.toFork].get(key.data, decode).expectDb() and success
|
||||||
|
|
||||||
proc getBlockSSZ*[
|
proc getBlockSSZ*[
|
||||||
|
@ -1034,8 +1026,8 @@ proc getBlockSSZ*[
|
||||||
let dataPtr = addr data # Short-lived
|
let dataPtr = addr data # Short-lived
|
||||||
var success = true
|
var success = true
|
||||||
func decode(data: openArray[byte]) =
|
func decode(data: openArray[byte]) =
|
||||||
try: dataPtr[] = decodeFramed(data)
|
dataPtr[] = decodeFramed(data, checkIntegrity = false)
|
||||||
except CatchableError: success = false
|
success = dataPtr[].len > 0
|
||||||
db.blocks[T.toFork].get(key.data, decode).expectDb() and success
|
db.blocks[T.toFork].get(key.data, decode).expectDb() and success
|
||||||
|
|
||||||
proc getBlockSSZ*(
|
proc getBlockSSZ*(
|
||||||
|
@ -1067,9 +1059,8 @@ proc getBlockSZ*(
|
||||||
let dataPtr = addr data # Short-lived
|
let dataPtr = addr data # Short-lived
|
||||||
var success = true
|
var success = true
|
||||||
func decode(data: openArray[byte]) =
|
func decode(data: openArray[byte]) =
|
||||||
try: dataPtr[] = snappy.encodeFramed(
|
dataPtr[] = snappy.encodeFramed(snappy.decode(data))
|
||||||
snappy.decode(data, maxDecompressedDbRecordSize))
|
success = dataPtr[].len > 0
|
||||||
except CatchableError: success = false
|
|
||||||
db.blocks[ConsensusFork.Phase0].get(key.data, decode).expectDb() and success or
|
db.blocks[ConsensusFork.Phase0].get(key.data, decode).expectDb() and success or
|
||||||
db.v0.getPhase0BlockSZ(key, data)
|
db.v0.getPhase0BlockSZ(key, data)
|
||||||
|
|
||||||
|
@ -1079,9 +1070,8 @@ proc getBlockSZ*(
|
||||||
let dataPtr = addr data # Short-lived
|
let dataPtr = addr data # Short-lived
|
||||||
var success = true
|
var success = true
|
||||||
func decode(data: openArray[byte]) =
|
func decode(data: openArray[byte]) =
|
||||||
try: dataPtr[] = snappy.encodeFramed(
|
dataPtr[] = snappy.encodeFramed(snappy.decode(data))
|
||||||
snappy.decode(data, maxDecompressedDbRecordSize))
|
success = dataPtr[].len > 0
|
||||||
except CatchableError: success = false
|
|
||||||
db.blocks[T.toFork].get(key.data, decode).expectDb() and success
|
db.blocks[T.toFork].get(key.data, decode).expectDb() and success
|
||||||
|
|
||||||
proc getBlockSZ*[
|
proc getBlockSZ*[
|
||||||
|
|
|
@ -106,12 +106,20 @@ proc getBlockSSZ*(
|
||||||
f: EraFile, slot: Slot, bytes: var seq[byte]): Result[void, string] =
|
f: EraFile, slot: Slot, bytes: var seq[byte]): Result[void, string] =
|
||||||
var tmp: seq[byte]
|
var tmp: seq[byte]
|
||||||
? f.getBlockSZ(slot, tmp)
|
? f.getBlockSZ(slot, tmp)
|
||||||
|
let
|
||||||
|
len = uncompressedLenFramed(tmp).valueOr:
|
||||||
|
return err("Cannot read uncompressed length, era file corrupt?")
|
||||||
|
|
||||||
try:
|
if len > int.high.uint64:
|
||||||
bytes = decodeFramed(tmp)
|
return err("Invalid uncompressed size")
|
||||||
|
|
||||||
|
bytes = newSeqUninitialized[byte](len)
|
||||||
|
|
||||||
|
# Where it matters, we will integrity-check the data with SSZ - no
|
||||||
|
# need to waste cycles on crc32
|
||||||
|
discard uncompressFramed(tmp, bytes, checkIntegrity = false).valueOr:
|
||||||
|
return err("Block failed to decompress, era file corrupt?")
|
||||||
ok()
|
ok()
|
||||||
except CatchableError as exc:
|
|
||||||
err(exc.msg)
|
|
||||||
|
|
||||||
proc getStateSZ*(
|
proc getStateSZ*(
|
||||||
f: EraFile, slot: Slot, bytes: var seq[byte]): Result[void, string] =
|
f: EraFile, slot: Slot, bytes: var seq[byte]): Result[void, string] =
|
||||||
|
@ -151,7 +159,10 @@ proc getStateSSZ*(
|
||||||
else: len
|
else: len
|
||||||
|
|
||||||
bytes = newSeqUninitialized[byte](wanted)
|
bytes = newSeqUninitialized[byte](wanted)
|
||||||
discard uncompressFramed(tmp, bytes).valueOr:
|
|
||||||
|
# Where it matters, we will integrity-check the data with SSZ - no
|
||||||
|
# need to waste cycles on crc32
|
||||||
|
discard uncompressFramed(tmp, bytes, checkIntegrity = false).valueOr:
|
||||||
return err("State failed to decompress, era file corrupt?")
|
return err("State failed to decompress, era file corrupt?")
|
||||||
|
|
||||||
ok()
|
ok()
|
||||||
|
|
|
@ -166,7 +166,8 @@ proc readRecord*(f: IoHandle, data: var seq[byte]): Result[Header, string] =
|
||||||
if header.len > 0:
|
if header.len > 0:
|
||||||
? f.checkBytesLeft(header.len)
|
? f.checkBytesLeft(header.len)
|
||||||
|
|
||||||
data.setLen(header.len)
|
if data.len != header.len:
|
||||||
|
data = newSeqUninitialized[byte](header.len)
|
||||||
|
|
||||||
? readFileExact(f, data)
|
? readFileExact(f, data)
|
||||||
|
|
||||||
|
|
|
@ -620,7 +620,7 @@ proc cmdImportEra(conf: DbConf, cfg: RuntimeConfig) =
|
||||||
|
|
||||||
if header.typ == SnappyBeaconBlock:
|
if header.typ == SnappyBeaconBlock:
|
||||||
withTimer(timers[tBlock]):
|
withTimer(timers[tBlock]):
|
||||||
let uncompressed = decodeFramed(data)
|
let uncompressed = decodeFramed(data, checkIntegrity = false)
|
||||||
let blck = try: readSszForkedSignedBeaconBlock(cfg, uncompressed)
|
let blck = try: readSszForkedSignedBeaconBlock(cfg, uncompressed)
|
||||||
except CatchableError as exc:
|
except CatchableError as exc:
|
||||||
error "Invalid snappy block", msg = exc.msg, file
|
error "Invalid snappy block", msg = exc.msg, file
|
||||||
|
|
|
@ -1 +1 @@
|
||||||
Subproject commit 6da3e98f5420a0e86a1844b928cf38c8b656e2bc
|
Subproject commit ecbcee1d100140db6cb9c13d753d739fb5102fa3
|
Loading…
Reference in New Issue