nimbus-eth2/beacon_chain/eth2_json_rpc_serialization...

111 lines
3.6 KiB
Nim
Raw Normal View History

import
# Standard library
std/[tables, json, typetraits],
# Nimble packages
stew/byteutils, ssz/types,
json_rpc/jsonmarshal,
# Local modules
spec/[datatypes, crypto, digest]
proc toJsonHex(data: openArray[byte]): string =
# Per the eth2 API spec, hex arrays are printed with leading 0x
"0x" & toHex(data)
proc fromJson*(n: JsonNode, argName: string, result: var ValidatorPubKey) =
n.kind.expect(JString, argName)
performance fixes (#2259) * performance fixes * don't mark tree cache as dirty on read-only List accesses * store only blob in memory for keys and signatures, parse blob lazily * compare public keys by blob instead of parsing / converting to raw * compare Eth2Digest using non-constant-time comparison * avoid some unnecessary validator copying This branch will in particular speed up deposit processing which has been slowing down block replay. Pre (mainnet, 1600 blocks): ``` All time are ms Average, StdDev, Min, Max, Samples, Test Validation is turned off meaning that no BLS operations are performed 3450.269, 0.000, 3450.269, 3450.269, 1, Initialize DB 0.417, 0.822, 0.036, 21.098, 1400, Load block from database 16.521, 0.000, 16.521, 16.521, 1, Load state from database 27.906, 50.846, 8.104, 1507.633, 1350, Apply block 52.617, 37.029, 20.640, 135.938, 50, Apply epoch block ``` Post: ``` 3502.715, 0.000, 3502.715, 3502.715, 1, Initialize DB 0.080, 0.560, 0.035, 21.015, 1400, Load block from database 17.595, 0.000, 17.595, 17.595, 1, Load state from database 15.706, 11.028, 8.300, 107.537, 1350, Apply block 33.217, 12.622, 17.331, 60.580, 50, Apply epoch block ``` * more perf fixes * load EpochRef cache into StateCache more aggressively * point out security concern with public key cache * reuse proposer index from state when processing block * avoid genericAssign in a few more places * don't parse key when signature is unparseable * fix `==` overload for Eth2Digest * preallocate validator list when getting active validators * speed up proposer index calculation a little bit * reuse cache when replaying blocks in ncli_db * avoid a few more copying loops ``` Average, StdDev, Min, Max, Samples, Test Validation is turned off meaning that no BLS operations are performed 3279.158, 0.000, 3279.158, 3279.158, 1, Initialize DB 0.072, 0.357, 0.035, 13.400, 1400, Load block from database 17.295, 0.000, 17.295, 17.295, 1, Load state from database 5.918, 9.896, 0.198, 98.028, 1350, Apply block 15.888, 10.951, 7.902, 39.535, 50, Apply epoch block 0.000, 0.000, 0.000, 0.000, 0, Database block store ``` * clear full balance cache before processing rewards and penalties ``` All time are ms Average, StdDev, Min, Max, Samples, Test Validation is turned off meaning that no BLS operations are performed 3947.901, 0.000, 3947.901, 3947.901, 1, Initialize DB 0.124, 0.506, 0.026, 202.370, 363345, Load block from database 97.614, 0.000, 97.614, 97.614, 1, Load state from database 0.186, 0.188, 0.012, 99.561, 357262, Advance slot, non-epoch 14.161, 5.966, 1.099, 395.511, 11524, Advance slot, epoch 1.372, 4.170, 0.017, 276.401, 363345, Apply block, no slot processing 0.000, 0.000, 0.000, 0.000, 0, Database block store ```
2021-01-25 12:04:18 +00:00
var tmp = ValidatorPubKey.fromHex(n.getStr()).tryGet()
if not tmp.loadWithCache().isSome():
raise (ref ValueError)(msg: "Invalid public BLS key")
result = tmp
proc `%`*(pubkey: ValidatorPubKey): JsonNode =
newJString(toJsonHex(toRaw(pubkey)))
proc fromJson*(n: JsonNode, argName: string, result: var List) =
fromJson(n, argName, asSeq result)
proc `%`*(list: List): JsonNode = %(asSeq(list))
proc fromJson*(n: JsonNode, argName: string, result: var BitList) =
n.kind.expect(JString, argName)
result = type(result)(hexToSeqByte(n.getStr()))
proc `%`*(bitlist: BitList): JsonNode =
newJString(toJsonHex(seq[byte](BitSeq(bitlist))))
proc fromJson*(n: JsonNode, argName: string, result: var ValidatorSig) =
n.kind.expect(JString, argName)
result = ValidatorSig.fromHex(n.getStr()).tryGet()
proc `%`*(value: ValidatorSig): JsonNode =
newJString(toJsonHex(toRaw(value)))
proc fromJson*(n: JsonNode, argName: string, result: var TrustedSig) =
n.kind.expect(JString, argName)
hexToByteArray(n.getStr(), result.data)
proc `%`*(value: TrustedSig): JsonNode =
newJString(toJsonHex(toRaw(value)))
proc fromJson*(n: JsonNode, argName: string, result: var Version) =
n.kind.expect(JString, argName)
hexToByteArray(n.getStr(), array[4, byte](result))
proc `%`*(value: Version): JsonNode =
newJString(toJsonHex(distinctBase(value)))
template genFromJsonForIntType(T: untyped) =
proc fromJson*(n: JsonNode, argName: string, result: var T) =
n.kind.expect(JInt, argName)
let asInt = n.getBiggestInt()
when T is Epoch:
if asInt == -1:
# TODO: This is a major hack here. Since the json library
# cannot handle properly 0xffffffff when serializing and
# deserializing uint64 values, we detect one known wrong
# result, appering in most `Validator` records. To fix
# this issue, we'll have to switch to nim-json-serialization
# in nim-json-rpc or work towards implementing a fix upstream.
result = FAR_FUTURE_EPOCH
return
if asInt < 0:
# signed -> unsigned conversions are unchecked
# https://github.com/nim-lang/RFCs/issues/175
raise newException(
ValueError, "JSON-RPC input is an unexpected negative value")
result = T(asInt)
genFromJsonForIntType(Epoch)
genFromJsonForIntType(Slot)
genFromJsonForIntType(CommitteeIndex)
genFromJsonForIntType(ValidatorIndex)
proc `%`*(value: GraffitiBytes): JsonNode =
newJString(toJsonHex(distinctBase(value)))
proc fromJson*(n: JsonNode, argName: string, value: var GraffitiBytes) =
n.kind.expect(JString, argName)
value = GraffitiBytes.init n.getStr()
proc `%`*(value: CommitteeIndex): JsonNode =
newJInt(value.BiggestInt)
proc `%`*(value: ValidatorIndex): JsonNode =
newJInt(value.BiggestInt)
proc `%`*(value: Eth2Digest): JsonNode =
newJString(toJsonHex(value.data))
proc fromJson*(n: JsonNode, argName: string, result: var Eth2Digest) =
n.kind.expect(JString, argName)
hexToByteArray(n.getStr(), result.data)
proc `%`*(value: BitSeq): JsonNode =
newJString(toJsonHex(value.bytes))
proc fromJson*(n: JsonNode, argName: string, result: var BitSeq) =
n.kind.expect(JString, argName)
result = BitSeq(hexToSeqByte(n.getStr()))