status-im
/
nimbus-eth2
mirror of https://github.com/status-im/nimbus-eth2.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

SSZ cleanup 

* be stricter about SSZ length prefix
* compute zeroHash list at compile time
* remove SSZ schema stuff
* move SSZ navigation to ncli
* cleanup a few leftover openArray uses
This commit is contained in:
Jacek Sieka 2020-05-27 13:36:02 +02:00 committed by zah
parent 49c33f0890
commit f53b55cbe0
9 changed files with 120 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Makefile
View File

@ -22,6 +22,7 @@ TOOLS := \
deposit_contract \
ncli_hash_tree_root \
ncli_pretty \
ncli_query \
ncli_transition \
process_dashboard \
stack_sizes \

24
beacon_chain/beacon_node.nim
View File

@ -24,7 +24,7 @@ import
attestation_pool, block_pool, eth2_network, eth2_discovery,
beacon_node_common, beacon_node_types,
nimbus_binary_common,
mainchain_monitor, version, ssz, ssz/dynamic_navigator,
mainchain_monitor, version, ssz,
sync_protocol, request_manager, validator_keygen, interop, statusbar,
sync_manager, state_transition,
validator_duties, validator_api
@ -1012,25 +1012,3 @@ programMain:
config.depositContractAddress,
config.depositPrivateKey,
delayGenerator)
of query:
case config.queryCmd
of QueryCmd.nimQuery:
# TODO: This will handle a simple subset of Nim using
# dot syntax and `[]` indexing.
echo "nim query: ", config.nimQueryExpression
of QueryCmd.get:
let pathFragments = config.getQueryPath.split('/', maxsplit = 1)
let bytes =
case pathFragments[0]
of "genesis_state":
readFile(config.dataDir/genesisFile).string.toBytes()
else:
stderr.write config.getQueryPath & " is not a valid path"
quit 1
let navigator = DynamicSszNavigator.init(bytes, BeaconState)
echo navigator.navigatePath(pathFragments[1 .. ^1]).toJson

21
beacon_chain/conf.nim
View File

@ -18,15 +18,10 @@ type
importValidator
createTestnet
makeDeposits
query
VCStartUpCmd* = enum
VCNoCommand
QueryCmd* = enum
nimQuery
get
Eth1Network* = enum
custom
mainnet
@ -270,22 +265,6 @@ type
desc: "Maximum possible delay between making two deposits (in seconds)"
name: "max-delay" }: float
of query:
case queryCmd* {.
defaultValue: nimQuery
command
desc: "Query the beacon node database and print the result" }: QueryCmd
of nimQuery:
nimQueryExpression* {.
argument
desc: "Nim expression to evaluate (using limited syntax)" }: string
of get:
getQueryPath* {.
argument
desc: "REST API path to evaluate" }: string
ValidatorClientConf* = object
logLevel* {.
defaultValue: "DEBUG"

4
beacon_chain/spec/state_transition_epoch.nim
View File

@ -384,7 +384,7 @@ func process_final_updates*(state: var BeaconState) {.nbench.}=
# Reset eth1 data votes
if next_epoch mod EPOCHS_PER_ETH1_VOTING_PERIOD == 0:
state.eth1_data_votes = type(state.eth1_data_votes) @[]
state.eth1_data_votes = default(type state.eth1_data_votes)
# Update effective balances with hysteresis
for index, validator in state.validators:
@ -420,7 +420,7 @@ func process_final_updates*(state: var BeaconState) {.nbench.}=
# Rotate current/previous epoch attestations
state.previous_epoch_attestations = state.current_epoch_attestations
state.current_epoch_attestations = typeof(state.current_epoch_attestations) @[]
state.current_epoch_attestations = default(type state.current_epoch_attestations)
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#epoch-processing
proc process_epoch*(state: var BeaconState, updateFlags: UpdateFlags)

71
beacon_chain/ssz.nim
View File

@ -29,12 +29,11 @@ export
when defined(serialization_tracing):
import
typetraits, stew/ranges/ptr_arith
typetraits
const
bytesPerChunk = 32
bitsPerChunk = bytesPerChunk * 8
defaultMaxObjectSize = 1 * 1024 * 1024
type
SszReader* = object
@ -63,6 +62,17 @@ serializationFormat SSZ,
Writer = SszWriter,
PreferedOutput = seq[byte]
template decode*(Format: type SSZ,
input: openarray[byte],
RecordType: distinct type): auto =
serialization.decode(SSZ, input, RecordType, maxObjectSize = input.len)
template loadFile*(Format: type SSZ,
file: string,
RecordType: distinct type): auto =
let bytes = readFile(file)
decode(SSZ, toOpenArrayByte(string bytes, 0, bytes.high), RecordType)
template bytes(x: BitSeq): untyped =
seq[byte](x)
@ -72,16 +82,12 @@ template sizePrefixed*[TT](x: TT): untyped =
proc init*(T: type SszReader,
stream: InputStream,
maxObjectSize = defaultMaxObjectSize): T {.raises: [Defect].} =
maxObjectSize: int): T {.raises: [Defect].} =
T(stream: stream, maxObjectSize: maxObjectSize)
proc mount*(F: type SSZ, stream: InputStream, T: type): T {.raises: [Defect].} =
mixin readValue
var reader = init(SszReader, stream)
reader.readValue(T)
method formatMsg*(err: ref SszSizeMismatchError, filename: string): string {.gcsafe, raises: [Defect].} =
# TODO: implement proper error string
method formatMsg*(
err: ref SszSizeMismatchError,
filename: string): string {.gcsafe, raises: [Defect].} =
try:
&"SSZ size mismatch, element {err.elementSize}, actual {err.actualSszSize}, type {err.deserializedType}, file {filename}"
except CatchableError:
@ -96,7 +102,7 @@ template toSszType*(x: auto): auto =
when x is Slot|Epoch|ValidatorIndex|enum: uint64(x)
elif x is Eth2Digest: x.data
elif x is BlsCurveType: toRaw(x)
elif x is ForkDigest|Version: array[4, byte](x)
elif x is ForkDigest|Version: distinctBase(x)
else: x
proc writeFixedSized(s: var (OutputStream|WriteCursor), x: auto) {.raises: [Defect, IOError].} =
@ -140,7 +146,7 @@ template enumerateSubFields(holder, fieldVar, body: untyped) =
when holder is array:
for fieldVar in holder: body
else:
enumInstanceSerializedFields(holder, _, fieldVar): body
enumInstanceSerializedFields(holder, _{.used.}, fieldVar): body
proc writeVarSizeType(w: var SszWriter, value: auto) {.gcsafe.}
@ -306,8 +312,8 @@ const
func hash(a, b: openArray[byte]): Eth2Digest =
result = withEth2Hash:
trs "MERGING BRANCHES "
trs a
trs b
trs toHex(a)
trs toHex(b)
h.update a
h.update b
@ -316,8 +322,8 @@ func hash(a, b: openArray[byte]): Eth2Digest =
func mergeBranches(existing: Eth2Digest, newData: openarray[byte]): Eth2Digest =
result = withEth2Hash:
trs "MERGING BRANCHES OPEN ARRAY"
trs existing.data
trs newData
trs toHex(existing.data)
trs toHex(newData)
h.update existing.data
h.update newData
@ -331,19 +337,12 @@ func mergeBranches(existing: Eth2Digest, newData: openarray[byte]): Eth2Digest =
template mergeBranches(a, b: Eth2Digest): Eth2Digest =
hash(a.data, b.data)
func computeZeroHashes: array[100, Eth2Digest] =
func computeZeroHashes: array[64, Eth2Digest] =
result[0] = Eth2Digest(data: zeroChunk)
for i in 1 .. result.high:
result[i] = mergeBranches(result[i - 1], result[i - 1])
let zeroHashes = computeZeroHashes()
template getZeroHashWithoutSideEffect(idx: int): Eth2Digest =
# TODO this is a work-around for the somewhat broken side
# effects analysis of Nim - reading from global let variables
# is considered a side-effect.
{.noSideEffect.}:
zeroHashes[idx]
const zeroHashes = computeZeroHashes()
func addChunk(merkleizer: var SszChunksMerkleizer, data: openarray[byte]) =
doAssert data.len > 0 and data.len <= bytesPerChunk
@ -381,7 +380,7 @@ template createMerkleizer(totalElements: static Limit): SszChunksMerkleizer =
func getFinalHash(merkleizer: var SszChunksMerkleizer): Eth2Digest =
if merkleizer.totalChunks == 0:
return getZeroHashWithoutSideEffect(merkleizer.topIndex)
return zeroHashes[merkleizer.topIndex]
let
bottomHashIdx = firstOne(merkleizer.totalChunks) - 1
@ -396,14 +395,14 @@ func getFinalHash(merkleizer: var SszChunksMerkleizer): Eth2Digest =
# Our tree is not finished. We must complete the work in progress
# branches and then extend the tree to the right height.
result = mergeBranches(merkleizer.combinedChunks[bottomHashIdx],
getZeroHashWithoutSideEffect(bottomHashIdx))
zeroHashes[bottomHashIdx])
for i in bottomHashIdx + 1 ..< topHashIdx:
if getBitLE(merkleizer.totalChunks, i):
result = mergeBranches(merkleizer.combinedChunks[i], result)
trs "COMBINED"
else:
result = mergeBranches(result, getZeroHashWithoutSideEffect(i))
result = mergeBranches(result, zeroHashes[i])
trs "COMBINED WITH ZERO"
elif bottomHashIdx == topHashIdx:
@ -413,10 +412,10 @@ func getFinalHash(merkleizer: var SszChunksMerkleizer): Eth2Digest =
# We have a perfect tree of user chunks, but we have more work to
# do - we must extend it to reach the desired height
result = mergeBranches(merkleizer.combinedChunks[bottomHashIdx],
getZeroHashWithoutSideEffect(bottomHashIdx))
zeroHashes[bottomHashIdx])
for i in bottomHashIdx + 1 ..< topHashIdx:
result = mergeBranches(result, getZeroHashWithoutSideEffect(i))
result = mergeBranches(result, zeroHashes[i])
func mixInLength(root: Eth2Digest, length: int): Eth2Digest =
var dataLen: array[32, byte]
@ -505,8 +504,8 @@ func bitListHashTreeRoot(merkleizer: var SszChunksMerkleizer, x: BitSeq): Eth2Di
if totalBytes == 1:
# This is an empty bit list.
# It should be hashed as a tree containing all zeros:
return mergeBranches(getZeroHashWithoutSideEffect(merkleizer.topIndex),
getZeroHashWithoutSideEffect(0)) # this is the mixed length
return mergeBranches(zeroHashes[merkleizer.topIndex],
zeroHashes[0]) # this is the mixed length
totalBytes -= 1
lastCorrectedByte = bytes(x)[^2]
@ -595,14 +594,14 @@ func hashTreeRootAux[T](x: T): Eth2Digest =
func hash_tree_root*(x: auto): Eth2Digest {.raises: [Defect], nbench.} =
trs "STARTING HASH TREE ROOT FOR TYPE ", name(type(x))
mixin toSszType
when x is List|BitList:
result = when x is List|BitList:
const maxLen = static(x.maxLen)
type T = type(x)
const limit = maxChunksCount(T, maxLen)
var merkleizer = createMerkleizer(limit)
when x is BitList:
result = merkleizer.bitListHashTreeRoot(BitSeq x)
merkleizer.bitListHashTreeRoot(BitSeq x)
else:
type E = ElemType(T)
let contentsHash = when E is BasicType:
@ -612,9 +611,9 @@ func hash_tree_root*(x: auto): Eth2Digest {.raises: [Defect], nbench.} =
let elemHash = hash_tree_root(elem)
merkleizer.addChunk(elemHash.data)
merkleizer.getFinalHash()
result = mixInLength(contentsHash, x.len)
mixInLength(contentsHash, x.len)
else:
result = hashTreeRootAux toSszType(x)
hashTreeRootAux toSszType(x)
trs "HASH TREE ROOT FOR ", name(type x), " = ", "0x", $result

17
beacon_chain/ssz/bytes_reader.nim
View File

@ -31,8 +31,7 @@ func fromSszBytes*(T: type UintN, data: openarray[byte]): T {.raisesssz.} =
T.fromBytesLE(data)
func fromSszBytes*(T: type bool, data: openarray[byte]): T {.raisesssz.} =
# TODO: spec doesn't say what to do if the value is >1 - we'll use the C
# definition for now, but maybe this should be a parse error instead?
# Strict: only allow 0 or 1
if data.len != 1 or byte(data[0]) > byte(1):
raise newException(MalformedSszError, "invalid boolean value")
data[0] == 1
@ -123,14 +122,14 @@ func readSszValue*(input: openarray[byte], T: type): T {.raisesssz.} =
checkForForbiddenBits(T, input, result.maxLen + 1)
elif result is List|array:
type ElemType = type result[0]
when ElemType is byte:
type E = type result[0]
when E is byte:
result.setOutputSize input.len
if input.len > 0:
copyMem(addr result[0], unsafeAddr input[0], input.len)
elif isFixedSize(ElemType):
const elemSize = fixedPortionSize(ElemType)
elif isFixedSize(E):
const elemSize = fixedPortionSize(E)
if input.len mod elemSize != 0:
var ex = new SszSizeMismatchError
ex.deserializedType = cstring typetraits.name(T)
@ -142,7 +141,7 @@ func readSszValue*(input: openarray[byte], T: type): T {.raisesssz.} =
for i in 0 ..< result.len:
trs "TRYING TO READ LIST ELEM ", i
let offset = i * elemSize
result[i] = readSszValue(input.toOpenArray(offset, offset + elemSize - 1), ElemType)
result[i] = readSszValue(input.toOpenArray(offset, offset + elemSize - 1), E)
trs "LIST READING COMPLETE"
else:
@ -171,10 +170,10 @@ func readSszValue*(input: openarray[byte], T: type): T {.raisesssz.} =
if nextOffset <= offset:
raise newException(MalformedSszError, "SSZ list element offsets are not monotonically increasing")
else:
result[i - 1] = readSszValue(input.toOpenArray(offset, nextOffset - 1), ElemType)
result[i - 1] = readSszValue(input.toOpenArray(offset, nextOffset - 1), E)
offset = nextOffset
result[resultLen - 1] = readSszValue(input.toOpenArray(offset, input.len - 1), ElemType)
result[resultLen - 1] = readSszValue(input.toOpenArray(offset, input.len - 1), E)
# TODO: Should be possible to remove BitArray from here
elif result is UintN|bool|enum:

117
beacon_chain/ssz/types.nim
View File

@ -30,48 +30,6 @@ type
actualSszSize*: int
elementSize*: int
SszChunksLimitExceeded* = object of SszError
SszSchema* = ref object
nodes*: seq[SszNode]
SszTypeKind* = enum
sszNull
sszUInt
sszBool
sszList
sszVector
sszBitList
sszBitVector
sszRecord
SszType* = ref object
case kind*: SszTypeKind
of sszUInt, sszBitVector:
bits*: int
of sszBool, sszNull, sszBitList:
discard
of sszVector:
size*: int
vectorElemType*: SszType
of sszList:
listElemType*: SszType
of sszRecord:
schema*: SszSchema
SszNodeKind* = enum
Field
Union
SszNode* = ref object
name*: string
typ*: SszType
case kind: SszNodeKind
of Union:
variants*: seq[SszSchema]
of Field:
discard
template asSeq*(x: List): auto = distinctBase(x)
template init*[T](L: type List, x: seq[T], N: static Limit): auto =
@ -81,21 +39,21 @@ template init*[T, N](L: type List[T, N], x: seq[T]): auto =
List[T, N](x)
template `$`*(x: List): auto = $(distinctBase x)
template add*(x: List, val: auto) = add(distinctBase x, val)
template add*(x: var List, val: auto) = add(distinctBase x, val)
template len*(x: List): auto = len(distinctBase x)
template setLen*(x: List, val: auto) = setLen(distinctBase x, val)
template setLen*(x: var List, val: auto) = setLen(distinctBase x, val)
template low*(x: List): auto = low(distinctBase x)
template high*(x: List): auto = high(distinctBase x)
template `[]`*(x: List, idx: auto): untyped = distinctBase(x)[idx]
template `[]=`*(x: List, idx: auto, val: auto) = distinctBase(x)[idx] = val
template `[]=`*(x: var List, idx: auto, val: auto) = distinctBase(x)[idx] = val
template `==`*(a, b: List): bool = asSeq(a) == distinctBase(b)
template `&`*(a, b: List): auto = (type(a)(distinctBase(a) & distinctBase(b)))
template items* (x: List): untyped = items(distinctBase x)
template pairs* (x: List): untyped = pairs(distinctBase x)
template mitems*(x: List): untyped = mitems(distinctBase x)
template mpairs*(x: List): untyped = mpairs(distinctBase x)
template mitems*(x: var List): untyped = mitems(distinctBase x)
template mpairs*(x: var List): untyped = mpairs(distinctBase x)
template init*(L: type BitList, x: seq[byte], N: static Limit): auto =
BitList[N](data: x)
@ -130,32 +88,27 @@ macro unsupported*(T: typed): untyped =
template ElemType*(T: type[array]): untyped =
type(default(T)[low(T)])
template ElemType*[T](A: type[openarray[T]]): untyped =
T
template ElemType*(T: type[seq|List]): untyped =
type(default(T)[0])
func isFixedSize*(T0: type): bool {.compileTime.} =
mixin toSszType, enumAllSerializedFields
when T0 is openarray:
return false
else:
type T = type toSszType(declval T0)
type T = type toSszType(declval T0)
when T is BasicType:
return true
elif T is array:
return isFixedSize(ElemType(T))
elif T is object|tuple:
enumAllSerializedFields(T):
when not isFixedSize(FieldType):
return false
return true
when T is BasicType:
return true
elif T is array:
return isFixedSize(ElemType(T))
elif T is object|tuple:
enumAllSerializedFields(T):
when not isFixedSize(FieldType):
return false
return true
func fixedPortionSize*(T0: type): int {.compileTime.} =
mixin enumAllSerializedFields, toSszType
type T = type toSszType(declval T0)
when T is BasicType: sizeof(T)
@ -163,7 +116,6 @@ func fixedPortionSize*(T0: type): int {.compileTime.} =
type E = ElemType(T)
when isFixedSize(E): len(T) * fixedPortionSize(E)
else: len(T) * offsetSize
elif T is seq|openarray: offsetSize
elif T is object|tuple:
enumAllSerializedFields(T):
when isFixedSize(FieldType):
@ -173,43 +125,6 @@ func fixedPortionSize*(T0: type): int {.compileTime.} =
else:
unsupported T0
func sszSchemaType*(T0: type): SszType {.compileTime.} =
mixin toSszType, enumAllSerializedFields
type T = type toSszType(declval T0)
when T is bool:
SszType(kind: sszBool)
elif T is uint8|char:
SszType(kind: sszUInt, bits: 8)
elif T is uint16:
SszType(kind: sszUInt, bits: 16)
elif T is uint32:
SszType(kind: sszUInt, bits: 32)
elif T is uint64:
SszType(kind: sszUInt, bits: 64)
elif T is seq:
SszType(kind: sszList, listElemType: sszSchemaType(ElemType(T)))
elif T is array:
SszType(kind: sszVector, vectorElemType: sszSchemaType(ElemType(T)))
elif T is BitArray:
SszType(kind: sszBitVector, bits: T.bits)
elif T is BitSeq:
SszType(kind: sszBitList)
elif T is object|tuple:
var recordSchema = SszSchema()
var caseBranches = initTable[string, SszSchema]()
caseBranches[""] = recordSchema
# TODO case objects are still not supported here.
# `recordFields` has to be refactored to properly
# report nested discriminator fields.
enumAllSerializedFields(T):
recordSchema.nodes.add SszNode(
name: fieldName,
typ: sszSchemaType(FieldType),
kind: Field)
else:
unsupported T0
# TODO This should have been an iterator, but the VM can't compile the
# code due to "too many registers required".
proc fieldInfos*(RecordType: type): seq[tuple[name: string,

56
ncli/ncli_query.nim Normal file
View File

@ -0,0 +1,56 @@
import
confutils, os, strutils, chronicles, json_serialization,
stew/byteutils,
../beacon_chain/spec/[crypto, datatypes, digest],
../beacon_chain/[ssz],
../beacon_chain/ssz/dynamic_navigator
type
QueryCmd* = enum
nimQuery
get
QueryConf = object
file* {.
defaultValue: ""
desc: "BeaconState ssz file"
name: "file" }: InputFile
case queryCmd* {.
defaultValue: nimQuery
command
desc: "Query the beacon node database and print the result" }: QueryCmd
of nimQuery:
nimQueryExpression* {.
argument
desc: "Nim expression to evaluate (using limited syntax)" }: string
of get:
getQueryPath* {.
argument
desc: "REST API path to evaluate" }: string
let
config = QueryConf.load()
case config.queryCmd
of QueryCmd.nimQuery:
# TODO: This will handle a simple subset of Nim using
# dot syntax and `[]` indexing.
echo "nim query: ", config.nimQueryExpression
of QueryCmd.get:
let pathFragments = config.getQueryPath.split('/', maxsplit = 1)
let bytes =
case pathFragments[0]
of "genesis_state":
readFile(config.file.string).string.toBytes()
else:
stderr.write config.getQueryPath & " is not a valid path"
quit 1
let navigator = DynamicSszNavigator.init(bytes, BeaconState)
echo navigator.navigatePath(pathFragments[1 .. ^1]).toJson

3
tests/official/fixtures_utils.nim
View File

@ -57,9 +57,8 @@ template readFileBytes*(path: string): seq[byte] =
proc sszDecodeEntireInput*(input: openarray[byte], Decoded: type): Decoded =
var stream = unsafeMemoryInput(input)
var reader = init(SszReader, stream)
var reader = init(SszReader, stream, input.len)
result = reader.readValue(Decoded)
if stream.readable:
raise newException(UnconsumedInput, "Remaining bytes in the input")