status-im
/
nimbus-eth2
mirror of https://github.com/status-im/nimbus-eth2.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

prefer endians2

This commit is contained in:
Jacek Sieka 2020-03-05 01:29:27 +01:00 committed by zah
parent 9a3db7a81f
commit 7902d070cd
4 changed files with 77 additions and 250 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
beacon_chain/spec/helpers.nim
View File

@ -9,8 +9,9 @@
import
# Standard lib
math, endians,
math,
# Third-party
stew/endians2,
blscurve, # defines Domain
# Internal
./datatypes, ./digest, ../ssz
@ -91,12 +92,12 @@ func bytes_to_int*(data: openarray[byte]): uint64 =
func int_to_bytes32*(x: uint64): array[32, byte] =
## Little-endian data representation
## TODO remove uint64 when those callers fade away
littleEndian64(result[0].addr, x.unsafeAddr)
result[0..<7] = x.toBytesLE()
func int_to_bytes32*(x: Epoch): array[32, byte] {.borrow.}
func int_to_bytes8*(x: uint64): array[8, byte] =
littleEndian64(result[0].addr, x.unsafeAddr)
x.toBytesLE()
func int_to_bytes1*(x: int): array[1, byte] =
doAssert x >= 0

30
beacon_chain/ssz.nim
View File

@ -9,8 +9,8 @@
# See https://github.com/ethereum/eth2.0-specs/blob/master/specs/simple-serialize.md
import
endians, stew/shims/macros, options, algorithm, options,
stew/[bitops2, bitseqs, objects, varints, ptrops, ranges/ptr_arith], stint,
stew/shims/macros, options, algorithm, options,
stew/[bitops2, bitseqs, endians2, objects, varints, ptrops, ranges/ptr_arith], stint,
faststreams/input_stream, serialization, serialization/testing/tracing,
nimcrypto/sha2, blscurve,
./spec/[crypto, datatypes, digest],
@ -117,20 +117,9 @@ func writeFixedSized(c: var WriteCursor, x: auto) =
elif x is bool|char:
c.append byte(ord(x))
elif x is SomeUnsignedInt:
when system.cpuEndian == bigEndian:
## Convert directly to bytes the size of the int. (e.g. ``uint16 = 2 bytes``)
## All integers are serialized as **little endian**.
var bytes: array[sizeof(x), byte]
when x.sizeof == 8: littleEndian64(addr bytes[0], x.unsafeAddr)
elif x.sizeof == 4: littleEndian32(addr bytes[0], x.unsafeAddr)
elif x.sizeof == 2: littleEndian16(addr bytes[0], x.unsafeAddr)
elif x.sizeof == 1: copyMem(addr bytes[0], x.unsafeAddr, sizeof(x))
else: unsupported x.type
c.append bytes
else:
let valueAddr {.used.} = unsafeAddr x
trs "APPENDING INT ", x, " = ", makeOpenArray(cast[ptr byte](valueAddr), sizeof(x))
c.appendMemCopy x
let value = x.toBytesLE()
trs "APPENDING INT ", x, " = ", value
c.appendMemCopy value
elif x is StUint:
c.appendMemCopy x # TODO: Is this always correct?
elif x is array|string|seq|openarray:
@ -426,8 +415,7 @@ func newSszHashingStream(merkelizer: SszChunksMerkelizer): ref OutputStream =
func mixInLength(root: Eth2Digest, length: int): Eth2Digest =
var dataLen: array[32, byte]
var lstLen = uint64(length)
littleEndian64(addr dataLen[0], addr lstLen)
dataLen[0..<8] = uint64(length).toBytesLE()
hash(root.data, dataLen)
func merkelizeSerializedChunks(merkelizer: SszChunksMerkelizer,
@ -516,11 +504,7 @@ func bitlistHashTreeRoot(merkelizer: SszChunksMerkelizer, x: BitSeq): Eth2Digest
func hashTreeRootImpl[T](x: T): Eth2Digest =
when T is uint64:
trs "UINT64; LITTLE-ENDIAN IDENTITY MAPPING"
when system.cpuEndian == bigEndian:
littleEndian64(addr result.data[0], x.unsafeAddr)
else:
let valueAddr = unsafeAddr x
result.data[0..7] = makeOpenArray(cast[ptr byte](valueAddr), 8)
result.data[0..<8] = x.toBytesLE()
elif (when T is array: ElemType(T) is byte and
sizeof(T) == sizeof(Eth2Digest) else: false):
# TODO is this sizeof comparison guranteed? it's whole structure vs field

16
beacon_chain/ssz/bytes_reader.nim
View File

@ -1,6 +1,6 @@
import
endians, typetraits, options,
stew/[objects, bitseqs], serialization/testing/tracing,
typetraits, options,
stew/[bitseqs, endians2, objects, bitseqs], serialization/testing/tracing,
../spec/[digest, datatypes], ./types
const
@ -28,17 +28,7 @@ func fromSszBytes*(T: type SomeInteger, data: openarray[byte]): T =
if data.len < sizeof(result):
raise newException(MalformedSszError, "SSZ input of insufficient size")
# TODO: any better way to get a suitably aligned buffer in nim???
# see also: https://github.com/nim-lang/Nim/issues/9206
var tmp: uint64
var alignedBuf = cast[ptr byte](tmp.addr)
copyMem(alignedBuf, unsafeAddr data[0], result.sizeof)
when result.sizeof == 8: littleEndian64(result.addr, alignedBuf)
elif result.sizeof == 4: littleEndian32(result.addr, alignedBuf)
elif result.sizeof == 2: littleEndian16(result.addr, alignedBuf)
elif result.sizeof == 1: copyMem(result.addr, alignedBuf, sizeof(result))
else: {.fatal: "Unsupported type deserialization: " & $(type(result)).name.}
T.fromBytesLE(data)
func fromSszBytes*(T: type bool, data: openarray[byte]): T =
# TODO: spec doesn't say what to do if the value is >1 - we'll use the C

256
nfuzz/libnfuzz.nim
View File

@ -1,7 +1,7 @@
import
endians, stew/ptrops, stew/ranges/ptr_arith,
stew/ptrops, stew/ranges/ptr_arith,
../beacon_chain/[ssz, state_transition],
../beacon_chain/spec/[datatypes, helpers, digest, validator, beaconstate,
../beacon_chain/spec/[datatypes, digest, validator, beaconstate,
state_transition_block],
# Required for deserialisation of ValidatorSig in Attestation due to
# https://github.com/nim-lang/Nim/issues/11225
@ -17,7 +17,7 @@ type
attesterSlashing: AttesterSlashing
BlockInput = object
state: BeaconState
beaconBlock: BeaconBlock
beaconBlock: SignedBeaconBlock
BlockHeaderInput = BlockInput
DepositInput = object
state: BeaconState
@ -27,7 +27,7 @@ type
proposerSlashing: ProposerSlashing
VoluntaryExitInput = object
state: BeaconState
exit: VoluntaryExit
exit: SignedVoluntaryExit
# This and AssertionError are raised to indicate programming bugs
# A wrapper to allow exception tracking to identify unexpected exceptions
FuzzCrashError = object of Exception
@ -35,10 +35,9 @@ type
# TODO: change ptr uint to ptr csize_t when available in newer Nim version.
proc copyState(state: BeaconState, output: ptr byte,
output_size: ptr uint): bool {.raises: [FuzzCrashError, Defect].} =
var resultState: seq[byte]
var resultState =
try:
resultState = SSZ.encode(state)
SSZ.encode(state)
except IOError as e:
# Shouldn't occur as the writer isn't a file
raise newException(FuzzCrashError, "Unexpected failure to serialize.", e)
@ -57,195 +56,77 @@ proc copyState(state: BeaconState, output: ptr byte,
copyMem(output, unsafeAddr resultState[0], output_size[])
result = true
proc nfuzz_attestation(input: openArray[byte], output: ptr byte,
output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
template decodeAndProcess(typ, process: untyped): bool =
let flags {.inject.} = if disable_bls: {skipBlsValidation} else: {}
var
data: AttestationInput
cache = get_empty_per_epoch_cache()
let flags = if disable_bls: {skipBlsValidation} else: {}
cache {.used, inject.} = get_empty_per_epoch_cache()
data {.inject.} =
try:
data = SSZ.decode(input, AttestationInput)
except MalformedSszError, SszSizeMismatchError:
let e = getCurrentException()
SSZ.decode(input, typ)
except MalformedSszError as e:
raise newException(
FuzzCrashError,
"SSZ deserialisation failed, likely bug in preprocessing.",
e,
)
"Malformed SSZ, likely bug in preprocessing.", e)
except SszSizeMismatchError as e:
raise newException(
FuzzCrashError,
"SSZ size mismatch, likely bug in preprocessing.", e)
let processOk =
try:
result = process_attestation(data.state, data.attestation,
flags, cache)
process
except IOError as e:
raise newException(
FuzzCrashError, "Unexpected (logging?) IOError in state transition", e,
)
except ValueError as e:
# These exceptions are expected to be raised by chronicles logging:
# See status-im/nim-chronicles#60
# TODO remove this when resolved
raise newException(
FuzzCrashError,
"Unexpected (logging?) error in attestation processing",
e
)
if result:
result = copyState(data.state, output, output_size)
proc nfuzz_attester_slashing(input: openArray[byte], output: ptr byte,
output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
var
data: AttesterSlashingInput
cache = get_empty_per_epoch_cache()
let flags = if disable_bls: {skipBlsValidation} else: {}
try:
data = SSZ.decode(input, AttesterSlashingInput)
except MalformedSszError, SszSizeMismatchError:
let e = getCurrentException()
raise newException(
FuzzCrashError,
"SSZ deserialisation failed, likely bug in preprocessing.",
e,
)
try:
# TODO flags
result = process_attester_slashing(data.state, data.attesterSlashing, flags, cache)
except ValueError as e:
# TODO remove when status-im/nim-chronicles#60 is resolved
raise newException(
FuzzCrashError,
"Unexpected (logging?) error in attester slashing",
e,
)
if result:
result = copyState(data.state, output, output_size)
proc nfuzz_block(input: openArray[byte], output: ptr byte,
output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
var data: BlockInput
let flags = if disable_bls: {skipBlsValidation} else: {}
try:
data = SSZ.decode(input, BlockInput)
except MalformedSszError, SszSizeMismatchError:
let e = getCurrentException()
raise newException(
FuzzCrashError,
"SSZ deserialisation failed, likely bug in preprocessing.",
e,
)
try:
result = state_transition(data.state, data.beaconBlock, flags)
except IOError, ValueError:
# TODO remove when status-im/nim-chronicles#60 is resolved
let e = getCurrentException()
raise newException(
FuzzCrashError,
"Unexpected (logging?) error in state transition",
e,
)
"Unexpected (logging?) IOError in state transition", e)
except Exception as e:
# TODO why an Exception?
# Lots of vendor code looks like it might raise a bare exception type
raise newException(FuzzCrashError, "Unexpected Exception in state transition", e)
if result:
result = copyState(data.state, output, output_size)
if processOk:
copyState(data.state, output, output_size)
else:
false
proc nfuzz_attestation(input: openArray[byte], output: ptr byte,
output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
decodeAndProcess(AttestationInput):
process_attestation(data.state, data.attestation, flags, cache)
proc nfuzz_attester_slashing(input: openArray[byte], output: ptr byte,
output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
decodeAndProcess(AttesterSlashingInput):
process_attester_slashing(data.state, data.attesterSlashing, flags, cache)
proc nfuzz_block(input: openArray[byte], output: ptr byte,
output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
decodeAndProcess(BlockInput):
state_transition(data.state, data.beaconBlock, flags)
proc nfuzz_block_header(input: openArray[byte], output: ptr byte,
output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
var
data: BlockHeaderInput
cache = get_empty_per_epoch_cache()
let flags = if disable_bls: {skipBlsValidation} else: {}
try:
data = SSZ.decode(input, BlockHeaderInput)
except MalformedSszError, SszSizeMismatchError:
let e = getCurrentException()
raise newException(
FuzzCrashError,
"SSZ deserialisation failed, likely bug in preprocessing.",
e,
)
try:
# TODO disable bls
result = process_block_header(data.state, data.beaconBlock, flags, cache)
except IOError, ValueError:
let e = getCurrentException()
# TODO remove when status-im/nim-chronicles#60 is resolved
raise newException(
FuzzCrashError,
"Unexpected IOError in block header processing",
e,
)
if result:
result = copyState(data.state, output, output_size)
decodeAndProcess(BlockHeaderInput):
process_block_header(data.state, data.beaconBlock.message, flags, cache)
proc nfuzz_deposit(input: openArray[byte], output: ptr byte,
output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
var
data: DepositInput
let flags = if disable_bls: {skipBlsValidation} else: {}
try:
data = SSZ.decode(input, DepositInput)
except MalformedSszError, SszSizeMismatchError:
let e = getCurrentException()
raise newException(
FuzzCrashError,
"SSZ deserialisation failed, likely bug in preprocessing.",
e,
)
try:
result = process_deposit(data.state, data.deposit, flags)
except IOError, ValueError:
let e = getCurrentException()
# TODO remove when status-im/nim-chronicles#60 is resolved
raise newException(
FuzzCrashError,
"Unexpected (logging?) error in deposit processing",
e,
)
if result:
result = copyState(data.state, output, output_size)
decodeAndProcess(DepositInput):
process_deposit(data.state, data.deposit, flags)
proc nfuzz_proposer_slashing(input: openArray[byte], output: ptr byte,
output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
var
data: ProposerSlashingInput
cache = get_empty_per_epoch_cache()
let flags = if disable_bls: {skipBlsValidation} else: {}
decodeAndProcess(ProposerSlashingInput):
process_proposer_slashing(data.state, data.proposerSlashing, flags, cache)
try:
data = SSZ.decode(input, ProposerSlashingInput)
except MalformedSszError, SszSizeMismatchError:
let e = getCurrentException()
raise newException(
FuzzCrashError,
"SSZ deserialisation failed, likely bug in preprocessing.",
e,
)
try:
result = process_proposer_slashing(data.state, data.proposerSlashing, flags, cache)
except ValueError as e:
# TODO remove when status-im/nim-chronicles#60 is resolved
raise newException(
FuzzCrashError,
"Unexpected (logging?) error in proposer slashing",
e,
)
if result:
result = copyState(data.state, output, output_size)
proc nfuzz_voluntary_exit(input: openArray[byte], output: ptr byte,
output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
decodeAndProcess(VoluntaryExitInput):
process_voluntary_exit(data.state, data.exit, flags)
# Note: Could also accept raw input pointer and access list_size + seed here.
# However, list_size needs to be known also outside this proc to allocate output.
@ -274,32 +155,3 @@ proc nfuzz_shuffle(input_seed: ptr byte, output: var openArray[uint64]): bool
sizeof(ValidatorIndex))
result = true
proc nfuzz_voluntary_exit(input: openArray[byte], output: ptr byte,
output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
var
data: VoluntaryExitInput
let flags = if disable_bls: {skipBlsValidation} else: {}
try:
data = SSZ.decode(input, VoluntaryExitInput)
except MalformedSszError, SszSizeMismatchError:
let e = getCurrentException()
raise newException(
FuzzCrashError,
"SSZ deserialisation failed, likely bug in preprocessing.",
e,
)
try:
result = process_voluntary_exit(data.state, data.exit, flags)
except ValueError as e:
# TODO remove when status-im/nim-chronicles#60 is resolved
raise newException(
FuzzCrashError,
"Unexpected (logging?) error in voluntary exit processing",
e,
)
if result:
result = copyState(data.state, output, output_size)