prefer endians2

beacon_chain/spec/helpers.nim

@@ -9,8 +9,9 @@
 
 import
   # Standard lib
-  math, endians,
+  math,
   # Third-party
+  stew/endians2,
   blscurve, # defines Domain
   # Internal
   ./datatypes, ./digest, ../ssz
@@ -91,12 +92,12 @@ func bytes_to_int*(data: openarray[byte]): uint64 =
 func int_to_bytes32*(x: uint64): array[32, byte] =
   ## Little-endian data representation
   ## TODO remove uint64 when those callers fade away
-  littleEndian64(result[0].addr, x.unsafeAddr)
+  result[0..<7] = x.toBytesLE()
 
 func int_to_bytes32*(x: Epoch): array[32, byte] {.borrow.}
 
 func int_to_bytes8*(x: uint64): array[8, byte] =
-  littleEndian64(result[0].addr, x.unsafeAddr)
+  x.toBytesLE()
 
 func int_to_bytes1*(x: int): array[1, byte] =
   doAssert x >= 0

beacon_chain/ssz.nim

@@ -9,8 +9,8 @@
 # See https://github.com/ethereum/eth2.0-specs/blob/master/specs/simple-serialize.md
 
 import
-  endians, stew/shims/macros, options, algorithm, options,
+  stew/shims/macros, options, algorithm, options,
-  stew/[bitops2, bitseqs, objects, varints, ptrops, ranges/ptr_arith], stint,
+  stew/[bitops2, bitseqs, endians2, objects, varints, ptrops, ranges/ptr_arith], stint,
   faststreams/input_stream, serialization, serialization/testing/tracing,
   nimcrypto/sha2, blscurve,
   ./spec/[crypto, datatypes, digest],
@@ -117,20 +117,9 @@ func writeFixedSized(c: var WriteCursor, x: auto) =
   elif x is bool|char:
     c.append byte(ord(x))
   elif x is SomeUnsignedInt:
-    when system.cpuEndian == bigEndian:
+    let value = x.toBytesLE()
-      ## Convert directly to bytes the size of the int. (e.g. ``uint16 = 2 bytes``)
+    trs "APPENDING INT ", x, " = ", value
-      ## All integers are serialized as **little endian**.
+    c.appendMemCopy value
-      var bytes: array[sizeof(x), byte]
-      when x.sizeof == 8: littleEndian64(addr bytes[0], x.unsafeAddr)
-      elif x.sizeof == 4: littleEndian32(addr bytes[0], x.unsafeAddr)
-      elif x.sizeof == 2: littleEndian16(addr bytes[0], x.unsafeAddr)
-      elif x.sizeof == 1: copyMem(addr bytes[0], x.unsafeAddr, sizeof(x))
-      else: unsupported x.type
-      c.append bytes
-    else:
-      let valueAddr {.used.} = unsafeAddr x
-      trs "APPENDING INT ", x, " = ", makeOpenArray(cast[ptr byte](valueAddr), sizeof(x))
-      c.appendMemCopy x
   elif x is StUint:
     c.appendMemCopy x # TODO: Is this always correct?
   elif x is array|string|seq|openarray:
@@ -426,8 +415,7 @@ func newSszHashingStream(merkelizer: SszChunksMerkelizer): ref OutputStream =
 
 func mixInLength(root: Eth2Digest, length: int): Eth2Digest =
   var dataLen: array[32, byte]
-  var lstLen = uint64(length)
+  dataLen[0..<8] = uint64(length).toBytesLE()
-  littleEndian64(addr dataLen[0], addr lstLen)
   hash(root.data, dataLen)
 
 func merkelizeSerializedChunks(merkelizer: SszChunksMerkelizer,
@@ -516,11 +504,7 @@ func bitlistHashTreeRoot(merkelizer: SszChunksMerkelizer, x: BitSeq): Eth2Digest
 func hashTreeRootImpl[T](x: T): Eth2Digest =
   when T is uint64:
     trs "UINT64; LITTLE-ENDIAN IDENTITY MAPPING"
-    when system.cpuEndian == bigEndian:
+    result.data[0..<8] = x.toBytesLE()
-      littleEndian64(addr result.data[0], x.unsafeAddr)
-    else:
-      let valueAddr = unsafeAddr x
-      result.data[0..7] = makeOpenArray(cast[ptr byte](valueAddr), 8)
   elif (when T is array: ElemType(T) is byte and
       sizeof(T) == sizeof(Eth2Digest) else: false):
     # TODO is this sizeof comparison guranteed? it's whole structure vs field

beacon_chain/ssz/bytes_reader.nim

@@ -1,6 +1,6 @@
 import
-  endians, typetraits, options,
+  typetraits, options,
-  stew/[objects, bitseqs], serialization/testing/tracing,
+  stew/[bitseqs, endians2, objects, bitseqs], serialization/testing/tracing,
   ../spec/[digest, datatypes], ./types
 
 const
@@ -28,17 +28,7 @@ func fromSszBytes*(T: type SomeInteger, data: openarray[byte]): T =
   if data.len < sizeof(result):
     raise newException(MalformedSszError, "SSZ input of insufficient size")
 
-  # TODO: any better way to get a suitably aligned buffer in nim???
+  T.fromBytesLE(data)
-  # see also: https://github.com/nim-lang/Nim/issues/9206
-  var tmp: uint64
-  var alignedBuf = cast[ptr byte](tmp.addr)
-  copyMem(alignedBuf, unsafeAddr data[0], result.sizeof)
-
-  when result.sizeof == 8: littleEndian64(result.addr, alignedBuf)
-  elif result.sizeof == 4: littleEndian32(result.addr, alignedBuf)
-  elif result.sizeof == 2: littleEndian16(result.addr, alignedBuf)
-  elif result.sizeof == 1: copyMem(result.addr, alignedBuf, sizeof(result))
-  else: {.fatal: "Unsupported type deserialization: " & $(type(result)).name.}
 
 func fromSszBytes*(T: type bool, data: openarray[byte]): T =
   # TODO: spec doesn't say what to do if the value is >1 - we'll use the C

nfuzz/libnfuzz.nim

@@ -1,7 +1,7 @@
 import
-  endians, stew/ptrops, stew/ranges/ptr_arith,
+  stew/ptrops, stew/ranges/ptr_arith,
   ../beacon_chain/[ssz, state_transition],
-  ../beacon_chain/spec/[datatypes, helpers, digest, validator, beaconstate,
+  ../beacon_chain/spec/[datatypes, digest, validator, beaconstate,
       state_transition_block],
   # Required for deserialisation of ValidatorSig in Attestation due to
 # https://github.com/nim-lang/Nim/issues/11225
@@ -17,7 +17,7 @@ type
     attesterSlashing: AttesterSlashing
   BlockInput = object
     state: BeaconState
-    beaconBlock: BeaconBlock
+    beaconBlock: SignedBeaconBlock
   BlockHeaderInput = BlockInput
   DepositInput = object
     state: BeaconState
@@ -27,7 +27,7 @@ type
     proposerSlashing: ProposerSlashing
   VoluntaryExitInput = object
     state: BeaconState
-    exit: VoluntaryExit
+    exit: SignedVoluntaryExit
   # This and AssertionError are raised to indicate programming bugs
   # A wrapper to allow exception tracking to identify unexpected exceptions
   FuzzCrashError = object of Exception
@@ -35,10 +35,9 @@ type
 # TODO: change ptr uint to ptr csize_t when available in newer Nim version.
 proc copyState(state: BeaconState, output: ptr byte,
     output_size: ptr uint): bool {.raises: [FuzzCrashError, Defect].} =
-  var resultState: seq[byte]
+  var resultState =
-
     try:
-    resultState = SSZ.encode(state)
+      SSZ.encode(state)
     except IOError as e:
       # Shouldn't occur as the writer isn't a file
       raise newException(FuzzCrashError, "Unexpected failure to serialize.", e)
@@ -57,195 +56,77 @@ proc copyState(state: BeaconState, output: ptr byte,
   copyMem(output, unsafeAddr resultState[0], output_size[])
   result = true
 
-proc nfuzz_attestation(input: openArray[byte], output: ptr byte,
+template decodeAndProcess(typ, process: untyped): bool =
-    output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
+  let flags {.inject.} = if disable_bls: {skipBlsValidation} else: {}
+
   var
-    data: AttestationInput
+    cache {.used, inject.} = get_empty_per_epoch_cache()
-    cache = get_empty_per_epoch_cache()
+    data {.inject.} =
-  let flags = if disable_bls: {skipBlsValidation} else: {}
-
       try:
-    data = SSZ.decode(input, AttestationInput)
+        SSZ.decode(input, typ)
-  except MalformedSszError, SszSizeMismatchError:
+      except MalformedSszError as e:
-    let e = getCurrentException()
         raise newException(
           FuzzCrashError,
-      "SSZ deserialisation failed, likely bug in preprocessing.",
+          "Malformed SSZ, likely bug in preprocessing.", e)
-      e,
+      except SszSizeMismatchError as e:
-    )
+        raise newException(
-
+          FuzzCrashError,
+          "SSZ size mismatch, likely bug in preprocessing.", e)
+  let processOk =
     try:
-    result = process_attestation(data.state, data.attestation,
+      process
-      flags, cache)
+    except IOError as e:
+      raise newException(
+        FuzzCrashError, "Unexpected  (logging?) IOError in state transition", e,
+      )
     except ValueError as e:
-    # These exceptions are expected to be raised by chronicles logging:
-    # See status-im/nim-chronicles#60
-    # TODO remove this when resolved
       raise newException(
         FuzzCrashError,
-      "Unexpected (logging?) error in attestation processing",
+        "Unexpected  (logging?) IOError in state transition", e)
-      e
-    )
-
-  if result:
-    result = copyState(data.state, output, output_size)
-
-proc nfuzz_attester_slashing(input: openArray[byte], output: ptr byte,
-    output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
-  var
-    data: AttesterSlashingInput
-    cache = get_empty_per_epoch_cache()
-  let flags = if disable_bls: {skipBlsValidation} else: {}
-
-  try:
-    data = SSZ.decode(input, AttesterSlashingInput)
-  except MalformedSszError, SszSizeMismatchError:
-    let e = getCurrentException()
-    raise newException(
-      FuzzCrashError,
-      "SSZ deserialisation failed, likely bug in preprocessing.",
-      e,
-    )
-
-  try:
-    # TODO flags
-    result = process_attester_slashing(data.state, data.attesterSlashing, flags, cache)
-  except ValueError as e:
-    # TODO remove when status-im/nim-chronicles#60 is resolved
-    raise newException(
-      FuzzCrashError,
-      "Unexpected (logging?) error in attester slashing",
-      e,
-    )
-
-  if result:
-    result = copyState(data.state, output, output_size)
-
-proc nfuzz_block(input: openArray[byte], output: ptr byte,
-    output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
-  var data: BlockInput
-  let flags = if disable_bls: {skipBlsValidation} else: {}
-
-  try:
-    data = SSZ.decode(input, BlockInput)
-  except MalformedSszError, SszSizeMismatchError:
-    let e = getCurrentException()
-    raise newException(
-      FuzzCrashError,
-      "SSZ deserialisation failed, likely bug in preprocessing.",
-      e,
-    )
-
-  try:
-    result = state_transition(data.state, data.beaconBlock, flags)
-  except IOError, ValueError:
-    # TODO remove when status-im/nim-chronicles#60 is resolved
-    let e = getCurrentException()
-    raise newException(
-      FuzzCrashError,
-      "Unexpected (logging?) error in state transition",
-      e,
-    )
     except Exception as e:
       # TODO why an Exception?
       # Lots of vendor code looks like it might raise a bare exception type
       raise newException(FuzzCrashError, "Unexpected Exception in state transition", e)
 
-  if result:
+  if processOk:
-    result = copyState(data.state, output, output_size)
+    copyState(data.state, output, output_size)
+  else:
+    false
+
+proc nfuzz_attestation(input: openArray[byte], output: ptr byte,
+    output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
+  decodeAndProcess(AttestationInput):
+    process_attestation(data.state, data.attestation, flags, cache)
+
+proc nfuzz_attester_slashing(input: openArray[byte], output: ptr byte,
+    output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
+  decodeAndProcess(AttesterSlashingInput):
+    process_attester_slashing(data.state, data.attesterSlashing, flags, cache)
+
+proc nfuzz_block(input: openArray[byte], output: ptr byte,
+    output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
+  decodeAndProcess(BlockInput):
+    state_transition(data.state, data.beaconBlock, flags)
 
 proc nfuzz_block_header(input: openArray[byte], output: ptr byte,
     output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
-  var
+  decodeAndProcess(BlockHeaderInput):
-    data: BlockHeaderInput
+    process_block_header(data.state, data.beaconBlock.message, flags, cache)
-    cache = get_empty_per_epoch_cache()
-  let flags = if disable_bls: {skipBlsValidation} else: {}
-
-  try:
-    data = SSZ.decode(input, BlockHeaderInput)
-  except MalformedSszError, SszSizeMismatchError:
-    let e = getCurrentException()
-    raise newException(
-      FuzzCrashError,
-      "SSZ deserialisation failed, likely bug in preprocessing.",
-      e,
-    )
-
-  try:
-    # TODO disable bls
-    result = process_block_header(data.state, data.beaconBlock, flags, cache)
-  except IOError, ValueError:
-    let e = getCurrentException()
-    # TODO remove when status-im/nim-chronicles#60 is resolved
-    raise newException(
-      FuzzCrashError,
-      "Unexpected IOError in block header processing",
-      e,
-    )
-
-  if result:
-    result = copyState(data.state, output, output_size)
-
 
 proc nfuzz_deposit(input: openArray[byte], output: ptr byte,
     output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
-  var
+  decodeAndProcess(DepositInput):
-    data: DepositInput
+    process_deposit(data.state, data.deposit, flags)
-  let flags = if disable_bls: {skipBlsValidation} else: {}
-
-  try:
-    data = SSZ.decode(input, DepositInput)
-  except MalformedSszError, SszSizeMismatchError:
-    let e = getCurrentException()
-    raise newException(
-      FuzzCrashError,
-      "SSZ deserialisation failed, likely bug in preprocessing.",
-      e,
-    )
-
-  try:
-    result = process_deposit(data.state, data.deposit, flags)
-  except IOError, ValueError:
-    let e = getCurrentException()
-    # TODO remove when status-im/nim-chronicles#60 is resolved
-    raise newException(
-      FuzzCrashError,
-      "Unexpected (logging?) error in deposit processing",
-      e,
-    )
-
-  if result:
-    result = copyState(data.state, output, output_size)
 
 proc nfuzz_proposer_slashing(input: openArray[byte], output: ptr byte,
     output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
-  var
+  decodeAndProcess(ProposerSlashingInput):
-    data: ProposerSlashingInput
+    process_proposer_slashing(data.state, data.proposerSlashing, flags, cache)
-    cache = get_empty_per_epoch_cache()
-  let flags = if disable_bls: {skipBlsValidation} else: {}
 
-  try:
+proc nfuzz_voluntary_exit(input: openArray[byte], output: ptr byte,
-    data = SSZ.decode(input, ProposerSlashingInput)
+    output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
-  except MalformedSszError, SszSizeMismatchError:
+  decodeAndProcess(VoluntaryExitInput):
-    let e = getCurrentException()
+    process_voluntary_exit(data.state, data.exit, flags)
-    raise newException(
-      FuzzCrashError,
-      "SSZ deserialisation failed, likely bug in preprocessing.",
-      e,
-    )
-
-  try:
-    result = process_proposer_slashing(data.state, data.proposerSlashing, flags, cache)
-  except ValueError as e:
-    # TODO remove when status-im/nim-chronicles#60 is resolved
-    raise newException(
-      FuzzCrashError,
-      "Unexpected (logging?) error in proposer slashing",
-      e,
-    )
-
-  if result:
-    result = copyState(data.state, output, output_size)
 
 # Note: Could also accept raw input pointer and access list_size + seed here.
 # However, list_size needs to be known also outside this proc to allocate output.
@@ -274,32 +155,3 @@ proc nfuzz_shuffle(input_seed: ptr byte, output: var openArray[uint64]): bool
       sizeof(ValidatorIndex))
 
   result = true
-
-proc nfuzz_voluntary_exit(input: openArray[byte], output: ptr byte,
-    output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
-  var
-    data: VoluntaryExitInput
-  let flags = if disable_bls: {skipBlsValidation} else: {}
-
-  try:
-    data = SSZ.decode(input, VoluntaryExitInput)
-  except MalformedSszError, SszSizeMismatchError:
-    let e = getCurrentException()
-    raise newException(
-      FuzzCrashError,
-      "SSZ deserialisation failed, likely bug in preprocessing.",
-      e,
-    )
-
-  try:
-    result = process_voluntary_exit(data.state, data.exit, flags)
-  except ValueError as e:
-    # TODO remove when status-im/nim-chronicles#60 is resolved
-    raise newException(
-      FuzzCrashError,
-      "Unexpected (logging?) error in voluntary exit processing",
-      e,
-    )
-
-  if result:
-    result = copyState(data.state, output, output_size)