nimbus-eth2/nfuzz/libnfuzz.nim

174 lines
6.9 KiB
Nim
Raw Permalink Normal View History

# Required for deserialisation of ValidatorSig in Attestation due to
# https://github.com/nim-lang/Nim/issues/11225
import
stew/ptrops, stew/ranges/ptr_arith, chronicles,
../beacon_chain/spec/datatypes/phase0,
../beacon_chain/spec/[
disentangle eth2 types from the ssz library (#2785) * reorganize ssz dependencies This PR continues the work in https://github.com/status-im/nimbus-eth2/pull/2646, https://github.com/status-im/nimbus-eth2/pull/2779 as well as past issues with serialization and type, to disentangle SSZ from eth2 and at the same time simplify imports and exports with a structured approach. The principal idea here is that when a library wants to introduce SSZ support, they do so via 3 files: * `ssz_codecs` which imports and reexports `codecs` - this covers the basic byte conversions and ensures no overloads get lost * `xxx_merkleization` imports and exports `merkleization` to specialize and get access to `hash_tree_root` and friends * `xxx_ssz_serialization` imports and exports `ssz_serialization` to specialize ssz for a specific library Those that need to interact with SSZ always import the `xxx_` versions of the modules and never `ssz` itself so as to keep imports simple and safe. This is similar to how the REST / JSON-RPC serializers are structured in that someone wanting to serialize spec types to REST-JSON will import `eth2_rest_serialization` and nothing else. * split up ssz into a core library that is independendent of eth2 types * rename `bytes_reader` to `codec` to highlight that it contains coding and decoding of bytes and native ssz types * remove tricky List init overload that causes compile issues * get rid of top-level ssz import * reenable merkleization tests * move some "standard" json serializers to spec * remove `ValidatorIndex` serialization for now * remove test_ssz_merkleization * add tests for over/underlong byte sequences * fix broken seq[byte] test - seq[byte] is not an SSZ type There are a few things this PR doesn't solve: * like #2646 this PR is weak on how to handle root and other dontSerialize fields that "sometimes" should be computed - the same problem appears in REST / JSON-RPC etc * Fix a build problem on macOS * Another way to fix the macOS builds Co-authored-by: Zahary Karadjov <zahary@gmail.com>
2021-08-18 18:57:58 +00:00
beaconstate, eth2_ssz_serialization, forks, validator, state_transition,
state_transition_block]
type
AttestationInput = object
state: phase0.BeaconState
attestation: Attestation
AttesterSlashingInput = object
state: phase0.BeaconState
attesterSlashing: AttesterSlashing
BlockInput = object
state: phase0.BeaconState
beaconBlock: phase0.SignedBeaconBlock
BlockHeaderInput = BlockInput
DepositInput = object
state: phase0.BeaconState
deposit: Deposit
ProposerSlashingInput = object
state: phase0.BeaconState
proposerSlashing: ProposerSlashing
VoluntaryExitInput = object
state: phase0.BeaconState
2020-03-05 00:29:27 +00:00
exit: SignedVoluntaryExit
# This and AssertionError are raised to indicate programming bugs
2020-01-08 01:03:33 +00:00
# A wrapper to allow exception tracking to identify unexpected exceptions
FuzzCrashError = object of CatchableError
2019-11-28 22:01:12 +00:00
# TODO: change ptr uint to ptr csize_t when available in newer Nim version.
proc copyState(state: phase0.BeaconState, xoutput: ptr byte,
xoutput_size: ptr uint): bool {.raises: [FuzzCrashError, Defect].} =
2020-03-05 00:29:27 +00:00
var resultState =
try:
SSZ.encode(state)
except IOError as e:
# Shouldn't occur as the writer isn't a file
raise newException(FuzzCrashError, "Unexpected failure to serialize.", e)
if unlikely(resultState.len.uint > xoutput_size[]):
2020-01-08 01:03:33 +00:00
let msg = (
"Not enough xoutput buffer provided to nimbus harness. Provided: " &
$(xoutput_size[]) &
2020-01-08 01:03:33 +00:00
"Required: " &
$resultState.len.uint
)
raise newException(FuzzCrashError, msg)
xoutput_size[] = resultState.len.uint
# TODO: improvement might be to write directly to buffer with xoutputStream
# and SszWriter (but then need to ensure length doesn't overflow)
copyMem(xoutput, unsafeAddr resultState[0], xoutput_size[])
result = true
2020-03-05 00:29:27 +00:00
template decodeAndProcess(typ, process: untyped): bool =
let flags {.inject.} = if disable_bls: {skipBlsValidation} else: {}
2020-03-05 00:29:27 +00:00
var
cache {.used, inject.} = StateCache()
data {.inject.} = newClone(
2020-03-05 00:29:27 +00:00
try:
SSZ.decode(input, typ)
except MalformedSszError as e:
raise newException(
FuzzCrashError,
"Malformed SSZ, likely bug in preprocessing.", e)
except SszSizeMismatchError as e:
raise newException(
FuzzCrashError,
"SSZ size mismatch, likely bug in preprocessing.", e)
)
2020-03-05 00:29:27 +00:00
let processOk =
try:
process
except IOError as e:
raise newException(
FuzzCrashError, "Unexpected (logging?) IOError in state transition", e,
)
except ValueError as e:
raise newException(
FuzzCrashError,
"Unexpected (logging?) IOError in state transition", e)
except Exception as e:
# TODO why an Exception?
# Lots of vendor code looks like it might raise a bare exception type
raise newException(FuzzCrashError, "Unexpected Exception in state transition", e)
if processOk:
copyState(data.state, xoutput, xoutput_size)
2020-03-05 00:29:27 +00:00
else:
false
proc nfuzz_attestation(input: openArray[byte], xoutput: ptr byte,
xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
2020-03-05 00:29:27 +00:00
decodeAndProcess(AttestationInput):
process_attestation(data.state, data.attestation, flags, 0.Gwei, cache).isOk
proc nfuzz_attester_slashing(input: openArray[byte], xoutput: ptr byte,
xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
2020-03-05 00:29:27 +00:00
decodeAndProcess(AttesterSlashingInput):
Implement split preset/config support (#2710) * Implement split preset/config support This is the initial bulk refactor to introduce runtime config values in a number of places, somewhat replacing the existing mechanism of loading network metadata. It still needs more work, this is the initial refactor that introduces runtime configuration in some of the places that need it. The PR changes the way presets and constants work, to match the spec. In particular, a "preset" now refers to the compile-time configuration while a "cfg" or "RuntimeConfig" is the dynamic part. A single binary can support either mainnet or minimal, but not both. Support for other presets has been removed completely (can be readded, in case there's need). There's a number of outstanding tasks: * `SECONDS_PER_SLOT` still needs fixing * loading custom runtime configs needs redoing * checking constants against YAML file * yeerongpilly support `build/nimbus_beacon_node --network=yeerongpilly --discv5:no --log-level=DEBUG` * load fork epoch from config * fix fork digest sent in status * nicer error string for request failures * fix tools * one more * fixup * fixup * fixup * use "standard" network definition folder in local testnet Files are loaded from their standard locations, including genesis etc, to conform to the format used in the `eth2-networks` repo. * fix launch scripts, allow unknown config values * fix base config of rest test * cleanups * bundle mainnet config using common loader * fix spec links and names * only include supported preset in binary * drop yeerongpilly, add altair-devnet-0, support boot_enr.yaml
2021-07-12 13:01:38 +00:00
process_attester_slashing(defaultRuntimeConfig, data.state, data.attesterSlashing, flags, cache).isOk
proc nfuzz_block(input: openArray[byte], xoutput: ptr byte,
xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
# There's not a perfect approach here, but it's not worth switching the rest
# and requiring HashedBeaconState (yet). So to keep consistent, puts wrapper
# only in one function.
proc state_transition(
Implement split preset/config support (#2710) * Implement split preset/config support This is the initial bulk refactor to introduce runtime config values in a number of places, somewhat replacing the existing mechanism of loading network metadata. It still needs more work, this is the initial refactor that introduces runtime configuration in some of the places that need it. The PR changes the way presets and constants work, to match the spec. In particular, a "preset" now refers to the compile-time configuration while a "cfg" or "RuntimeConfig" is the dynamic part. A single binary can support either mainnet or minimal, but not both. Support for other presets has been removed completely (can be readded, in case there's need). There's a number of outstanding tasks: * `SECONDS_PER_SLOT` still needs fixing * loading custom runtime configs needs redoing * checking constants against YAML file * yeerongpilly support `build/nimbus_beacon_node --network=yeerongpilly --discv5:no --log-level=DEBUG` * load fork epoch from config * fix fork digest sent in status * nicer error string for request failures * fix tools * one more * fixup * fixup * fixup * use "standard" network definition folder in local testnet Files are loaded from their standard locations, including genesis etc, to conform to the format used in the `eth2-networks` repo. * fix launch scripts, allow unknown config values * fix base config of rest test * cleanups * bundle mainnet config using common loader * fix spec links and names * only include supported preset in binary * drop yeerongpilly, add altair-devnet-0, support boot_enr.yaml
2021-07-12 13:01:38 +00:00
cfg: RuntimeConfig, data: auto, blck: auto, flags: auto,
rollback: RollbackForkedHashedProc): auto =
2020-11-13 12:57:48 +00:00
var
fhState = (ref ForkedHashedBeaconState)(
phase0Data: phase0.HashedBeaconState(
data: data.state, root: hash_tree_root(data.state)),
kind: BeaconStateFork.Phase0)
2020-11-13 12:57:48 +00:00
cache = StateCache()
info = ForkedEpochInfo()
2020-11-13 12:57:48 +00:00
result =
state_transition(
cfg, fhState[], blck, cache, info, flags, rollback)
data.state = fhState.phase0Data.data
2020-03-05 00:29:27 +00:00
decodeAndProcess(BlockInput):
Implement split preset/config support (#2710) * Implement split preset/config support This is the initial bulk refactor to introduce runtime config values in a number of places, somewhat replacing the existing mechanism of loading network metadata. It still needs more work, this is the initial refactor that introduces runtime configuration in some of the places that need it. The PR changes the way presets and constants work, to match the spec. In particular, a "preset" now refers to the compile-time configuration while a "cfg" or "RuntimeConfig" is the dynamic part. A single binary can support either mainnet or minimal, but not both. Support for other presets has been removed completely (can be readded, in case there's need). There's a number of outstanding tasks: * `SECONDS_PER_SLOT` still needs fixing * loading custom runtime configs needs redoing * checking constants against YAML file * yeerongpilly support `build/nimbus_beacon_node --network=yeerongpilly --discv5:no --log-level=DEBUG` * load fork epoch from config * fix fork digest sent in status * nicer error string for request failures * fix tools * one more * fixup * fixup * fixup * use "standard" network definition folder in local testnet Files are loaded from their standard locations, including genesis etc, to conform to the format used in the `eth2-networks` repo. * fix launch scripts, allow unknown config values * fix base config of rest test * cleanups * bundle mainnet config using common loader * fix spec links and names * only include supported preset in binary * drop yeerongpilly, add altair-devnet-0, support boot_enr.yaml
2021-07-12 13:01:38 +00:00
state_transition(defaultRuntimeConfig, data, data.beaconBlock, flags, noRollback)
proc nfuzz_block_header(input: openArray[byte], xoutput: ptr byte,
xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
2020-03-05 00:29:27 +00:00
decodeAndProcess(BlockHeaderInput):
process_block_header(data.state, data.beaconBlock.message, flags, cache).isOk
proc nfuzz_deposit(input: openArray[byte], xoutput: ptr byte,
xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
2020-03-05 00:29:27 +00:00
decodeAndProcess(DepositInput):
Implement split preset/config support (#2710) * Implement split preset/config support This is the initial bulk refactor to introduce runtime config values in a number of places, somewhat replacing the existing mechanism of loading network metadata. It still needs more work, this is the initial refactor that introduces runtime configuration in some of the places that need it. The PR changes the way presets and constants work, to match the spec. In particular, a "preset" now refers to the compile-time configuration while a "cfg" or "RuntimeConfig" is the dynamic part. A single binary can support either mainnet or minimal, but not both. Support for other presets has been removed completely (can be readded, in case there's need). There's a number of outstanding tasks: * `SECONDS_PER_SLOT` still needs fixing * loading custom runtime configs needs redoing * checking constants against YAML file * yeerongpilly support `build/nimbus_beacon_node --network=yeerongpilly --discv5:no --log-level=DEBUG` * load fork epoch from config * fix fork digest sent in status * nicer error string for request failures * fix tools * one more * fixup * fixup * fixup * use "standard" network definition folder in local testnet Files are loaded from their standard locations, including genesis etc, to conform to the format used in the `eth2-networks` repo. * fix launch scripts, allow unknown config values * fix base config of rest test * cleanups * bundle mainnet config using common loader * fix spec links and names * only include supported preset in binary * drop yeerongpilly, add altair-devnet-0, support boot_enr.yaml
2021-07-12 13:01:38 +00:00
process_deposit(defaultRuntimeConfig, data.state, data.deposit, flags).isOk
proc nfuzz_proposer_slashing(input: openArray[byte], xoutput: ptr byte,
xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
2020-03-05 00:29:27 +00:00
decodeAndProcess(ProposerSlashingInput):
Implement split preset/config support (#2710) * Implement split preset/config support This is the initial bulk refactor to introduce runtime config values in a number of places, somewhat replacing the existing mechanism of loading network metadata. It still needs more work, this is the initial refactor that introduces runtime configuration in some of the places that need it. The PR changes the way presets and constants work, to match the spec. In particular, a "preset" now refers to the compile-time configuration while a "cfg" or "RuntimeConfig" is the dynamic part. A single binary can support either mainnet or minimal, but not both. Support for other presets has been removed completely (can be readded, in case there's need). There's a number of outstanding tasks: * `SECONDS_PER_SLOT` still needs fixing * loading custom runtime configs needs redoing * checking constants against YAML file * yeerongpilly support `build/nimbus_beacon_node --network=yeerongpilly --discv5:no --log-level=DEBUG` * load fork epoch from config * fix fork digest sent in status * nicer error string for request failures * fix tools * one more * fixup * fixup * fixup * use "standard" network definition folder in local testnet Files are loaded from their standard locations, including genesis etc, to conform to the format used in the `eth2-networks` repo. * fix launch scripts, allow unknown config values * fix base config of rest test * cleanups * bundle mainnet config using common loader * fix spec links and names * only include supported preset in binary * drop yeerongpilly, add altair-devnet-0, support boot_enr.yaml
2021-07-12 13:01:38 +00:00
process_proposer_slashing(defaultRuntimeConfig, data.state, data.proposerSlashing, flags, cache).isOk
proc nfuzz_voluntary_exit(input: openArray[byte], xoutput: ptr byte,
xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
2020-03-05 00:29:27 +00:00
decodeAndProcess(VoluntaryExitInput):
Implement split preset/config support (#2710) * Implement split preset/config support This is the initial bulk refactor to introduce runtime config values in a number of places, somewhat replacing the existing mechanism of loading network metadata. It still needs more work, this is the initial refactor that introduces runtime configuration in some of the places that need it. The PR changes the way presets and constants work, to match the spec. In particular, a "preset" now refers to the compile-time configuration while a "cfg" or "RuntimeConfig" is the dynamic part. A single binary can support either mainnet or minimal, but not both. Support for other presets has been removed completely (can be readded, in case there's need). There's a number of outstanding tasks: * `SECONDS_PER_SLOT` still needs fixing * loading custom runtime configs needs redoing * checking constants against YAML file * yeerongpilly support `build/nimbus_beacon_node --network=yeerongpilly --discv5:no --log-level=DEBUG` * load fork epoch from config * fix fork digest sent in status * nicer error string for request failures * fix tools * one more * fixup * fixup * fixup * use "standard" network definition folder in local testnet Files are loaded from their standard locations, including genesis etc, to conform to the format used in the `eth2-networks` repo. * fix launch scripts, allow unknown config values * fix base config of rest test * cleanups * bundle mainnet config using common loader * fix spec links and names * only include supported preset in binary * drop yeerongpilly, add altair-devnet-0, support boot_enr.yaml
2021-07-12 13:01:38 +00:00
process_voluntary_exit(defaultRuntimeConfig, data.state, data.exit, flags, cache).isOk
# Note: Could also accept raw input pointer and access list_size + seed here.
# However, list_size needs to be known also outside this proc to allocate xoutput.
# TODO: rework to copy immediatly in an uint8 openArray, considering we have to
# go over the list anyhow?
proc nfuzz_shuffle(input_seed: ptr byte, xoutput: var openArray[uint64]): bool
2020-01-08 01:03:33 +00:00
{.exportc, raises: [Defect].} =
var seed: Eth2Digest
2019-11-28 22:01:12 +00:00
# Should be OK as max 2 bytes are passed by the framework.
let list_size = xoutput.len
2019-11-28 22:01:12 +00:00
copyMem(addr(seed.data), input_seed, sizeof(seed.data))
2019-11-28 22:01:12 +00:00
var shuffled_seq: seq[ValidatorIndex]
for i in 0..<list_size:
shuffled_seq.add i.ValidatorIndex
shuffle_list(shuffled_seq, seed)
for i in 0..<list_size:
# ValidatorIndex is currently wrongly uint32 so we copy this 1 by 1,
# assumes passed xoutput is zeroed.
copyMem(offset(addr xoutput, i), shuffled_seq[i].unsafeAddr,
sizeof(ValidatorIndex))
2019-11-28 22:01:12 +00:00
true