Merge pull request #616 from status-im/devel

Testnet0 Release 2019-12-03

Makefile

@@ -90,3 +90,15 @@ testnet1-no-clean: | build deps
 
 clean: | clean-common
 	rm -rf build/{$(TOOLS_CSV),all_tests,*_node}
+
+libnfuzz.so: | build deps-common beacon_chain.nims
+	echo -e $(BUILD_MSG) "build/$@" && \
+		$(ENV_SCRIPT) nim c -d:release --app:lib --noMain --nimcache:nimcache/libnfuzz $(NIM_PARAMS) -o:build/$@.0 nfuzz/libnfuzz.nim && \
+		rm -f build/$@ && \
+		ln -s $@.0 build/$@
+
+libnfuzz.a: | build deps-common beacon_chain.nims
+	echo -e $(BUILD_MSG) "build/$@" && \
+		rm -f build/$@ && \
+		$(ENV_SCRIPT) nim c -d:release --app:staticlib --noMain --nimcache:nimcache/libnfuzz_static $(NIM_PARAMS) -o:build/$@ nfuzz/libnfuzz.nim && \
+		[[ -e "$@" ]] && mv "$@" build/ # workaround for https://github.com/nim-lang/Nim/issues/12745

beacon_chain.nimble

@@ -9,6 +9,7 @@ author        = "Status Research & Development GmbH"
 description   = "Eth2.0 research implementation of the beacon chain"
 license       = "MIT or Apache License 2.0"
 installDirs   = @["beacon_chain", "research"]
+skipDirs      = @["nfuzz"]
 bin           = @[
   "beacon_chain/beacon_node",
   "research/serialized_sizes",

beacon_chain/beacon_node.nim

@@ -373,6 +373,12 @@ proc sendAttestation(node: BeaconNode,
 
   node.network.broadcast(topicAttestations, attestation)
 
+  if node.config.dump:
+    SSZ.saveFile(
+      node.config.dumpDir / "att-" & $attestationData.slot & "-" &
+      $attestationData.index & "-" & validator.pubKey.shortLog &
+      ".ssz", attestation)
+
   info "Attestation sent",
     attestationData = shortLog(attestationData),
     validator = shortLog(validator),
@@ -482,6 +488,15 @@ proc proposeBlock(node: BeaconNode,
     validator = shortLog(validator),
     cat = "consensus"
 
+  if node.config.dump:
+    SSZ.saveFile(
+      node.config.dumpDir / "block-" & $newBlock.slot & "-" &
+      shortLog(newBlockRef.root) & ".ssz", newBlock)
+    SSZ.saveFile(
+      node.config.dumpDir / "state-" & $tmpState.data.slot & "-" &
+      shortLog(newBlockRef.root) & "-"  & shortLog(tmpState.root) & ".ssz",
+      tmpState.data)
+
   node.network.broadcast(topicBeaconBlocks, newBlock)
 
   beacon_blocks_proposed.inc()

beacon_chain/block_pool.nim

@@ -81,7 +81,7 @@ proc init*(T: type BlockPool, db: BeaconChainDB): BlockPool =
         link(newRef, curRef)
         curRef = curRef.parent
       blocks[curRef.root] = curRef
-      debug "Populating block pool", key = curRef.root, val = curRef
+      trace "Populating block pool", key = curRef.root, val = curRef
 
       if latestStateRoot.isNone() and db.containsState(blck.state_root):
         latestStateRoot = some(blck.state_root)
@@ -169,7 +169,7 @@ proc addResolvedBlock(
   link(parent, blockRef)
 
   pool.blocks[blockRoot] = blockRef
-  debug "Populating block pool", key = blockRoot, val = blockRef
+  trace "Populating block pool", key = blockRoot, val = blockRef
 
   pool.addSlotMapping(blockRef)
 

beacon_chain/conf.nim

@@ -136,6 +136,11 @@ type
         desc: "Listening HTTP port of the metrics server."
         name: "metrics-server-port" }: uint16
 
+      dump* {.
+        defaultValue: false
+        desc: "Write SSZ dumps of blocks, attestations and states to data dir"
+        .}: bool
+
     of createTestnet:
       validatorsDir* {.
         desc: "Directory containing validator descriptors named 'vXXXXXXX.deposit.json'."
@@ -243,3 +248,6 @@ proc defaultDataDir*(conf: BeaconNodeConf): string =
 proc validatorFileBaseName*(validatorIdx: int): string =
   # there can apparently be tops 4M validators so we use 7 digits..
   fmt"v{validatorIdx:07}"
+
+func dumpDir*(conf: BeaconNodeConf): string =
+  conf.dataDir / "dump"

beacon_chain/fork_choice.nim

@@ -1,16 +1,28 @@
 import
   deques, options, sequtils, tables,
-  chronicles,
   ./spec/[datatypes, crypto, helpers],
   ./attestation_pool, ./beacon_node_types, ./ssz
 
 func get_ancestor(blck: BlockRef, slot: Slot): BlockRef =
-  if blck.slot == slot:
-    blck
-  elif blck.slot < slot:
-    nil
-  else:
-    get_ancestor(blck.parent, slot)
+  var blck = blck
+
+  var depth = 0
+  const maxDepth = (100'i64 * 365 * 24 * 60 * 60 div SECONDS_PER_SLOT.int)
+
+  while true:
+    if blck.slot == slot:
+      return blck
+
+    if blck.slot < slot:
+      return nil
+
+    if blck.parent == nil:
+      return nil
+
+    doAssert depth < maxDepth
+    depth += 1
+
+    blck = blck.parent
 
 # https://github.com/ethereum/eth2.0-specs/blob/v0.8.4/specs/core/0_fork-choice.md
 # The structure of this code differs from the spec since we use a different

beacon_chain/libp2p_daemon_backend.nim

@@ -590,10 +590,13 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend =
     msg.defineThunk quote do:
       proc `thunkName`(`daemonVar`: `DaemonAPI`,
                        `streamVar`: `P2PStream`) {.async, gcsafe.} =
-        when chronicles.runtimeFilteringEnabled:
-          setLogLevel(LogLevel.TRACE)
-          defer: setLogLevel(LogLevel.DEBUG)
-          trace "incoming " & `msgNameLit` & " stream"
+        ## Uncomment this to enable tracing on all incoming requests
+        ## You can include `msgNameLit` in the condition to select
+        ## more specific requests:
+        # when chronicles.runtimeFilteringEnabled:
+        #   setLogLevel(LogLevel.TRACE)
+        #   defer: setLogLevel(LogLevel.DEBUG)
+        #   trace "incoming " & `msgNameLit` & " stream"
 
         defer:
           `await` safeClose(`streamVar`)

beacon_chain/spec/validator.nim

@@ -13,7 +13,7 @@ import
 # TODO: Proceed to renaming and signature changes
 # https://github.com/ethereum/eth2.0-specs/blob/v0.9.2/specs/core/0_beacon-chain.md#compute_shuffled_index
 # https://github.com/ethereum/eth2.0-specs/blob/v0.9.2/specs/core/0_beacon-chain.md#compute_committee
-func get_shuffled_seq(seed: Eth2Digest,
+func get_shuffled_seq*(seed: Eth2Digest,
                       list_size: uint64,
                       ): seq[ValidatorIndex] =
   ## Via https://github.com/protolambda/eth2-shuffle/blob/master/shuffle.go

beacon_chain/validator_keygen.nim

@@ -52,19 +52,20 @@ proc generateDeposits*(totalValidators: int,
 proc sendDeposits*(
     deposits: seq[Deposit],
     depositWeb3Url, depositContractAddress, privateKey: string) {.async.} =
+
   var web3 = await newWeb3(depositWeb3Url)
   if privateKey.len != 0:
     web3.privateKey = initPrivateKey(privateKey)
-
-  let eth1Addresses = await web3.provider.eth_accounts()
-  if eth1Addresses.len == 0:
-    error "Eth1 account rejected"
-    return
+  else:
+    let accounts = await web3.provider.eth_accounts()
+    if accounts.len == 0:
+      error "No account offered by the web3 provider", web3url = depositWeb3Url
+      return
+    web3.defaultAccount = accounts[0]
 
   let contractAddress = Address.fromHex(depositContractAddress)
 
   for i, dp in deposits:
-    web3.defaultAccount = eth1Addresses[0]
     let depositContract = web3.contractSender(DepositContract, contractAddress)
     discard await depositContract.deposit(
       Bytes48(dp.data.pubKey.getBytes()),

docker/Dockerfile

@@ -26,7 +26,7 @@ RUN cd /root/nim-beacon-chain \
  && git fetch \
  && git reset --hard ${GIT_REVISION} \
  && make -j$(nproc) update \
- && make LOG_LEVEL=TRACE NIMFLAGS="-d:debug -d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS}" beacon_node
+ && make LOG_LEVEL=DEBUG NIMFLAGS="-d:debug -d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS}" beacon_node
 
 # --------------------------------- #
 # Starting new image to reduce size #

nfuzz/README.md

@@ -0,0 +1,40 @@
+# Introduction
+
+`libnfuzz` is a wrapper library that exports to C, a set of fuzzing test cases
+written in Nim and making use of nim-beacon-chain.
+
+
+# Building
+
+To build the wrapper library (for more details follow first the instructions from
+[nim-beacon-chain](../README.md)):
+
+```bash
+git clone https://github.com/status-im/nim-beacon-chain.git
+cd nim-beacon-chain
+make
+# static library
+make libnfuzz.a
+# dynamic loaded library
+make libnfuzz.so
+```
+
+For the library to be useful for fuzzing with libFuzzer (e.g. for
+integration with [beacon-fuzz](https://github.com/sigp/beacon-fuzz)) we can pass
+additional Nim arguments, e.g.:
+
+```bash
+make libnfuzz.a NIMFLAGS="--cc:clang --passC:'-fsanitize=fuzzer' --passL='-fsanitize=fuzzer'"
+```
+
+Other useful options might include: `--clang.path:<path>`, `--clang.exe:<exe>`, `--clang.linkerexe:<exe>`.
+
+It might also deem useful to lower the log level, e.g. by adding `-d:chronicles_log_level=fatal`.
+
+# Usage
+There is a `libnfuzz.h` file provided for easy including in C or C++ projects.
+
+It is most important that before any of the exported tests are called, the
+`NimMain()` call is done first. Additionally, all following library calls need
+to be done from the same thread as from where the original `NimMain()` call was
+done.

nfuzz/libnfuzz.h

@@ -0,0 +1,25 @@
+#ifndef __LIBNFUZZ_H__
+#define __LIBNFUZZ_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** Initialize Nim & Garbage Collector. Must be called before anything else
+ * of the API. Also, all following calls must come from the same thread as from
+ * which this call was done.
+ */
+void NimMain();
+
+/** Supported fuzzing tests */
+bool nfuzz_block(uint8_t* input_ptr, size_t input_size,
+  uint8_t* output_ptr, size_t* output_size);
+bool nfuzz_attestation(uint8_t* input_ptr, size_t input_size,
+  uint8_t* output_ptr, size_t* output_size);
+bool nfuzz_shuffle(uint8_t* seed_ptr, uint64_t* output_ptr, size_t output_size);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif //__LIBNFUZZ_H__

nfuzz/libnfuzz.nim

@@ -0,0 +1,99 @@
+import
+  endians, stew/ptrops, stew/ranges/ptr_arith,
+  ../beacon_chain/[ssz, state_transition],
+  ../beacon_chain/spec/[datatypes, helpers, digest, validator, beaconstate],
+# Required for deserialisation of ValidatorSig in Attestation due to
+# https://github.com/nim-lang/Nim/issues/11225
+  ../beacon_chain/spec/crypto
+
+type
+  BlockInput = object
+    state: BeaconState
+    beaconBlock: BeaconBlock
+  AttestationInput = object
+    state: BeaconState
+    attestation: Attestation
+
+# TODO: change ptr uint to ptr csize_t when available in newer Nim version.
+proc copyState(state: BeaconState, output: ptr byte,
+    output_size: ptr uint): bool {.raises:[].} =
+  var resultState: seq[byte]
+
+  try:
+    resultState = SSZ.encode(state)
+  except IOError, Defect:
+    return false
+
+  if resultState.len.uint <= output_size[]:
+    output_size[] = resultState.len.uint
+    # TODO: improvement might be to write directly to buffer with OutputStream
+    # and SszWriter
+    copyMem(output, unsafeAddr resultState[0], output_size[])
+    result = true
+
+proc nfuzz_block(input: openArray[byte], output: ptr byte,
+    output_size: ptr uint): bool {.exportc, raises:[].} =
+  var data: BlockInput
+
+  try:
+    data = SSZ.decode(input, BlockInput)
+  except MalformedSszError, SszSizeMismatchError, RangeError:
+    return false
+
+  try:
+    result = state_transition(data.state, data.beaconBlock, flags = {})
+  except ValueError, RangeError, Exception:
+    discard
+
+  if result:
+    result = copyState(data.state, output, output_size)
+
+proc nfuzz_attestation(input: openArray[byte], output: ptr byte,
+    output_size: ptr uint): bool {.exportc, raises:[].} =
+  var
+    data: AttestationInput
+    cache = get_empty_per_epoch_cache()
+
+  try:
+    data = SSZ.decode(input, AttestationInput)
+  except MalformedSszError, SszSizeMismatchError, RangeError:
+    return false
+
+  try:
+    result = process_attestation(data.state, data.attestation,
+      flags = {}, cache)
+  except ValueError, RangeError:
+    discard
+
+  if result:
+    result = copyState(data.state, output, output_size)
+
+# Note: Could also accept raw input pointer and access list_size + seed here.
+# However, list_size needs to be known also outside this proc to allocate output.
+# TODO: rework to copy immediatly in an uint8 openArray, considering we have to
+# go over the list anyhow?
+proc nfuzz_shuffle(input_seed: ptr byte, output: var openArray[uint64]): bool
+    {.exportc, raises:[].} =
+  var seed: Eth2Digest
+  # Should be OK as max 2 bytes are passed by the framework.
+  let list_size = output.len.uint64
+
+  copyMem(addr(seed.data), input_seed, sizeof(seed.data))
+
+  var shuffled_seq: seq[ValidatorIndex]
+  try:
+    shuffled_seq = get_shuffled_seq(seed, list_size)
+  except RangeError:
+    return false
+
+  # TODO: Hah! AssertionError doesn't get picked up by raises. Do we let them
+  # slip or shall we wrap one big try/except AssertionError around the calls?
+  doAssert(list_size == shuffled_seq.len.uint64)
+
+  for i in 0..<list_size:
+    # ValidatorIndex is currently wrongly uint32 so we copy this 1 by 1,
+    # assumes passed output is zeroed.
+    copyMem(offset(addr output, i.int), shuffled_seq[i.int].unsafeAddr,
+      sizeof(ValidatorIndex))
+
+  result = true

scripts/connect_to_testnet.nims

@@ -56,6 +56,7 @@ cli do (testnetName {.argument.}: string):
     dataDirName = testnetName.replace("/", "_")
     dataDir = buildDir / "data" / dataDirName
     validatorsDir = dataDir / "validators"
+    dumpDir = dataDir / "dump"
     beaconNodeBinary = buildDir / "beacon_node_" & dataDirName
     nimFlags = "-d:chronicles_log_level=DEBUG " & getEnv("NIM_PARAMS")
 
@@ -75,6 +76,8 @@ cli do (testnetName {.argument.}: string):
   cd rootDir
   exec &"""nim c {nimFlags} -d:"const_preset={preset}" -o:"{beaconNodeBinary}" beacon_chain/beacon_node.nim"""
 
+  mkDir dumpDir
+
   if depositContractOpt.len > 0 and not system.dirExists(validatorsDir):
     mode = Silent
     echo "Would you like to become a validator (you'll need access to 32 GoETH)? [Yn]"
@@ -101,6 +104,7 @@ cli do (testnetName {.argument.}: string):
   mode = Verbose
   exec replace(&"""{beaconNodeBinary}
     --data-dir="{dataDir}"
+    --dump=true
     --bootstrap-file="{testnetDir/bootstrapFile}"
     --state-snapshot="{testnetDir/genesisFile}" """ & depositContractOpt, "\n", " ")
 

tests/simulation/run_node.sh

@@ -44,6 +44,9 @@ if [[ $NODE_ID -lt $TOTAL_NODES ]]; then
   popd >/dev/null
 fi
 
+rm -rf "$DATA_DIR/dump"
+mkdir -p "$DATA_DIR/dump"
+
 $BEACON_NODE_BIN \
   --bootstrap-file=$BOOTSTRAP_ADDRESS_FILE \
   --data-dir=$DATA_DIR \