Merge pull request #1052 from status-im/devel

merge devel into master

Jenkinsfile

@@ -11,9 +11,12 @@ def runStages() {
 			[$class: "ArbitraryFileCache", excludes: "", includes: "**/*", path: "${WORKSPACE}/jsonTestsCache"]
 		]) {
 			stage("Build") {
-				sh "make -j${env.NPROC} update" /* to allow a newer Nim version to be detected */
-				sh "make -j${env.NPROC} deps" /* to allow the following parallel stages */
-				sh "V=1 ./scripts/setup_official_tests.sh jsonTestsCache"
+				sh """#!/bin/bash
+				set -e
+				make -j${env.NPROC} update # to allow a newer Nim version to be detected
+				make -j${env.NPROC} deps # to allow the following parallel stages
+				V=1 ./scripts/setup_official_tests.sh jsonTestsCache
+				"""
 			}
 		}
 
@@ -21,8 +24,11 @@ def runStages() {
 			parallel(
 				"tools": {
 					stage("Tools") {
-						sh "make -j${env.NPROC}"
-						sh "make -j${env.NPROC} LOG_LEVEL=TRACE NIMFLAGS='-d:testnet_servers_image'"
+						sh """#!/bin/bash
+						set -e
+						make -j${env.NPROC}
+						make -j${env.NPROC} LOG_LEVEL=TRACE NIMFLAGS='-d:testnet_servers_image'
+						"""
 					}
 				},
 				"test suite": {
@@ -32,8 +38,11 @@ def runStages() {
 					if ("${NODE_NAME}" ==~ /linux.*/) {
 						stage("testnet finalization") {
 							// EXECUTOR_NUMBER will be 0 or 1, since we have 2 executors per Jenkins node
-							sh "timeout -k 20s 10m ./scripts/launch_local_testnet.sh --testnet 0 --nodes 4 --log-level INFO --disable-htop --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --stop-at-epoch=5"
-							sh "timeout -k 20s 40m ./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --log-level INFO --disable-htop --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --stop-at-epoch=5"
+							sh """#!/bin/bash
+							set -e
+							timeout -k 20s 10m ./scripts/launch_local_testnet.sh --testnet 0 --nodes 4 --log-level INFO --disable-htop --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --stop-at-epoch=5
+							timeout -k 20s 40m ./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --log-level INFO --disable-htop --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --stop-at-epoch=5
+							"""
 						}
 					}
 				}

beacon_chain/eth2_network.nim

@@ -4,8 +4,9 @@ import
   options as stdOptions, net as stdNet,
 
   # Status libs
-  stew/[varints, base58, bitseqs, results], stew/shims/[macros, tables],
-  stint, faststreams/[inputs, outputs, buffers], snappy, snappy/framing,
+  stew/[varints, base58, bitseqs, endians2, results],
+  stew/shims/[macros, tables],
+  faststreams/[inputs, outputs, buffers], snappy, snappy/framing,
   json_serialization, json_serialization/std/[net, options],
   chronos, chronicles, metrics,
   # TODO: create simpler to use libp2p modules that use re-exports
@@ -719,11 +720,14 @@ proc start*(node: Eth2Node) {.async.} =
   traceAsyncErrors node.discoveryLoop
 
 proc stop*(node: Eth2Node) {.async.} =
-  # ignore errors in futures, since we're shutting down
-  await allFutures(@[
+  # Ignore errors in futures, since we're shutting down.
+  # Use a timer to avoid hangups.
+  discard await one(sleepAsync(5.seconds),
+                    allFutures(@[
                       node.discovery.closeWait(),
                       node.switch.stop(),
                     ])
+    )
 
 proc init*(T: type Peer, network: Eth2Node, info: PeerInfo): Peer =
   new result

beacon_chain/ssz.nim

@@ -14,7 +14,7 @@
 
 import
   options, algorithm, options, strformat, typetraits,
-  stint, stew/[bitops2, bitseqs, objects, varints, ptrops],
+  stew/[bitops2, bitseqs, endians2, objects, varints, ptrops],
   stew/ranges/[ptr_arith, stackarrays], stew/shims/macros,
   faststreams/[inputs, outputs, buffers],
   serialization, serialization/testing/tracing,
@@ -44,7 +44,7 @@ type
   SszWriter* = object
     stream: OutputStream
 
-  BasicType = byte|char|bool|SomeUnsignedInt|StUint
+  BasicType = byte|char|bool|SomeUnsignedInt
 
   SszChunksMerkleizer = object
     combinedChunks: StackArray[Eth2Digest]
@@ -105,7 +105,7 @@ proc writeFixedSized(s: var (OutputStream|WriteCursor), x: auto) {.raises: [Defe
     s.write x
   elif x is bool|char:
     s.write byte(ord(x))
-  elif x is SomeUnsignedInt|StUint:
+  elif x is SomeUnsignedInt:
     when cpuEndian == bigEndian:
       s.write toBytesLE(x)
     else:
@@ -418,7 +418,7 @@ template merkleizeFields(totalElements: int, body: untyped): Eth2Digest =
   getFinalHash(merkleizer)
 
 template writeBytesLE(chunk: var array[bytesPerChunk, byte], atParam: int,
-                      val: SomeUnsignedInt|StUint) =
+                      val: SomeUnsignedInt) =
   let at = atParam
   chunk[at ..< at + sizeof(val)] = toBytesLE(val)
 
@@ -448,7 +448,7 @@ func chunkedHashTreeRootForBasicTypes[T](merkleizer: var SszChunksMerkleizer,
 
   else:
     static:
-      assert T is SomeUnsignedInt|StUInt
+      assert T is SomeUnsignedInt
       assert bytesPerChunk mod sizeof(Т) == 0
 
     const valuesPerChunk = bytesPerChunk div sizeof(Т)
@@ -539,7 +539,7 @@ func hashTreeRootAux[T](x: T): Eth2Digest =
     unsupported T # Blocks are identified by htr(BeaconBlock) so we avoid these
   elif T is bool|char:
     result.data[0] = byte(x)
-  elif T is SomeUnsignedInt|StUint:
+  elif T is SomeUnsignedInt:
     when cpuEndian == bigEndian:
       result.data[0..<sizeof(x)] = toBytesLE(x)
     else:

tests/helpers/debug_ssz.nim

@@ -1,82 +0,0 @@
-# beacon_chain
-# Copyright (c) 2018-Present Status Research & Development GmbH
-# Licensed and distributed under either of
-#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
-#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
-# at your option. This file may not be copied, modified, or distributed except according to those terms.
-
-import
-  # Standard library
-  os, unittest, strutils, streams,
-  # Status libraries
-  stint, stew/bitseqs,
-  # Third-party
-  yaml,
-  # Beacon chain internals
-  ../../beacon_chain/spec/[datatypes, digest, crypto],
-  ../../beacon_chain/ssz
-
-# Config
-# ---------------------------------------------
-
-const
-  FixturesDir = currentSourcePath.rsplit(DirSep, 1)[0] / ".." / "official"/"fixtures"
-  SSZDir = FixturesDir/"tests-v0.9.1"/const_preset/"phase0"/"ssz_static"
-  UnitTestDir = SSZDir/"Validator"/"ssz_zero"/"case_0"
-
-type
-  SSZHashTreeRoot = object
-    # The test files have the values at the "root"
-    # so we **must** use "root" as a field name
-    root: string
-    # Some have a signing_root field
-    signing_root: string
-
-# Make signing root optional
-setDefaultValue(SSZHashTreeRoot, signing_root, "")
-
-# Parsing + Test
-# ---------------------------------------------
-
-type Skip = enum
-  SkipNone
-  SkipHashTreeRoot
-  SkipSigningRoot
-
-proc checkSSZ(T: typedesc, dir: string, expectedHash: SSZHashTreeRoot, skip = SkipNone) =
-  # Deserialize into a ref object to not fill Nim stack
-  var deserialized: ref T
-  new deserialized
-  deserialized[] = SSZ.loadFile(dir/"serialized.ssz", T)
-
-  echo "\n\nObject: ", T
-  echo "---------------------------------------"
-  echo deserialized[]
-
-  if not(skip == SkipHashTreeRoot):
-    check: expectedHash.root == "0x" & toLowerASCII($deserialized.hashTreeRoot())
-  if expectedHash.signing_root != "" and not(skip == SkipSigningRoot):
-    check: expectedHash.signing_root == "0x" & toLowerASCII($deserialized[].signingRoot())
-
-proc loadExpectedHashTreeRoot(dir: string): SSZHashTreeRoot =
-  var s = openFileStream(dir/"roots.yaml")
-  yaml.load(s, result)
-  s.close()
-
-# Manual checks
-# ---------------------------------------------
-
-# Compile with -d:ssz_testing for consensus objects
-# as they are always an Opaque Blob even if they might seem like a valid BLS signature
-
-echo "Current preset: ", const_preset
-
-let hash = loadExpectedHashTreeRoot(UnitTestDir)
-checkSSZ(Validator, UnitTestDir, hash)
-
-echo "\n\n"
-var deserialized: ref Validator
-new deserialized
-deserialized[] = SSZ.loadFile(UnitTestDir/"serialized.ssz", Validator)
-
-echo deserialized[].hashTreeRoot()

tests/official/test_fixture_ssz_consensus_objects.nim

@@ -10,7 +10,7 @@ import
   os, unittest, strutils, streams, strformat,
   macros, sets,
   # Status libraries
-  stint, stew/bitseqs,
+  stew/bitseqs,
   # Third-party
   yaml,
   # Beacon chain internals

tests/official/test_fixture_ssz_generic_types.nim

@@ -10,7 +10,7 @@ import
   os, unittest, strutils, streams, strformat, strscans,
   macros, typetraits,
   # Status libraries
-  faststreams, stint, stew/bitseqs, ../testutil,
+  faststreams, stew/bitseqs, ../testutil,
   # Third-party
   yaml,
   # Beacon chain internals

tests/test_honest_validator.nim

@@ -1,7 +1,7 @@
 {.used.}
 
 import
-  unittest, stint, ./testutil,
+  unittest, ./testutil,
   ../beacon_chain/spec/[datatypes, network]
 
 suiteReport "Honest validator":

tests/test_ssz.nim

@@ -9,7 +9,7 @@
 
 import
   unittest, options, json_serialization,
-  stint, nimcrypto, eth/common, serialization/testing/generic_suite,
+  nimcrypto, eth/common, serialization/testing/generic_suite,
   ./testutil,
   ../beacon_chain/spec/[datatypes, digest],
   ../beacon_chain/ssz, ../beacon_chain/ssz/[navigator, dynamic_navigator]
@@ -20,7 +20,6 @@ type
 
   Simple = object
     flag: bool
-    # count: StUint[256]
     # ignored {.dontSerialize.}: string
     # data: array[256, bool]
 

vendor/nimbus-build-system

@@ -1 +1 @@
-Subproject commit 1c9feb0d66fbd8e55560543aee96d3e0d33c6c0f
+Subproject commit 252af3e779416c627158ba39b1df97cde51e5981