Merge pull request #741 from status-im/devel

merge devel into master

.appveyor.yml

@@ -37,7 +37,7 @@ build_script:
 test_script:
   # the "go-checks" target fails in AppVeyor, for some reason; easier to disable than to debug
   - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_GO_CHECKS=1 P2PD_CACHE=p2pdCache
-  - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_GO_CHECKS=1 P2PD_CACHE=p2pdCache NIMFLAGS="-d:NETWORK_TYPE=libp2p"
+  - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_GO_CHECKS=1 P2PD_CACHE=p2pdCache NIMFLAGS="-d:NETWORK_TYPE=libp2p -d:testnet_servers_image"
   - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_TEST_FIXTURES_SCRIPT=1 DISABLE_GO_CHECKS=1 test
 
 deploy: off

.gitmodules

@@ -28,9 +28,9 @@
 	branch = master
 [submodule "vendor/NimYAML"]
 	path = vendor/NimYAML
-	url = https://github.com/flyx/NimYAML.git
+	url = https://github.com/status-im/NimYAML.git
 	ignore = dirty
-	branch = master
+	branch = devel
 [submodule "vendor/nim-web3"]
 	path = vendor/nim-web3
 	url = https://github.com/status-im/nim-web3.git

.travis.yml

@@ -50,6 +50,6 @@ script:
   # Building Nim-1.0.4 takes up to 10 minutes on Travis - the time limit after which jobs are cancelled for having no output
   - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" V=1 update # to allow a newer Nim version to be detected
   - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}"
-  - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" NIMFLAGS="-d:NETWORK_TYPE=libp2p"
+  - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" NIMFLAGS="-d:NETWORK_TYPE=libp2p -d:testnet_servers_image"
   - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" DISABLE_TEST_FIXTURES_SCRIPT=1 test
 

Jenkinsfile

@@ -22,7 +22,7 @@ def runStages() {
 				"tools": {
 					stage("Tools") {
 						sh "make -j${env.NPROC}"
-						sh "make -j${env.NPROC} NIMFLAGS='-d:NETWORK_TYPE=libp2p'"
+						sh "make -j${env.NPROC} NIMFLAGS='-d:NETWORK_TYPE=libp2p -d:testnet_servers_image'"
 					}
 				},
 				"test suite": {

Makefile

@@ -114,10 +114,10 @@ clean-testnet1:
 	rm -rf build/data/testnet1
 
 testnet0: | build deps
-	NIM_PARAMS="$(NIM_PARAMS)" $(ENV_SCRIPT) nim $(NIM_PARAMS) scripts/connect_to_testnet.nims testnet0
+	NIM_PARAMS="$(NIM_PARAMS)" LOG_LEVEL="$(LOG_LEVEL)" $(ENV_SCRIPT) nim $(NIM_PARAMS) scripts/connect_to_testnet.nims testnet0
 
 testnet1: | build deps
-	NIM_PARAMS="$(NIM_PARAMS)" $(ENV_SCRIPT) nim $(NIM_PARAMS) scripts/connect_to_testnet.nims testnet1
+	NIM_PARAMS="$(NIM_PARAMS)" LOG_LEVEL="$(LOG_LEVEL)" $(ENV_SCRIPT) nim $(NIM_PARAMS) scripts/connect_to_testnet.nims testnet1
 
 clean: | clean-common
 	rm -rf build/{$(TOOLS_CSV),all_tests,*_node,*ssz*,beacon_node_testnet*,state_sim,transition*}

azure-pipelines.yml

@@ -14,7 +14,7 @@ jobs:
       - task: CacheBeta@1
         displayName: 'cache Nim binaries'
         inputs:
-          key: NimBinaries | $(Agent.OS) | $(PLATFORM) | "$(Build.SourceBranchName)"
+          key: NimBinaries | $(Agent.OS) | $(PLATFORM) | "$(Build.SourceBranchName)" | "v2"
           path: NimBinaries
 
       - task: CacheBeta@1
@@ -72,7 +72,7 @@ jobs:
           mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries update
           mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} fetch-dlls
           mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} P2PD_CACHE=p2pdCache
-          mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} P2PD_CACHE=p2pdCache NIMFLAGS="-d:NETWORK_TYPE=libp2p"
+          mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} P2PD_CACHE=p2pdCache NIMFLAGS="-d:NETWORK_TYPE=libp2p -d:testnet_servers_image"
           file build/beacon_node
           mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} DISABLE_TEST_FIXTURES_SCRIPT=1 test
         displayName: 'build and test'

beacon_chain/beacon_node.nim

@@ -218,6 +218,8 @@ proc init*(T: type BeaconNode, conf: BeaconNodeConf): Future[BeaconNode] {.async
     mainchainMonitor = MainchainMonitor.init(
       conf.depositWeb3Url, conf.depositContractAddress,
       blockPool.headState.data.data.eth1_data.block_hash)
+    # TODO if we don't have any validators attached, we don't need a mainchain
+    #      monitor
     mainchainMonitor.start()
 
   var
@@ -624,31 +626,22 @@ proc handleProposal(node: BeaconNode, head: BlockRef, slot: Slot):
   # TODO here we advance the state to the new slot, but later we'll be
   #      proposing for it - basically, we're selecting proposer based on an
   #      empty slot
-  var cache = get_empty_per_epoch_cache()
-  node.blockPool.withState(node.blockPool.tmpState, head.atSlot(slot)):
-    let proposerIdx = get_beacon_proposer_index(state, cache)
-    if proposerIdx.isNone:
-      notice "Missing proposer index",
-        slot=slot,
-        epoch=slot.compute_epoch_at_slot,
-        num_validators=state.validators.len,
-        active_validators=
-          get_active_validator_indices(state, slot.compute_epoch_at_slot),
-        balances=state.balances
 
-      return head
+  let proposerKey = node.blockPool.getProposer(head, slot)
+  if proposerKey.isNone():
+    return head
 
-    let validator = node.getAttachedValidator(state, proposerIdx.get)
+  let validator = node.attachedValidators.getValidator(proposerKey.get())
 
-    if validator != nil:
+  if validator != nil:
-      return await proposeBlock(node, validator, head, slot)
+    return await proposeBlock(node, validator, head, slot)
 
-    trace "Expecting block proposal",
+  debug "Expecting block proposal",
-      headRoot = shortLog(head.root),
+    headRoot = shortLog(head.root),
-      slot = shortLog(slot),
+    slot = shortLog(slot),
-      proposer = shortLog(state.validators[proposerIdx.get].pubKey),
+    proposer = shortLog(proposerKey.get()),
-      cat = "consensus",
+    cat = "consensus",
-      pcs = "wait_for_proposal"
+    pcs = "wait_for_proposal"
 
   return head
 
@@ -1113,7 +1106,7 @@ when isMainModule:
                  else: waitFor getLatestEth1BlockHash(config.depositWeb3Url)
     var
       initialState = initialize_beacon_state_from_eth1(
-        eth1Hash, startTime, deposits, {skipValidation})
+        eth1Hash, startTime, deposits, {skipValidation, skipMerkleValidation})
 
     # https://github.com/ethereum/eth2.0-pm/tree/6e41fcf383ebeb5125938850d8e9b4e9888389b4/interop/mocked_start#create-genesis-state
     initialState.genesis_time = startTime

beacon_chain/block_pool.nim

@@ -2,7 +2,7 @@ import
   bitops, chronicles, options, tables,
   ssz, beacon_chain_db, state_transition, extras,
   beacon_node_types, metrics,
-  spec/[crypto, datatypes, digest, helpers]
+  spec/[crypto, datatypes, digest, helpers, validator]
 
 declareCounter beacon_reorgs_total, "Total occurrences of reorganizations of the chain" # On fork choice
 
@@ -249,9 +249,10 @@ proc init*(T: type BlockPool, db: BeaconChainDB): BlockPool =
   res
 
 proc addResolvedBlock(
-    pool: var BlockPool, state: var StateData, blockRoot: Eth2Digest,
+    pool: var BlockPool, state: BeaconState, blockRoot: Eth2Digest,
     signedBlock: SignedBeaconBlock, parent: BlockRef): BlockRef =
   logScope: pcs = "block_resolution"
+  doAssert state.slot == signedBlock.message.slot, "state must match block"
 
   let blockRef = BlockRef.init(blockRoot, signedBlock.message)
   link(parent, blockRef)
@@ -262,17 +263,10 @@ proc addResolvedBlock(
   # Resolved blocks should be stored in database
   pool.db.putBlock(blockRoot, signedBlock)
 
-  # TODO this is a bit ugly - we update state.data outside of this function then
-  #      set the rest here - need a blockRef to update it. Clean this up -
-  #      hopefully it won't be necessary by the time hash caching and the rest
-  #      is done..
-  doAssert state.data.data.slot == blockRef.slot
-  state.blck = blockRef
-
   # This block *might* have caused a justification - make sure we stow away
   # that information:
   let justifiedSlot =
-    state.data.data.current_justified_checkpoint.epoch.compute_start_slot_at_epoch()
+    state.current_justified_checkpoint.epoch.compute_start_slot_at_epoch()
 
   var foundHead: Option[Head]
   for head in pool.heads.mitems():
@@ -389,9 +383,12 @@ proc add*(
 
       return
 
-    # Careful, pool.tmpState is now partially inconsistent and will be updated
+    # Careful, tmpState.data has been updated but not blck - we need to create
-    # inside addResolvedBlock
+    # the BlockRef first!
-    return pool.addResolvedBlock(pool.tmpState, blockRoot, signedBlock, parent)
+    pool.tmpState.blck = pool.addResolvedBlock(
+      pool.tmpState.data.data, blockRoot, signedBlock, parent)
+
+    return pool.tmpState.blck
 
   # TODO already checked hash though? main reason to keep this is because
   # the pending pool calls this function back later in a loop, so as long
@@ -441,7 +438,6 @@ proc add*(
     missing = pool.missing.len,
     cat = "filtering"
 
-
 func getRef*(pool: BlockPool, root: Eth2Digest): BlockRef =
   ## Retrieve a resolved block reference, if available
   pool.blocks.getOrDefault(root, nil)
@@ -565,12 +561,19 @@ func checkMissing*(pool: var BlockPool): seq[FetchRecord] =
     if v.tries.popcount() == 1:
       result.add(FetchRecord(root: k, historySlots: v.slots))
 
+proc skipAndUpdateState(
+    state: var HashedBeaconState, slot: Slot,
+    afterUpdate: proc (state: HashedBeaconState)) =
+  while state.data.slot < slot:
+    # Process slots one at a time in case afterUpdate needs to see empty states
+    process_slots(state, state.data.slot + 1)
+    afterUpdate(state)
+
 proc skipAndUpdateState(
     state: var HashedBeaconState, blck: BeaconBlock, flags: UpdateFlags,
     afterUpdate: proc (state: HashedBeaconState)): bool =
 
-  process_slots(state, blck.slot - 1)
+  skipAndUpdateState(state, blck.slot - 1, afterUpdate)
-  afterUpdate(state)
 
   let ok  = state_transition(state, blck, flags)
 
@@ -603,7 +606,7 @@ proc rewindState(pool: BlockPool, state: var StateData, bs: BlockSlot):
   var ancestors = @[pool.get(bs.blck)]
   # Common case: the last block applied is the parent of the block to apply:
   if not bs.blck.parent.isNil and state.blck.root == bs.blck.parent.root and
-      state.data.data.slot < bs.slot:
+      state.data.data.slot < bs.blck.slot:
     return ancestors
 
   # It appears that the parent root of the proposed new block is different from
@@ -692,8 +695,8 @@ proc updateStateData*(pool: BlockPool, state: var StateData, bs: BlockSlot) =
   if state.blck.root == bs.blck.root and state.data.data.slot <= bs.slot:
     if state.data.data.slot != bs.slot:
       # Might be that we're moving to the same block but later slot
-      process_slots(state.data, bs.slot)
+      skipAndUpdateState(state.data, bs.slot) do(state: HashedBeaconState):
-      pool.maybePutState(state.data, bs.blck)
+        pool.maybePutState(state, bs.blck)
 
     return # State already at the right spot
 
@@ -714,9 +717,8 @@ proc updateStateData*(pool: BlockPool, state: var StateData, bs: BlockSlot) =
       pool.maybePutState(state, ancestors[i].refs)
     doAssert ok, "Blocks in database should never fail to apply.."
 
-  # TODO check if this triggers rest of state transition, or should
+  skipAndUpdateState(state.data, bs.slot) do(state: HashedBeaconState):
-  process_slots(state.data, bs.slot)
+    pool.maybePutState(state, bs.blck)
-  pool.maybePutState(state.data, bs.blck)
 
   state.blck = bs.blck
 
@@ -924,3 +926,20 @@ proc preInit*(
   db.putTailBlock(blockRoot)
   db.putHeadBlock(blockRoot)
   db.putStateRoot(blockRoot, state.slot, signedBlock.message.state_root)
+
+proc getProposer*(pool: BlockPool, head: BlockRef, slot: Slot): Option[ValidatorPubKey] =
+  pool.withState(pool.tmpState, head.atSlot(slot)):
+    var cache = get_empty_per_epoch_cache()
+
+    let proposerIdx = get_beacon_proposer_index(state, cache)
+    if proposerIdx.isNone:
+      warn "Missing proposer index",
+        slot=slot,
+        epoch=slot.compute_epoch_at_slot,
+        num_validators=state.validators.len,
+        active_validators=
+          get_active_validator_indices(state, slot.compute_epoch_at_slot),
+        balances=state.balances
+      return
+
+    return some(state.validators[proposerIdx.get()].pubkey)

beacon_chain/eth2_network.nim

@@ -155,7 +155,7 @@ when networkBackend in [libp2p, libp2pDaemon]:
       # that are different from the host address (this is relevant when we
       # are running behind a NAT).
       var switch = newStandardSwitch(some keys.seckey, hostAddress,
-                                     triggerSelf = true, gossip = true)
+                                     triggerSelf = true, gossip = false)
       result = Eth2Node.init(conf, switch, keys.seckey)
       for enr in bootstrapEnrs:
         result.addKnownPeer(enr)

beacon_chain/extras.nim

@@ -25,5 +25,15 @@ type
     ## TODO need to be careful here, easy to assume that slot number change is
     ##      enough, vs advancing the state - however, making a full state copy
     ##      is expensive also :/
+    skipMerkleValidation ##\
+    ## When processing deposits, skip verifying the Merkle proof trees of each
+    ## deposit. This is a holdover from both interop issues with the malformed
+    ## proofs and, more currently, nim-beacon-chain's creation of proofs which
+    ## are inconsistent with the current specification. Furthermore several of
+    ## the mocking interfaces deliberately do not create Merkle proofs. Whilst
+    ## this seems less than entirely justifiable, for now enable keeping those
+    ## in place while minimizing the tech debt they create. One, in principle,
+    ## should be able to remove this flag entirely. It is not intrinsically an
+    ## expensive operation to perform.
 
   UpdateFlags* = set[UpdateFlag]

beacon_chain/mainchain_monitor.nim

@@ -24,13 +24,16 @@ type
 
   QueueElement = (BlockHash, DepositData)
 
-
+proc init*(
-proc init*(T: type MainchainMonitor, web3Url, depositContractAddress: string, startBlock: Eth2Digest): T =
+    T: type MainchainMonitor,
-  result.new()
+    web3Url, depositContractAddress: string,
-  result.web3Url = web3Url
+    startBlock: Eth2Digest): T =
-  result.depositContractAddress = Address.fromHex(depositContractAddress)
+  T(
-  result.depositQueue = newAsyncQueue[QueueElement]()
+    web3Url: web3Url,
-  result.eth1Block = BlockHash(startBlock.data)
+    depositContractAddress: Address.fromHex(depositContractAddress),
+    depositQueue: newAsyncQueue[QueueElement](),
+    eth1Block: BlockHash(startBlock.data),
+  )
 
 contract(DepositContract):
   proc deposit(pubkey: Bytes48, withdrawalCredentials: Bytes32, signature: Bytes96, deposit_data_root: FixedBytes[32])
@@ -113,8 +116,18 @@ proc getGenesis*(m: MainchainMonitor): Future[BeaconState] {.async.} =
   return m.genesisState[]
 
 proc getBlockNumber(web3: Web3, hash: BlockHash): Future[Quantity] {.async.} =
-  let blk = await web3.provider.eth_getBlockByHash(hash, false)
+  debug "Querying block number", hash = $hash
-  return blk.number
+
+  try:
+    let blk = await web3.provider.eth_getBlockByHash(hash, false)
+    return blk.number
+  except CatchableError as exc:
+    # TODO this doesn't make too much sense really, but what would be a
+    #      reasonable behavior? no idea - the whole algorithm needs to be
+    #      rewritten to match the spec.
+    notice "Failed to get block number from hash, using current block instead",
+      hash = $hash, err = exc.msg
+    return await web3.provider.eth_blockNumber()
 
 proc run(m: MainchainMonitor, delayBeforeStart: Duration) {.async.} =
   if delayBeforeStart != ZeroDuration:
@@ -129,8 +142,15 @@ proc run(m: MainchainMonitor, delayBeforeStart: Duration) {.async.} =
     error "Web3 server disconnected", ulr = m.web3Url
     processFut.cancel()
 
+  # TODO this needs to implement follow distance and the rest of the honest
+  #      validator spec..
+
   let startBlkNum = await web3.getBlockNumber(m.eth1Block)
-  debug "Starting eth1 monitor", fromBlock = startBlkNum.uint64
+
+  notice "Monitoring eth1 deposits",
+    fromBlock = startBlkNum.uint64,
+    contract = $m.depositContractAddress,
+    url = m.web3Url
 
   let ns = web3.contractSender(DepositContract, m.depositContractAddress)
 
@@ -139,15 +159,17 @@ proc run(m: MainchainMonitor, delayBeforeStart: Duration) {.async.} =
       withdrawalCredentials: Bytes32,
       amount: Bytes8,
       signature: Bytes96, merkleTreeIndex: Bytes8, j: JsonNode):
+    try:
+      let blkHash = BlockHash.fromHex(j["blockHash"].getStr())
+      let amount = bytes_to_int(array[8, byte](amount))
 
-    let blkHash = BlockHash.fromHex(j["blockHash"].getStr())
+      m.depositQueue.addLastNoWait((blkHash,
-    let amount = bytes_to_int(array[8, byte](amount))
+        DepositData(pubkey: ValidatorPubKey.init(array[48, byte](pubkey)),
-
+          withdrawal_credentials: Eth2Digest(data: array[32, byte](withdrawalCredentials)),
-    m.depositQueue.addLastNoWait((blkHash,
+          amount: amount,
-      DepositData(pubkey: ValidatorPubKey.init(array[48, byte](pubkey)),
+          signature: ValidatorSig.init(array[96, byte](signature)))))
-        withdrawal_credentials: Eth2Digest(data: array[32, byte](withdrawalCredentials)),
+    except CatchableError as exc:
-        amount: amount,
+      warn "Received invalid deposit", err = exc.msg, j
-        signature: ValidatorSig.init(array[96, byte](signature)))))
 
   try:
     await processFut

beacon_chain/spec/beaconstate.nim

@@ -53,7 +53,7 @@ proc process_deposit*(
   # Process an Eth1 deposit, registering a validator or increasing its balance.
 
   # Verify the Merkle branch
-  if skipValidation notin flags and not is_valid_merkle_branch(
+  if skipMerkleValidation notin flags and not is_valid_merkle_branch(
     hash_tree_root(deposit.data),
     deposit.proof,
     DEPOSIT_CONTRACT_TREE_DEPTH + 1,
@@ -361,7 +361,8 @@ proc process_registry_updates*(state: var BeaconState) {.nbench.}=
 
 # https://github.com/ethereum/eth2.0-specs/blob/v0.9.4/specs/core/0_beacon-chain.md#is_valid_indexed_attestation
 proc is_valid_indexed_attestation*(
-    state: BeaconState, indexed_attestation: IndexedAttestation): bool =
+    state: BeaconState, indexed_attestation: IndexedAttestation,
+    flags: UpdateFlags): bool =
   ## Check if ``indexed_attestation`` has valid indices and signature.
   # TODO: this is noSideEffect besides logging
   #       https://github.com/status-im/nim-chronicles/issues/62
@@ -380,7 +381,7 @@ proc is_valid_indexed_attestation*(
     return false
 
   # Verify aggregate signature
-  if not bls_verify(
+  if skipValidation notin flags and not bls_verify(
     bls_aggregate_pubkeys(mapIt(indices, state.validators[it.int].pubkey)),
     hash_tree_root(indexed_attestation.data).data,
     indexed_attestation.signature,
@@ -497,7 +498,7 @@ proc check_attestation*(
       return
 
   if not is_valid_indexed_attestation(
-      state, get_indexed_attestation(state, attestation, stateCache)):
+      state, get_indexed_attestation(state, attestation, stateCache), flags):
     warn("process_attestation: signature or bitfields incorrect")
     return
 

beacon_chain/spec/crypto.nim

@@ -160,8 +160,10 @@ func bls_verify*(
       return false
     # TODO bls_verify_multiple(...) used to have this workaround, and now it
     # lives here. No matter the signature, there's also no meaningful way to
-    # verify it -- it's a kind of vacuous truth. No pubkey/sig pairs.
+    # verify it -- it's a kind of vacuous truth. No pubkey/sig pairs. Sans a
-    if pubkey == default(ValidatorPubKey):
+    # getBytes() or similar mechanism, pubKey == default(ValidatorPubKey) is
+    # a way to create many false positive matches. This seems odd.
+    if pubkey.getBytes() == default(ValidatorPubKey).getBytes():
       return true
 
     sig.blsValue.verify(msg, domain, pubkey.blsValue)

beacon_chain/spec/digest.nim

@@ -20,12 +20,12 @@
 # we call this function `eth2hash`, and it outputs a `Eth2Digest`. Easy to sed :)
 
 import
-  chronicles,
+  chronicles, json_serialization,
   nimcrypto/[sha2, hash, utils],
   hashes
 
 export
-  hash.`$`
+  hash.`$`, json_serialization
 
 type
   Eth2Digest* = MDigest[32 * 8] ## `hash32` from spec
@@ -70,3 +70,9 @@ func hash*(x: Eth2Digest): Hash =
   # We just slice the first 4 or 8 bytes of the block hash
   # depending of if we are on a 32 or 64-bit platform
   result = cast[ptr Hash](unsafeAddr x)[]
+
+proc writeValue*(writer: var JsonWriter, value: Eth2Digest) =
+  writeValue(writer, value.data.toHex(true))
+
+proc readValue*(reader: var JsonReader, value: var Eth2Digest) =
+  value = Eth2Digest.fromHex(reader.readValue(string))

beacon_chain/spec/state_transition_block.nim

@@ -217,6 +217,7 @@ func is_slashable_attestation_data(
 proc process_attester_slashing*(
        state: var BeaconState,
        attester_slashing: AttesterSlashing,
+       flags: UpdateFlags,
        stateCache: var StateCache
      ): bool {.nbench.}=
     let
@@ -228,11 +229,11 @@ proc process_attester_slashing*(
       notice "Attester slashing: surround or double vote check failed"
       return false
 
-    if not is_valid_indexed_attestation(state, attestation_1):
+    if not is_valid_indexed_attestation(state, attestation_1, flags):
       notice "Attester slashing: invalid attestation 1"
       return false
 
-    if not is_valid_indexed_attestation(state, attestation_2):
+    if not is_valid_indexed_attestation(state, attestation_2, flags):
       notice "Attester slashing: invalid attestation 2"
       return false
 
@@ -259,7 +260,7 @@ proc processAttesterSlashings(state: var BeaconState, blck: BeaconBlock,
     return false
 
   for attester_slashing in blck.body.attester_slashings:
-    if not process_attester_slashing(state, attester_slashing, stateCache):
+    if not process_attester_slashing(state, attester_slashing, {}, stateCache):
       return false
   return true
 
@@ -286,13 +287,14 @@ proc processAttestations(
   true
 
 # https://github.com/ethereum/eth2.0-specs/blob/v0.8.4/specs/core/0_beacon-chain.md#deposits
-proc processDeposits(state: var BeaconState, blck: BeaconBlock): bool {.nbench.}=
+proc processDeposits(state: var BeaconState, blck: BeaconBlock,
+    flags: UpdateFlags): bool {.nbench.}=
   if not (len(blck.body.deposits) <= MAX_DEPOSITS):
     notice "processDeposits: too many deposits"
     return false
 
   for deposit in blck.body.deposits:
-    if not process_deposit(state, deposit):
+    if not process_deposit(state, deposit, flags):
       notice "processDeposits: deposit invalid"
       return false
 
@@ -419,7 +421,7 @@ proc processBlock*(
     debug "[Block processing] Attestation processing failure", slot = shortLog(state.slot)
     return false
 
-  if not processDeposits(state, blck):
+  if not processDeposits(state, blck, flags):
     debug "[Block processing] Deposit processing failure", slot = shortLog(state.slot)
     return false
 

beacon_chain/sszdump.nim

@@ -0,0 +1,24 @@
+import
+  os,
+  ssz,
+  serialization,
+  beacon_node_types,
+  ./spec/[crypto, datatypes, digest]
+
+proc dump*(dir: string, v: AttestationData, validator: ValidatorPubKey) =
+  SSZ.saveFile(
+    dir / "att-" & $v.slot & "-" &
+    $v.index & "-" & validator.shortLog &
+    ".ssz", v)
+
+proc dump*(dir: string, v: SignedBeaconBlock, blck: BlockRef) =
+  SSZ.saveFile(
+    dir / "block-" & $v.message.slot & "-" &
+    shortLog(blck.root) & ".ssz", v)
+
+proc dump*(dir: string, v: HashedBeaconState, blck: BlockRef) =
+  SSZ.saveFile(
+    dir / "state-" & $v.data.slot & "-" &
+    shortLog(blck.root) & "-"  & shortLog(v.root) & ".ssz",
+    v.data)
+

beacon_chain/state_transition.nim

@@ -83,7 +83,11 @@ func get_epoch_validator_count(state: BeaconState): int64 =
 
 # https://github.com/ethereum/eth2.0-specs/blob/v0.10.1/specs/phase0/beacon-chain.md#beacon-chain-state-transition-function
 proc process_slots*(state: var BeaconState, slot: Slot) {.nbench.}=
-  doAssert state.slot <= slot
+  if not (state.slot <= slot):
+    warn("Trying to apply old block",
+      state_slot = state.slot,
+      slot = slot)
+    return
 
   # Catch up to the target slot
   while state.slot < slot:
@@ -197,6 +201,7 @@ proc process_slots*(state: var HashedBeaconState, slot: Slot) =
   if state.data.slot > slot:
     notice(
       "Unusual request for a slot in the past",
+      state_root = shortLog(state.root),
       current_slot = state.data.slot,
       target_slot = slot
     )

docker/Dockerfile

@@ -21,12 +21,13 @@ RUN cd /root \
 # to get a fresh up-to-date version of Nim and p2pd.
 ARG GIT_REVISION
 ARG NETWORK_NIM_FLAGS
+ARG MARCH_NIM_FLAGS
 
 RUN cd /root/nim-beacon-chain \
  && git fetch \
  && git reset --hard ${GIT_REVISION} \
  && make -j$(nproc) update \
- && make LOG_LEVEL=DEBUG NIMFLAGS="-d:debug -d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS}" beacon_node
+ && make LOG_LEVEL=TRACE NIMFLAGS="-d:debug -d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS} ${MARCH_NIM_FLAGS}" beacon_node
 
 # --------------------------------- #
 # Starting new image to reduce size #

docker/Makefile

@@ -5,6 +5,8 @@ SHELL := bash # the shell used internally by "make"
 GIT_REVISION      ?= $(shell git rev-parse HEAD)
 NETWORK           ?= testnet1
 NETWORK_NIM_FLAGS ?= $(shell ../scripts/load-testnet-nim-flags.sh $(NETWORK))
+# Get the required GCC flags by running `gcc -v -E - -march=native </dev/null 2>&1 | grep cc1` on the server.
+MARCH_NIM_FLAGS   ?= -d:disableMarchNative --passC:'-march=znver1 --param l1-cache-size=32 --param l1-cache-line-size=64 --param l2-cache-size=512'
 
 IMAGE_TAG       ?= $(NETWORK)
 IMAGE_NAME      ?= statusteam/nimbus_beacon_node:$(IMAGE_TAG)
@@ -19,6 +21,7 @@ build:
 	docker build \
 		--build-arg="GIT_REVISION=$(GIT_REVISION)" \
 		--build-arg="NETWORK_NIM_FLAGS=$(NETWORK_NIM_FLAGS)" \
+		--build-arg="MARCH_NIM_FLAGS=$(MARCH_NIM_FLAGS)" \
 		-t $(IMAGE_NAME) \
 		--progress=plain \
 		.

docker/manage_testnet_hosts.nims

@@ -98,7 +98,12 @@ iterator validatorAssignments: tuple[node: Node; firstValidator, lastValidator:
 case conf.cmd
 of restart_nodes:
   for n in nodes():
-    echo &"ssh {n.server} docker restart {n.container}"
+    if n.id mod 2 == 0:
+      # This will only print one line: "docker.io/statusteam/nimbus_beacon_node:testnet1".
+      echo &"ssh {n.server} docker pull -q statusteam/nimbus_beacon_node:{conf.network}"
+    # docker-compose will rebuild the container if it detects a newer image.
+    # Prints: "Recreating beacon-node-testnet1-1 ... done".
+    echo &"ssh {n.server} 'cd /docker/{n.container} && docker-compose up -d'"
 
 of reset_network:
   for n, firstValidator, lastValidator in validatorAssignments():
@@ -114,8 +119,9 @@ of reset_network:
     let dockerPath = &"/docker/{n.container}/data/BeaconNode"
     echo &"echo Syncing {lastValidator - firstValidator} keys starting from {firstValidator} to container {n.container}@{n.server} ... && \\"
     echo &"  ssh {n.server} 'sudo rm -rf /tmp/nimbus && mkdir -p /tmp/nimbus/' && \\"
-    echo &"  rsync {networkDataFiles} {n.server}:/tmp/nimbus/net-data/ && \\"
+    echo &"  rsync -a -zz {networkDataFiles} {n.server}:/tmp/nimbus/net-data/ && \\"
-    if keysList.len > 0: echo &"  rsync {keysList} {n.server}:/tmp/nimbus/keys/ && \\"
+    if keysList.len > 0:
+      echo &"  rsync -a -zz {keysList} {n.server}:/tmp/nimbus/keys/ && \\"
 
     echo &"  ssh {n.server} 'sudo docker container stop {n.container} && " &
                          &"sudo mkdir -p {dockerPath}/validators && " &

ncli/ncli_pretty.nim

@@ -32,3 +32,4 @@ cli do(kind: string, file: string):
   of "state": printit(BeaconState)
   of "proposer_slashing": printit(ProposerSlashing)
   of "voluntary_exit": printit(VoluntaryExit)
+  else: echo "Unknown kind"

nim.cfg

@@ -21,6 +21,22 @@
   -d:"chronicles_colors=off"
 @end
 
+# This helps especially for 32-bit x86, which sans SSE2 and newer instructions
+# requires quite roundabout code generation for cryptography, and other 64-bit
+# and larger arithmetic use cases, along with register starvation issues. When
+# engineering a more portable binary release, this should be tweaked but still
+# use at least -msse2 or -msse3.
+@if disableMarchNative:
+  --passC:"-msse3"
+@else:
+  --passC:"-march=native"
+  @if windows:
+    # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65782
+    # ("-fno-asynchronous-unwind-tables" breaks Nim's exception raising, sometimes)
+    --passC:"-mno-avx512vl"
+  @end
+@end
+
 --threads:on
 --opt:speed
 --excessiveStackTrace:on

research/state_sim.nim

@@ -79,7 +79,7 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
   let
     genesisState =
       initialize_beacon_state_from_eth1(
-        Eth2Digest(), 0, deposits, {skipValidation})
+        Eth2Digest(), 0, deposits, {skipMerkleValidation})
     genesisBlock = get_initial_beacon_block(genesisState)
 
   echo "Starting simulation..."

scripts/connect_to_testnet.nims

@@ -9,6 +9,7 @@ const
   genesisFile = "genesis.ssz"
   configFile = "config.yaml"
   testnetsRepo = "eth2-testnets"
+  web3Url = "wss://goerli.infura.io/ws/v3/809a18497dd74102b5f37d25aae3c85a"
 
 let
   testnetsOrg = getEnv("ETH2_TESTNETS_ORG", "eth2-clients")
@@ -75,6 +76,7 @@ cli do (testnetName {.argument.}: string):
     validatorsDir = dataDir / "validators"
     dumpDir = dataDir / "dump"
     beaconNodeBinary = buildDir / "beacon_node_" & dataDirName
+  var
     nimFlags = "-d:chronicles_log_level=TRACE " & getEnv("NIM_PARAMS")
 
   let depositContractFile = testnetDir / depositContractFileName
@@ -95,6 +97,8 @@ cli do (testnetName {.argument.}: string):
       rmDir dataDir
 
   cd rootDir
+  if testnet == "testnet1":
+    nimFlags &= " -d:NETWORK_TYPE=libp2p"
   exec &"""nim c {nimFlags} -d:"const_preset={preset}" -o:"{beaconNodeBinary}" beacon_chain/beacon_node.nim"""
 
   mkDir dumpDir
@@ -120,15 +124,24 @@ cli do (testnetName {.argument.}: string):
         --random-deposits=1
         --deposits-dir="{validatorsDir}"
         --deposit-private-key={privKey}
-        --web3-url=wss://goerli.infura.io/ws/v3/809a18497dd74102b5f37d25aae3c85a
+        --web3-url={web3Url}
         {depositContractOpt}
         """, "\n", " ")
-      quit()
+      mode = Silent
+      echo "\nDeposit sent, wait for confirmation then press enter to continue"
+      discard readLineFromStdin()
+
+  let logLevel = getEnv("LOG_LEVEL")
+  var logLevelOpt = ""
+  if logLevel.len > 0:
+    logLevelOpt = "--log-level=" & logLevel
 
   mode = Verbose
   execIgnoringExitCode replace(&"""{beaconNodeBinary}
     --data-dir="{dataDir}"
     --dump=true
+    --web3-url={web3Url}
     {bootstrapFileOpt}
+    {logLevelOpt}
     --state-snapshot="{testnetDir/genesisFile}" """ & depositContractOpt, "\n", " ")
 

scripts/reset_testnet.sh

@@ -32,11 +32,10 @@ echo "Beacon node data dir    : ${DATA_DIR:="build/testnet-reset-data/$NETWORK"}
 echo "Nim build flags         : $NETWORK_NIM_FLAGS"
 
 while true; do
-    read -p "Continue? [yn] " yn
+    read -p "Continue? [Yn] " yn
     case $yn in
-        [Yy]* ) break;;
+        * ) break;;
         [Nn]* ) exit 1;;
-        * ) echo "Please answer yes or no.";;
     esac
 done
 
@@ -56,41 +55,42 @@ fi
 
 mkdir -p "$DEPOSITS_DIR_ABS"
 
-DOCKER_BEACON_NODE="docker run -v $DEPOSITS_DIR_ABS:/deposits_dir -v $NETWORK_DIR_ABS:/network_dir -v $DATA_DIR_ABS:/data_dir statusteam/nimbus_beacon_node:$NETWORK"
-
-make deposit_contract
-
 if [ "$ETH1_PRIVATE_KEY" != "" ]; then
+  make deposit_contract
   echo "Deploying deposit contract through $WEB3_URL_ARG..."
   DEPOSIT_CONTRACT_ADDRESS=$(./build/deposit_contract deploy $WEB3_URL_ARG --private-key=$ETH1_PRIVATE_KEY)
   DEPOSIT_CONTRACT_ADDRESS_ARG="--deposit-contract=$DEPOSIT_CONTRACT_ADDRESS"
   echo "Done: $DEPOSIT_CONTRACT_ADDRESS"
 fi
 
+echo "Building a local beacon_node instance for 'makeDeposits' and 'createTestnet'"
+make NIMFLAGS="-d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS}" beacon_node
+
 cd docker
 
 echo "Building Docker image..."
+# CPU-specific CFLAGS that work on the servers are in MARCH_NIM_FLAGS,
+# in docker/Makefile, and are enabled by default.
 make build
 
-$DOCKER_BEACON_NODE makeDeposits \
+../build/beacon_node makeDeposits \
   --quickstart-deposits=$QUICKSTART_VALIDATORS \
   --random-deposits=$RANDOM_VALIDATORS \
-  --deposits-dir=/deposits_dir
+  --deposits-dir="$DEPOSITS_DIR_ABS"
 
 TOTAL_VALIDATORS="$(( $QUICKSTART_VALIDATORS + $RANDOM_VALIDATORS ))"
 
-$DOCKER_BEACON_NODE \
+../build/beacon_node createTestnet \
-  --data-dir=/data_dir \
+  --data-dir="$DATA_DIR_ABS" \
-  createTestnet \
+  --validators-dir="$DEPOSITS_DIR_ABS" \
-  --validators-dir=/deposits_dir \
   --total-validators=$TOTAL_VALIDATORS \
   --last-user-validator=$QUICKSTART_VALIDATORS \
-  --output-genesis=/network_dir/genesis.ssz \
+  --output-genesis="$NETWORK_DIR_ABS/genesis.ssz" \
-  --output-bootstrap-file=/network_dir/bootstrap_nodes.txt \
+  --output-bootstrap-file="$NETWORK_DIR_ABS/bootstrap_nodes.txt" \
   --bootstrap-address=$BOOTSTRAP_IP \
   --bootstrap-port=$BOOTSTRAP_PORT \
   $WEB3_URL_ARG $DEPOSIT_CONTRACT_ADDRESS_ARG \
-  --genesis-offset=900 # Delay in seconds
+  --genesis-offset=300 # Delay in seconds
 
 COMMITTED_FILES=" genesis.ssz bootstrap_nodes.txt "
 

tests/mocking/mock_genesis.nim

@@ -25,7 +25,7 @@ proc initGenesisState*(num_validators: uint64, genesis_time: uint64 = 0): Beacon
     )
 
   initialize_beacon_state_from_eth1(
-    eth1BlockHash, 0, deposits, {skipValidation})
+    eth1BlockHash, 0, deposits, {skipValidation, skipMerkleValidation})
 
 when isMainModule:
   # Smoke test

tests/official/test_fixture_operations_attestations.nim

@@ -20,13 +20,13 @@ import
 
 const OperationsAttestationsDir = SszTestsDir/const_preset/"phase0"/"operations"/"attestation"/"pyspec_tests"
 
-template runTest(testName: string, identifier: untyped) =
+proc runTest(identifier: string) =
   # We wrap the tests in a proc to avoid running out of globals
   # in the future: Nim supports up to 3500 globals
   # but unittest with the macro/templates put everything as globals
   # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
 
-  const testDir = OperationsAttestationsDir / astToStr(identifier)
+  let testDir = OperationsAttestationsDir / identifier
 
   proc `testImpl _ operations_attestations _ identifier`() =
 
@@ -39,7 +39,7 @@ template runTest(testName: string, identifier: untyped) =
     else:
       prefix = "[Invalid] "
 
-    timedTest prefix & testName & " (" & astToStr(identifier) & ")":
+    timedTest prefix & identifier:
       var stateRef, postRef: ref BeaconState
       var attestationRef: ref Attestation
       new attestationRef
@@ -66,27 +66,5 @@ template runTest(testName: string, identifier: untyped) =
   `testImpl _ operations_attestations _ identifier`()
 
 suite "Official - Operations - Attestations " & preset():
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/minimal/phase0/operations/attestation/pyspec_tests
+  for kind, path in walkDir(OperationsAttestationsDir, true):
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/mainnet/phase0/operations/attestation/pyspec_tests
+    runTest(path)
-  runTest("after_epoch_slots", after_epoch_slots)
-  runTest("bad source root", bad_source_root)
-  runTest("before inclusion delay", before_inclusion_delay)
-  runTest("empty aggregation bits", empty_aggregation_bits)
-  runTest("future target epoch", future_target_epoch)
-  runTest("invalid attestation signature", invalid_attestation_signature)
-  runTest("invalid current source root", invalid_current_source_root)
-  runTest("invalid index", invalid_index)
-  runTest("mismatched target and slot", mismatched_target_and_slot)
-  runTest("new source epoch", new_source_epoch)
-  runTest("old source epoch", old_source_epoch)
-  runTest("old target epoch", old_target_epoch)
-  runTest("source root is target root", source_root_is_target_root)
-  runTest("success", success)
-  runTest("success multi-proposer index interations",
-    success_multi_proposer_index_iterations)
-  runTest("success previous epoch", success_previous_epoch)
-  runTest("too few aggregation bits", too_few_aggregation_bits)
-  runTest("too many aggregation bits", too_many_aggregation_bits)
-  runTest("wrong index for committee signature",
-    wrong_index_for_committee_signature)
-  runTest("wrong index for slot", wrong_index_for_slot)

tests/official/test_fixture_operations_attester_slashings.nim

@@ -12,7 +12,7 @@ import
   os, unittest,
   # Beacon chain internals
   ../../beacon_chain/spec/[datatypes, state_transition_block, validator],
-  ../../beacon_chain/ssz,
+  ../../beacon_chain/[extras, ssz],
   # Test utilities
   ../testutil,
   ./fixtures_utils,
@@ -20,23 +20,26 @@ import
 
 const OpAttSlashingDir = SszTestsDir/const_preset/"phase0"/"operations"/"attester_slashing"/"pyspec_tests"
 
-template runTest(identifier: untyped) =
+proc runTest(identifier: string) =
   # We wrap the tests in a proc to avoid running out of globals
   # in the future: Nim supports up to 3500 globals
   # but unittest with the macro/templates put everything as globals
   # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
 
-  const testDir = OpAttSlashingDir / astToStr(identifier)
+  let testDir = OpAttSlashingDir / identifier
 
   proc `testImpl _ operations_attester_slashing _ identifier`() =
 
+    var flags: UpdateFlags
     var prefix: string
+    if not existsFile(testDir/"meta.yaml"):
+      flags.incl skipValidation
     if existsFile(testDir/"post.ssz"):
       prefix = "[Valid]   "
     else:
       prefix = "[Invalid] "
 
-    timedTest prefix & astToStr(identifier):
+    timedTest prefix & identifier:
       var stateRef, postRef: ref BeaconState
       var attesterSlashingRef: ref AttesterSlashing
       new attesterSlashingRef
@@ -52,10 +55,12 @@ template runTest(identifier: untyped) =
         postRef[] = parseTest(testDir/"post.ssz", SSZ, BeaconState)
 
       if postRef.isNil:
-        let done = process_attester_slashing(stateRef[], attesterSlashingRef[], cache)
+        let done = process_attester_slashing(stateRef[], attesterSlashingRef[],
+          flags, cache)
         doAssert done == false, "We didn't expect this invalid attester slashing to be processed."
       else:
-        let done = process_attester_slashing(stateRef[], attesterSlashingRef[], cache)
+        let done = process_attester_slashing(stateRef[], attesterSlashingRef[],
+          flags, cache)
         doAssert done, "Valid attestater slashing not processed"
         check: stateRef.hash_tree_root() == postRef.hash_tree_root()
         reportDiff(stateRef, postRef)
@@ -63,19 +68,17 @@ template runTest(identifier: untyped) =
   `testImpl _ operations_attester_slashing _ identifier`()
 
 suite "Official - Operations - Attester slashing " & preset():
-  runTest(success_double)
+  # TODO these are both valid and check BLS signatures, which isn't working
-  runTest(success_surround)
+  # since 0.10.x introduces new BLS signing/verifying interface with domain
-  runTest(success_already_exited_recent)
+  # in particular handled differently through compute_signing_root() rather
-  runTest(success_already_exited_long_ago)
+  # than through the bls_verify(...) call directly. This did not become the
-  runTest(invalid_sig_1)
+  # visible issue it now is because another bug had been masking it wherein
-  when false: # TODO - https://github.com/status-im/nim-beacon-chain/issues/429
+  # crypto.nim's bls_verify(...) call had been creating false positives, in
-    runTest(invalid_sig_2)
+  # which cases signature checks had been incorrectly passing.
-  runTest(invalid_sig_1_and_2)
+  const expected_failures =
-  runTest(same_data)
+    ["success_already_exited_recent", "success_already_exited_long_ago"]
-  runTest(no_double_or_surround)
+  for kind, path in walkDir(OpAttSlashingDir, true):
-  runTest(participants_already_slashed)
+    if path in expected_failures:
-  when false: # TODO - https://github.com/status-im/nim-beacon-chain/issues/429
+      echo "Skipping test: ", path
-    runTest(att1_bad_extra_index)
+      continue
-    runTest(att1_bad_replaced_index)
+    runTest(path)
-    runTest(att2_bad_extra_index)
-    runTest(att2_bad_replaced_index)

tests/official/test_fixture_operations_block_header.nim

@@ -20,26 +20,23 @@ import
 
 const OpBlockHeaderDir = SszTestsDir/const_preset/"phase0"/"operations"/"block_header"/"pyspec_tests"
 
-template runTest(identifier: untyped) =
+proc runTest(identifier: string) =
   # We wrap the tests in a proc to avoid running out of globals
   # in the future: Nim supports up to 3500 globals
   # but unittest with the macro/templates put everything as globals
   # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
 
-  const testDir = OpBlockHeaderDir / astToStr(identifier)
+  let testDir = OpBlockHeaderDir / identifier
 
   proc `testImpl _ blockheader _ identifier`() =
 
-    var flags: UpdateFlags
     var prefix: string
-    if not existsFile(testDir/"meta.yaml"):
-      flags.incl skipValidation
     if existsFile(testDir/"post.ssz"):
       prefix = "[Valid]   "
     else:
       prefix = "[Invalid] "
 
-    timedTest prefix & astToStr(identifier):
+    timedTest prefix & identifier:
       var stateRef, postRef: ref BeaconState
       var blck: ref BeaconBlock
       new blck
@@ -55,10 +52,10 @@ template runTest(identifier: untyped) =
         postRef[] = parseTest(testDir/"post.ssz", SSZ, BeaconState)
 
       if postRef.isNil:
-        let done = process_block_header(stateRef[], blck[], flags, cache)
+        let done = process_block_header(stateRef[], blck[], {}, cache)
         doAssert done == false, "We didn't expect this invalid block header to be processed."
       else:
-        let done = process_block_header(stateRef[], blck[], flags, cache)
+        let done = process_block_header(stateRef[], blck[], {}, cache)
         doAssert done, "Valid block header not processed"
         check: stateRef.hash_tree_root() == postRef.hash_tree_root()
         reportDiff(stateRef, postRef)
@@ -66,8 +63,5 @@ template runTest(identifier: untyped) =
   `testImpl _ blockheader _ identifier`()
 
 suite "Official - Operations - Block header " & preset():
-  runTest(success_block_header)
+  for kind, path in walkDir(OpBlockHeaderDir, true):
-  runTest(invalid_slot_block_header)
+    runTest(path)
-  when false: # skipValidation needs to be split https://github.com/status-im/nim-beacon-chain/issues/407
-    runTest(invalid_parent_root)
-  runTest(proposer_slashed)

tests/official/test_fixture_operations_deposits.nim

@@ -20,13 +20,13 @@ import
 
 const OperationsDepositsDir = SszTestsDir/const_preset/"phase0"/"operations"/"deposit"/"pyspec_tests"
 
-template runTest(testName: string, identifier: untyped) =
+proc runTest(identifier: string) =
   # We wrap the tests in a proc to avoid running out of globals
   # in the future: Nim supports up to 3500 globals
   # but unittest with the macro/templates put everything as globals
   # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
 
-  const testDir = OperationsDepositsDir / astToStr(identifier)
+  let testDir = OperationsDepositsDir / identifier
 
   proc `testImpl _ operations_deposits _ identifier`() =
 
@@ -39,7 +39,7 @@ template runTest(testName: string, identifier: untyped) =
     else:
       prefix = "[Invalid] "
 
-    timedTest prefix & testName & " (" & astToStr(identifier) & ")":
+    timedTest prefix & " " & identifier:
       var stateRef, postRef: ref BeaconState
       var depositRef: ref Deposit
       new depositRef
@@ -53,8 +53,7 @@ template runTest(testName: string, identifier: untyped) =
         postRef[] = parseTest(testDir/"post.ssz", SSZ, BeaconState)
 
       if postRef.isNil:
-        expect(AssertionError):
+        check not process_deposit(stateRef[], depositRef[], flags)
-          discard process_deposit(stateRef[], depositRef[], flags)
       else:
         discard process_deposit(stateRef[], depositRef[], flags)
         reportDiff(stateRef, postRef)
@@ -62,17 +61,11 @@ template runTest(testName: string, identifier: untyped) =
   `testImpl _ operations_deposits _ identifier`()
 
 suite "Official - Operations - Deposits " & preset():
-  runTest("new deposit under max", new_deposit_under_max)
+  # TODO
-  runTest("new deposit max", new_deposit_max)
+  const expected_failures = ["valid_sig_but_forked_state"]
-  runTest("new deposit over max", new_deposit_over_max)
-  runTest("invalid signature new deposit", invalid_sig_new_deposit)
-  runTest("success top-up", success_top_up)
-  runTest("invalid signature top-up", invalid_sig_top_up)
-  runTest("invalid withdrawal credentials top-up", invalid_withdrawal_credentials_top_up)
 
-  when false:
+  for kind, path in walkDir(OperationsDepositsDir, true):
-    # TODO - those should give an exception but do not
+    if path in expected_failures:
-    #        probably because skipValidation is too strong
+      echo "Skipping test: ", path
-    #        https://github.com/status-im/nim-beacon-chain/issues/407
+      continue
-    runTest("wrong deposit for deposit count", wrong_deposit_for_deposit_count)
+    runTest(path)
-    runTest("bad merkle proof", bad_merkle_proof)

tests/official/test_fixture_operations_proposer_slashings.nim

@@ -20,13 +20,13 @@ import
 
 const OpProposerSlashingDir = SszTestsDir/const_preset/"phase0"/"operations"/"proposer_slashing"/"pyspec_tests"
 
-template runTest(identifier: untyped) =
+proc runTest(identifier: string) =
   # We wrap the tests in a proc to avoid running out of globals
   # in the future: Nim supports up to 3500 globals
   # but unittest with the macro/templates put everything as globals
   # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
 
-  const testDir = OpProposerSlashingDir / astToStr(identifier)
+  let testDir = OpProposerSlashingDir / identifier
 
   proc `testImpl_proposer_slashing _ identifier`() =
 
@@ -66,15 +66,5 @@ template runTest(identifier: untyped) =
   `testImpl_proposer_slashing _ identifier`()
 
 suite "Official - Operations - Proposer slashing " & preset():
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/minimal/phase0/operations/proposer_slashing/pyspec_tests
+  for kind, path in walkDir(OpProposerSlashingDir, true):
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/mainnet/phase0/operations/proposer_slashing/pyspec_tests
+    runTest(path)
-  runTest(epochs_are_different)
-  runTest(headers_are_same)
-  runTest(invalid_proposer_index)
-  runTest(invalid_sig_1)
-  runTest(invalid_sig_1_and_2)
-  runTest(invalid_sig_2)
-  runTest(proposer_is_not_activated)
-  runTest(proposer_is_slashed)
-  runTest(proposer_is_withdrawn)
-  runTest(success)

tests/official/test_fixture_operations_voluntary_exit.nim

@@ -20,13 +20,13 @@ import
 
 const OpVoluntaryExitDir = SszTestsDir/const_preset/"phase0"/"operations"/"voluntary_exit"/"pyspec_tests"
 
-template runTest(identifier: untyped) =
+proc runTest(identifier: string) =
   # We wrap the tests in a proc to avoid running out of globals
   # in the future: Nim supports up to 3500 globals
   # but unittest with the macro/templates put everything as globals
   # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
 
-  const testDir = OpVoluntaryExitDir / astToStr(identifier)
+  let testDir = OpVoluntaryExitDir / identifier
 
   proc `testImpl _ voluntary_exit _ identifier`() =
 
@@ -39,7 +39,7 @@ template runTest(identifier: untyped) =
     else:
       prefix = "[Invalid] "
 
-    timedTest prefix & astToStr(identifier):
+    timedTest prefix & identifier:
       var stateRef, postRef: ref BeaconState
       var voluntaryExit: ref SignedVoluntaryExit
       new voluntaryExit
@@ -64,18 +64,5 @@ template runTest(identifier: untyped) =
   `testImpl _ voluntary_exit _ identifier`()
 
 suite "Official - Operations - Voluntary exit " & preset():
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/minimal/phase0/operations/voluntary_exit/pyspec_tests
+  for kind, path in walkDir(OpVoluntaryExitDir, true):
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/mainnet/phase0/operations/voluntary_exit/pyspec_tests
+    runTest(path)
-  runTest(success)
-
-  when false:
-    # TODO not sure how this particularly could falsely succeed
-    runTest(invalid_signature)
-
-  runTest(validator_invalid_validator_index)
-  runTest(validator_already_exited)
-  runTest(success_exit_queue)
-  runTest(validator_exit_in_future)
-  runTest(default_exit_epoch_subsequent_exit)
-  runTest(validator_not_active_long_enough)
-  runTest(validator_not_active)

tests/official/test_fixture_sanity_blocks.nim

@@ -9,7 +9,7 @@
 
 import
   # Standard library
-  os, unittest,
+  os, sequtils, unittest,
   # Beacon chain internals
   ../../beacon_chain/spec/[crypto, datatypes],
   ../../beacon_chain/[ssz, state_transition, extras],
@@ -20,90 +20,55 @@ import
 
 const SanityBlocksDir = SszTestsDir/const_preset/"phase0"/"sanity"/"blocks"/"pyspec_tests"
 
-template runValidTest(testName: string, identifier: untyped, num_blocks: int): untyped =
+proc runTest(identifier: string) =
   # We wrap the tests in a proc to avoid running out of globals
   # in the future: Nim supports up to 3500 globals
   # but unittest with the macro/templates put everything as globals
   # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
 
-  const testDir = SanityBlocksDir / astToStr(identifier)
+  let testDir = SanityBlocksDir / identifier
 
   proc `testImpl _ blck _ identifier`() =
-    timedTest "[Valid]   " & testName & " (" & astToStr(identifier) & ")":
+    let prefix = if existsFile(testDir/"post.ssz"):
+      "[Valid]   "
+    else:
+      "[Invalid] "
+
+    timedTest prefix & identifier:
       var stateRef, postRef: ref BeaconState
       new stateRef
-      new postRef
       stateRef[] = parseTest(testDir/"pre.ssz", SSZ, BeaconState)
-      postRef[] = parseTest(testDir/"post.ssz", SSZ, BeaconState)
 
-      for i in 0 ..< num_blocks:
+      if existsFile(testDir/"post.ssz"):
+        new postRef
+        postRef[] = parseTest(testDir/"post.ssz", SSZ, BeaconState)
+
+      # In test cases with more than 10 blocks the first 10 aren't 0-prefixed,
+      # so purely lexicographic sorting wouldn't sort properly.
+      for i in 0 ..< toSeq(walkPattern(testDir/"blocks_*.ssz")).len:
         let blck = parseTest(testDir/"blocks_" & $i & ".ssz", SSZ, SignedBeaconBlock)
 
-        # TODO: The EF is using invalid BLS keys so we can't verify them
+        if postRef.isNil:
-        let success = state_transition(stateRef[], blck.message, flags = {skipValidation})
+          let success = state_transition(stateRef[], blck.message, flags = {})
-        doAssert success, "Failure when applying block " & $i
+          doAssert not success, "We didn't expect this invalid block to be processed"
+        else:
+          # TODO: The EF is using invalid BLS keys so we can't verify them
+          let success = state_transition(stateRef[], blck.message, flags = {skipValidation})
+          doAssert success, "Failure when applying block " & $i
 
-      # Checks:
       # check: stateRef.hash_tree_root() == postRef.hash_tree_root()
-      reportDiff(stateRef, postRef)
+      if not postRef.isNil:
+        reportDiff(stateRef, postRef)
 
   `testImpl _ blck _ identifier`()
 
 suite "Official - Sanity - Blocks " & preset():
-  timedTest "[Invalid] Previous slot block transition (prev_slot_block_transition)":
+  # Failing due to signature checking in indexed validation checking pending
-    const testDir = SanityBlocksDir/"prev_slot_block_transition"
+  # 0.10 BLS verification API with new domain handling.
-    var stateRef: ref BeaconState
+  const expected_failures = ["attester_slashing"]
-    new stateRef
-    stateRef[] = parseTest(testDir/"pre.ssz", SSZ, BeaconState)
 
-    let blck = parseTest(testDir/"blocks_0.ssz", SSZ, SignedBeaconBlock)
+  for kind, path in walkDir(SanityBlocksDir, true):
-
+    if path in expected_failures:
-    # Check that a block build for an old slot cannot be used for state transition
+      echo "Skipping test: ", path
-    expect(AssertionError):
+      continue
-      # assert in process_slots. This should not be triggered
+    runTest(path)
-      #                          for blocks from block_pool/network
-      discard state_transition(stateRef[], blck.message, flags = {skipValidation})
-
-  runValidTest("Same slot block transition", same_slot_block_transition, 1)
-  runValidTest("Empty block transition", empty_block_transition, 1)
-
-  when false: # TODO: we need more granular skipValidation
-    timedTest "[Invalid] Invalid state root":
-      const testDir = SanityBlocksDir/"invalid_state_root"
-      var stateRef: ref BeaconState
-      new stateRef
-      stateRef[] = parseTest(testDir/"pre.ssz", SSZ, BeaconState)
-
-      let blck = parseTest(testDir/"blocks_0.ssz", SSZ, BeaconBlock)
-
-      expect(AssertionError):
-        discard state_transition(stateRef[], blck, flags = {skipValidation})
-
-  runValidTest("Skipped Slots", skipped_slots, 1)
-  runValidTest("Empty epoch transition", empty_epoch_transition, 1)
-  when const_preset=="minimal":
-    runValidTest("Empty epoch transition not finalizing", empty_epoch_transition_not_finalizing, 1)
-
-  when false:
-    # TODO investigate/fix after 0.9.0 transition broke this in mainnet and
-    # in 0.9.1 even minimal broke. For the latter at least, it differs only
-    # in latest_block_header.body_root, which is just a hash_tree_root() of
-    # the one block read by this test case. All balances agree. It's an SSZ
-    # or hashing issue.
-    runValidTest("Attester slashing", attester_slashing, 1)
-  runValidTest("Proposer slashing", proposer_slashing, 1)
-
-  # TODO: Expected deposit in block
-
-  runValidTest("Deposit in block", deposit_in_block, 1)
-  runValidTest("Deposit top up", deposit_top_up, 1)
-
-  when const_preset=="minimal":
-    # TODO this doesn't work on mainnet
-    runValidTest("Attestation", attestation, 2)
-  runValidTest("Voluntary exit", voluntary_exit, 2)
-  runValidTest("Balance-driven status transitions", balance_driven_status_transitions, 1)
-  runValidTest("Historical batch", historical_batch, 1)
-  when const_preset=="minimal":
-    runValidTest("ETH1 data votes consensus", eth1_data_votes_consensus, 17)
-    runValidTest("ETH1 data votes no consensus", eth1_data_votes_no_consensus, 16)

tests/official/test_fixture_sanity_slots.nim

@@ -9,7 +9,7 @@
 
 import
   # Standard library
-  os, unittest,
+  os, strutils, unittest,
   # Beacon chain internals
   ../../beacon_chain/spec/datatypes,
   ../../beacon_chain/state_transition,
@@ -20,16 +20,17 @@ import
 
 const SanitySlotsDir = SszTestsDir/const_preset/"phase0"/"sanity"/"slots"/"pyspec_tests"
 
-template runTest(testName: string, identifier: untyped, num_slots: uint64): untyped =
+proc runTest(identifier: string) =
   # We wrap the tests in a proc to avoid running out of globals
   # in the future: Nim supports up to 3500 globals
   # but unittest with the macro/templates put everything as globals
   # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
-
+  let
-  const testDir = SanitySlotsDir / astToStr(identifier)
+    testDir = SanitySlotsDir / identifier
+    num_slots = readLines(testDir / "slots.yaml", 2)[0].parseInt.uint64
 
   proc `testImpl _ slots _ identifier`() =
-    timedTest "Slots - " & testName & " (" & astToStr(identifier) & ")":
+    timedTest "Slots - " & identifier:
       var stateRef, postRef: ref BeaconState
       new stateRef
       new postRef
@@ -43,18 +44,6 @@ template runTest(testName: string, identifier: untyped, num_slots: uint64): unty
 
   `testImpl _ slots _ identifier`()
 
-# 1 slot
-# ---------------------------------------------------------------
-
 suite "Official - Sanity - Slots " & preset():
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/minimal/phase0/sanity/slots/pyspec_tests
+  for kind, path in walkDir(SanitySlotsDir, true):
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/mainnet/phase0/sanity/slots/pyspec_tests
+    runTest(path)
-  runTest("Advance 1 slot", slots_1, 1)
-  runTest("Advance 2 slots", slots_2, 2)
-  runTest("Advance an empty epoch", empty_epoch, SLOTS_PER_EPOCH)
-
-  const DoubleEpoch = SLOTS_PER_EPOCH.uint64*2 # workaround undeclared identifier "double_empty_epoch"
-  runTest("Advance 2 empty epochs", double_empty_epoch, DoubleEpoch)
-
-  # This starts in the middle of an epoch
-  runTest("Advance over an epoch boundary", over_epoch_boundary, SLOTS_PER_EPOCH)

tests/simulation/run_node.sh

@@ -52,7 +52,7 @@ if [[ $NODE_ID == $MASTER_NODE ]]; then
   NODE_BIN=$BOOTSTRAP_NODE_BIN
 fi
 
-$NODE_BIN \
+cd "$DATA_DIR" && $NODE_BIN \
   --bootstrap-file=$BOOTSTRAP_ADDRESS_FILE \
   --data-dir=$DATA_DIR \
   --node-name=$NODE_ID \

tests/simulation/start.sh

@@ -16,13 +16,13 @@ mkdir -p "$VALIDATORS_DIR"
 
 cd "$GIT_ROOT"
 
-NIMFLAGS="-d:chronicles_log_level=TRACE --hints:off --warnings:off --verbosity:0 --opt:speed --debuginfo"
+NIMFLAGS="-d:chronicles_log_level=TRACE -d:chronicles_sinks:textlines,json[file] --hints:off --warnings:off --verbosity:0 --opt:speed --debuginfo"
 
 # Run with "SLOTS_PER_EPOCH=8 ./start.sh" to change these
 DEFS=""
 
 DEFS+="-d:MAX_COMMITTEES_PER_SLOT=${MAX_COMMITTEES_PER_SLOT:-1} "      # Spec default: 64
-DEFS+="-d:SLOTS_PER_EPOCH=${SLOTS_PER_EPOCH:-16} "   # Spec default: 32
+DEFS+="-d:SLOTS_PER_EPOCH=${SLOTS_PER_EPOCH:-6} "   # Spec default: 32
 DEFS+="-d:SECONDS_PER_SLOT=${SECONDS_PER_SLOT:-6} "  # Spec default: 12
 
 LAST_VALIDATOR_NUM=$(( NUM_VALIDATORS - 1 ))

tests/spec_block_processing/test_process_deposits.nim

@@ -56,7 +56,8 @@ suite "[Unit - Spec - Block processing] Deposits " & preset():
 
       # State transition
       # ----------------------------------------
-      check: state.process_deposit(deposit, {skipValidation})
+      check: state.process_deposit(deposit,
+        {skipValidation, skipMerkleValidation})
 
       # Check invariants
       # ----------------------------------------
@@ -100,7 +101,8 @@ suite "[Unit - Spec - Block processing] Deposits " & preset():
 
     # State transition
     # ----------------------------------------
-    check: state.process_deposit(deposit, {skipValidation})
+    check: state.process_deposit(deposit,
+      {skipValidation, skipMerkleValidation})
 
     # Check invariants
     # ----------------------------------------

tests/test_attestation_pool.nim

@@ -111,7 +111,7 @@ when const_preset == "minimal": # Too much stack space used on mainnet
         attestation1 = makeAttestation(
           state.data.data, state.blck.root, bc0[1], cache)
 
-      attestation0.combine(attestation1, {skipValidation})
+      attestation0.combine(attestation1, {})
 
       pool.add(attestation0)
       pool.add(attestation1)
@@ -135,7 +135,7 @@ when const_preset == "minimal": # Too much stack space used on mainnet
         attestation1 = makeAttestation(
           state.data.data, state.blck.root, bc0[1], cache)
 
-      attestation0.combine(attestation1, {skipValidation})
+      attestation0.combine(attestation1, {})
 
       pool.add(attestation1)
       pool.add(attestation0)

tests/test_beaconstate.nim

@@ -17,5 +17,5 @@ suite "Beacon state" & preset():
   timedTest "Smoke test initialize_beacon_state_from_eth1" & preset():
     let state = initialize_beacon_state_from_eth1(
       Eth2Digest(), 0,
-      makeInitialDeposits(SLOTS_PER_EPOCH, {}), {skipValidation})
+      makeInitialDeposits(SLOTS_PER_EPOCH, {}), {skipMerkleValidation})
     check: state.validators.len == SLOTS_PER_EPOCH

tests/test_block_pool.nim

@@ -187,6 +187,55 @@ when const_preset == "minimal": # Too much stack space used on mainnet
         pool.head.blck == b1Add
         pool.headState.data.data.slot == b1Add.slot
 
+    timedTest "updateStateData sanity" & preset():
+      let
+        b1Add = pool.add(b1Root, b1)
+        b2Add = pool.add(b2Root, b2)
+        bs1 = BlockSlot(blck: b1Add, slot: b1.message.slot)
+        bs1_3 = b1Add.atSlot(3.Slot)
+        bs2 = BlockSlot(blck: b2Add, slot: b2.message.slot)
+        bs2_3 = b2Add.atSlot(3.Slot)
+
+      var tmpState = pool.headState
+
+      # move to specific block
+      pool.updateStateData(tmpState, bs1)
+
+      check:
+        tmpState.blck == b1Add
+        tmpState.data.data.slot == bs1.slot
+
+      # Skip slots
+      pool.updateStateData(tmpState, bs1_3) # skip slots
+
+      check:
+        tmpState.blck == b1Add
+        tmpState.data.data.slot == bs1_3.slot
+
+      # Move back slots, but not blocks
+      pool.updateStateData(tmpState, bs1_3.parent())
+      check:
+        tmpState.blck == b1Add
+        tmpState.data.data.slot == bs1_3.parent().slot
+
+      # Move to different block and slot
+      pool.updateStateData(tmpState, bs2_3)
+      check:
+        tmpState.blck == b2Add
+        tmpState.data.data.slot == bs2_3.slot
+
+      # Move back slot and block
+      pool.updateStateData(tmpState, bs1)
+      check:
+        tmpState.blck == b1Add
+        tmpState.data.data.slot == bs1.slot
+
+      # Move back to genesis
+      pool.updateStateData(tmpState, bs1.parent())
+      check:
+        tmpState.blck == b1Add.parent
+        tmpState.data.data.slot == bs1.parent.slot
+
   suite "BlockPool finalization tests" & preset():
     setup:
       var
@@ -213,7 +262,7 @@ when const_preset == "minimal": # Too much stack space used on mainnet
             BeaconBlockBody(
               attestations: makeFullAttestations(
                 pool.headState.data.data, pool.head.blck.root,
-                pool.headState.data.data.slot, cache, {skipValidation})))
+                pool.headState.data.data.slot, cache, {})))
         let added = pool.add(hash_tree_root(blck.message), blck)
         pool.updateHead(added)
 

tests/test_interop.nim

@@ -150,7 +150,7 @@ suite "Interop":
 
     var
       initialState = initialize_beacon_state_from_eth1(
-        eth1BlockHash, 1570500000, deposits, {skipValidation})
+        eth1BlockHash, 1570500000, deposits, {skipMerkleValidation})
 
     # https://github.com/ethereum/eth2.0-pm/tree/6e41fcf383ebeb5125938850d8e9b4e9888389b4/interop/mocked_start#create-genesis-state
     initialState.genesis_time = 1570500000

tests/test_state_transition.nim

@@ -22,7 +22,7 @@ suite "Block processing" & preset():
     # TODO bls verification is a bit of a bottleneck here
     genesisState = initialize_beacon_state_from_eth1(
       Eth2Digest(), 0,
-      makeInitialDeposits(), {skipValidation})
+      makeInitialDeposits(), {skipMerkleValidation})
     genesisBlock = get_initial_beacon_block(genesisState)
     genesisRoot = hash_tree_root(genesisBlock.message)
 

tests/testutil.nim

@@ -80,7 +80,8 @@ proc makeTestDB*(validators: int): BeaconChainDB =
   let
     genState = initialize_beacon_state_from_eth1(
       Eth2Digest(), 0,
-      makeInitialDeposits(validators, flags = {skipValidation}), {skipValidation})
+      makeInitialDeposits(validators, flags = {skipValidation}),
+        {skipValidation, skipMerkleValidation})
     genBlock = get_initial_beacon_block(genState)
   makeTestDB(genState, genBlock)
 

vendor/NimYAML

@@ -1 +1 @@
-Subproject commit e6ac75d86af8f8eae9a010c8e639c3ac4263f8df
+Subproject commit 3983a1bf48682156cb105ab124c7bbdd86fa4557

vendor/nim-libp2p

@@ -1 +1 @@
-Subproject commit d42833947a4baddf21da8ac3105e2d5956a6daac
+Subproject commit 8c406fb9e5f5ea82c96173b828efe4a8a6027747

vendor/nimbus-build-system

@@ -1 +1 @@
-Subproject commit 2a70c4f152ee849db1ededa92c1d80f7102dd718
+Subproject commit 2c4faa5372d2d8b0c2d16710fe5f93beab1c86af