From d634eba3fdf0047f79cf38d2502f3556029348db Mon Sep 17 00:00:00 2001
From: Zahary Karadjov <zahary@gmail.com>
Date: Tue, 4 Feb 2020 16:03:39 +0100
Subject: [PATCH 01/31] Produce a json log file in the data dir of each node in
 the local network sim

---
 tests/simulation/run_node.sh | 2 +-
 tests/simulation/start.sh    | 2 +-
 vendor/nim-libp2p            | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/simulation/run_node.sh b/tests/simulation/run_node.sh
index 4def0a452..8bfdc1356 100755
--- a/tests/simulation/run_node.sh
+++ b/tests/simulation/run_node.sh
@@ -52,7 +52,7 @@ if [[ $NODE_ID == $MASTER_NODE ]]; then
   NODE_BIN=$BOOTSTRAP_NODE_BIN
 fi
 
-$NODE_BIN \
+cd "$DATA_DIR" && $NODE_BIN \
   --bootstrap-file=$BOOTSTRAP_ADDRESS_FILE \
   --data-dir=$DATA_DIR \
   --node-name=$NODE_ID \
diff --git a/tests/simulation/start.sh b/tests/simulation/start.sh
index 3c4d63fd4..7a63946ca 100755
--- a/tests/simulation/start.sh
+++ b/tests/simulation/start.sh
@@ -16,7 +16,7 @@ mkdir -p "$VALIDATORS_DIR"
 
 cd "$GIT_ROOT"
 
-NIMFLAGS="-d:chronicles_log_level=TRACE --hints:off --warnings:off --verbosity:0 --opt:speed --debuginfo"
+NIMFLAGS="-d:chronicles_log_level=TRACE -d:chronicles_sinks:textlines,json[file] --hints:off --warnings:off --verbosity:0 --opt:speed --debuginfo"
 
 # Run with "SLOTS_PER_EPOCH=8 ./start.sh" to change these
 DEFS=""
diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p
index d42833947..7bd305471 160000
--- a/vendor/nim-libp2p
+++ b/vendor/nim-libp2p
@@ -1 +1 @@
-Subproject commit d42833947a4baddf21da8ac3105e2d5956a6daac
+Subproject commit 7bd305471c0ab1a0306e6734915952f236cc88eb

From 1ffc2df23dc43aa4d6e087d42b894aeb5282c709 Mon Sep 17 00:00:00 2001
From: Dustin Brody <tersec@users.noreply.github.com>
Date: Thu, 30 Jan 2020 22:03:47 +0100
Subject: [PATCH 02/31] add a couple new deposit tests; fix the false-positive
 BLS verifications while keeping all but two tests working, despite mismatched
 0.9/0.10 BLS standards; better-factor the skipping of BLS validation and
 Merkle tree validation

---
 beacon_chain/beacon_node.nim                  |  2 +-
 beacon_chain/extras.nim                       | 10 ++++++++
 beacon_chain/spec/beaconstate.nim             |  9 ++++---
 beacon_chain/spec/crypto.nim                  |  6 +++--
 beacon_chain/spec/state_transition_block.nim  | 14 ++++++-----
 research/state_sim.nim                        |  2 +-
 tests/mocking/mock_genesis.nim                |  2 +-
 ..._fixture_operations_attester_slashings.nim | 23 +++++++++++++----
 .../test_fixture_operations_deposits.nim      | 25 ++++++++++---------
 .../test_process_deposits.nim                 |  6 +++--
 tests/test_attestation_pool.nim               |  4 +--
 tests/test_beaconstate.nim                    |  2 +-
 tests/test_block_pool.nim                     |  2 +-
 tests/test_interop.nim                        |  2 +-
 tests/test_state_transition.nim               |  2 +-
 tests/testutil.nim                            |  3 ++-
 16 files changed, 73 insertions(+), 41 deletions(-)

diff --git a/beacon_chain/beacon_node.nim b/beacon_chain/beacon_node.nim
index 3b5a6def2..ea693a0c5 100644
--- a/beacon_chain/beacon_node.nim
+++ b/beacon_chain/beacon_node.nim
@@ -1113,7 +1113,7 @@ when isMainModule:
                  else: waitFor getLatestEth1BlockHash(config.depositWeb3Url)
     var
       initialState = initialize_beacon_state_from_eth1(
-        eth1Hash, startTime, deposits, {skipValidation})
+        eth1Hash, startTime, deposits, {skipValidation, skipMerkleValidation})
 
     # https://github.com/ethereum/eth2.0-pm/tree/6e41fcf383ebeb5125938850d8e9b4e9888389b4/interop/mocked_start#create-genesis-state
     initialState.genesis_time = startTime
diff --git a/beacon_chain/extras.nim b/beacon_chain/extras.nim
index b99682986..203e43cf4 100644
--- a/beacon_chain/extras.nim
+++ b/beacon_chain/extras.nim
@@ -25,5 +25,15 @@ type
     ## TODO need to be careful here, easy to assume that slot number change is
     ##      enough, vs advancing the state - however, making a full state copy
     ##      is expensive also :/
+    skipMerkleValidation ##\
+    ## When processing deposits, skip verifying the Merkle proof trees of each
+    ## deposit. This is a holdover from both interop issues with the malformed
+    ## proofs and, more currently, nim-beacon-chain's creation of proofs which
+    ## are inconsistent with the current specification. Furthermore several of
+    ## the mocking interfaces deliberately do not create Merkle proofs. Whilst
+    ## this seems less than entirely justifiable, for now enable keeping those
+    ## in place while minimizing the tech debt they create. One, in principle,
+    ## should be able to remove this flag entirely. It is not intrinsically an
+    ## expensive operation to perform.
 
   UpdateFlags* = set[UpdateFlag]
diff --git a/beacon_chain/spec/beaconstate.nim b/beacon_chain/spec/beaconstate.nim
index 14f44fbe9..a166df490 100644
--- a/beacon_chain/spec/beaconstate.nim
+++ b/beacon_chain/spec/beaconstate.nim
@@ -53,7 +53,7 @@ proc process_deposit*(
   # Process an Eth1 deposit, registering a validator or increasing its balance.
 
   # Verify the Merkle branch
-  if skipValidation notin flags and not is_valid_merkle_branch(
+  if skipMerkleValidation notin flags and not is_valid_merkle_branch(
     hash_tree_root(deposit.data),
     deposit.proof,
     DEPOSIT_CONTRACT_TREE_DEPTH + 1,
@@ -361,7 +361,8 @@ proc process_registry_updates*(state: var BeaconState) {.nbench.}=
 
 # https://github.com/ethereum/eth2.0-specs/blob/v0.9.4/specs/core/0_beacon-chain.md#is_valid_indexed_attestation
 proc is_valid_indexed_attestation*(
-    state: BeaconState, indexed_attestation: IndexedAttestation): bool =
+    state: BeaconState, indexed_attestation: IndexedAttestation,
+    flags: UpdateFlags): bool =
   ## Check if ``indexed_attestation`` has valid indices and signature.
   # TODO: this is noSideEffect besides logging
   #       https://github.com/status-im/nim-chronicles/issues/62
@@ -380,7 +381,7 @@ proc is_valid_indexed_attestation*(
     return false
 
   # Verify aggregate signature
-  if not bls_verify(
+  if skipValidation notin flags and not bls_verify(
     bls_aggregate_pubkeys(mapIt(indices, state.validators[it.int].pubkey)),
     hash_tree_root(indexed_attestation.data).data,
     indexed_attestation.signature,
@@ -497,7 +498,7 @@ proc check_attestation*(
       return
 
   if not is_valid_indexed_attestation(
-      state, get_indexed_attestation(state, attestation, stateCache)):
+      state, get_indexed_attestation(state, attestation, stateCache), flags):
     warn("process_attestation: signature or bitfields incorrect")
     return
 
diff --git a/beacon_chain/spec/crypto.nim b/beacon_chain/spec/crypto.nim
index 7eda6e6b0..e5f71d1d8 100644
--- a/beacon_chain/spec/crypto.nim
+++ b/beacon_chain/spec/crypto.nim
@@ -160,8 +160,10 @@ func bls_verify*(
       return false
     # TODO bls_verify_multiple(...) used to have this workaround, and now it
     # lives here. No matter the signature, there's also no meaningful way to
-    # verify it -- it's a kind of vacuous truth. No pubkey/sig pairs.
-    if pubkey == default(ValidatorPubKey):
+    # verify it -- it's a kind of vacuous truth. No pubkey/sig pairs. Sans a
+    # getBytes() or similar mechanism, pubKey == default(ValidatorPubKey) is
+    # a way to create many false positive matches. This seems odd.
+    if pubkey.getBytes() == default(ValidatorPubKey).getBytes():
       return true
 
     sig.blsValue.verify(msg, domain, pubkey.blsValue)
diff --git a/beacon_chain/spec/state_transition_block.nim b/beacon_chain/spec/state_transition_block.nim
index b568a62f6..0ff0daec8 100644
--- a/beacon_chain/spec/state_transition_block.nim
+++ b/beacon_chain/spec/state_transition_block.nim
@@ -217,6 +217,7 @@ func is_slashable_attestation_data(
 proc process_attester_slashing*(
        state: var BeaconState,
        attester_slashing: AttesterSlashing,
+       flags: UpdateFlags,
        stateCache: var StateCache
      ): bool {.nbench.}=
     let
@@ -228,11 +229,11 @@ proc process_attester_slashing*(
       notice "Attester slashing: surround or double vote check failed"
       return false
 
-    if not is_valid_indexed_attestation(state, attestation_1):
+    if not is_valid_indexed_attestation(state, attestation_1, flags):
       notice "Attester slashing: invalid attestation 1"
       return false
 
-    if not is_valid_indexed_attestation(state, attestation_2):
+    if not is_valid_indexed_attestation(state, attestation_2, flags):
       notice "Attester slashing: invalid attestation 2"
       return false
 
@@ -259,7 +260,7 @@ proc processAttesterSlashings(state: var BeaconState, blck: BeaconBlock,
     return false
 
   for attester_slashing in blck.body.attester_slashings:
-    if not process_attester_slashing(state, attester_slashing, stateCache):
+    if not process_attester_slashing(state, attester_slashing, {}, stateCache):
       return false
   return true
 
@@ -286,13 +287,14 @@ proc processAttestations(
   true
 
 # https://github.com/ethereum/eth2.0-specs/blob/v0.8.4/specs/core/0_beacon-chain.md#deposits
-proc processDeposits(state: var BeaconState, blck: BeaconBlock): bool {.nbench.}=
+proc processDeposits(state: var BeaconState, blck: BeaconBlock,
+    flags: UpdateFlags): bool {.nbench.}=
   if not (len(blck.body.deposits) <= MAX_DEPOSITS):
     notice "processDeposits: too many deposits"
     return false
 
   for deposit in blck.body.deposits:
-    if not process_deposit(state, deposit):
+    if not process_deposit(state, deposit, flags):
       notice "processDeposits: deposit invalid"
       return false
 
@@ -419,7 +421,7 @@ proc processBlock*(
     debug "[Block processing] Attestation processing failure", slot = shortLog(state.slot)
     return false
 
-  if not processDeposits(state, blck):
+  if not processDeposits(state, blck, flags):
     debug "[Block processing] Deposit processing failure", slot = shortLog(state.slot)
     return false
 
diff --git a/research/state_sim.nim b/research/state_sim.nim
index 361caf8b7..2022ecca6 100644
--- a/research/state_sim.nim
+++ b/research/state_sim.nim
@@ -79,7 +79,7 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
   let
     genesisState =
       initialize_beacon_state_from_eth1(
-        Eth2Digest(), 0, deposits, {skipValidation})
+        Eth2Digest(), 0, deposits, {skipMerkleValidation})
     genesisBlock = get_initial_beacon_block(genesisState)
 
   echo "Starting simulation..."
diff --git a/tests/mocking/mock_genesis.nim b/tests/mocking/mock_genesis.nim
index 2b874002a..1cfe1dbe5 100644
--- a/tests/mocking/mock_genesis.nim
+++ b/tests/mocking/mock_genesis.nim
@@ -25,7 +25,7 @@ proc initGenesisState*(num_validators: uint64, genesis_time: uint64 = 0): Beacon
     )
 
   initialize_beacon_state_from_eth1(
-    eth1BlockHash, 0, deposits, {skipValidation})
+    eth1BlockHash, 0, deposits, {skipValidation, skipMerkleValidation})
 
 when isMainModule:
   # Smoke test
diff --git a/tests/official/test_fixture_operations_attester_slashings.nim b/tests/official/test_fixture_operations_attester_slashings.nim
index 5aa44a0df..f0b874d10 100644
--- a/tests/official/test_fixture_operations_attester_slashings.nim
+++ b/tests/official/test_fixture_operations_attester_slashings.nim
@@ -12,7 +12,7 @@ import
   os, unittest,
   # Beacon chain internals
   ../../beacon_chain/spec/[datatypes, state_transition_block, validator],
-  ../../beacon_chain/ssz,
+  ../../beacon_chain/[extras, ssz],
   # Test utilities
   ../testutil,
   ./fixtures_utils,
@@ -30,7 +30,10 @@ template runTest(identifier: untyped) =
 
   proc `testImpl _ operations_attester_slashing _ identifier`() =
 
+    var flags: UpdateFlags
     var prefix: string
+    if not existsFile(testDir/"meta.yaml"):
+      flags.incl skipValidation
     if existsFile(testDir/"post.ssz"):
       prefix = "[Valid]   "
     else:
@@ -52,10 +55,12 @@ template runTest(identifier: untyped) =
         postRef[] = parseTest(testDir/"post.ssz", SSZ, BeaconState)
 
       if postRef.isNil:
-        let done = process_attester_slashing(stateRef[], attesterSlashingRef[], cache)
+        let done = process_attester_slashing(stateRef[], attesterSlashingRef[],
+          flags, cache)
         doAssert done == false, "We didn't expect this invalid attester slashing to be processed."
       else:
-        let done = process_attester_slashing(stateRef[], attesterSlashingRef[], cache)
+        let done = process_attester_slashing(stateRef[], attesterSlashingRef[],
+          flags, cache)
         doAssert done, "Valid attestater slashing not processed"
         check: stateRef.hash_tree_root() == postRef.hash_tree_root()
         reportDiff(stateRef, postRef)
@@ -65,8 +70,16 @@ template runTest(identifier: untyped) =
 suite "Official - Operations - Attester slashing " & preset():
   runTest(success_double)
   runTest(success_surround)
-  runTest(success_already_exited_recent)
-  runTest(success_already_exited_long_ago)
+  when false:
+    # TODO these are both valid and check BLS signatures, which isn't working
+    # since 0.10.x introduces new BLS signing/verifying interface with domain
+    # in particular handled differently through compute_signing_root() rather
+    # than through the bls_verify(...) call directly. This did not become the
+    # visible issue it now is because another bug had been masking it wherein
+    # crypto.nim's bls_verify(...) call had been creating false positives, in
+    # which cases signature checks had been incorrectly passing.
+    runTest(success_already_exited_recent)
+    runTest(success_already_exited_long_ago)
   runTest(invalid_sig_1)
   when false: # TODO - https://github.com/status-im/nim-beacon-chain/issues/429
     runTest(invalid_sig_2)
diff --git a/tests/official/test_fixture_operations_deposits.nim b/tests/official/test_fixture_operations_deposits.nim
index 324caa469..3603b41a6 100644
--- a/tests/official/test_fixture_operations_deposits.nim
+++ b/tests/official/test_fixture_operations_deposits.nim
@@ -53,8 +53,7 @@ template runTest(testName: string, identifier: untyped) =
         postRef[] = parseTest(testDir/"post.ssz", SSZ, BeaconState)
 
       if postRef.isNil:
-        expect(AssertionError):
-          discard process_deposit(stateRef[], depositRef[], flags)
+        check not process_deposit(stateRef[], depositRef[], flags)
       else:
         discard process_deposit(stateRef[], depositRef[], flags)
         reportDiff(stateRef, postRef)
@@ -62,17 +61,19 @@ template runTest(testName: string, identifier: untyped) =
   `testImpl _ operations_deposits _ identifier`()
 
 suite "Official - Operations - Deposits " & preset():
-  runTest("new deposit under max", new_deposit_under_max)
+  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/minimal/phase0/operations/deposit/pyspec_tests
+  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/mainnet/phase0/operations/deposit/pyspec_tests
+  runTest("bad merkle proof", bad_merkle_proof)
+  runTest("invalid signature new deposit", invalid_sig_new_deposit)
+  runTest("invalid signature other version", invalid_sig_other_version)
+  runTest("invalid signature top-up", invalid_sig_top_up)
+  runTest("invalid withdrawal credentials top-up",
+    invalid_withdrawal_credentials_top_up)
   runTest("new deposit max", new_deposit_max)
   runTest("new deposit over max", new_deposit_over_max)
-  runTest("invalid signature new deposit", invalid_sig_new_deposit)
+  runTest("new deposit under max", new_deposit_under_max)
   runTest("success top-up", success_top_up)
-  runTest("invalid signature top-up", invalid_sig_top_up)
-  runTest("invalid withdrawal credentials top-up", invalid_withdrawal_credentials_top_up)
-
   when false:
-    # TODO - those should give an exception but do not
-    #        probably because skipValidation is too strong
-    #        https://github.com/status-im/nim-beacon-chain/issues/407
-    runTest("wrong deposit for deposit count", wrong_deposit_for_deposit_count)
-    runTest("bad merkle proof", bad_merkle_proof)
+    # TODO
+    runTest("valid signature but forked state", valid_sig_but_forked_state)
+  runTest("wrong deposit for deposit count", wrong_deposit_for_deposit_count)
diff --git a/tests/spec_block_processing/test_process_deposits.nim b/tests/spec_block_processing/test_process_deposits.nim
index f75988997..5a120aedc 100644
--- a/tests/spec_block_processing/test_process_deposits.nim
+++ b/tests/spec_block_processing/test_process_deposits.nim
@@ -56,7 +56,8 @@ suite "[Unit - Spec - Block processing] Deposits " & preset():
 
       # State transition
       # ----------------------------------------
-      check: state.process_deposit(deposit, {skipValidation})
+      check: state.process_deposit(deposit,
+        {skipValidation, skipMerkleValidation})
 
       # Check invariants
       # ----------------------------------------
@@ -100,7 +101,8 @@ suite "[Unit - Spec - Block processing] Deposits " & preset():
 
     # State transition
     # ----------------------------------------
-    check: state.process_deposit(deposit, {skipValidation})
+    check: state.process_deposit(deposit,
+      {skipValidation, skipMerkleValidation})
 
     # Check invariants
     # ----------------------------------------
diff --git a/tests/test_attestation_pool.nim b/tests/test_attestation_pool.nim
index a78288fb1..1a44ddad1 100644
--- a/tests/test_attestation_pool.nim
+++ b/tests/test_attestation_pool.nim
@@ -111,7 +111,7 @@ when const_preset == "minimal": # Too much stack space used on mainnet
         attestation1 = makeAttestation(
           state.data.data, state.blck.root, bc0[1], cache)
 
-      attestation0.combine(attestation1, {skipValidation})
+      attestation0.combine(attestation1, {})
 
       pool.add(attestation0)
       pool.add(attestation1)
@@ -135,7 +135,7 @@ when const_preset == "minimal": # Too much stack space used on mainnet
         attestation1 = makeAttestation(
           state.data.data, state.blck.root, bc0[1], cache)
 
-      attestation0.combine(attestation1, {skipValidation})
+      attestation0.combine(attestation1, {})
 
       pool.add(attestation1)
       pool.add(attestation0)
diff --git a/tests/test_beaconstate.nim b/tests/test_beaconstate.nim
index 26f81a8c1..13edaa804 100644
--- a/tests/test_beaconstate.nim
+++ b/tests/test_beaconstate.nim
@@ -17,5 +17,5 @@ suite "Beacon state" & preset():
   timedTest "Smoke test initialize_beacon_state_from_eth1" & preset():
     let state = initialize_beacon_state_from_eth1(
       Eth2Digest(), 0,
-      makeInitialDeposits(SLOTS_PER_EPOCH, {}), {skipValidation})
+      makeInitialDeposits(SLOTS_PER_EPOCH, {}), {skipMerkleValidation})
     check: state.validators.len == SLOTS_PER_EPOCH
diff --git a/tests/test_block_pool.nim b/tests/test_block_pool.nim
index 3874431bf..4578e1f9b 100644
--- a/tests/test_block_pool.nim
+++ b/tests/test_block_pool.nim
@@ -213,7 +213,7 @@ when const_preset == "minimal": # Too much stack space used on mainnet
             BeaconBlockBody(
               attestations: makeFullAttestations(
                 pool.headState.data.data, pool.head.blck.root,
-                pool.headState.data.data.slot, cache, {skipValidation})))
+                pool.headState.data.data.slot, cache, {})))
         let added = pool.add(hash_tree_root(blck.message), blck)
         pool.updateHead(added)
 
diff --git a/tests/test_interop.nim b/tests/test_interop.nim
index 3050375cf..ca6596561 100644
--- a/tests/test_interop.nim
+++ b/tests/test_interop.nim
@@ -150,7 +150,7 @@ suite "Interop":
 
     var
       initialState = initialize_beacon_state_from_eth1(
-        eth1BlockHash, 1570500000, deposits, {skipValidation})
+        eth1BlockHash, 1570500000, deposits, {skipMerkleValidation})
 
     # https://github.com/ethereum/eth2.0-pm/tree/6e41fcf383ebeb5125938850d8e9b4e9888389b4/interop/mocked_start#create-genesis-state
     initialState.genesis_time = 1570500000
diff --git a/tests/test_state_transition.nim b/tests/test_state_transition.nim
index 566ac7e06..5f3cc7bc4 100644
--- a/tests/test_state_transition.nim
+++ b/tests/test_state_transition.nim
@@ -22,7 +22,7 @@ suite "Block processing" & preset():
     # TODO bls verification is a bit of a bottleneck here
     genesisState = initialize_beacon_state_from_eth1(
       Eth2Digest(), 0,
-      makeInitialDeposits(), {skipValidation})
+      makeInitialDeposits(), {skipMerkleValidation})
     genesisBlock = get_initial_beacon_block(genesisState)
     genesisRoot = hash_tree_root(genesisBlock.message)
 
diff --git a/tests/testutil.nim b/tests/testutil.nim
index 0ee30dc35..1c3eb89d2 100644
--- a/tests/testutil.nim
+++ b/tests/testutil.nim
@@ -80,7 +80,8 @@ proc makeTestDB*(validators: int): BeaconChainDB =
   let
     genState = initialize_beacon_state_from_eth1(
       Eth2Digest(), 0,
-      makeInitialDeposits(validators, flags = {skipValidation}), {skipValidation})
+      makeInitialDeposits(validators, flags = {skipValidation}),
+        {skipValidation, skipMerkleValidation})
     genBlock = get_initial_beacon_block(genState)
   makeTestDB(genState, genBlock)
 

From ba9c90c0a1f5f37a2f5e4c218cb18963a3eaba3a Mon Sep 17 00:00:00 2001
From: Dustin Brody <tersec@users.noreply.github.com>
Date: Tue, 4 Feb 2020 19:34:33 +0100
Subject: [PATCH 03/31] switch attestations, proposer slashings, and slots
 sanity tests to automatically iterate across all available test vectors
 rather than hard-coding them

---
 .../test_fixture_operations_attestations.nim  | 32 +++----------------
 ..._fixture_operations_proposer_slashings.nim | 18 +++--------
 tests/official/test_fixture_sanity_slots.nim  | 27 +++++-----------
 3 files changed, 17 insertions(+), 60 deletions(-)

diff --git a/tests/official/test_fixture_operations_attestations.nim b/tests/official/test_fixture_operations_attestations.nim
index 8a9d6ff6e..e01e38e9f 100644
--- a/tests/official/test_fixture_operations_attestations.nim
+++ b/tests/official/test_fixture_operations_attestations.nim
@@ -20,13 +20,13 @@ import
 
 const OperationsAttestationsDir = SszTestsDir/const_preset/"phase0"/"operations"/"attestation"/"pyspec_tests"
 
-template runTest(testName: string, identifier: untyped) =
+proc runTest(identifier: string) =
   # We wrap the tests in a proc to avoid running out of globals
   # in the future: Nim supports up to 3500 globals
   # but unittest with the macro/templates put everything as globals
   # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
 
-  const testDir = OperationsAttestationsDir / astToStr(identifier)
+  let testDir = OperationsAttestationsDir / identifier
 
   proc `testImpl _ operations_attestations _ identifier`() =
 
@@ -39,7 +39,7 @@ template runTest(testName: string, identifier: untyped) =
     else:
       prefix = "[Invalid] "
 
-    timedTest prefix & testName & " (" & astToStr(identifier) & ")":
+    timedTest prefix & identifier:
       var stateRef, postRef: ref BeaconState
       var attestationRef: ref Attestation
       new attestationRef
@@ -66,27 +66,5 @@ template runTest(testName: string, identifier: untyped) =
   `testImpl _ operations_attestations _ identifier`()
 
 suite "Official - Operations - Attestations " & preset():
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/minimal/phase0/operations/attestation/pyspec_tests
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/mainnet/phase0/operations/attestation/pyspec_tests
-  runTest("after_epoch_slots", after_epoch_slots)
-  runTest("bad source root", bad_source_root)
-  runTest("before inclusion delay", before_inclusion_delay)
-  runTest("empty aggregation bits", empty_aggregation_bits)
-  runTest("future target epoch", future_target_epoch)
-  runTest("invalid attestation signature", invalid_attestation_signature)
-  runTest("invalid current source root", invalid_current_source_root)
-  runTest("invalid index", invalid_index)
-  runTest("mismatched target and slot", mismatched_target_and_slot)
-  runTest("new source epoch", new_source_epoch)
-  runTest("old source epoch", old_source_epoch)
-  runTest("old target epoch", old_target_epoch)
-  runTest("source root is target root", source_root_is_target_root)
-  runTest("success", success)
-  runTest("success multi-proposer index interations",
-    success_multi_proposer_index_iterations)
-  runTest("success previous epoch", success_previous_epoch)
-  runTest("too few aggregation bits", too_few_aggregation_bits)
-  runTest("too many aggregation bits", too_many_aggregation_bits)
-  runTest("wrong index for committee signature",
-    wrong_index_for_committee_signature)
-  runTest("wrong index for slot", wrong_index_for_slot)
+  for kind, path in walkDir(OperationsAttestationsDir, true):
+    runTest(path)
diff --git a/tests/official/test_fixture_operations_proposer_slashings.nim b/tests/official/test_fixture_operations_proposer_slashings.nim
index 50375bc1f..716919069 100644
--- a/tests/official/test_fixture_operations_proposer_slashings.nim
+++ b/tests/official/test_fixture_operations_proposer_slashings.nim
@@ -20,13 +20,13 @@ import
 
 const OpProposerSlashingDir = SszTestsDir/const_preset/"phase0"/"operations"/"proposer_slashing"/"pyspec_tests"
 
-template runTest(identifier: untyped) =
+proc runTest(identifier: string) =
   # We wrap the tests in a proc to avoid running out of globals
   # in the future: Nim supports up to 3500 globals
   # but unittest with the macro/templates put everything as globals
   # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
 
-  const testDir = OpProposerSlashingDir / astToStr(identifier)
+  let testDir = OpProposerSlashingDir / identifier
 
   proc `testImpl_proposer_slashing _ identifier`() =
 
@@ -66,15 +66,5 @@ template runTest(identifier: untyped) =
   `testImpl_proposer_slashing _ identifier`()
 
 suite "Official - Operations - Proposer slashing " & preset():
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/minimal/phase0/operations/proposer_slashing/pyspec_tests
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/mainnet/phase0/operations/proposer_slashing/pyspec_tests
-  runTest(epochs_are_different)
-  runTest(headers_are_same)
-  runTest(invalid_proposer_index)
-  runTest(invalid_sig_1)
-  runTest(invalid_sig_1_and_2)
-  runTest(invalid_sig_2)
-  runTest(proposer_is_not_activated)
-  runTest(proposer_is_slashed)
-  runTest(proposer_is_withdrawn)
-  runTest(success)
+  for kind, path in walkDir(OpProposerSlashingDir, true):
+    runTest(path)
diff --git a/tests/official/test_fixture_sanity_slots.nim b/tests/official/test_fixture_sanity_slots.nim
index a20479750..a0ef59e98 100644
--- a/tests/official/test_fixture_sanity_slots.nim
+++ b/tests/official/test_fixture_sanity_slots.nim
@@ -9,7 +9,7 @@
 
 import
   # Standard library
-  os, unittest,
+  os, strutils, unittest,
   # Beacon chain internals
   ../../beacon_chain/spec/datatypes,
   ../../beacon_chain/state_transition,
@@ -20,16 +20,17 @@ import
 
 const SanitySlotsDir = SszTestsDir/const_preset/"phase0"/"sanity"/"slots"/"pyspec_tests"
 
-template runTest(testName: string, identifier: untyped, num_slots: uint64): untyped =
+proc runTest(identifier: string) =
   # We wrap the tests in a proc to avoid running out of globals
   # in the future: Nim supports up to 3500 globals
   # but unittest with the macro/templates put everything as globals
   # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
-
-  const testDir = SanitySlotsDir / astToStr(identifier)
+  let
+    testDir = SanitySlotsDir / identifier
+    num_slots = readLines(testDir / "slots.yaml", 2)[0].parseInt.uint64
 
   proc `testImpl _ slots _ identifier`() =
-    timedTest "Slots - " & testName & " (" & astToStr(identifier) & ")":
+    timedTest "Slots - " & identifier:
       var stateRef, postRef: ref BeaconState
       new stateRef
       new postRef
@@ -43,18 +44,6 @@ template runTest(testName: string, identifier: untyped, num_slots: uint64): unty
 
   `testImpl _ slots _ identifier`()
 
-# 1 slot
-# ---------------------------------------------------------------
-
 suite "Official - Sanity - Slots " & preset():
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/minimal/phase0/sanity/slots/pyspec_tests
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/mainnet/phase0/sanity/slots/pyspec_tests
-  runTest("Advance 1 slot", slots_1, 1)
-  runTest("Advance 2 slots", slots_2, 2)
-  runTest("Advance an empty epoch", empty_epoch, SLOTS_PER_EPOCH)
-
-  const DoubleEpoch = SLOTS_PER_EPOCH.uint64*2 # workaround undeclared identifier "double_empty_epoch"
-  runTest("Advance 2 empty epochs", double_empty_epoch, DoubleEpoch)
-
-  # This starts in the middle of an epoch
-  runTest("Advance over an epoch boundary", over_epoch_boundary, SLOTS_PER_EPOCH)
+  for kind, path in walkDir(SanitySlotsDir, true):
+    runTest(path)

From b39f36b49bef25d07e98f0912db1beca85743fdb Mon Sep 17 00:00:00 2001
From: Dmitriy Ryajov <dryajov@gmail.com>
Date: Wed, 5 Feb 2020 17:19:52 +0100
Subject: [PATCH 04/31] use floodsub instead of gossipsub

---
 beacon_chain/eth2_network.nim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/beacon_chain/eth2_network.nim b/beacon_chain/eth2_network.nim
index 4c3a93046..0824e1404 100644
--- a/beacon_chain/eth2_network.nim
+++ b/beacon_chain/eth2_network.nim
@@ -155,7 +155,7 @@ when networkBackend in [libp2p, libp2pDaemon]:
       # that are different from the host address (this is relevant when we
       # are running behind a NAT).
       var switch = newStandardSwitch(some keys.seckey, hostAddress,
-                                     triggerSelf = true, gossip = true)
+                                     triggerSelf = true, gossip = false)
       result = Eth2Node.init(conf, switch, keys.seckey)
       for enr in bootstrapEnrs:
         result.addKnownPeer(enr)

From 4d487cea1fd210bf7d57156628609aec7fd309ca Mon Sep 17 00:00:00 2001
From: Dustin Brody <tersec@users.noreply.github.com>
Date: Wed, 5 Feb 2020 18:20:05 +0100
Subject: [PATCH 05/31] build with -march=native by default

---
 nim.cfg | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/nim.cfg b/nim.cfg
index e2a06a516..156531911 100644
--- a/nim.cfg
+++ b/nim.cfg
@@ -21,6 +21,13 @@
   -d:"chronicles_colors=off"
 @end
 
+# This helps especially for 32-bit x86, which sans SSE2 and newer instructions
+# requires quite roundabout code generation for cryptography, and other 64-bit
+# and larger arithmetic use cases, along with register starvation issues. When
+# engineering a more portable binary release, this should be tweaked but still
+# use at least -msse2 or -msse3.
+--passC:"-march=native"
+
 --threads:on
 --opt:speed
 --excessiveStackTrace:on

From fb9c4fabf49ec7dd56034a89d847a31d9fa61fce Mon Sep 17 00:00:00 2001
From: Jacek Sieka <jacek@status.im>
Date: Wed, 5 Feb 2020 12:41:46 +0100
Subject: [PATCH 06/31] fix state rewind

* rewind fast path comparison was not taking skipped slots into account
properly
* less messy blockref creation
---
 beacon_chain/block_pool.nim | 25 +++++++++----------
 tests/test_block_pool.nim   | 49 +++++++++++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+), 13 deletions(-)

diff --git a/beacon_chain/block_pool.nim b/beacon_chain/block_pool.nim
index 49137d7eb..f4c47987e 100644
--- a/beacon_chain/block_pool.nim
+++ b/beacon_chain/block_pool.nim
@@ -249,9 +249,10 @@ proc init*(T: type BlockPool, db: BeaconChainDB): BlockPool =
   res
 
 proc addResolvedBlock(
-    pool: var BlockPool, state: var StateData, blockRoot: Eth2Digest,
+    pool: var BlockPool, state: BeaconState, blockRoot: Eth2Digest,
     signedBlock: SignedBeaconBlock, parent: BlockRef): BlockRef =
   logScope: pcs = "block_resolution"
+  doAssert state.slot == signedBlock.message.slot, "state must match block"
 
   let blockRef = BlockRef.init(blockRoot, signedBlock.message)
   link(parent, blockRef)
@@ -262,17 +263,10 @@ proc addResolvedBlock(
   # Resolved blocks should be stored in database
   pool.db.putBlock(blockRoot, signedBlock)
 
-  # TODO this is a bit ugly - we update state.data outside of this function then
-  #      set the rest here - need a blockRef to update it. Clean this up -
-  #      hopefully it won't be necessary by the time hash caching and the rest
-  #      is done..
-  doAssert state.data.data.slot == blockRef.slot
-  state.blck = blockRef
-
   # This block *might* have caused a justification - make sure we stow away
   # that information:
   let justifiedSlot =
-    state.data.data.current_justified_checkpoint.epoch.compute_start_slot_at_epoch()
+    state.current_justified_checkpoint.epoch.compute_start_slot_at_epoch()
 
   var foundHead: Option[Head]
   for head in pool.heads.mitems():
@@ -389,9 +383,12 @@ proc add*(
 
       return
 
-    # Careful, pool.tmpState is now partially inconsistent and will be updated
-    # inside addResolvedBlock
-    return pool.addResolvedBlock(pool.tmpState, blockRoot, signedBlock, parent)
+    # Careful, tmpState.data has been updated but not blck - we need to create
+    # the BlockRef first!
+    pool.tmpState.blck = pool.addResolvedBlock(
+      pool.tmpState.data.data, blockRoot, signedBlock, parent)
+
+    return pool.tmpState.blck
 
   # TODO already checked hash though? main reason to keep this is because
   # the pending pool calls this function back later in a loop, so as long
@@ -603,7 +600,7 @@ proc rewindState(pool: BlockPool, state: var StateData, bs: BlockSlot):
   var ancestors = @[pool.get(bs.blck)]
   # Common case: the last block applied is the parent of the block to apply:
   if not bs.blck.parent.isNil and state.blck.root == bs.blck.parent.root and
-      state.data.data.slot < bs.slot:
+      state.data.data.slot < bs.blck.slot:
     return ancestors
 
   # It appears that the parent root of the proposed new block is different from
@@ -693,6 +690,7 @@ proc updateStateData*(pool: BlockPool, state: var StateData, bs: BlockSlot) =
     if state.data.data.slot != bs.slot:
       # Might be that we're moving to the same block but later slot
       process_slots(state.data, bs.slot)
+      # TODO we will not save if multiple slots are skipped here
       pool.maybePutState(state.data, bs.blck)
 
     return # State already at the right spot
@@ -715,6 +713,7 @@ proc updateStateData*(pool: BlockPool, state: var StateData, bs: BlockSlot) =
     doAssert ok, "Blocks in database should never fail to apply.."
 
   # TODO check if this triggers rest of state transition, or should
+  # TODO we will not save if multiple slots are skipped here
   process_slots(state.data, bs.slot)
   pool.maybePutState(state.data, bs.blck)
 
diff --git a/tests/test_block_pool.nim b/tests/test_block_pool.nim
index 4578e1f9b..c28de7280 100644
--- a/tests/test_block_pool.nim
+++ b/tests/test_block_pool.nim
@@ -187,6 +187,55 @@ when const_preset == "minimal": # Too much stack space used on mainnet
         pool.head.blck == b1Add
         pool.headState.data.data.slot == b1Add.slot
 
+    timedTest "updateStateData sanity" & preset():
+      let
+        b1Add = pool.add(b1Root, b1)
+        b2Add = pool.add(b2Root, b2)
+        bs1 = BlockSlot(blck: b1Add, slot: b1.message.slot)
+        bs1_3 = b1Add.atSlot(3.Slot)
+        bs2 = BlockSlot(blck: b2Add, slot: b2.message.slot)
+        bs2_3 = b2Add.atSlot(3.Slot)
+
+      var tmpState = pool.headState
+
+      # move to specific block
+      pool.updateStateData(tmpState, bs1)
+
+      check:
+        tmpState.blck == b1Add
+        tmpState.data.data.slot == bs1.slot
+
+      # Skip slots
+      pool.updateStateData(tmpState, bs1_3) # skip slots
+
+      check:
+        tmpState.blck == b1Add
+        tmpState.data.data.slot == bs1_3.slot
+
+      # Move back slots, but not blocks
+      pool.updateStateData(tmpState, bs1_3.parent())
+      check:
+        tmpState.blck == b1Add
+        tmpState.data.data.slot == bs1_3.parent().slot
+
+      # Move to different block and slot
+      pool.updateStateData(tmpState, bs2_3)
+      check:
+        tmpState.blck == b2Add
+        tmpState.data.data.slot == bs2_3.slot
+
+      # Move back slot and block
+      pool.updateStateData(tmpState, bs1)
+      check:
+        tmpState.blck == b1Add
+        tmpState.data.data.slot == bs1.slot
+
+      # Move back to genesis
+      pool.updateStateData(tmpState, bs1.parent())
+      check:
+        tmpState.blck == b1Add.parent
+        tmpState.data.data.slot == bs1.parent.slot
+
   suite "BlockPool finalization tests" & preset():
     setup:
       var

From 7efd113a757a654af9298c5f7f71d4031f0504b0 Mon Sep 17 00:00:00 2001
From: Jacek Sieka <jacek@status.im>
Date: Wed, 5 Feb 2020 13:04:22 +0100
Subject: [PATCH 07/31] store empty slot states also

---
 beacon_chain/block_pool.nim       | 23 +++++++++++++----------
 beacon_chain/state_transition.nim |  1 +
 2 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/beacon_chain/block_pool.nim b/beacon_chain/block_pool.nim
index f4c47987e..ea8a8adaa 100644
--- a/beacon_chain/block_pool.nim
+++ b/beacon_chain/block_pool.nim
@@ -438,7 +438,6 @@ proc add*(
     missing = pool.missing.len,
     cat = "filtering"
 
-
 func getRef*(pool: BlockPool, root: Eth2Digest): BlockRef =
   ## Retrieve a resolved block reference, if available
   pool.blocks.getOrDefault(root, nil)
@@ -562,12 +561,19 @@ func checkMissing*(pool: var BlockPool): seq[FetchRecord] =
     if v.tries.popcount() == 1:
       result.add(FetchRecord(root: k, historySlots: v.slots))
 
+proc skipAndUpdateState(
+    state: var HashedBeaconState, slot: Slot,
+    afterUpdate: proc (state: HashedBeaconState)) =
+  while state.data.slot < slot:
+    # Process slots one at a time in case afterUpdate needs to see empty states
+    process_slots(state, state.data.slot + 1)
+    afterUpdate(state)
+
 proc skipAndUpdateState(
     state: var HashedBeaconState, blck: BeaconBlock, flags: UpdateFlags,
     afterUpdate: proc (state: HashedBeaconState)): bool =
 
-  process_slots(state, blck.slot - 1)
-  afterUpdate(state)
+  skipAndUpdateState(state, blck.slot - 1, afterUpdate)
 
   let ok  = state_transition(state, blck, flags)
 
@@ -689,9 +695,8 @@ proc updateStateData*(pool: BlockPool, state: var StateData, bs: BlockSlot) =
   if state.blck.root == bs.blck.root and state.data.data.slot <= bs.slot:
     if state.data.data.slot != bs.slot:
       # Might be that we're moving to the same block but later slot
-      process_slots(state.data, bs.slot)
-      # TODO we will not save if multiple slots are skipped here
-      pool.maybePutState(state.data, bs.blck)
+      skipAndUpdateState(state.data, bs.slot) do(state: HashedBeaconState):
+        pool.maybePutState(state, bs.blck)
 
     return # State already at the right spot
 
@@ -712,10 +717,8 @@ proc updateStateData*(pool: BlockPool, state: var StateData, bs: BlockSlot) =
       pool.maybePutState(state, ancestors[i].refs)
     doAssert ok, "Blocks in database should never fail to apply.."
 
-  # TODO check if this triggers rest of state transition, or should
-  # TODO we will not save if multiple slots are skipped here
-  process_slots(state.data, bs.slot)
-  pool.maybePutState(state.data, bs.blck)
+  skipAndUpdateState(state.data, bs.slot) do(state: HashedBeaconState):
+    pool.maybePutState(state, bs.blck)
 
   state.blck = bs.blck
 
diff --git a/beacon_chain/state_transition.nim b/beacon_chain/state_transition.nim
index 25f1b064e..e16a30daa 100644
--- a/beacon_chain/state_transition.nim
+++ b/beacon_chain/state_transition.nim
@@ -197,6 +197,7 @@ proc process_slots*(state: var HashedBeaconState, slot: Slot) =
   if state.data.slot > slot:
     notice(
       "Unusual request for a slot in the past",
+      state_root = shortLog(state.root),
       current_slot = state.data.slot,
       target_slot = slot
     )

From 0bc16518edfe3daed8379d63f424c1d958f32343 Mon Sep 17 00:00:00 2001
From: Dmitriy Ryajov <dryajov@gmail.com>
Date: Wed, 5 Feb 2020 20:59:32 +0100
Subject: [PATCH 08/31] bumping libp2p to latest

---
 vendor/nim-libp2p | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p
index 7bd305471..88a030d8f 160000
--- a/vendor/nim-libp2p
+++ b/vendor/nim-libp2p
@@ -1 +1 @@
-Subproject commit 7bd305471c0ab1a0306e6734915952f236cc88eb
+Subproject commit 88a030d8fbd76023354f14ff10ba740786eb46a4

From cd1c8155bf0df0e67b7586d108dc71720eb00f7d Mon Sep 17 00:00:00 2001
From: Jacek Sieka <jacek@status.im>
Date: Thu, 6 Feb 2020 12:11:51 +0100
Subject: [PATCH 09/31] shorter epochs in sim

---
 tests/simulation/start.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/simulation/start.sh b/tests/simulation/start.sh
index 7a63946ca..caf46d0ad 100755
--- a/tests/simulation/start.sh
+++ b/tests/simulation/start.sh
@@ -22,7 +22,7 @@ NIMFLAGS="-d:chronicles_log_level=TRACE -d:chronicles_sinks:textlines,json[file]
 DEFS=""
 
 DEFS+="-d:MAX_COMMITTEES_PER_SLOT=${MAX_COMMITTEES_PER_SLOT:-1} "      # Spec default: 64
-DEFS+="-d:SLOTS_PER_EPOCH=${SLOTS_PER_EPOCH:-16} "   # Spec default: 32
+DEFS+="-d:SLOTS_PER_EPOCH=${SLOTS_PER_EPOCH:-6} "   # Spec default: 32
 DEFS+="-d:SECONDS_PER_SLOT=${SECONDS_PER_SLOT:-6} "  # Spec default: 12
 
 LAST_VALIDATOR_NUM=$(( NUM_VALIDATORS - 1 ))

From 989559cd2dd6e7eae18620956b168bff349fbfea Mon Sep 17 00:00:00 2001
From: tersec <tersec@users.noreply.github.com>
Date: Thu, 6 Feb 2020 11:41:06 +0000
Subject: [PATCH 10/31] set block header tests to automatically enumerate all
 relevant tests (#717)

* set block header tests to automatically enumerate all relevant tests

* add several attester slashing tests and run all tests by default, rather than only explicitly listed test vectors

* print something relating to skipped tests

* set voluntary exits to automatically enumerate all tests
---
 ..._fixture_operations_attester_slashings.nim | 44 +++++++------------
 .../test_fixture_operations_block_header.nim  | 20 +++------
 ...test_fixture_operations_voluntary_exit.nim | 23 +++-------
 3 files changed, 29 insertions(+), 58 deletions(-)

diff --git a/tests/official/test_fixture_operations_attester_slashings.nim b/tests/official/test_fixture_operations_attester_slashings.nim
index f0b874d10..46fb82c8c 100644
--- a/tests/official/test_fixture_operations_attester_slashings.nim
+++ b/tests/official/test_fixture_operations_attester_slashings.nim
@@ -20,13 +20,13 @@ import
 
 const OpAttSlashingDir = SszTestsDir/const_preset/"phase0"/"operations"/"attester_slashing"/"pyspec_tests"
 
-template runTest(identifier: untyped) =
+proc runTest(identifier: string) =
   # We wrap the tests in a proc to avoid running out of globals
   # in the future: Nim supports up to 3500 globals
   # but unittest with the macro/templates put everything as globals
   # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
 
-  const testDir = OpAttSlashingDir / astToStr(identifier)
+  let testDir = OpAttSlashingDir / identifier
 
   proc `testImpl _ operations_attester_slashing _ identifier`() =
 
@@ -39,7 +39,7 @@ template runTest(identifier: untyped) =
     else:
       prefix = "[Invalid] "
 
-    timedTest prefix & astToStr(identifier):
+    timedTest prefix & identifier:
       var stateRef, postRef: ref BeaconState
       var attesterSlashingRef: ref AttesterSlashing
       new attesterSlashingRef
@@ -68,27 +68,17 @@ template runTest(identifier: untyped) =
   `testImpl _ operations_attester_slashing _ identifier`()
 
 suite "Official - Operations - Attester slashing " & preset():
-  runTest(success_double)
-  runTest(success_surround)
-  when false:
-    # TODO these are both valid and check BLS signatures, which isn't working
-    # since 0.10.x introduces new BLS signing/verifying interface with domain
-    # in particular handled differently through compute_signing_root() rather
-    # than through the bls_verify(...) call directly. This did not become the
-    # visible issue it now is because another bug had been masking it wherein
-    # crypto.nim's bls_verify(...) call had been creating false positives, in
-    # which cases signature checks had been incorrectly passing.
-    runTest(success_already_exited_recent)
-    runTest(success_already_exited_long_ago)
-  runTest(invalid_sig_1)
-  when false: # TODO - https://github.com/status-im/nim-beacon-chain/issues/429
-    runTest(invalid_sig_2)
-  runTest(invalid_sig_1_and_2)
-  runTest(same_data)
-  runTest(no_double_or_surround)
-  runTest(participants_already_slashed)
-  when false: # TODO - https://github.com/status-im/nim-beacon-chain/issues/429
-    runTest(att1_bad_extra_index)
-    runTest(att1_bad_replaced_index)
-    runTest(att2_bad_extra_index)
-    runTest(att2_bad_replaced_index)
+  # TODO these are both valid and check BLS signatures, which isn't working
+  # since 0.10.x introduces new BLS signing/verifying interface with domain
+  # in particular handled differently through compute_signing_root() rather
+  # than through the bls_verify(...) call directly. This did not become the
+  # visible issue it now is because another bug had been masking it wherein
+  # crypto.nim's bls_verify(...) call had been creating false positives, in
+  # which cases signature checks had been incorrectly passing.
+  const expected_failures =
+    ["success_already_exited_recent", "success_already_exited_long_ago"]
+  for kind, path in walkDir(OpAttSlashingDir, true):
+    if path in expected_failures:
+      echo "Skipping test: ", path
+      continue
+    runTest(path)
diff --git a/tests/official/test_fixture_operations_block_header.nim b/tests/official/test_fixture_operations_block_header.nim
index 92c86dfe7..3cc68467d 100644
--- a/tests/official/test_fixture_operations_block_header.nim
+++ b/tests/official/test_fixture_operations_block_header.nim
@@ -20,26 +20,23 @@ import
 
 const OpBlockHeaderDir = SszTestsDir/const_preset/"phase0"/"operations"/"block_header"/"pyspec_tests"
 
-template runTest(identifier: untyped) =
+proc runTest(identifier: string) =
   # We wrap the tests in a proc to avoid running out of globals
   # in the future: Nim supports up to 3500 globals
   # but unittest with the macro/templates put everything as globals
   # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
 
-  const testDir = OpBlockHeaderDir / astToStr(identifier)
+  let testDir = OpBlockHeaderDir / identifier
 
   proc `testImpl _ blockheader _ identifier`() =
 
-    var flags: UpdateFlags
     var prefix: string
-    if not existsFile(testDir/"meta.yaml"):
-      flags.incl skipValidation
     if existsFile(testDir/"post.ssz"):
       prefix = "[Valid]   "
     else:
       prefix = "[Invalid] "
 
-    timedTest prefix & astToStr(identifier):
+    timedTest prefix & identifier:
       var stateRef, postRef: ref BeaconState
       var blck: ref BeaconBlock
       new blck
@@ -55,10 +52,10 @@ template runTest(identifier: untyped) =
         postRef[] = parseTest(testDir/"post.ssz", SSZ, BeaconState)
 
       if postRef.isNil:
-        let done = process_block_header(stateRef[], blck[], flags, cache)
+        let done = process_block_header(stateRef[], blck[], {}, cache)
         doAssert done == false, "We didn't expect this invalid block header to be processed."
       else:
-        let done = process_block_header(stateRef[], blck[], flags, cache)
+        let done = process_block_header(stateRef[], blck[], {}, cache)
         doAssert done, "Valid block header not processed"
         check: stateRef.hash_tree_root() == postRef.hash_tree_root()
         reportDiff(stateRef, postRef)
@@ -66,8 +63,5 @@ template runTest(identifier: untyped) =
   `testImpl _ blockheader _ identifier`()
 
 suite "Official - Operations - Block header " & preset():
-  runTest(success_block_header)
-  runTest(invalid_slot_block_header)
-  when false: # skipValidation needs to be split https://github.com/status-im/nim-beacon-chain/issues/407
-    runTest(invalid_parent_root)
-  runTest(proposer_slashed)
+  for kind, path in walkDir(OpBlockHeaderDir, true):
+    runTest(path)
diff --git a/tests/official/test_fixture_operations_voluntary_exit.nim b/tests/official/test_fixture_operations_voluntary_exit.nim
index 943921e6d..f93fe27e0 100644
--- a/tests/official/test_fixture_operations_voluntary_exit.nim
+++ b/tests/official/test_fixture_operations_voluntary_exit.nim
@@ -20,13 +20,13 @@ import
 
 const OpVoluntaryExitDir = SszTestsDir/const_preset/"phase0"/"operations"/"voluntary_exit"/"pyspec_tests"
 
-template runTest(identifier: untyped) =
+proc runTest(identifier: string) =
   # We wrap the tests in a proc to avoid running out of globals
   # in the future: Nim supports up to 3500 globals
   # but unittest with the macro/templates put everything as globals
   # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
 
-  const testDir = OpVoluntaryExitDir / astToStr(identifier)
+  let testDir = OpVoluntaryExitDir / identifier
 
   proc `testImpl _ voluntary_exit _ identifier`() =
 
@@ -39,7 +39,7 @@ template runTest(identifier: untyped) =
     else:
       prefix = "[Invalid] "
 
-    timedTest prefix & astToStr(identifier):
+    timedTest prefix & identifier:
       var stateRef, postRef: ref BeaconState
       var voluntaryExit: ref SignedVoluntaryExit
       new voluntaryExit
@@ -64,18 +64,5 @@ template runTest(identifier: untyped) =
   `testImpl _ voluntary_exit _ identifier`()
 
 suite "Official - Operations - Voluntary exit " & preset():
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/minimal/phase0/operations/voluntary_exit/pyspec_tests
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/mainnet/phase0/operations/voluntary_exit/pyspec_tests
-  runTest(success)
-
-  when false:
-    # TODO not sure how this particularly could falsely succeed
-    runTest(invalid_signature)
-
-  runTest(validator_invalid_validator_index)
-  runTest(validator_already_exited)
-  runTest(success_exit_queue)
-  runTest(validator_exit_in_future)
-  runTest(default_exit_epoch_subsequent_exit)
-  runTest(validator_not_active_long_enough)
-  runTest(validator_not_active)
+  for kind, path in walkDir(OpVoluntaryExitDir, true):
+    runTest(path)

From a37aa3b86d68f23d0aadb4dc3e72cd25f4d112f5 Mon Sep 17 00:00:00 2001
From: Stefan Talpalaru <stefantalpalaru@yahoo.com>
Date: Thu, 6 Feb 2020 12:44:11 +0100
Subject: [PATCH 11/31] `make testnet1`: switch the local node to the Nim
 libp2p

so we can debug it without having to also consider Go-Nim libp2p
interoperability issues
---
 scripts/connect_to_testnet.nims | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/connect_to_testnet.nims b/scripts/connect_to_testnet.nims
index c90aeb975..ae167f4dc 100644
--- a/scripts/connect_to_testnet.nims
+++ b/scripts/connect_to_testnet.nims
@@ -75,6 +75,7 @@ cli do (testnetName {.argument.}: string):
     validatorsDir = dataDir / "validators"
     dumpDir = dataDir / "dump"
     beaconNodeBinary = buildDir / "beacon_node_" & dataDirName
+  var
     nimFlags = "-d:chronicles_log_level=TRACE " & getEnv("NIM_PARAMS")
 
   let depositContractFile = testnetDir / depositContractFileName
@@ -95,6 +96,8 @@ cli do (testnetName {.argument.}: string):
       rmDir dataDir
 
   cd rootDir
+  if testnet == "testnet1":
+    nimFlags &= " -d:NETWORK_TYPE=libp2p"
   exec &"""nim c {nimFlags} -d:"const_preset={preset}" -o:"{beaconNodeBinary}" beacon_chain/beacon_node.nim"""
 
   mkDir dumpDir

From 662debf008dadaff66057f581e151a40b325b451 Mon Sep 17 00:00:00 2001
From: Stefan Talpalaru <stefantalpalaru@yahoo.com>
Date: Thu, 6 Feb 2020 17:51:45 +0100
Subject: [PATCH 12/31] testnet: change remote log level to TRACE

---
 docker/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index aef697883..b0b51d68b 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -26,7 +26,7 @@ RUN cd /root/nim-beacon-chain \
  && git fetch \
  && git reset --hard ${GIT_REVISION} \
  && make -j$(nproc) update \
- && make LOG_LEVEL=DEBUG NIMFLAGS="-d:debug -d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS}" beacon_node
+ && make LOG_LEVEL=TRACE NIMFLAGS="-d:debug -d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS}" beacon_node
 
 # --------------------------------- #
 # Starting new image to reduce size #

From 09d735212ddb1f176e28b23949d2bcc54586e003 Mon Sep 17 00:00:00 2001
From: tersec <tersec@users.noreply.github.com>
Date: Fri, 7 Feb 2020 07:11:26 +0000
Subject: [PATCH 13/31] initial refactoring of block sanity test runner, with
 several new tests (#736)

* initial refactoring of block sanity test runner, with several new tests enabled

* remove trailing whitespace
---
 beacon_chain/state_transition.nim             |   6 +-
 tests/official/test_fixture_sanity_blocks.nim | 111 ++++++++----------
 2 files changed, 54 insertions(+), 63 deletions(-)

diff --git a/beacon_chain/state_transition.nim b/beacon_chain/state_transition.nim
index e16a30daa..fa1f1698d 100644
--- a/beacon_chain/state_transition.nim
+++ b/beacon_chain/state_transition.nim
@@ -83,7 +83,11 @@ func get_epoch_validator_count(state: BeaconState): int64 =
 
 # https://github.com/ethereum/eth2.0-specs/blob/v0.10.1/specs/phase0/beacon-chain.md#beacon-chain-state-transition-function
 proc process_slots*(state: var BeaconState, slot: Slot) {.nbench.}=
-  doAssert state.slot <= slot
+  if not (state.slot <= slot):
+    warn("Trying to apply old block",
+      state_slot = state.slot,
+      slot = slot)
+    return
 
   # Catch up to the target slot
   while state.slot < slot:
diff --git a/tests/official/test_fixture_sanity_blocks.nim b/tests/official/test_fixture_sanity_blocks.nim
index 49af5461c..3746a9a19 100644
--- a/tests/official/test_fixture_sanity_blocks.nim
+++ b/tests/official/test_fixture_sanity_blocks.nim
@@ -20,90 +20,77 @@ import
 
 const SanityBlocksDir = SszTestsDir/const_preset/"phase0"/"sanity"/"blocks"/"pyspec_tests"
 
-template runValidTest(testName: string, identifier: untyped, num_blocks: int): untyped =
+template runTest(identifier: string, num_blocks: int): untyped =
   # We wrap the tests in a proc to avoid running out of globals
   # in the future: Nim supports up to 3500 globals
   # but unittest with the macro/templates put everything as globals
   # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
 
-  const testDir = SanityBlocksDir / astToStr(identifier)
+  const testDir = SanityBlocksDir / identifier
 
   proc `testImpl _ blck _ identifier`() =
-    timedTest "[Valid]   " & testName & " (" & astToStr(identifier) & ")":
+    let prefix = if existsFile(testDir/"post.ssz"):
+      "[Valid]   "
+    else:
+      "[Invalid] "
+
+    timedTest prefix & identifier:
       var stateRef, postRef: ref BeaconState
       new stateRef
-      new postRef
       stateRef[] = parseTest(testDir/"pre.ssz", SSZ, BeaconState)
-      postRef[] = parseTest(testDir/"post.ssz", SSZ, BeaconState)
+
+      if existsFile(testDir/"post.ssz"):
+        new postRef
+        postRef[] = parseTest(testDir/"post.ssz", SSZ, BeaconState)
 
       for i in 0 ..< num_blocks:
         let blck = parseTest(testDir/"blocks_" & $i & ".ssz", SSZ, SignedBeaconBlock)
 
-        # TODO: The EF is using invalid BLS keys so we can't verify them
-        let success = state_transition(stateRef[], blck.message, flags = {skipValidation})
-        doAssert success, "Failure when applying block " & $i
+        if postRef.isNil:
+          let success = state_transition(stateRef[], blck.message, flags = {})
+          doAssert not success, "We didn't expect this invalid block to be processed"
+        else:
+          # TODO: The EF is using invalid BLS keys so we can't verify them
+          let success = state_transition(stateRef[], blck.message, flags = {skipValidation})
+          doAssert success, "Failure when applying block " & $i
 
-      # Checks:
-      # check: stateRef.hash_tree_root() == postRef.hash_tree_root()
-      reportDiff(stateRef, postRef)
+          # Checks:
+          # check: stateRef.hash_tree_root() == postRef.hash_tree_root()
+          if i == num_blocks - 1: reportDiff(stateRef, postRef)
 
   `testImpl _ blck _ identifier`()
 
 suite "Official - Sanity - Blocks " & preset():
-  timedTest "[Invalid] Previous slot block transition (prev_slot_block_transition)":
-    const testDir = SanityBlocksDir/"prev_slot_block_transition"
-    var stateRef: ref BeaconState
-    new stateRef
-    stateRef[] = parseTest(testDir/"pre.ssz", SSZ, BeaconState)
-
-    let blck = parseTest(testDir/"blocks_0.ssz", SSZ, SignedBeaconBlock)
-
-    # Check that a block build for an old slot cannot be used for state transition
-    expect(AssertionError):
-      # assert in process_slots. This should not be triggered
-      #                          for blocks from block_pool/network
-      discard state_transition(stateRef[], blck.message, flags = {skipValidation})
-
-  runValidTest("Same slot block transition", same_slot_block_transition, 1)
-  runValidTest("Empty block transition", empty_block_transition, 1)
-
-  when false: # TODO: we need more granular skipValidation
-    timedTest "[Invalid] Invalid state root":
-      const testDir = SanityBlocksDir/"invalid_state_root"
-      var stateRef: ref BeaconState
-      new stateRef
-      stateRef[] = parseTest(testDir/"pre.ssz", SSZ, BeaconState)
-
-      let blck = parseTest(testDir/"blocks_0.ssz", SSZ, BeaconBlock)
-
-      expect(AssertionError):
-        discard state_transition(stateRef[], blck, flags = {skipValidation})
-
-  runValidTest("Skipped Slots", skipped_slots, 1)
-  runValidTest("Empty epoch transition", empty_epoch_transition, 1)
-  when const_preset=="minimal":
-    runValidTest("Empty epoch transition not finalizing", empty_epoch_transition_not_finalizing, 1)
+  const expected_failures = ["attester_slashing"]
+  runTest("attestation", 2)
 
   when false:
-    # TODO investigate/fix after 0.9.0 transition broke this in mainnet and
-    # in 0.9.1 even minimal broke. For the latter at least, it differs only
-    # in latest_block_header.body_root, which is just a hash_tree_root() of
-    # the one block read by this test case. All balances agree. It's an SSZ
-    # or hashing issue.
-    runValidTest("Attester slashing", attester_slashing, 1)
-  runValidTest("Proposer slashing", proposer_slashing, 1)
+    # Failing due to signature checking in indexed validation checking pending
+    # 0.10 BLS verification API with new domain handling.
+    runTest("attester_slashing", 1)
+  echo "Skipping test: attester_slashing"
 
-  # TODO: Expected deposit in block
-
-  runValidTest("Deposit in block", deposit_in_block, 1)
-  runValidTest("Deposit top up", deposit_top_up, 1)
+  runTest("balance_driven_status_transitions", 1)
+  runTest("deposit_in_block", 1)
+  runTest("deposit_top_up", 1)
+  runTest("empty_block_transition", 1)
+  runTest("empty_epoch_transition", 1)
 
   when const_preset=="minimal":
-    # TODO this doesn't work on mainnet
-    runValidTest("Attestation", attestation, 2)
-  runValidTest("Voluntary exit", voluntary_exit, 2)
-  runValidTest("Balance-driven status transitions", balance_driven_status_transitions, 1)
-  runValidTest("Historical batch", historical_batch, 1)
+    runTest("empty_epoch_transition_not_finalizing", 1)
+    runTest("eth1_data_votes_consensus", 17)
+    runTest("eth1_data_votes_no_consensus", 16)
+
+  runTest("expected_deposit_in_block", 1)
+  runTest("high_proposer_index", 1)
+  runTest("historical_batch", 1)
+  runTest("invalid_block_sig", 1)
+  runTest("invalid_state_root", 1)
+  runTest("prev_slot_block_transition", 1)
+  runTest("proposer_after_inactive_index", 1)
+  runTest("proposer_slashing", 1)
+  runTest("same_slot_block_transition", 1)
+  runTest("skipped_slots", 1)
+  runTest("voluntary_exit", 2)
   when const_preset=="minimal":
-    runValidTest("ETH1 data votes consensus", eth1_data_votes_consensus, 17)
-    runValidTest("ETH1 data votes no consensus", eth1_data_votes_no_consensus, 16)
+    runTest("zero_block_sig", 1)

From 521b0ed6ba5ea8dd1de578bd65da562a8228014d Mon Sep 17 00:00:00 2001
From: Jacek Sieka <jacek@status.im>
Date: Fri, 7 Feb 2020 08:13:38 +0100
Subject: [PATCH 14/31] Clean 20200205 (#729)

* beacon node code cleanup
* rudimentary error checking on mainnet monitor
* start client even when sending deposit
* work around missing block number exception
* connect to testnet with web3 url
* pretty-print digests in json
---
 beacon_chain/beacon_node.nim       | 35 ++++++++----------
 beacon_chain/block_pool.nim        | 19 +++++++++-
 beacon_chain/mainchain_monitor.nim | 58 ++++++++++++++++++++----------
 beacon_chain/spec/digest.nim       | 10 ++++--
 beacon_chain/sszdump.nim           | 24 +++++++++++++
 ncli/ncli_pretty.nim               |  1 +
 scripts/connect_to_testnet.nims    |  8 +++--
 7 files changed, 111 insertions(+), 44 deletions(-)
 create mode 100644 beacon_chain/sszdump.nim

diff --git a/beacon_chain/beacon_node.nim b/beacon_chain/beacon_node.nim
index ea693a0c5..7baf216c1 100644
--- a/beacon_chain/beacon_node.nim
+++ b/beacon_chain/beacon_node.nim
@@ -218,6 +218,8 @@ proc init*(T: type BeaconNode, conf: BeaconNodeConf): Future[BeaconNode] {.async
     mainchainMonitor = MainchainMonitor.init(
       conf.depositWeb3Url, conf.depositContractAddress,
       blockPool.headState.data.data.eth1_data.block_hash)
+    # TODO if we don't have any validators attached, we don't need a mainchain
+    #      monitor
     mainchainMonitor.start()
 
   var
@@ -624,31 +626,22 @@ proc handleProposal(node: BeaconNode, head: BlockRef, slot: Slot):
   # TODO here we advance the state to the new slot, but later we'll be
   #      proposing for it - basically, we're selecting proposer based on an
   #      empty slot
-  var cache = get_empty_per_epoch_cache()
-  node.blockPool.withState(node.blockPool.tmpState, head.atSlot(slot)):
-    let proposerIdx = get_beacon_proposer_index(state, cache)
-    if proposerIdx.isNone:
-      notice "Missing proposer index",
-        slot=slot,
-        epoch=slot.compute_epoch_at_slot,
-        num_validators=state.validators.len,
-        active_validators=
-          get_active_validator_indices(state, slot.compute_epoch_at_slot),
-        balances=state.balances
 
-      return head
+  let proposerKey = node.blockPool.getProposer(head, slot)
+  if proposerKey.isNone():
+    return head
 
-    let validator = node.getAttachedValidator(state, proposerIdx.get)
+  let validator = node.attachedValidators.getValidator(proposerKey.get())
 
-    if validator != nil:
-      return await proposeBlock(node, validator, head, slot)
+  if validator != nil:
+    return await proposeBlock(node, validator, head, slot)
 
-    trace "Expecting block proposal",
-      headRoot = shortLog(head.root),
-      slot = shortLog(slot),
-      proposer = shortLog(state.validators[proposerIdx.get].pubKey),
-      cat = "consensus",
-      pcs = "wait_for_proposal"
+  debug "Expecting block proposal",
+    headRoot = shortLog(head.root),
+    slot = shortLog(slot),
+    proposer = shortLog(proposerKey.get()),
+    cat = "consensus",
+    pcs = "wait_for_proposal"
 
   return head
 
diff --git a/beacon_chain/block_pool.nim b/beacon_chain/block_pool.nim
index ea8a8adaa..1e556061e 100644
--- a/beacon_chain/block_pool.nim
+++ b/beacon_chain/block_pool.nim
@@ -2,7 +2,7 @@ import
   bitops, chronicles, options, tables,
   ssz, beacon_chain_db, state_transition, extras,
   beacon_node_types, metrics,
-  spec/[crypto, datatypes, digest, helpers]
+  spec/[crypto, datatypes, digest, helpers, validator]
 
 declareCounter beacon_reorgs_total, "Total occurrences of reorganizations of the chain" # On fork choice
 
@@ -926,3 +926,20 @@ proc preInit*(
   db.putTailBlock(blockRoot)
   db.putHeadBlock(blockRoot)
   db.putStateRoot(blockRoot, state.slot, signedBlock.message.state_root)
+
+proc getProposer*(pool: BlockPool, head: BlockRef, slot: Slot): Option[ValidatorPubKey] =
+  pool.withState(pool.tmpState, head.atSlot(slot)):
+    var cache = get_empty_per_epoch_cache()
+
+    let proposerIdx = get_beacon_proposer_index(state, cache)
+    if proposerIdx.isNone:
+      warn "Missing proposer index",
+        slot=slot,
+        epoch=slot.compute_epoch_at_slot,
+        num_validators=state.validators.len,
+        active_validators=
+          get_active_validator_indices(state, slot.compute_epoch_at_slot),
+        balances=state.balances
+      return
+
+    return some(state.validators[proposerIdx.get()].pubkey)
diff --git a/beacon_chain/mainchain_monitor.nim b/beacon_chain/mainchain_monitor.nim
index fc681a044..711193edf 100644
--- a/beacon_chain/mainchain_monitor.nim
+++ b/beacon_chain/mainchain_monitor.nim
@@ -24,13 +24,16 @@ type
 
   QueueElement = (BlockHash, DepositData)
 
-
-proc init*(T: type MainchainMonitor, web3Url, depositContractAddress: string, startBlock: Eth2Digest): T =
-  result.new()
-  result.web3Url = web3Url
-  result.depositContractAddress = Address.fromHex(depositContractAddress)
-  result.depositQueue = newAsyncQueue[QueueElement]()
-  result.eth1Block = BlockHash(startBlock.data)
+proc init*(
+    T: type MainchainMonitor,
+    web3Url, depositContractAddress: string,
+    startBlock: Eth2Digest): T =
+  T(
+    web3Url: web3Url,
+    depositContractAddress: Address.fromHex(depositContractAddress),
+    depositQueue: newAsyncQueue[QueueElement](),
+    eth1Block: BlockHash(startBlock.data),
+  )
 
 contract(DepositContract):
   proc deposit(pubkey: Bytes48, withdrawalCredentials: Bytes32, signature: Bytes96, deposit_data_root: FixedBytes[32])
@@ -113,8 +116,18 @@ proc getGenesis*(m: MainchainMonitor): Future[BeaconState] {.async.} =
   return m.genesisState[]
 
 proc getBlockNumber(web3: Web3, hash: BlockHash): Future[Quantity] {.async.} =
-  let blk = await web3.provider.eth_getBlockByHash(hash, false)
-  return blk.number
+  debug "Querying block number", hash
+
+  try:
+    let blk = await web3.provider.eth_getBlockByHash(hash, false)
+    return blk.number
+  except CatchableError as exc:
+    # TODO this doesn't make too much sense really, but what would be a
+    #      reasonable behavior? no idea - the whole algorithm needs to be
+    #      rewritten to match the spec.
+    notice "Failed to get block number from hash, using current block instead",
+      hash, err = exc.msg
+    return await web3.provider.eth_blockNumber()
 
 proc run(m: MainchainMonitor, delayBeforeStart: Duration) {.async.} =
   if delayBeforeStart != ZeroDuration:
@@ -129,8 +142,15 @@ proc run(m: MainchainMonitor, delayBeforeStart: Duration) {.async.} =
     error "Web3 server disconnected", ulr = m.web3Url
     processFut.cancel()
 
+  # TODO this needs to implement follow distance and the rest of the honest
+  #      validator spec..
+
   let startBlkNum = await web3.getBlockNumber(m.eth1Block)
-  debug "Starting eth1 monitor", fromBlock = startBlkNum.uint64
+
+  notice "Monitoring eth1 deposits",
+    fromBlock = startBlkNum.uint64,
+    contract = $m.depositContractAddress,
+    url = m.web3Url
 
   let ns = web3.contractSender(DepositContract, m.depositContractAddress)
 
@@ -139,15 +159,17 @@ proc run(m: MainchainMonitor, delayBeforeStart: Duration) {.async.} =
       withdrawalCredentials: Bytes32,
       amount: Bytes8,
       signature: Bytes96, merkleTreeIndex: Bytes8, j: JsonNode):
+    try:
+      let blkHash = BlockHash.fromHex(j["blockHash"].getStr())
+      let amount = bytes_to_int(array[8, byte](amount))
 
-    let blkHash = BlockHash.fromHex(j["blockHash"].getStr())
-    let amount = bytes_to_int(array[8, byte](amount))
-
-    m.depositQueue.addLastNoWait((blkHash,
-      DepositData(pubkey: ValidatorPubKey.init(array[48, byte](pubkey)),
-        withdrawal_credentials: Eth2Digest(data: array[32, byte](withdrawalCredentials)),
-        amount: amount,
-        signature: ValidatorSig.init(array[96, byte](signature)))))
+      m.depositQueue.addLastNoWait((blkHash,
+        DepositData(pubkey: ValidatorPubKey.init(array[48, byte](pubkey)),
+          withdrawal_credentials: Eth2Digest(data: array[32, byte](withdrawalCredentials)),
+          amount: amount,
+          signature: ValidatorSig.init(array[96, byte](signature)))))
+    except CatchableError as exc:
+      warn "Received invalid deposit", err = exc.msg, j
 
   try:
     await processFut
diff --git a/beacon_chain/spec/digest.nim b/beacon_chain/spec/digest.nim
index fe27bebeb..6fb8321b6 100644
--- a/beacon_chain/spec/digest.nim
+++ b/beacon_chain/spec/digest.nim
@@ -20,12 +20,12 @@
 # we call this function `eth2hash`, and it outputs a `Eth2Digest`. Easy to sed :)
 
 import
-  chronicles,
+  chronicles, json_serialization,
   nimcrypto/[sha2, hash, utils],
   hashes
 
 export
-  hash.`$`
+  hash.`$`, json_serialization
 
 type
   Eth2Digest* = MDigest[32 * 8] ## `hash32` from spec
@@ -70,3 +70,9 @@ func hash*(x: Eth2Digest): Hash =
   # We just slice the first 4 or 8 bytes of the block hash
   # depending of if we are on a 32 or 64-bit platform
   result = cast[ptr Hash](unsafeAddr x)[]
+
+proc writeValue*(writer: var JsonWriter, value: Eth2Digest) =
+  writeValue(writer, value.data.toHex(true))
+
+proc readValue*(reader: var JsonReader, value: var Eth2Digest) =
+  value = Eth2Digest.fromHex(reader.readValue(string))
diff --git a/beacon_chain/sszdump.nim b/beacon_chain/sszdump.nim
new file mode 100644
index 000000000..ec252d9da
--- /dev/null
+++ b/beacon_chain/sszdump.nim
@@ -0,0 +1,24 @@
+import
+  os,
+  ssz,
+  serialization,
+  beacon_node_types,
+  ./spec/[crypto, datatypes, digest]
+
+proc dump*(dir: string, v: AttestationData, validator: ValidatorPubKey) =
+  SSZ.saveFile(
+    dir / "att-" & $v.slot & "-" &
+    $v.index & "-" & validator.shortLog &
+    ".ssz", v)
+
+proc dump*(dir: string, v: SignedBeaconBlock, blck: BlockRef) =
+  SSZ.saveFile(
+    dir / "block-" & $v.message.slot & "-" &
+    shortLog(blck.root) & ".ssz", v)
+
+proc dump*(dir: string, v: HashedBeaconState, blck: BlockRef) =
+  SSZ.saveFile(
+    dir / "state-" & $v.data.slot & "-" &
+    shortLog(blck.root) & "-"  & shortLog(v.root) & ".ssz",
+    v.data)
+
diff --git a/ncli/ncli_pretty.nim b/ncli/ncli_pretty.nim
index 8749f31ea..8131d4c12 100644
--- a/ncli/ncli_pretty.nim
+++ b/ncli/ncli_pretty.nim
@@ -32,3 +32,4 @@ cli do(kind: string, file: string):
   of "state": printit(BeaconState)
   of "proposer_slashing": printit(ProposerSlashing)
   of "voluntary_exit": printit(VoluntaryExit)
+  else: echo "Unknown kind"
diff --git a/scripts/connect_to_testnet.nims b/scripts/connect_to_testnet.nims
index ae167f4dc..0813940b4 100644
--- a/scripts/connect_to_testnet.nims
+++ b/scripts/connect_to_testnet.nims
@@ -9,6 +9,7 @@ const
   genesisFile = "genesis.ssz"
   configFile = "config.yaml"
   testnetsRepo = "eth2-testnets"
+  web3Url = "wss://goerli.infura.io/ws/v3/809a18497dd74102b5f37d25aae3c85a"
 
 let
   testnetsOrg = getEnv("ETH2_TESTNETS_ORG", "eth2-clients")
@@ -123,15 +124,18 @@ cli do (testnetName {.argument.}: string):
         --random-deposits=1
         --deposits-dir="{validatorsDir}"
         --deposit-private-key={privKey}
-        --web3-url=wss://goerli.infura.io/ws/v3/809a18497dd74102b5f37d25aae3c85a
+        --web3-url={web3Url}
         {depositContractOpt}
         """, "\n", " ")
-      quit()
+      mode = Silent
+      echo "\nDeposit sent, wait for confirmation then press enter to continue"
+      discard readLineFromStdin()
 
   mode = Verbose
   execIgnoringExitCode replace(&"""{beaconNodeBinary}
     --data-dir="{dataDir}"
     --dump=true
+    --web3-url={web3Url}
     {bootstrapFileOpt}
     --state-snapshot="{testnetDir/genesisFile}" """ & depositContractOpt, "\n", " ")
 

From 7feaa113720be40bf1598f4f6c80a8905c3f8d52 Mon Sep 17 00:00:00 2001
From: Stefan Talpalaru <stefantalpalaru@yahoo.com>
Date: Sat, 8 Feb 2020 00:57:48 +0100
Subject: [PATCH 15/31] make testnetX: propagate LOG_LEVEL to beacon_node
 runtime [skip ci]

This allows running `make LOG_LEVEL=TRACE testnet1` and having that log
level also enabled at runtime, not just at compile time.
---
 Makefile                        | 4 ++--
 scripts/connect_to_testnet.nims | 6 ++++++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index ea7aeb0c8..776fee0c2 100644
--- a/Makefile
+++ b/Makefile
@@ -114,10 +114,10 @@ clean-testnet1:
 	rm -rf build/data/testnet1
 
 testnet0: | build deps
-	NIM_PARAMS="$(NIM_PARAMS)" $(ENV_SCRIPT) nim $(NIM_PARAMS) scripts/connect_to_testnet.nims testnet0
+	NIM_PARAMS="$(NIM_PARAMS)" LOG_LEVEL="$(LOG_LEVEL)" $(ENV_SCRIPT) nim $(NIM_PARAMS) scripts/connect_to_testnet.nims testnet0
 
 testnet1: | build deps
-	NIM_PARAMS="$(NIM_PARAMS)" $(ENV_SCRIPT) nim $(NIM_PARAMS) scripts/connect_to_testnet.nims testnet1
+	NIM_PARAMS="$(NIM_PARAMS)" LOG_LEVEL="$(LOG_LEVEL)" $(ENV_SCRIPT) nim $(NIM_PARAMS) scripts/connect_to_testnet.nims testnet1
 
 clean: | clean-common
 	rm -rf build/{$(TOOLS_CSV),all_tests,*_node,*ssz*,beacon_node_testnet*,state_sim,transition*}
diff --git a/scripts/connect_to_testnet.nims b/scripts/connect_to_testnet.nims
index 0813940b4..4e2142be9 100644
--- a/scripts/connect_to_testnet.nims
+++ b/scripts/connect_to_testnet.nims
@@ -131,11 +131,17 @@ cli do (testnetName {.argument.}: string):
       echo "\nDeposit sent, wait for confirmation then press enter to continue"
       discard readLineFromStdin()
 
+  let logLevel = getEnv("LOG_LEVEL")
+  var logLevelOpt = ""
+  if logLevel.len > 0:
+    logLevelOpt = "--log-level=" & logLevel
+
   mode = Verbose
   execIgnoringExitCode replace(&"""{beaconNodeBinary}
     --data-dir="{dataDir}"
     --dump=true
     --web3-url={web3Url}
     {bootstrapFileOpt}
+    {logLevelOpt}
     --state-snapshot="{testnetDir/genesisFile}" """ & depositContractOpt, "\n", " ")
 

From d99ce1bcf0adbd73f6ba05c1ab1f46095b39e3ae Mon Sep 17 00:00:00 2001
From: Jacek Sieka <jacek@status.im>
Date: Sat, 8 Feb 2020 19:42:45 +0100
Subject: [PATCH 16/31] fix json compile

---
 beacon_chain/mainchain_monitor.nim | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/beacon_chain/mainchain_monitor.nim b/beacon_chain/mainchain_monitor.nim
index 711193edf..ad459b54a 100644
--- a/beacon_chain/mainchain_monitor.nim
+++ b/beacon_chain/mainchain_monitor.nim
@@ -116,7 +116,7 @@ proc getGenesis*(m: MainchainMonitor): Future[BeaconState] {.async.} =
   return m.genesisState[]
 
 proc getBlockNumber(web3: Web3, hash: BlockHash): Future[Quantity] {.async.} =
-  debug "Querying block number", hash
+  debug "Querying block number", hash = $hash
 
   try:
     let blk = await web3.provider.eth_getBlockByHash(hash, false)
@@ -126,7 +126,7 @@ proc getBlockNumber(web3: Web3, hash: BlockHash): Future[Quantity] {.async.} =
     #      reasonable behavior? no idea - the whole algorithm needs to be
     #      rewritten to match the spec.
     notice "Failed to get block number from hash, using current block instead",
-      hash, err = exc.msg
+      hash = $hash, err = exc.msg
     return await web3.provider.eth_blockNumber()
 
 proc run(m: MainchainMonitor, delayBeforeStart: Duration) {.async.} =

From 5cea471c4b4205a049983eb27d9e6fd09d378879 Mon Sep 17 00:00:00 2001
From: Jacek Sieka <jacek@status.im>
Date: Sat, 8 Feb 2020 20:19:33 +0100
Subject: [PATCH 17/31] bump libp2p

---
 vendor/nim-libp2p | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p
index 88a030d8f..8c406fb9e 160000
--- a/vendor/nim-libp2p
+++ b/vendor/nim-libp2p
@@ -1 +1 @@
-Subproject commit 88a030d8fbd76023354f14ff10ba740786eb46a4
+Subproject commit 8c406fb9e5f5ea82c96173b828efe4a8a6027747

From a58fadea6d682e054e27efecd22bfc9b76a73242 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= <stefantalpalaru@yahoo.com>
Date: Sat, 8 Feb 2020 23:20:45 +0100
Subject: [PATCH 18/31] CI: add "-d:testnet_servers_image"

---
 .appveyor.yml       | 2 +-
 .travis.yml         | 2 +-
 Jenkinsfile         | 2 +-
 azure-pipelines.yml | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.appveyor.yml b/.appveyor.yml
index af7e52d38..6d8ac806d 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -37,7 +37,7 @@ build_script:
 test_script:
   # the "go-checks" target fails in AppVeyor, for some reason; easier to disable than to debug
   - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_GO_CHECKS=1 P2PD_CACHE=p2pdCache
-  - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_GO_CHECKS=1 P2PD_CACHE=p2pdCache NIMFLAGS="-d:NETWORK_TYPE=libp2p"
+  - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_GO_CHECKS=1 P2PD_CACHE=p2pdCache NIMFLAGS="-d:NETWORK_TYPE=libp2p -d:testnet_servers_image"
   - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_TEST_FIXTURES_SCRIPT=1 DISABLE_GO_CHECKS=1 test
 
 deploy: off
diff --git a/.travis.yml b/.travis.yml
index d7c9e0696..9fab9b125 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -50,6 +50,6 @@ script:
   # Building Nim-1.0.4 takes up to 10 minutes on Travis - the time limit after which jobs are cancelled for having no output
   - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" V=1 update # to allow a newer Nim version to be detected
   - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}"
-  - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" NIMFLAGS="-d:NETWORK_TYPE=libp2p"
+  - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" NIMFLAGS="-d:NETWORK_TYPE=libp2p -d:testnet_servers_image"
   - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" DISABLE_TEST_FIXTURES_SCRIPT=1 test
 
diff --git a/Jenkinsfile b/Jenkinsfile
index e1ec60f10..0f59e82fb 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -22,7 +22,7 @@ def runStages() {
 				"tools": {
 					stage("Tools") {
 						sh "make -j${env.NPROC}"
-						sh "make -j${env.NPROC} NIMFLAGS='-d:NETWORK_TYPE=libp2p'"
+						sh "make -j${env.NPROC} NIMFLAGS='-d:NETWORK_TYPE=libp2p -d:testnet_servers_image'"
 					}
 				},
 				"test suite": {
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 969fc54a2..1fc333cf7 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -72,7 +72,7 @@ jobs:
           mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries update
           mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} fetch-dlls
           mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} P2PD_CACHE=p2pdCache
-          mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} P2PD_CACHE=p2pdCache NIMFLAGS="-d:NETWORK_TYPE=libp2p"
+          mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} P2PD_CACHE=p2pdCache NIMFLAGS="-d:NETWORK_TYPE=libp2p -d:testnet_servers_image"
           file build/beacon_node
           mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} DISABLE_TEST_FIXTURES_SCRIPT=1 test
         displayName: 'build and test'

From 6007b01d47047f72a9f2689d75952039f03a6107 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= <stefantalpalaru@yahoo.com>
Date: Sun, 9 Feb 2020 03:22:42 +0100
Subject: [PATCH 19/31] testnet: don't use -march=native when building the
 Docker image [skip ci]

(because we run that Docker image both locally and remotely, we can't
use the server's "-march" either)
- reduce the genesis offset to 5 minutes
---
 docker/Dockerfile        |  3 ++-
 docker/Makefile          |  3 +++
 nim.cfg                  |  6 +++++-
 scripts/reset_testnet.sh | 13 ++++++-------
 4 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index b0b51d68b..a1fcc4c90 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -21,12 +21,13 @@ RUN cd /root \
 # to get a fresh up-to-date version of Nim and p2pd.
 ARG GIT_REVISION
 ARG NETWORK_NIM_FLAGS
+ARG MARCH_NIM_FLAGS
 
 RUN cd /root/nim-beacon-chain \
  && git fetch \
  && git reset --hard ${GIT_REVISION} \
  && make -j$(nproc) update \
- && make LOG_LEVEL=TRACE NIMFLAGS="-d:debug -d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS}" beacon_node
+ && make LOG_LEVEL=TRACE NIMFLAGS="-d:debug -d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS} ${MARCH_NIM_FLAGS}" beacon_node
 
 # --------------------------------- #
 # Starting new image to reduce size #
diff --git a/docker/Makefile b/docker/Makefile
index 1b2d5d9bf..069bf5e93 100644
--- a/docker/Makefile
+++ b/docker/Makefile
@@ -5,6 +5,8 @@ SHELL := bash # the shell used internally by "make"
 GIT_REVISION      ?= $(shell git rev-parse HEAD)
 NETWORK           ?= testnet1
 NETWORK_NIM_FLAGS ?= $(shell ../scripts/load-testnet-nim-flags.sh $(NETWORK))
+# Get the required GCC flags by running `gcc -v -E - -march=native </dev/null 2>&1 | grep cc1` on the server.
+MARCH_NIM_FLAGS   ?= -d:disableMarchNative --passC:'-march=znver1 --param l1-cache-size=32 --param l1-cache-line-size=64 --param l2-cache-size=512'
 
 IMAGE_TAG       ?= $(NETWORK)
 IMAGE_NAME      ?= statusteam/nimbus_beacon_node:$(IMAGE_TAG)
@@ -19,6 +21,7 @@ build:
 	docker build \
 		--build-arg="GIT_REVISION=$(GIT_REVISION)" \
 		--build-arg="NETWORK_NIM_FLAGS=$(NETWORK_NIM_FLAGS)" \
+		--build-arg="MARCH_NIM_FLAGS=$(MARCH_NIM_FLAGS)" \
 		-t $(IMAGE_NAME) \
 		--progress=plain \
 		.
diff --git a/nim.cfg b/nim.cfg
index 156531911..eee42dc2f 100644
--- a/nim.cfg
+++ b/nim.cfg
@@ -26,7 +26,11 @@
 # and larger arithmetic use cases, along with register starvation issues. When
 # engineering a more portable binary release, this should be tweaked but still
 # use at least -msse2 or -msse3.
---passC:"-march=native"
+@if disableMarchNative:
+  --passC:"-msse3"
+@else:
+  --passC:"-march=native"
+@end
 
 --threads:on
 --opt:speed
diff --git a/scripts/reset_testnet.sh b/scripts/reset_testnet.sh
index 44c75e3c4..27c06053f 100755
--- a/scripts/reset_testnet.sh
+++ b/scripts/reset_testnet.sh
@@ -32,11 +32,10 @@ echo "Beacon node data dir    : ${DATA_DIR:="build/testnet-reset-data/$NETWORK"}
 echo "Nim build flags         : $NETWORK_NIM_FLAGS"
 
 while true; do
-    read -p "Continue? [yn] " yn
+    read -p "Continue? [Yn] " yn
     case $yn in
-        [Yy]* ) break;;
+        * ) break;;
         [Nn]* ) exit 1;;
-        * ) echo "Please answer yes or no.";;
     esac
 done
 
@@ -70,7 +69,8 @@ fi
 cd docker
 
 echo "Building Docker image..."
-make build
+# we're running this Docker image both locally and on the servers
+make MARCH_NIM_FLAGS="-d:disableMarchNative" build
 
 $DOCKER_BEACON_NODE makeDeposits \
   --quickstart-deposits=$QUICKSTART_VALIDATORS \
@@ -79,9 +79,8 @@ $DOCKER_BEACON_NODE makeDeposits \
 
 TOTAL_VALIDATORS="$(( $QUICKSTART_VALIDATORS + $RANDOM_VALIDATORS ))"
 
-$DOCKER_BEACON_NODE \
+$DOCKER_BEACON_NODE createTestnet \
   --data-dir=/data_dir \
-  createTestnet \
   --validators-dir=/deposits_dir \
   --total-validators=$TOTAL_VALIDATORS \
   --last-user-validator=$QUICKSTART_VALIDATORS \
@@ -90,7 +89,7 @@ $DOCKER_BEACON_NODE \
   --bootstrap-address=$BOOTSTRAP_IP \
   --bootstrap-port=$BOOTSTRAP_PORT \
   $WEB3_URL_ARG $DEPOSIT_CONTRACT_ADDRESS_ARG \
-  --genesis-offset=900 # Delay in seconds
+  --genesis-offset=300 # Delay in seconds
 
 COMMITTED_FILES=" genesis.ssz bootstrap_nodes.txt "
 

From ebc8630d5aa27240e363fe18301da91994760d5f Mon Sep 17 00:00:00 2001
From: Dustin Brody <tersec@users.noreply.github.com>
Date: Mon, 10 Feb 2020 13:56:53 +0100
Subject: [PATCH 20/31] finish conversion of manual iteration to automatic
 iteration over EF test vectors per section

---
 .../test_fixture_operations_deposits.nim      | 29 ++++------
 tests/official/test_fixture_sanity_blocks.nim | 53 ++++++-------------
 2 files changed, 25 insertions(+), 57 deletions(-)

diff --git a/tests/official/test_fixture_operations_deposits.nim b/tests/official/test_fixture_operations_deposits.nim
index 3603b41a6..10c3f0710 100644
--- a/tests/official/test_fixture_operations_deposits.nim
+++ b/tests/official/test_fixture_operations_deposits.nim
@@ -20,13 +20,13 @@ import
 
 const OperationsDepositsDir = SszTestsDir/const_preset/"phase0"/"operations"/"deposit"/"pyspec_tests"
 
-template runTest(testName: string, identifier: untyped) =
+proc runTest(identifier: string) =
   # We wrap the tests in a proc to avoid running out of globals
   # in the future: Nim supports up to 3500 globals
   # but unittest with the macro/templates put everything as globals
   # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
 
-  const testDir = OperationsDepositsDir / astToStr(identifier)
+  let testDir = OperationsDepositsDir / identifier
 
   proc `testImpl _ operations_deposits _ identifier`() =
 
@@ -39,7 +39,7 @@ template runTest(testName: string, identifier: untyped) =
     else:
       prefix = "[Invalid] "
 
-    timedTest prefix & testName & " (" & astToStr(identifier) & ")":
+    timedTest prefix & " " & identifier:
       var stateRef, postRef: ref BeaconState
       var depositRef: ref Deposit
       new depositRef
@@ -61,19 +61,10 @@ template runTest(testName: string, identifier: untyped) =
   `testImpl _ operations_deposits _ identifier`()
 
 suite "Official - Operations - Deposits " & preset():
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/minimal/phase0/operations/deposit/pyspec_tests
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/mainnet/phase0/operations/deposit/pyspec_tests
-  runTest("bad merkle proof", bad_merkle_proof)
-  runTest("invalid signature new deposit", invalid_sig_new_deposit)
-  runTest("invalid signature other version", invalid_sig_other_version)
-  runTest("invalid signature top-up", invalid_sig_top_up)
-  runTest("invalid withdrawal credentials top-up",
-    invalid_withdrawal_credentials_top_up)
-  runTest("new deposit max", new_deposit_max)
-  runTest("new deposit over max", new_deposit_over_max)
-  runTest("new deposit under max", new_deposit_under_max)
-  runTest("success top-up", success_top_up)
-  when false:
-    # TODO
-    runTest("valid signature but forked state", valid_sig_but_forked_state)
-  runTest("wrong deposit for deposit count", wrong_deposit_for_deposit_count)
+  # TODO
+  const expected_failures = ["valid_sig_but_forked_state"]
+
+  for kind, path in walkDir(OperationsDepositsDir, true):
+    if path in expected_failures:
+      continue
+    runTest(path)
diff --git a/tests/official/test_fixture_sanity_blocks.nim b/tests/official/test_fixture_sanity_blocks.nim
index 3746a9a19..988d6fc5c 100644
--- a/tests/official/test_fixture_sanity_blocks.nim
+++ b/tests/official/test_fixture_sanity_blocks.nim
@@ -9,7 +9,7 @@
 
 import
   # Standard library
-  os, unittest,
+  os, sequtils, unittest,
   # Beacon chain internals
   ../../beacon_chain/spec/[crypto, datatypes],
   ../../beacon_chain/[ssz, state_transition, extras],
@@ -20,13 +20,13 @@ import
 
 const SanityBlocksDir = SszTestsDir/const_preset/"phase0"/"sanity"/"blocks"/"pyspec_tests"
 
-template runTest(identifier: string, num_blocks: int): untyped =
+proc runTest(identifier: string) =
   # We wrap the tests in a proc to avoid running out of globals
   # in the future: Nim supports up to 3500 globals
   # but unittest with the macro/templates put everything as globals
   # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
 
-  const testDir = SanityBlocksDir / identifier
+  let testDir = SanityBlocksDir / identifier
 
   proc `testImpl _ blck _ identifier`() =
     let prefix = if existsFile(testDir/"post.ssz"):
@@ -43,7 +43,9 @@ template runTest(identifier: string, num_blocks: int): untyped =
         new postRef
         postRef[] = parseTest(testDir/"post.ssz", SSZ, BeaconState)
 
-      for i in 0 ..< num_blocks:
+      # In test cases with more than 10 blocks the first 10 aren't 0-prefixed,
+      # so purely lexicographic sorting wouldn't sort properly.
+      for i in 0 ..< toSeq(walkPattern(testDir/"blocks_*.ssz")).len:
         let blck = parseTest(testDir/"blocks_" & $i & ".ssz", SSZ, SignedBeaconBlock)
 
         if postRef.isNil:
@@ -54,43 +56,18 @@ template runTest(identifier: string, num_blocks: int): untyped =
           let success = state_transition(stateRef[], blck.message, flags = {skipValidation})
           doAssert success, "Failure when applying block " & $i
 
-          # Checks:
-          # check: stateRef.hash_tree_root() == postRef.hash_tree_root()
-          if i == num_blocks - 1: reportDiff(stateRef, postRef)
+      # check: stateRef.hash_tree_root() == postRef.hash_tree_root()
+      if not postRef.isNil:
+        reportDiff(stateRef, postRef)
 
   `testImpl _ blck _ identifier`()
 
 suite "Official - Sanity - Blocks " & preset():
+  # Failing due to signature checking in indexed validation checking pending
+  # 0.10 BLS verification API with new domain handling.
   const expected_failures = ["attester_slashing"]
-  runTest("attestation", 2)
 
-  when false:
-    # Failing due to signature checking in indexed validation checking pending
-    # 0.10 BLS verification API with new domain handling.
-    runTest("attester_slashing", 1)
-  echo "Skipping test: attester_slashing"
-
-  runTest("balance_driven_status_transitions", 1)
-  runTest("deposit_in_block", 1)
-  runTest("deposit_top_up", 1)
-  runTest("empty_block_transition", 1)
-  runTest("empty_epoch_transition", 1)
-
-  when const_preset=="minimal":
-    runTest("empty_epoch_transition_not_finalizing", 1)
-    runTest("eth1_data_votes_consensus", 17)
-    runTest("eth1_data_votes_no_consensus", 16)
-
-  runTest("expected_deposit_in_block", 1)
-  runTest("high_proposer_index", 1)
-  runTest("historical_batch", 1)
-  runTest("invalid_block_sig", 1)
-  runTest("invalid_state_root", 1)
-  runTest("prev_slot_block_transition", 1)
-  runTest("proposer_after_inactive_index", 1)
-  runTest("proposer_slashing", 1)
-  runTest("same_slot_block_transition", 1)
-  runTest("skipped_slots", 1)
-  runTest("voluntary_exit", 2)
-  when const_preset=="minimal":
-    runTest("zero_block_sig", 1)
+  for kind, path in walkDir(SanityBlocksDir, true):
+    if path in expected_failures:
+      continue
+    runTest(path)

From 04f63da2cc7d303dbbd596e8e02ee847b64f4962 Mon Sep 17 00:00:00 2001
From: Dustin Brody <tersec@users.noreply.github.com>
Date: Mon, 10 Feb 2020 14:20:35 +0100
Subject: [PATCH 21/31] print messages for skipped tests

---
 tests/official/test_fixture_operations_deposits.nim | 1 +
 tests/official/test_fixture_sanity_blocks.nim       | 1 +
 2 files changed, 2 insertions(+)

diff --git a/tests/official/test_fixture_operations_deposits.nim b/tests/official/test_fixture_operations_deposits.nim
index 10c3f0710..641f68a7c 100644
--- a/tests/official/test_fixture_operations_deposits.nim
+++ b/tests/official/test_fixture_operations_deposits.nim
@@ -66,5 +66,6 @@ suite "Official - Operations - Deposits " & preset():
 
   for kind, path in walkDir(OperationsDepositsDir, true):
     if path in expected_failures:
+      echo "Skipping test: ", path
       continue
     runTest(path)
diff --git a/tests/official/test_fixture_sanity_blocks.nim b/tests/official/test_fixture_sanity_blocks.nim
index 988d6fc5c..ad993a2e0 100644
--- a/tests/official/test_fixture_sanity_blocks.nim
+++ b/tests/official/test_fixture_sanity_blocks.nim
@@ -69,5 +69,6 @@ suite "Official - Sanity - Blocks " & preset():
 
   for kind, path in walkDir(SanityBlocksDir, true):
     if path in expected_failures:
+      echo "Skipping test: ", path
       continue
     runTest(path)

From e54f738715ce535566c3a38f285396507053b130 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= <stefantalpalaru@yahoo.com>
Date: Tue, 11 Feb 2020 17:45:52 +0100
Subject: [PATCH 22/31] testnets: replace watchtower with direct container
 management [skip ci]

---
 docker/manage_testnet_hosts.nims | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/docker/manage_testnet_hosts.nims b/docker/manage_testnet_hosts.nims
index f1a5c9b0d..2fd919df2 100644
--- a/docker/manage_testnet_hosts.nims
+++ b/docker/manage_testnet_hosts.nims
@@ -98,7 +98,12 @@ iterator validatorAssignments: tuple[node: Node; firstValidator, lastValidator:
 case conf.cmd
 of restart_nodes:
   for n in nodes():
-    echo &"ssh {n.server} docker restart {n.container}"
+    if n.id mod 2 == 0:
+      # This will only print one line: "docker.io/statusteam/nimbus_beacon_node:testnet1".
+      echo &"ssh {n.server} docker pull -q statusteam/nimbus_beacon_node:{conf.network}"
+    # docker-compose will rebuild the container if it detects a newer image.
+    # Prints: "Recreating beacon-node-testnet1-1 ... done".
+    echo &"ssh {n.server} 'cd /docker/{n.container} && docker-compose up -d'"
 
 of reset_network:
   for n, firstValidator, lastValidator in validatorAssignments():
@@ -114,8 +119,9 @@ of reset_network:
     let dockerPath = &"/docker/{n.container}/data/BeaconNode"
     echo &"echo Syncing {lastValidator - firstValidator} keys starting from {firstValidator} to container {n.container}@{n.server} ... && \\"
     echo &"  ssh {n.server} 'sudo rm -rf /tmp/nimbus && mkdir -p /tmp/nimbus/' && \\"
-    echo &"  rsync {networkDataFiles} {n.server}:/tmp/nimbus/net-data/ && \\"
-    if keysList.len > 0: echo &"  rsync {keysList} {n.server}:/tmp/nimbus/keys/ && \\"
+    echo &"  rsync -a -zz {networkDataFiles} {n.server}:/tmp/nimbus/net-data/ && \\"
+    if keysList.len > 0:
+      echo &"  rsync -a -zz {keysList} {n.server}:/tmp/nimbus/keys/ && \\"
 
     echo &"  ssh {n.server} 'sudo docker container stop {n.container} && " &
                          &"sudo mkdir -p {dockerPath}/validators && " &

From 8df447d497d65f22d3efcfb526a0544a2f743ac7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= <stefantalpalaru@yahoo.com>
Date: Tue, 11 Feb 2020 18:41:25 +0100
Subject: [PATCH 23/31] Docker: optimise container beacon_node for server CPU
 [skip ci]

This means that we can no longer do "docker run" locally, so we compile
a local beacon_node with the same flags as the container one.
---
 scripts/reset_testnet.sh | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/scripts/reset_testnet.sh b/scripts/reset_testnet.sh
index 27c06053f..c10255dc8 100755
--- a/scripts/reset_testnet.sh
+++ b/scripts/reset_testnet.sh
@@ -55,37 +55,40 @@ fi
 
 mkdir -p "$DEPOSITS_DIR_ABS"
 
-DOCKER_BEACON_NODE="docker run -v $DEPOSITS_DIR_ABS:/deposits_dir -v $NETWORK_DIR_ABS:/network_dir -v $DATA_DIR_ABS:/data_dir statusteam/nimbus_beacon_node:$NETWORK"
-
-make deposit_contract
+#DOCKER_BEACON_NODE="docker run -v $DEPOSITS_DIR_ABS:/deposits_dir -v $NETWORK_DIR_ABS:/network_dir -v $DATA_DIR_ABS:/data_dir statusteam/nimbus_beacon_node:$NETWORK"
 
 if [ "$ETH1_PRIVATE_KEY" != "" ]; then
+  make deposit_contract
   echo "Deploying deposit contract through $WEB3_URL_ARG..."
   DEPOSIT_CONTRACT_ADDRESS=$(./build/deposit_contract deploy $WEB3_URL_ARG --private-key=$ETH1_PRIVATE_KEY)
   DEPOSIT_CONTRACT_ADDRESS_ARG="--deposit-contract=$DEPOSIT_CONTRACT_ADDRESS"
   echo "Done: $DEPOSIT_CONTRACT_ADDRESS"
 fi
 
+echo "Building a local beacon_node instance for 'makeDeposits' and 'createTestnet'"
+make NIMFLAGS="-d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS}" beacon_node
+
 cd docker
 
 echo "Building Docker image..."
-# we're running this Docker image both locally and on the servers
-make MARCH_NIM_FLAGS="-d:disableMarchNative" build
+# CPU-specific CFLAGS that work on the servers are in MARCH_NIM_FLAGS,
+# in docker/Makefile, and are enabled by default.
+make build
 
-$DOCKER_BEACON_NODE makeDeposits \
+../build/beacon_node makeDeposits \
   --quickstart-deposits=$QUICKSTART_VALIDATORS \
   --random-deposits=$RANDOM_VALIDATORS \
-  --deposits-dir=/deposits_dir
+  --deposits-dir="$DEPOSITS_DIR_ABS"
 
 TOTAL_VALIDATORS="$(( $QUICKSTART_VALIDATORS + $RANDOM_VALIDATORS ))"
 
-$DOCKER_BEACON_NODE createTestnet \
-  --data-dir=/data_dir \
-  --validators-dir=/deposits_dir \
+../build/beacon_node createTestnet \
+  --data-dir="$DATA_DIR_ABS" \
+  --validators-dir="$DEPOSITS_DIR_ABS" \
   --total-validators=$TOTAL_VALIDATORS \
   --last-user-validator=$QUICKSTART_VALIDATORS \
-  --output-genesis=/network_dir/genesis.ssz \
-  --output-bootstrap-file=/network_dir/bootstrap_nodes.txt \
+  --output-genesis="$NETWORK_DIR_ABS/genesis.ssz" \
+  --output-bootstrap-file="$NETWORK_DIR_ABS/bootstrap_nodes.txt" \
   --bootstrap-address=$BOOTSTRAP_IP \
   --bootstrap-port=$BOOTSTRAP_PORT \
   $WEB3_URL_ARG $DEPOSIT_CONTRACT_ADDRESS_ARG \

From 82447e22b29a866e110bae23241c4d6fbf4c779a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= <stefantalpalaru@yahoo.com>
Date: Tue, 11 Feb 2020 18:43:57 +0100
Subject: [PATCH 24/31] cleanup [skip ci]

---
 scripts/reset_testnet.sh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scripts/reset_testnet.sh b/scripts/reset_testnet.sh
index c10255dc8..07df8cf7a 100755
--- a/scripts/reset_testnet.sh
+++ b/scripts/reset_testnet.sh
@@ -55,8 +55,6 @@ fi
 
 mkdir -p "$DEPOSITS_DIR_ABS"
 
-#DOCKER_BEACON_NODE="docker run -v $DEPOSITS_DIR_ABS:/deposits_dir -v $NETWORK_DIR_ABS:/network_dir -v $DATA_DIR_ABS:/data_dir statusteam/nimbus_beacon_node:$NETWORK"
-
 if [ "$ETH1_PRIVATE_KEY" != "" ]; then
   make deposit_contract
   echo "Deploying deposit contract through $WEB3_URL_ARG..."

From b2b284dd25a609bbd433033596ce8859048644df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= <stefantalpalaru@yahoo.com>
Date: Tue, 11 Feb 2020 22:21:07 +0100
Subject: [PATCH 25/31] fork NimYAML

because of https://github.com/flyx/NimYAML/issues/77
---
 .gitmodules    | 4 ++--
 vendor/NimYAML | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.gitmodules b/.gitmodules
index 14fab9c25..98461881d 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -28,9 +28,9 @@
 	branch = master
 [submodule "vendor/NimYAML"]
 	path = vendor/NimYAML
-	url = https://github.com/flyx/NimYAML.git
+	url = https://github.com/status-im/NimYAML.git
 	ignore = dirty
-	branch = master
+	branch = devel
 [submodule "vendor/nim-web3"]
 	path = vendor/nim-web3
 	url = https://github.com/status-im/nim-web3.git
diff --git a/vendor/NimYAML b/vendor/NimYAML
index e6ac75d86..ef6cd224c 160000
--- a/vendor/NimYAML
+++ b/vendor/NimYAML
@@ -1 +1 @@
-Subproject commit e6ac75d86af8f8eae9a010c8e639c3ac4263f8df
+Subproject commit ef6cd224cc038e58a8cabb3138ac2a9b9e3ea4ea

From 649d691081539ed551d1550bb28631bbba640f50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= <stefantalpalaru@yahoo.com>
Date: Tue, 11 Feb 2020 23:16:02 +0100
Subject: [PATCH 26/31] bump submodules

---
 vendor/NimYAML             | 2 +-
 vendor/nimbus-build-system | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/vendor/NimYAML b/vendor/NimYAML
index ef6cd224c..3983a1bf4 160000
--- a/vendor/NimYAML
+++ b/vendor/NimYAML
@@ -1 +1 @@
-Subproject commit ef6cd224cc038e58a8cabb3138ac2a9b9e3ea4ea
+Subproject commit 3983a1bf48682156cb105ab124c7bbdd86fa4557
diff --git a/vendor/nimbus-build-system b/vendor/nimbus-build-system
index 2a70c4f15..acfe8c4bf 160000
--- a/vendor/nimbus-build-system
+++ b/vendor/nimbus-build-system
@@ -1 +1 @@
-Subproject commit 2a70c4f152ee849db1ededa92c1d80f7102dd718
+Subproject commit acfe8c4bfe603a40e191c950aef1bfe149dbc623

From d8a8078a65729e2e9f1fba0cd67286ccfdec88ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= <stefantalpalaru@yahoo.com>
Date: Tue, 11 Feb 2020 23:36:20 +0100
Subject: [PATCH 27/31] debug Azure Pipelines 64-bit failure

---
 azure-pipelines.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 1fc333cf7..7bbf88165 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -69,7 +69,7 @@ jobs:
           git config --global core.autocrlf false
           git submodule --quiet update --init --recursive
           scripts/setup_official_tests.sh jsonTestsCache
-          mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries update
+          mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries V=1 update
           mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} fetch-dlls
           mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} P2PD_CACHE=p2pdCache
           mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} P2PD_CACHE=p2pdCache NIMFLAGS="-d:NETWORK_TYPE=libp2p -d:testnet_servers_image"

From 18234c031bad0c85558a2fe5cde678c94199da88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= <stefantalpalaru@yahoo.com>
Date: Tue, 11 Feb 2020 23:52:20 +0100
Subject: [PATCH 28/31] Mingw-w64 workaround

---
 nim.cfg | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/nim.cfg b/nim.cfg
index eee42dc2f..04e640745 100644
--- a/nim.cfg
+++ b/nim.cfg
@@ -14,6 +14,9 @@
   @if i386:
     # set the IMAGE_FILE_LARGE_ADDRESS_AWARE flag so we can use PAE, if enabled, and access more than 2 GiB of RAM
     --passL:"-Wl,--large-address-aware"
+  @else
+    # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65782
+    --passC:"-fno-asynchronous-unwind-tables"
   @end
 
   # The dynamic Chronicles output currently prevents us from using colors on Windows

From 1ce6d6bf31868819806018a6878c8a4ef0aca4fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= <stefantalpalaru@yahoo.com>
Date: Wed, 12 Feb 2020 00:36:54 +0100
Subject: [PATCH 29/31] fix Azure Pipelines 64-bit build

---
 azure-pipelines.yml        | 2 +-
 nim.cfg                    | 7 ++++---
 vendor/nimbus-build-system | 2 +-
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 7bbf88165..1fc333cf7 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -69,7 +69,7 @@ jobs:
           git config --global core.autocrlf false
           git submodule --quiet update --init --recursive
           scripts/setup_official_tests.sh jsonTestsCache
-          mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries V=1 update
+          mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries update
           mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} fetch-dlls
           mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} P2PD_CACHE=p2pdCache
           mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} P2PD_CACHE=p2pdCache NIMFLAGS="-d:NETWORK_TYPE=libp2p -d:testnet_servers_image"
diff --git a/nim.cfg b/nim.cfg
index 04e640745..efd5151a2 100644
--- a/nim.cfg
+++ b/nim.cfg
@@ -14,9 +14,6 @@
   @if i386:
     # set the IMAGE_FILE_LARGE_ADDRESS_AWARE flag so we can use PAE, if enabled, and access more than 2 GiB of RAM
     --passL:"-Wl,--large-address-aware"
-  @else
-    # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65782
-    --passC:"-fno-asynchronous-unwind-tables"
   @end
 
   # The dynamic Chronicles output currently prevents us from using colors on Windows
@@ -33,6 +30,10 @@
   --passC:"-msse3"
 @else:
   --passC:"-march=native"
+  @if windows:
+    # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65782
+    --passC:"-fno-asynchronous-unwind-tables"
+  @end
 @end
 
 --threads:on
diff --git a/vendor/nimbus-build-system b/vendor/nimbus-build-system
index acfe8c4bf..2c4faa537 160000
--- a/vendor/nimbus-build-system
+++ b/vendor/nimbus-build-system
@@ -1 +1 @@
-Subproject commit acfe8c4bfe603a40e191c950aef1bfe149dbc623
+Subproject commit 2c4faa5372d2d8b0c2d16710fe5f93beab1c86af

From 39c57a49b12860d913d2ad20411ed4b74b0fb6b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= <stefantalpalaru@yahoo.com>
Date: Wed, 12 Feb 2020 12:05:04 +0100
Subject: [PATCH 30/31] Azure Pipelines: discard old NimBinaries cache

---
 azure-pipelines.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 1fc333cf7..a6374979c 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -14,7 +14,7 @@ jobs:
       - task: CacheBeta@1
         displayName: 'cache Nim binaries'
         inputs:
-          key: NimBinaries | $(Agent.OS) | $(PLATFORM) | "$(Build.SourceBranchName)"
+          key: NimBinaries | $(Agent.OS) | $(PLATFORM) | "$(Build.SourceBranchName)" | "v2"
           path: NimBinaries
 
       - task: CacheBeta@1

From 812386edd2c6724120d808dd0497ea4c85a358f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= <stefantalpalaru@yahoo.com>
Date: Wed, 12 Feb 2020 14:23:49 +0100
Subject: [PATCH 31/31] replace "-fno-asynchronous-unwind-tables" with
 "-mno-avx512vl"

---
 nim.cfg | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/nim.cfg b/nim.cfg
index efd5151a2..72db90c87 100644
--- a/nim.cfg
+++ b/nim.cfg
@@ -32,7 +32,8 @@
   --passC:"-march=native"
   @if windows:
     # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65782
-    --passC:"-fno-asynchronous-unwind-tables"
+    # ("-fno-asynchronous-unwind-tables" breaks Nim's exception raising, sometimes)
+    --passC:"-mno-avx512vl"
   @end
 @end