Don't import beacon_chain_db in consensus tests. [skip CI] Pass:

- test_ssz
- test_process_attestation
-test_process_deposits

beacon_chain/ssz/bytes_reader.nim

@@ -258,7 +258,7 @@ func readSszValue*[T](input: openArray[byte],
           type(field),
           input.toOpenArray(int(startOffset), int(endOffset - 1)))
 
-    when val is SignedBeaconBlock | TrustedSignedBeaconBlock:
+    when val is SignedBeaconBlock:
       if updateRoot:
         val.root = hash_tree_root(val.message)
   else:

tests/mocking/mock_attestations.nim

@@ -56,7 +56,7 @@ proc mockAttestationData(
     epoch: target_epoch, root: epoch_boundary_root
   )
 
-proc signMockAttestation*(state: BeaconState, attestation: var Attestation) =
+proc signMockAttestation*(state: BeaconState, attestation: var Attestation[Unchecked]) =
   var cache = StateCache()
   let participants = get_attesting_indices(
     state,
@@ -85,7 +85,7 @@ proc signMockAttestation*(state: BeaconState, attestation: var Attestation) =
 proc mockAttestationImpl(
        state: BeaconState,
        slot: Slot,
-       flags: UpdateFlags): Attestation =
+       flags: UpdateFlags): Attestation[Unchecked] =
 
   var cache = StateCache()
 
@@ -110,16 +110,16 @@ proc mockAttestationImpl(
 
 proc mockAttestation*(
        state: BeaconState,
-       flags: UpdateFlags = {}): Attestation =
+       flags: UpdateFlags = {}): Attestation[Unchecked] =
   mockAttestationImpl(state, state.slot, flags)
 
 proc mockAttestation*(
        state: BeaconState,
        slot: Slot,
-       flags: UpdateFlags = {}): Attestation =
+       flags: UpdateFlags = {}): Attestation[Unchecked] =
   mockAttestationImpl(state, slot, flags)
 
-func fillAggregateAttestation*(state: BeaconState, attestation: var Attestation) =
+func fillAggregateAttestation*(state: BeaconState, attestation: var Attestation[Unchecked]) =
   var cache = StateCache()
   let beacon_committee = get_beacon_committee(
     state,
@@ -130,7 +130,7 @@ func fillAggregateAttestation*(state: BeaconState, attestation: var Attestation)
   for i in 0 ..< beacon_committee.len:
     attestation.aggregation_bits[i] = true
 
-proc add*(state: var HashedBeaconState, attestation: Attestation, slot: Slot) =
+proc add*(state: var HashedBeaconState, attestation: Attestation[Unchecked], slot: Slot) =
   var
     signedBlock = mockBlockForNextSlot(state.data)
     cache = StateCache()

tests/mocking/mock_blocks.nim

@@ -40,7 +40,7 @@ proc signMockBlock*(state: BeaconState, signedBlock: var SignedBeaconBlock) =
 proc mockBlock(
     state: BeaconState,
     slot: Slot,
-    flags: UpdateFlags = {}): SignedBeaconBlock =
+    flags: UpdateFlags = {}): SignedBeaconBlock[Unchecked] =
   ## TODO don't do this gradual construction, for exception safety
   ## Mock a BeaconBlock for the specific slot
   ## Skip signature creation if block should not be signed (skipBlsValidation present)
@@ -60,5 +60,5 @@ proc mockBlock(
     signMockBlock(state, result)
 
 proc mockBlockForNextSlot*(state: BeaconState, flags: UpdateFlags = {}):
-    SignedBeaconBlock =
+    SignedBeaconBlock[Unchecked] =
   mockBlock(state, state.slot + 1, flags)

tests/spec_block_processing/test_process_attestation.nim

@@ -19,7 +19,7 @@ import
   ../../beacon_chain/spec/[beaconstate, datatypes, helpers],
   # Mock helpers
   ../mocking/[mock_genesis, mock_attestations, mock_state],
-  ../testutil
+  ../testreportutils
 
 suiteReport "[Unit - Spec - Block processing] Attestations " & preset():
 

tests/spec_block_processing/test_process_deposits.nim

@@ -21,7 +21,7 @@ import
   ../../beacon_chain/[ssz, extras],
   # Mock helpers
   ../mocking/[mock_deposits, mock_genesis],
-  ../testutil, ../helpers/math_helpers
+  ../testreportutils, ../helpers/math_helpers
 
 suiteReport "[Unit - Spec - Block processing] Deposits " & preset():
 

tests/test_ssz.nim

@@ -10,7 +10,7 @@
 import
   unittest, options, json_serialization,
   serialization/testing/generic_suite,
-  ./testutil,
+  ./testreportutils,
   ../beacon_chain/spec/[datatypes, digest],
   ../beacon_chain/ssz, ../beacon_chain/ssz/[navigator, dynamic_navigator]
 

tests/test_ssz_merkleization.nim

@@ -265,4 +265,3 @@ proc testMerkleizer =
     doAssert deposits[i].proof == depositsCopy[i].proof
 
 testMerkleizer()
-

tests/test_state_transition.nim

@@ -9,7 +9,7 @@
 
 import
   unittest, chronicles,
-  ./testutil, ./testblockutil,
+  ./testreportutils, ./testblockutil,
   ../beacon_chain/spec/[beaconstate, datatypes, digest, crypto,
                         validator, state_transition, presets],
   ../beacon_chain/ssz

tests/testblockutil.nim

@@ -7,11 +7,14 @@
 
 import
   options, stew/endians2,
-  ../beacon_chain/[extras, validator_pool],
+  ../beacon_chain/extras,
   ../beacon_chain/ssz/merkleization,
   ../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, presets,
                         helpers, validator, signatures, state_transition]
 
+# Keep only spec+SSZ types import in this file
+# To ensure we don't involve non-spec primitives in some tests like beaconchainDB
+
 func makeFakeValidatorPrivKey(i: int): ValidatorPrivKey =
   # 0 is not a valid BLS private key - 1000 helps interop with rust BLS library,
   # lighthouse.
@@ -58,10 +61,10 @@ proc makeInitialDeposits*(
     result.add makeDeposit(i, flags)
 
 func signBlock*(
-    fork: Fork, genesis_validators_root: Eth2Digest, blck: BeaconBlock,
+    fork: Fork, genesis_validators_root: Eth2Digest, blck: BeaconBlock[Unchecked],
-    privKey: ValidatorPrivKey, flags: UpdateFlags = {}): SignedBeaconBlock =
+    privKey: ValidatorPrivKey, flags: UpdateFlags = {}): SignedBeaconBlock[Unchecked] =
   var root = hash_tree_root(blck)
-  SignedBeaconBlock(
+  SignedBeaconBlock[Unchecked](
     message: blck,
     root: root,
     signature:
@@ -77,10 +80,10 @@ proc addTestBlock*(
     parent_root: Eth2Digest,
     cache: var StateCache,
     eth1_data = Eth1Data(),
-    attestations = newSeq[Attestation](),
+    attestations = newSeq[Attestation[Unchecked]](),
     deposits = newSeq[Deposit](),
     graffiti = default(GraffitiBytes),
-    flags: set[UpdateFlag] = {}): SignedBeaconBlock =
+    flags: set[UpdateFlag] = {}): SignedBeaconBlock[Unchecked] =
   # Create and add a block to state - state will advance by one slot!
   doAssert process_slots(state, state.data.slot + 1, cache, flags)
 
@@ -89,8 +92,12 @@ proc addTestBlock*(
     privKey = hackPrivKey(state.data.validators[proposer_index.get])
     randao_reveal =
       if skipBlsValidation notin flags:
-        privKey.genRandaoReveal(
+        # genRandaoReveal without importing validator_pool
-          state.data.fork, state.data.genesis_validators_root, state.data.slot)
+        get_epoch_signature(
+          state.data.fork,
+          state.data.genesis_validators_root,
+          state.data.slot.compute_epoch_at_slot(),
+          privKey)
       else:
         ValidatorSig()
 
@@ -129,10 +136,10 @@ proc makeTestBlock*(
     parent_root: Eth2Digest,
     cache: var StateCache,
     eth1_data = Eth1Data(),
-    attestations = newSeq[Attestation](),
+    attestations = newSeq[Attestation[Unchecked]](),
     deposits = newSeq[Deposit](),
     graffiti = default(GraffitiBytes),
-    flags: set[UpdateFlag] = {}): SignedBeaconBlock =
+    flags: set[UpdateFlag] = {}): SignedBeaconBlock[Unchecked] =
   # Create a block for `state.slot + 1` - like a block proposer would do!
   # It's a bit awkward - in order to produce a block for N+1, we need to
   # calculate what the state will look like after that block has been applied,
@@ -146,7 +153,7 @@ proc makeAttestation*(
     state: BeaconState, beacon_block_root: Eth2Digest,
     committee: seq[ValidatorIndex], slot: Slot, index: CommitteeIndex,
     validator_index: ValidatorIndex, cache: var StateCache,
-    flags: UpdateFlags = {}): Attestation =
+    flags: UpdateFlags = {}): Attestation[Unchecked] =
   # Avoids state_sim silliness; as it's responsible for all validators,
   # transforming, from monotonic enumerable index -> committee index ->
   # montonoic enumerable index, is wasteful and slow. Most test callers
@@ -169,7 +176,7 @@ proc makeAttestation*(
       else:
         ValidatorSig()
 
-  Attestation(
+  Attestation[Unchecked](
     data: data,
     aggregation_bits: aggregation_bits,
     signature: sig
@@ -193,7 +200,7 @@ proc find_beacon_committee(
 proc makeAttestation*(
     state: BeaconState, beacon_block_root: Eth2Digest,
     validator_index: ValidatorIndex, cache: var StateCache,
-    flags: UpdateFlags = {}): Attestation =
+    flags: UpdateFlags = {}): Attestation[Unchecked] =
   let (committee, slot, index) =
     find_beacon_committee(state, validator_index, cache)
   makeAttestation(state, beacon_block_root, committee, slot, index,
@@ -202,7 +209,7 @@ proc makeAttestation*(
 proc makeFullAttestations*(
     state: BeaconState, beacon_block_root: Eth2Digest, slot: Slot,
     cache: var StateCache,
-    flags: UpdateFlags = {}): seq[Attestation] =
+    flags: UpdateFlags = {}): seq[Attestation[Unchecked]] =
   # Create attestations in which the full committee participates for each shard
   # that should be attested to during a particular slot
   let committees_per_slot =
@@ -216,7 +223,7 @@ proc makeFullAttestations*(
 
     doAssert committee.len() >= 1
     # Initial attestation
-    var attestation = Attestation(
+    var attestation = Attestation[Unchecked](
       aggregation_bits: CommitteeValidatorsBits.init(committee.len),
       data: data,
       signature: get_attestation_signature(

tests/testdbutils.nim

@@ -0,0 +1,32 @@
+# beacon_chain
+# Copyright (c) 2018-2019 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+import
+  os, algorithm, strformat, stats, times, tables, std/monotimes, stew/endians2,
+  testutils/markdown_reports, chronicles,
+  ../beacon_chain/[beacon_chain_db, extras, ssz],
+  ../beacon_chain/spec/[digest, beaconstate, datatypes, presets],
+  ../beacon_chain/block_pools/chain_dag,
+  eth/db/[kvstore, kvstore_sqlite3],
+  testblockutil
+
+export testblockutil
+
+proc makeTestDB*(tailState: BeaconState, tailBlock: SignedBeaconBlock): BeaconChainDB =
+  result = BeaconChainDB.init(defaultRuntimePreset, "", inMemory = true)
+  ChainDAGRef.preInit(result, tailState, tailState, tailBlock)
+
+proc makeTestDB*(validators: Natural): BeaconChainDB =
+  let
+    genState = initialize_beacon_state_from_eth1(
+      defaultRuntimePreset,
+      Eth2Digest(),
+      0,
+      makeInitialDeposits(validators.uint64, flags = {skipBlsValidation}),
+      {skipBlsValidation})
+    genBlock = get_initial_beacon_block(genState[])
+  makeTestDB(genState[], genBlock)

tests/testreportutils.nim

@@ -0,0 +1,96 @@
+# beacon_chain
+# Copyright (c) 2018-2019 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+import
+  os, algorithm, strformat, stats, times, tables, std/monotimes, stew/endians2,
+  testutils/markdown_reports, chronicles,
+  ../beacon_chain/spec/presets
+
+type
+  TestDuration = tuple[duration: float, label: string]
+
+func preset*(): string =
+  " [Preset: " & const_preset & ']'
+
+# For state_sim
+template withTimer*(stats: var RunningStat, body: untyped) =
+  let start = getMonoTime()
+
+  block:
+    body
+
+  let stop = getMonoTime()
+  stats.push (stop - start).inMicroseconds.float / 1000000.0
+
+template withTimer*(duration: var float, body: untyped) =
+  let start = getMonoTime()
+
+  block:
+    body
+
+  duration = (getMonoTime() - start).inMicroseconds.float / 1000000.0
+
+# For state_sim
+template withTimerRet*(stats: var RunningStat, body: untyped): untyped =
+  let start = getMonoTime()
+  let tmp = block:
+    body
+  let stop = getMonoTime()
+  stats.push (stop - start).inMicroseconds.float / 1000000.0
+
+  tmp
+
+var testTimes: seq[TestDuration]
+var status = initOrderedTable[string, OrderedTable[string, Status]]()
+var last: string
+
+proc summarizeLongTests*(name: string) =
+  # TODO clean-up and make machine-readable/storable the output
+  # TODO this is too hard-coded and mostly a demo for using the
+  # timedTest wrapper template for unittest
+  sort(testTimes, system.cmp, SortOrder.Descending)
+
+  echo ""
+  echo "10 longest individual test durations"
+  echo "------------------------------------"
+  for i, item in testTimes:
+    echo &"{item.duration:6.2f}s for {item.label}"
+    if i >= 10:
+      break
+
+  status.sort do (a: (string, OrderedTable[string, Status]),
+                  b: (string, OrderedTable[string, Status])) -> int: cmp(a[0], b[0])
+
+  generateReport(name & "-" & const_preset, status, width=90)
+
+template suiteReport*(name, body) =
+  last = name
+  status[last] = initOrderedTable[string, Status]()
+  block: # namespacing
+    proc runSuite() =
+      suite name:
+        body
+    runSuite()
+
+template timedTest*(name, body) =
+  var f: float
+  test name:
+    status[last][name] = Status.Fail
+
+    withTimer f:
+      body
+
+    status[last][name] = case testStatusIMPL
+                         of OK: Status.OK
+                         of FAILED: Status.Fail
+                         of SKIPPED: Status.Skip
+
+  # TODO reached for a failed test; maybe defer or similar
+  # TODO noto thread-safe as-is
+  testTimes.add (f, name)
+
+export inMicroseconds

tests/testutil.nim

@@ -5,111 +5,5 @@
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
-import
+import testdbutils, testreportutils
-  os, algorithm, strformat, stats, times, tables, std/monotimes, stew/endians2,
+export testdbutils, testreportutils
-  testutils/markdown_reports, chronicles,
-  ../beacon_chain/[beacon_chain_db, extras, ssz],
-  ../beacon_chain/spec/[digest, beaconstate, datatypes, presets],
-  ../beacon_chain/block_pools/chain_dag,
-  eth/db/[kvstore, kvstore_sqlite3],
-  testblockutil
-
-type
-  TestDuration = tuple[duration: float, label: string]
-
-func preset*(): string =
-  " [Preset: " & const_preset & ']'
-
-# For state_sim
-template withTimer*(stats: var RunningStat, body: untyped) =
-  let start = getMonoTime()
-
-  block:
-    body
-
-  let stop = getMonoTime()
-  stats.push (stop - start).inMicroseconds.float / 1000000.0
-
-template withTimer*(duration: var float, body: untyped) =
-  let start = getMonoTime()
-
-  block:
-    body
-
-  duration = (getMonoTime() - start).inMicroseconds.float / 1000000.0
-
-# For state_sim
-template withTimerRet*(stats: var RunningStat, body: untyped): untyped =
-  let start = getMonoTime()
-  let tmp = block:
-    body
-  let stop = getMonoTime()
-  stats.push (stop - start).inMicroseconds.float / 1000000.0
-
-  tmp
-
-var testTimes: seq[TestDuration]
-var status = initOrderedTable[string, OrderedTable[string, Status]]()
-var last: string
-
-proc summarizeLongTests*(name: string) =
-  # TODO clean-up and make machine-readable/storable the output
-  # TODO this is too hard-coded and mostly a demo for using the
-  # timedTest wrapper template for unittest
-  sort(testTimes, system.cmp, SortOrder.Descending)
-
-  echo ""
-  echo "10 longest individual test durations"
-  echo "------------------------------------"
-  for i, item in testTimes:
-    echo &"{item.duration:6.2f}s for {item.label}"
-    if i >= 10:
-      break
-
-  status.sort do (a: (string, OrderedTable[string, Status]),
-                  b: (string, OrderedTable[string, Status])) -> int: cmp(a[0], b[0])
-
-  generateReport(name & "-" & const_preset, status, width=90)
-
-template suiteReport*(name, body) =
-  last = name
-  status[last] = initOrderedTable[string, Status]()
-  block: # namespacing
-    proc runSuite() =
-      suite name:
-        body
-    runSuite()
-
-template timedTest*(name, body) =
-  var f: float
-  test name:
-    status[last][name] = Status.Fail
-
-    withTimer f:
-      body
-
-    status[last][name] = case testStatusIMPL
-                         of OK: Status.OK
-                         of FAILED: Status.Fail
-                         of SKIPPED: Status.Skip
-
-  # TODO reached for a failed test; maybe defer or similar
-  # TODO noto thread-safe as-is
-  testTimes.add (f, name)
-
-proc makeTestDB*(tailState: BeaconState, tailBlock: SignedBeaconBlock): BeaconChainDB =
-  result = BeaconChainDB.init(defaultRuntimePreset, "", inMemory = true)
-  ChainDAGRef.preInit(result, tailState, tailState, tailBlock)
-
-proc makeTestDB*(validators: Natural): BeaconChainDB =
-  let
-    genState = initialize_beacon_state_from_eth1(
-      defaultRuntimePreset,
-      Eth2Digest(),
-      0,
-      makeInitialDeposits(validators.uint64, flags = {skipBlsValidation}),
-      {skipBlsValidation})
-    genBlock = get_initial_beacon_block(genState[])
-  makeTestDB(genState[], genBlock)
-
-export inMicroseconds