V0.8.3 tests part2 - BLS + Shuffling + State transition epoch (#372)

* Update BLS fixtures to 0.8.3

* Bump fixtures with shuffling with minimal preset

* Update shuffling tests

* parseTest generic over file format (Json or SSZ)

* Initial crosslink parsing commit for debugging Nim crash

* Workaround https://github.com/status-im/nim-beacon-chain/issues/369

* Crosslink test works for minimal - https://github.com/status-im/nim-beacon-chain/issues/369 is back on mainnet

* Use ref objects to workaround https://github.com/status-im/nim-beacon-chain/issues/369

* Generalize state transition epoch test to all epoch tests

* Fix slashing (potential uint64 overflow in previous spec)

* Add a state debugging macro to deeply inspect the wrong fields

* make reportDiff visible

* Improve the debug state macro for containers

beacon_chain/spec/beaconstate.nim

@@ -323,6 +323,7 @@ func get_total_balance*(state: BeaconState, validators: auto): Gwei =
     foldl(validators, a + state.validators[b].effective_balance, 0'u64)
   )
 
+# XXX: Move to state_transition_epoch.nim?
 # https://github.com/ethereum/eth2.0-specs/blob/v0.8.3/specs/core/0_beacon-chain.md#registry-updates
 func process_registry_updates*(state: var BeaconState) =
   ## Process activation eligibility and ejections

beacon_chain/spec/state_transition_epoch.nim

@@ -444,7 +444,7 @@ func process_rewards_and_penalties(
     decrease_balance(state, i.ValidatorIndex, penalties1[i] + penalties2[i])
 
 # https://github.com/ethereum/eth2.0-specs/blob/v0.8.3/specs/core/0_beacon-chain.md#slashings
-func process_slashings(state: var BeaconState) =
+func process_slashings*(state: var BeaconState) =
   let
     epoch = get_current_epoch(state)
     total_balance = get_total_active_balance(state)
@@ -452,13 +452,16 @@ func process_slashings(state: var BeaconState) =
   for index, validator in state.validators:
     if validator.slashed and epoch + EPOCHS_PER_SLASHINGS_VECTOR div 2 ==
         validator.withdrawable_epoch:
-      let penalty =
+      let increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from penalty
-        validator.effective_balance *
+                                                  # numerator to avoid uint64 overflow
-          min(sum(state.slashings) * 3, total_balance) div total_balance
+      let penalty_numerator =
+        validator.effective_balance div increment *
+          min(sum(state.slashings) * 3, total_balance)
+      let penalty = penalty_numerator div total_balance * increment
       decrease_balance(state, index.ValidatorIndex, penalty)
 
 # https://github.com/ethereum/eth2.0-specs/blob/v0.8.3/specs/core/0_beacon-chain.md#final-updates
-proc process_final_updates(state: var BeaconState) =
+proc process_final_updates*(state: var BeaconState) =
   let
     current_epoch = get_current_epoch(state)
     next_epoch = current_epoch + 1
@@ -482,7 +485,7 @@ proc process_final_updates(state: var BeaconState) =
   let
     index_epoch = next_epoch + ACTIVATION_EXIT_DELAY
     index_root_position = index_epoch mod EPOCHS_PER_HISTORICAL_VECTOR
-    indices_list = get_active_validator_indices(state, index_epoch)
+    indices_list = sszList(get_active_validator_indices(state, index_epoch), VALIDATOR_REGISTRY_LIMIT)
   state.active_index_roots[index_root_position] = hash_tree_root(indices_list)
 
   # Set committees root

tests/helpers/debug_state.nim

@@ -0,0 +1,154 @@
+# beacon_chain
+# Copyright (c) 2018-2019 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+import
+  macros,
+  ../../beacon_chain/spec/[datatypes, crypto, digest]
+  # digest is necessary for them to be printed as hex
+
+# Define comparison of object variants for BLSValue
+# https://github.com/nim-lang/Nim/issues/6676
+# ----------------------------------------------------------------
+
+proc processNode(arg, a,b, result: NimNode) =
+  case arg.kind
+  of nnkIdentDefs:
+    let field = arg[0]
+    result.add quote do:
+      if `a`.`field` != `b`.`field`:
+        return false
+  of nnkRecCase:
+    let kindField = arg[0][0]
+    processNode(arg[0], a,b, result)
+    let caseStmt = nnkCaseStmt.newTree(newDotExpr(a, kindField))
+    for i in 1 ..< arg.len:
+      let inputBranch = arg[i]
+      let outputBranch = newTree(inputBranch.kind)
+      let body = newStmtList()
+      if inputBranch.kind == nnkOfBranch:
+        outputBranch.add inputBranch[0]
+        processNode(inputBranch[1], a,b, body)
+      else:
+        inputBranch.expectKind nnkElse
+        processNode(inputBranch[0], a,b, body)
+      outputBranch.add body
+      caseStmt.add outputBranch
+    result.add caseStmt
+  of nnkRecList:
+    for child in arg:
+      child.expectKind {nnkIdentDefs, nnkRecCase}
+      processNode(child, a,b, result)
+  else:
+    arg.expectKind {nnkIdentDefs, nnkRecCase, nnkRecList}
+
+macro myCompareImpl(a,b: typed): untyped =
+  a.expectKind nnkSym
+  b.expectKind nnkSym
+  assert sameType(a, b)
+
+  let typeImpl = a.getTypeImpl
+  var checks = newSeq[NimNode]()
+
+  # uncomment to debug
+  # echo typeImpl.treeRepr
+
+  result = newStmtList()
+  processNode(typeImpl[2], a, b, result)
+
+  result.add quote do:
+    return true
+
+  # uncomment to debug
+  # echo result.repr
+
+proc `==`*[T](a,b: BlsValue[T]): bool =
+  myCompareImpl(a,b)
+# ---------------------------------------------------------------------
+
+# This tool inspects and compare 2 instances of a type recursively
+# highlighting the differences
+
+const builtinTypes = [
+  "int", "int8", "int16", "int32", "int64",
+  "uint", "uint8", "uint16", "uint32", "uint64",
+  "byte", "float32", "float64",
+  # "array", "seq", # wrapped in nnkBracketExpr
+  "char", "string"
+]
+
+proc compareStmt(xSubField, ySubField: NimNode, stmts: var NimNode) =
+  let xStr = $xSubField.toStrLit
+  let yStr = $ySubField.toStrLit
+
+  stmts.add quote do:
+    doAssert(
+      `xSubField` == `ySubField`,
+      "\nDiff: " & `xStr` & " = " & $`xSubField` & "\n" &
+      "and   " & `yStr` & " = " & $`ySubField` & "\n"
+    )
+
+proc compareContainerStmt(xSubField, ySubField: NimNode, stmts: var NimNode) =
+  let xStr = $xSubField.toStrLit
+  let yStr = $ySubField.toStrLit
+
+
+  stmts.add quote do:
+    doAssert(
+      `xSubField`.len == `ySubField`.len,
+        "\nDiff: " & `xStr` & ".len = " & $`xSubField`.len & "\n" &
+        "and   " & `yStr` & ".len = " & $`ySubField`.len & "\n"
+    )
+    for idx in `xSubField`.low .. `xSubField`.high:
+      doAssert(
+        `xSubField`[idx] == `ySubField`[idx],
+        "\nDiff: " & `xStr` & "[" & $idx & "] = " & $`xSubField`[idx] & "\n" &
+        "and   " & `yStr` & "[" & $idx & "] = " & $`ySubField`[idx] & "\n"
+      )
+
+proc inspectType(tImpl, xSubField, ySubField: NimNode, stmts: var NimNode) =
+  # echo "kind: " & $tImpl.kind
+  # echo "  -- field: " & $xSubField.toStrLit
+  case tImpl.kind
+  of nnkObjectTy:
+    # pass the records
+    let records = tImpl[2]
+    records.expectKind(nnkRecList)
+    for decl in records:
+      inspectType(
+        decl[1], # field type
+        nnkDotExpr.newTree(xSubField, decl[0]), # Accessor
+        nnkDotExpr.newTree(ySubField, decl[0]),
+        stmts
+      )
+  of {nnkRefTy, nnkDistinctTy}:
+    inspectType(tImpl[0], xSubField, ySubField, stmts)
+  of {nnkSym, nnkBracketExpr}:
+    if tImpl.kind == nnkBracketExpr:
+      assert tImpl[0].eqIdent"seq" or tImpl[0].eqIdent"array", "Error: unsupported generic type: " & $tImpl[0]
+      compareContainerStmt(xSubField, ySubField, stmts)
+    elif $tImpl in builtinTypes:
+      compareStmt(xSubField, ySubField, stmts)
+    elif $tImpl in ["ValidatorSig", "ValidatorPubKey"]:
+      # Workaround BlsValue being a case object
+      compareStmt(xSubField, ySubField, stmts)
+    else:
+      inspectType(tImpl.getTypeImpl(), xSubField, ySubField, stmts)
+  else:
+    error "Unsupported kind: " & $tImpl.kind &
+      " for field \"" & $xSubField.toStrLit &
+      "\" of type \"" & tImpl.repr
+
+macro reportDiff*(x, y: typed{`var`|`let`|`const`}): untyped =
+  assert sameType(x, y)
+  result = newStmtList()
+
+  let typeImpl = x.getTypeImpl
+  inspectType(typeImpl, x, y, result)
+
+  # echo result.toStrLit
+
+# -----------------------------------------

tests/official/fixtures

@@ -1 +1 @@
-Subproject commit f4fb3ab5558916ae4f2834542719f5a1555ee096
+Subproject commit 59830c90e11291c886c71b8cdf83a81651d5fbc0

tests/official/fixtures_utils.nim

@@ -0,0 +1,50 @@
+# beacon_chain
+# Copyright (c) 2018-Present Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+import
+  # Standard library
+  os, strutils, typetraits,
+  # Internals
+  ../../beacon_chain/ssz,
+  # Status libs
+  stew/byteutils,
+  serialization, json_serialization
+
+export  # Workaround:
+  #   - https://github.com/status-im/nim-serialization/issues/4
+  #   - https://github.com/status-im/nim-serialization/issues/5
+  #   - https://github.com/nim-lang/Nim/issues/11225
+  serialization.readValue,
+  Json, ssz
+
+# Process current EF test format (up to 0.8.2+)
+# ---------------------------------------------
+
+# #######################
+# JSON deserialization
+
+proc readValue*(r: var JsonReader, a: var seq[byte]) {.inline.} =
+  ## Custom deserializer for seq[byte]
+  a = hexToSeqByte(r.readValue(string))
+
+# #######################
+# Test helpers
+
+const
+  FixturesDir* = currentSourcePath.rsplit(DirSep, 1)[0] / "fixtures"
+  JsonTestsDir* = FixturesDir/"json_tests_v0.8.3"
+  SszTestsDir* = FixturesDir/"eth2.0-spec-tests"/"tests"
+
+proc parseTest*(path: string, Format: typedesc[Json or SSZ], T: typedesc): T =
+  try:
+    # debugEcho "          [Debug] Loading file: \"", path, '\"'
+    result = Format.loadFile(path, T)
+  except SerializationError as err:
+    writeStackTrace()
+    stderr.write $Format & " load issue for file \"", path, "\"\n"
+    stderr.write err.formatMsg(path), "\n"
+    quit 1

tests/official/fixtures_utils_v0_8_1.nim

@@ -1,3 +1,10 @@
+# beacon_chain
+# Copyright (c) 2018-Present Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
 import
   # Standard library
   os, strutils,

tests/official/test_fixture_bls.nim

@@ -7,31 +7,15 @@
 
 import
   # Standard libs
-  ospaths, unittest, endians,
+  os, unittest, endians,
   # Status libs
   blscurve, stew/byteutils,
   # Beacon chain internals
   ../../beacon_chain/spec/crypto,
   # Test utilities
-  ./fixtures_utils_v0_8_1
+  ./fixtures_utils
 
 type
-  # # TODO - but already tested in nim-blscurve
-  # BLSUncompressedG2 = object
-  #   input*: tuple[
-  #     message: seq[byte],
-  #     domain: array[1, byte]
-  #     ]
-  #   output*: ECP2_BLS381
-
-  # # TODO - but already tested in nim-blscurve
-  # BLSCompressedG2 = object
-  #   input*: tuple[
-  #     message: seq[byte],
-  #     domain: array[1, byte]
-  #     ]
-  #   output*: ECP2_BLS381
-
   Domain = distinct uint64
     ## Domains have custom hex serialization
 
@@ -65,28 +49,18 @@ proc readValue*(r: var JsonReader, a: var Domain) {.inline.} =
   let be_uint = hexToPaddedByteArray[8](r.readValue(string))
   bigEndian64(a.addr, be_uint.unsafeAddr)
 
-const TestsPath = JsonTestsDir / "bls"
+const BLSDir = JsonTestsDir/"general"/"phase0"/"bls"
-
-var
-  blsPrivToPubTests: Tests[BLSPrivToPub]
-  blsSignMsgTests: Tests[BLSSignMsg]
-  blsAggSigTests: Tests[BLSAggSig]
-  blsAggPubKeyTests: Tests[BLSAggPubKey]
 
 suite "Official - BLS tests":
-  test "Parsing the official BLS tests":
-    blsPrivToPubTests = parseTests(TestsPath / "priv_to_pub" / "priv_to_pub.json", BLSPrivToPub)
-    blsSignMsgTests = parseTests(TestsPath / "sign_msg" / "sign_msg.json", BLSSignMsg)
-    blsAggSigTests = parseTests(TestsPath / "aggregate_sigs" / "aggregate_sigs.json", BLSAggSig)
-    blsAggPubKeyTests = parseTests(TestsPath / "aggregate_pubkeys" / "aggregate_pubkeys.json", BLSAggPubKey)
-
   test "Private to public key conversion":
-    for t in blsPrivToPubTests.test_cases:
+    for file in walkDirRec(BLSDir/"priv_to_pub"):
+      let t = parseTest(file, Json, BLSPrivToPub)
       let implResult = t.input.pubkey()
       check: implResult == t.output
 
   test "Message signing":
-    for t in blsSignMsgTests.test_cases:
+    for file in walkDirRec(BLSDir/"sign_msg"):
+      let t = parseTest(file, Json, BLSSignMsg)
       let implResult = t.input.privkey.bls_sign(
         t.input.message,
         uint64(t.input.domain)
@@ -94,11 +68,15 @@ suite "Official - BLS tests":
       check: implResult == t.output
 
   test "Aggregating signatures":
-    for t in blsAggSigTests.test_cases:
+    for file in walkDirRec(BLSDir/"aggregate_sigs"):
+      let t = parseTest(file, Json, BLSAggSig)
       let implResult = t.input.combine()
       check: implResult == t.output
 
   test "Aggregating public keys":
-    for t in blsAggPubKeyTests.test_cases:
+    for file in walkDirRec(BLSDir/"aggregate_pubkeys"):
+      let t = parseTest(file, Json, BLSAggPubKey)
       let implResult = t.input.combine()
       check: implResult == t.output
+
+  # TODO: msg_hash_compressed and uncompressed

tests/official/test_fixture_shuffling.nim

@@ -1,5 +1,5 @@
 # beacon_chain
-# Copyright (c) 2018 Status Research & Development GmbH
+# Copyright (c) 2018-Present Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
@@ -7,31 +7,24 @@
 
 import
   # Standard library
-  ospaths, unittest,
+  os, unittest,
   # Beacon chain internals
   ../../beacon_chain/spec/[datatypes, validator, digest],
   # Test utilities
   ../testutil,
-  ./fixtures_utils_v0_8_1
+  ./fixtures_utils
 
 type
   Shuffling* = object
     seed*: Eth2Digest
     count*: uint64
-    shuffled*: seq[ValidatorIndex]
+    mapping*: seq[ValidatorIndex]
 
-when const_preset == "mainnet":
+const ShufflingDir = JsonTestsDir/const_preset/"phase0"/"shuffling"/"core"/"shuffle"
-  const TestsPath = JsonTestsDir / "shuffling" / "core" / "shuffling_full.json"
-elif const_preset == "minimal":
-  const TestsPath = JsonTestsDir / "shuffling" / "core" / "shuffling_minimal.json"
-
-var shufflingTests: Tests[Shuffling]
 
 suite "Official - Shuffling tests [Preset: " & preset():
-  test "Parsing the official shuffling tests [Preset: " & preset():
-    shufflingTests = parseTests(TestsPath, Shuffling)
-
   test "Shuffling a sequence of N validators" & preset():
-    for t in shufflingTests.test_cases:
+    for file in walkDirRec(ShufflingDir):
+      let t = parseTest(file, Json, Shuffling)
       let implResult = get_shuffled_seq(t.seed, t.count)
-      check: implResult == t.shuffled
+      check: implResult == t.mapping

tests/official/test_fixture_state_transition_epoch.nim

@@ -0,0 +1,88 @@
+# beacon_chain
+# Copyright (c) 2018-Present Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+import
+  # Standard library
+  os, unittest, strutils,
+  # Beacon chain internals
+  ../../beacon_chain/spec/[datatypes, validator, state_transition_epoch],
+  # Test utilities
+  ../testutil,
+  ./fixtures_utils
+
+from ../../beacon_chain/spec/beaconstate import process_registry_updates
+  # XXX: move to state_transition_epoch?
+
+# TODO: parsing SSZ
+#       can overwrite the calling function stack
+#       https://github.com/status-im/nim-beacon-chain/issues/369
+#
+# We store the state on the heap to avoid that
+
+template runSuite(suiteDir, testName: string, transitionProc: untyped{ident}, useCache: static bool): untyped =
+  # We wrap the tests in a proc to avoid running out of globals
+  # in the future: Nim supports up to 3500 globals
+  # but unittest with the macro/templates put everything as globals
+  # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
+
+  proc `suiteImpl _ transitionProc`() =
+    suite "Official - Epoch Processing - " & testName & " [Preset: " & preset():
+      for testDir in walkDirRec(suiteDir, yieldFilter = {pcDir}):
+
+        let unitTestName = testDir.rsplit(DirSep, 1)[1]
+        test testName & " - " & unitTestName & preset():
+          var stateRef, postRef: ref BeaconState
+          new stateRef
+          new postRef
+          stateRef[] = parseTest(testDir/"pre.ssz", SSZ, BeaconState)
+          postRef[] = parseTest(testDir/"post.ssz", SSZ, BeaconState)
+
+          when useCache:
+            var cache = get_empty_per_epoch_cache()
+            transitionProc(stateRef[], cache)
+          else:
+            transitionProc(stateRef[])
+
+          check: stateRef.hash_tree_root() == postRef.hash_tree_root()
+
+  `suiteImpl _ transitionProc`()
+
+# Justification & Finalization
+# ---------------------------------------------------------------
+
+const JustificationFinalizationDir = SszTestsDir/const_preset/"phase0"/"epoch_processing"/"justification_and_finalization"/"pyspec_tests"
+runSuite(JustificationFinalizationDir, "Justification & Finalization",  process_justification_and_finalization, useCache = true)
+
+# Crosslinks
+# ---------------------------------------------------------------
+
+const CrosslinksDir = SszTestsDir/const_preset/"phase0"/"epoch_processing"/"crosslinks"/"pyspec_tests"
+runSuite(CrosslinksDir, "Crosslinks",  process_crosslinks, useCache = true)
+
+# Rewards & Penalties
+# ---------------------------------------------------------------
+
+# No test upstream
+
+# Registry updates
+# ---------------------------------------------------------------
+
+const RegistryUpdatesDir = SszTestsDir/const_preset/"phase0"/"epoch_processing"/"registry_updates"/"pyspec_tests"
+runSuite(RegistryUpdatesDir, "Registry updates",  process_registry_updates, useCache = false)
+
+# Slashings
+# ---------------------------------------------------------------
+
+const SlashingsDir = SszTestsDir/const_preset/"phase0"/"epoch_processing"/"slashings"/"pyspec_tests"
+runSuite(SlashingsDir, "Slashings",  process_slashings, useCache = false)
+
+when false: # TODO: Failing
+  # Final updates
+  # ---------------------------------------------------------------
+
+  const FinalUpdatesDir = SszTestsDir/const_preset/"phase0"/"epoch_processing"/"final_updates"/"pyspec_tests"
+  runSuite(FinalUpdatesDir, "Final updates",  process_final_updates, useCache = false)

tests/official/test_fixture_state_transition_epoch.nim.cfg

@@ -0,0 +1 @@
+-d:"ssz_testing"