Merge pull request #423 from status-im/tester_skip_table

fixes #421, each test suite have their own skip table

GeneralStateTests.md

@@ -387,7 +387,7 @@ OK: 3/3 Fail: 0/3 Skip: 0/3
 + CreateMessageRevertedOOGInInit.json                             OK
 + RevertDepthCreate2OOG.json                                      OK
 + RevertDepthCreateAddressCollision.json                          OK
-  RevertInCreateInInitCreate2.json                                Skip
++ RevertInCreateInInitCreate2.json                                OK
 + RevertOpcodeCreate.json                                         OK
 + RevertOpcodeInCreateReturnsCreate2.json                         OK
 + call_outsize_then_create2_successful_then_returndatasize.json   OK
@@ -413,7 +413,7 @@ OK: 3/3 Fail: 0/3 Skip: 0/3
 + returndatacopy_following_successful_create.json                 OK
 + returndatasize_following_successful_create.json                 OK
 ```
-OK: 41/44 Fail: 0/44 Skip: 3/44
+OK: 42/44 Fail: 0/44 Skip: 2/44
 ## stCreateTest
 ```diff
 + CREATE_AcreateB_BSuicide_BStore.json                            OK
@@ -1686,7 +1686,7 @@ OK: 38/38 Fail: 0/38 Skip: 0/38
 + RevertDepthCreateAddressCollision.json                          OK
 + RevertDepthCreateOOG.json                                       OK
 + RevertInCallCode.json                                           OK
-  RevertInCreateInInit.json                                       Skip
++ RevertInCreateInInit.json                                       OK
 + RevertInDelegateCall.json                                       OK
 + RevertInStaticCall.json                                         OK
 + RevertOnEmptyStack.json                                         OK
@@ -1722,10 +1722,10 @@ OK: 38/38 Fail: 0/38 Skip: 0/38
 + TouchToEmptyAccountRevert2.json                                 OK
 + TouchToEmptyAccountRevert3.json                                 OK
 ```
-OK: 40/46 Fail: 0/46 Skip: 6/46
+OK: 41/46 Fail: 0/46 Skip: 5/46
 ## stSStoreTest
 ```diff
-  InitCollision.json                                              Skip
++ InitCollision.json                                              OK
 + InitCollisionNonZeroNonce.json                                  OK
 + SstoreCallToSelfSubRefundBelowZero.json                         OK
 + sstore_0to0.json                                                OK
@@ -1755,7 +1755,7 @@ OK: 40/46 Fail: 0/46 Skip: 6/46
 + sstore_combinations_initial1.json                               OK
 + sstore_combinations_initial2.json                               OK
 ```
-OK: 28/29 Fail: 0/29 Skip: 1/29
+OK: 29/29 Fail: 0/29 Skip: 0/29
 ## stShift
 ```diff
 + sar00.json                                                      OK
@@ -2645,4 +2645,4 @@ OK: 133/133 Fail: 0/133 Skip: 0/133
 OK: 130/130 Fail: 0/130 Skip: 0/130
 
 ---TOTAL---
-OK: 2340/2447 Fail: 0/2447 Skip: 107/2447
+OK: 2343/2447 Fail: 0/2447 Skip: 104/2447

PrecompileTests.md

@@ -6,12 +6,12 @@ PrecompileTests
 + bn256mul.json                                                   OK
 + ecrecover.json                                                  OK
 + identity.json                                                   OK
-  modexp.json                                                     Skip
++ modexp.json                                                     OK
 + pairing.json                                                    OK
 + ripemd160.json                                                  OK
 + sha256.json                                                     OK
 ```
-OK: 7/8 Fail: 0/8 Skip: 1/8
+OK: 8/8 Fail: 0/8 Skip: 0/8
 
 ---TOTAL---
-OK: 7/8 Fail: 0/8 Skip: 1/8
+OK: 8/8 Fail: 0/8 Skip: 0/8

tests/test_allowed_to_fail.nim

@@ -0,0 +1,188 @@
+# Nimbus
+# Copyright (c) 2018 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+# XXX: when all but a relative few dozen, say, GeneralStateTests run, remove this,
+# but for now, this enables some CI use before that to prevent regressions. In the
+# separate file here because it would otherwise just distract. Could use all sorts
+# of O(1) or O(log n) lookup structures, or be more careful to only initialize the
+# table once, but notion's that it should shrink reasonable quickly and disappear,
+# being mostly used for short-term regression prevention.
+
+import strutils
+
+func slowGSTTests(folder: string, name: string): bool =
+  result = folder == "stQuadraticComplexityTest" or
+    name in @["randomStatetest352.json", "randomStatetest1.json",
+             "randomStatetest32.json", "randomStatetest347.json",
+             "randomStatetest393.json", "randomStatetest626.json",
+             "CALLCODE_Bounds.json", "DELEGATECALL_Bounds3.json",
+             "CALLCODE_Bounds4.json", "CALL_Bounds.json",
+             "DELEGATECALL_Bounds2.json", "CALL_Bounds3.json",
+             "CALLCODE_Bounds2.json", "CALLCODE_Bounds3.json",
+             "DELEGATECALL_Bounds.json", "CALL_Bounds2a.json",
+             "CALL_Bounds2.json",
+             "CallToNameRegistratorMemOOGAndInsufficientBalance.json",
+             "CallToNameRegistratorTooMuchMemory0.json",
+
+              # all these tests below actually pass
+              # but they are very slow
+
+              # constantinople slow tests
+              "Create2Recursive.json",
+
+              # byzantium slow tests
+              "LoopCallsDepthThenRevert3.json",
+              "LoopCallsDepthThenRevert2.json",
+              "LoopCallsDepthThenRevert.json",
+              "static_Call50000.json",
+              "static_Call50000_ecrec.json",
+              "static_Call50000_identity.json",
+              "static_Call50000_identity2.json",
+              "static_Call50000_rip160.json",
+              "static_Call50000_sha256.json",
+              "LoopCallsThenRevert.json",
+              "LoopDelegateCallsDepthThenRevert.json",
+              "recursiveCreateReturnValue.json",
+              "static_Call1024PreCalls2.json",
+              "Callcode1024BalanceTooLow.json",
+              "static_Call1024BalanceTooLow.json",
+              "static_Call1024BalanceTooLow2.json",
+              "static_Call1024OOG.json",
+              "static_Call1024PreCalls3.json",
+              "static_Call1024PreCalls.json",
+              "static_Call1MB1024Calldepth.json",
+
+              # Homestead recursives
+              "ContractCreationSpam.json",
+              "Call1024OOG.json",
+              "Call1024PreCalls.json",
+              "CallRecursiveBombPreCall.json",
+              "Delegatecall1024.json",
+              "Delegatecall1024OOG.json",
+              "JUMPDEST_Attack.json",
+              "JUMPDEST_AttackwithJump.json",
+              "ABAcalls1.json",
+              "ABAcalls2.json",
+              "CallRecursiveBomb0.json",
+              "CallRecursiveBomb0_OOG_atMaxCallDepth.json",
+              "CallRecursiveBomb1.json",
+              "CallRecursiveBomb2.json",
+              "CallRecursiveBombLog.json",
+              "CallRecursiveBombLog2.json",
+              "Call1024BalanceTooLow.json",
+
+              # Frontier recursives
+              "Callcode1024OOG.json",
+              "callcallcodecall_ABCB_RECURSIVE.json",
+              "callcallcodecallcode_ABCB_RECURSIVE.json",
+              "callcodecallcall_ABCB_RECURSIVE.json",
+              "callcodecallcallcode_ABCB_RECURSIVE.json",
+              "callcodecallcodecall_ABCB_RECURSIVE.json",
+              "callcodecallcodecallcode_ABCB_RECURSIVE.json",
+              "callcallcallcode_ABCB_RECURSIVE.json"
+              ]
+
+func failIn32Bits(folder, name: string): bool =
+  return name in @[
+    # crash with OOM
+    "static_Return50000_2.json",
+    "randomStatetest185.json",
+    "randomStatetest159.json",
+    "randomStatetest48.json",
+
+    # OOM in AppVeyor, not on my machine
+    "randomStatetest36.json"
+  ]
+
+func allowedFailingGeneralStateTest(folder, name: string): bool =
+  let allowedFailingGeneralStateTests = @[
+    # conflicts between native int and big int.
+    # gasFee calculation in modexp precompiled
+    # contracts
+    "modexp.json",
+    # perhaps a design flaw with create/create2 opcode.
+    # a conflict between balance checker and
+    # static call context checker
+    "create2noCash.json",
+
+    # Failure once spotted on Travis CI Linux AMD64:
+    # "out of memorysubtest no: 7 failed"
+    # "randomStatetest159.json",
+  ]
+  result = name in allowedFailingGeneralStateTests
+
+func allowedFailInCurrentBuild(folder, name: string): bool =
+  when sizeof(int) == 4:
+    if failIn32Bits(folder, name):
+      return true
+  return allowedFailingGeneralStateTest(folder, name)
+
+func skipGSTTests*(folder: string, name: string): bool =
+  # we skip tests that are slow or expected to fail for now
+  if slowGSTTests(folder, name):
+    return true
+  result = allowedFailInCurrentBuild(folder, name)
+
+func skipNewGSTTests*(folder: string, name: string): bool =
+  # share the same slow and failing tests
+  if skipGSTTests(folder, name):
+    return true
+
+  result = name in @[
+    # skip slow tests
+    "CALLBlake2f_MaxRounds.json",
+
+    # py-evm claims these tests are incorrect
+    # nimbus also agree
+    "RevertInCreateInInit.json",
+    "RevertInCreateInInitCreate2.json",
+    "InitCollision.json"
+  ]
+
+func skipVMTests*(folder: string, name: string): bool =
+  when sizeof(int) == 4:
+    if name == "sha3_bigSize.json":
+      return true
+  result = (folder == "vmPerformance" and "loop" in name)
+
+func skipBCTests*(folder: string, name: string): bool =
+  let allowedFailingBCTests = @[
+    # BlockChain slow tests
+    "SuicideIssue.json",
+
+    # BC OOM tests
+    "randomStatetest94.json", # pre istanbul
+
+    # pre istanbul failing
+    "SuicidesMixingCoinbase.json",
+    "suicideCoinbase.json",
+    "suicideCoinbaseState.json"
+  ]
+
+  result =  name in allowedFailingBCTests
+
+func skipNewBCTests*(folder: string, name: string): bool =
+  let allowedFailingBCTests = @[
+    # Istanbul bc tests
+    # py-evm claims these tests are incorrect
+    # nimbus also agree
+    "RevertInCreateInInit.json",
+    "RevertInCreateInInitCreate2.json",
+    "InitCollision.json",
+
+    # BC OOM
+    "static_Return50000_2.json" # istanbul
+  ]
+
+  result = name in allowedFailingBCTests
+
+func skipTxTests*(folder: string, name: string): bool =
+  # from test_transaction_json
+  when sizeof(int) == 4:
+    result = name == "RLPHeaderSizeOverflowInt32.json"
+  else:
+    false

tests/test_blockchain_json.nim

@@ -10,7 +10,8 @@ import
   options,
   eth/[common, rlp, bloom], eth/trie/[db, trie_defs],
   ethash, stew/endians2, nimcrypto,
-  ./test_helpers, ../premix/parser, test_config,
+  ./test_helpers, , ./test_allowed_to_fail,
+  ../premix/parser, test_config,
   ../nimbus/vm/interpreter/vm_forks,
   ../nimbus/[vm_state, utils, vm_types, errors, transaction, constants],
   ../nimbus/db/[db_chain, state_db],
@@ -668,9 +669,9 @@ proc main() =
   if paramCount() == 0:
     # run all test fixtures
     suite "block chain json tests":
-      jsonTest("BlockchainTests", testFixture)
+      jsonTest("BlockchainTests", testFixture, skipBCTests)
     suite "new block chain json tests":
-      jsonTest("newBlockchainTests", testFixture)
+      jsonTest("newBlockchainTests", testFixture, skipNewBCTests)
   else:
     # execute single test in debug mode
     let config = getConfiguration()

tests/test_generalstate_json.nim

@@ -9,7 +9,8 @@ import
   unittest2, strformat, strutils, tables, json, times, os,
   stew/ranges/typedranges, nimcrypto, options,
   eth/[rlp, common], eth/trie/[db, trie_defs], chronicles,
-  ./test_helpers, ../nimbus/p2p/executor, test_config,
+  ./test_helpers, ./test_allowed_to_fail,
+  ../nimbus/p2p/executor, test_config,
   ../nimbus/transaction,
   ../nimbus/[vm_state, vm_types, vm_state_transactions, utils],
   ../nimbus/vm/interpreter,
@@ -178,10 +179,10 @@ proc testFixture(fixtures: JsonNode, testStatusIMPL: var TestStatus,
 proc generalStateJsonMain*(debugMode = false) =
   if paramCount() == 0 or not debugMode:
     # run all test fixtures
-    suite "generalstate json tests":
+    #suite "generalstate json tests":
-      jsonTest("GeneralStateTests", testFixture)
+      #jsonTest("GeneralStateTests", testFixture, skipGSTTests)
     suite "new generalstate json tests":
-      jsonTest("newGeneralStateTests", testFixture)
+      jsonTest("newGeneralStateTests", testFixture, skipNewGSTTests)
   else:
     # execute single test in debug mode
     let config = getConfiguration()

tests/test_helpers.nim

@@ -10,8 +10,7 @@ import
   stew/byteutils, stew/ranges/typedranges, net, eth/[common, keys, rlp, p2p], unittest2,
   ../nimbus/[vm_state, config, transaction, utils, errors],
   ../nimbus/db/[db_chain, state_db],
-  ../nimbus/vm/interpreter/vm_forks,
+  ../nimbus/vm/interpreter/vm_forks
-  ../tests/test_generalstate_failing
 
 func revmap(x: Table[Fork, string]): Table[string, Fork] =
   result = initTable[string, Fork]()
@@ -44,112 +43,7 @@ const
 type
   Status* {.pure.} = enum OK, Fail, Skip
 
-func slowTest*(folder: string, name: string): bool =
+func skipNothing*(folder: string, name: string): bool = false
-  result =
-    (folder == "vmPerformance" and "loop" in name) or
-    folder == "stQuadraticComplexityTest" or
-    name in @["randomStatetest352.json", "randomStatetest1.json",
-             "randomStatetest32.json", "randomStatetest347.json",
-             "randomStatetest393.json", "randomStatetest626.json",
-             "CALLCODE_Bounds.json", "DELEGATECALL_Bounds3.json",
-             "CALLCODE_Bounds4.json", "CALL_Bounds.json",
-             "DELEGATECALL_Bounds2.json", "CALL_Bounds3.json",
-             "CALLCODE_Bounds2.json", "CALLCODE_Bounds3.json",
-             "DELEGATECALL_Bounds.json", "CALL_Bounds2a.json",
-             "CALL_Bounds2.json",
-             "CallToNameRegistratorMemOOGAndInsufficientBalance.json",
-             "CallToNameRegistratorTooMuchMemory0.json",
-
-              # all these tests below actually pass
-              # but they are very slow
-
-              # constantinople slow tests
-              "Create2Recursive.json",
-
-              # byzantium slow tests
-              "LoopCallsDepthThenRevert3.json",
-              "LoopCallsDepthThenRevert2.json",
-              "LoopCallsDepthThenRevert.json",
-              "static_Call50000.json",
-              "static_Call50000_ecrec.json",
-              "static_Call50000_identity.json",
-              "static_Call50000_identity2.json",
-              "static_Call50000_rip160.json",
-              "static_Call50000_sha256.json",
-              "LoopCallsThenRevert.json",
-              "LoopDelegateCallsDepthThenRevert.json",
-              "recursiveCreateReturnValue.json",
-              "static_Call1024PreCalls2.json",
-              "Callcode1024BalanceTooLow.json",
-              "static_Call1024BalanceTooLow.json",
-              "static_Call1024BalanceTooLow2.json",
-              "static_Call1024OOG.json",
-              "static_Call1024PreCalls3.json",
-              "static_Call1024PreCalls.json",
-              "static_Call1MB1024Calldepth.json",
-
-              # Homestead recursives
-              "ContractCreationSpam.json",
-              "Call1024OOG.json",
-              "Call1024PreCalls.json",
-              "CallRecursiveBombPreCall.json",
-              "Delegatecall1024.json",
-              "Delegatecall1024OOG.json",
-              "JUMPDEST_Attack.json",
-              "JUMPDEST_AttackwithJump.json",
-              "ABAcalls1.json",
-              "ABAcalls2.json",
-              "CallRecursiveBomb0.json",
-              "CallRecursiveBomb0_OOG_atMaxCallDepth.json",
-              "CallRecursiveBomb1.json",
-              "CallRecursiveBomb2.json",
-              "CallRecursiveBombLog.json",
-              "CallRecursiveBombLog2.json",
-              "Call1024BalanceTooLow.json",
-
-              # Frontier recursives
-              "Callcode1024OOG.json",
-              "callcallcodecall_ABCB_RECURSIVE.json",
-              "callcallcodecallcode_ABCB_RECURSIVE.json",
-              "callcodecallcall_ABCB_RECURSIVE.json",
-              "callcodecallcallcode_ABCB_RECURSIVE.json",
-              "callcodecallcodecall_ABCB_RECURSIVE.json",
-              "callcodecallcodecallcode_ABCB_RECURSIVE.json",
-              "callcallcallcode_ABCB_RECURSIVE.json",
-
-              # BlockChain slow tests
-              "SuicideIssue.json",
-              "CALLBlake2f_MaxRounds.json"
-              ]
-
-func failIn32Bits(folder, name: string): bool =
-  return name in @[
-    "sha3_bigSize.json", # from vm_test
-
-    # crash with OOM
-    "static_Return50000_2.json",
-    "randomStatetest185.json",
-    "randomStatetest159.json",
-    "randomStatetest48.json",
-
-    # OOM in AppVeyor, not on my machine
-    "randomStatetest36.json",
-
-    # from test_transaction_json
-    "RLPHeaderSizeOverflowInt32.json"
-  ]
-
-func allowedFailInCurrentBuild(folder, name: string): bool =
-  when sizeof(int) == 4:
-    if failIn32Bits(folder, name):
-      return true
-  return allowedFailingGeneralStateTest(folder, name)
-
-func validTest*(folder: string, name: string): bool =
-  # we skip tests that are slow or expected to fail for now
-  result =
-    not slowTest(folder, name) and
-    not allowedFailInCurrentBuild(folder, name)
 
 proc lacksSupportedForks*(fixtures: JsonNode): bool =
   # XXX: Until Nimbus supports all forks, some of the GeneralStateTests won't work.
@@ -171,7 +65,7 @@ proc lacksSupportedForks*(fixtures: JsonNode): bool =
 
 var status = initOrderedTable[string, OrderedTable[string, Status]]()
 
-macro jsonTest*(s: static[string], handler: untyped): untyped =
+macro jsonTest*(s: static[string], handler: untyped, skipTest: untyped = skipNothing): untyped =
   let
     testStatusIMPL = ident("testStatusIMPL")
     testName = ident("testName")
@@ -191,8 +85,9 @@ macro jsonTest*(s: static[string], handler: untyped): untyped =
       if not status.hasKey(last):
         status[last] = initOrderedTable[string, Status]()
       status[last][name] = Status.Skip
-      if last.validTest(name):
+      if `skipTest`(last, name):
-        filenames.add(filename)
+        continue
+      filenames.add(filename)
     for fname in filenames:
       test fname:
         {.gcsafe.}:

tests/test_persistblock_json.nim

@@ -41,4 +41,3 @@ proc testFixture(node: JsonNode, testStatusIMPL: var TestStatus) =
 proc persistBlockJsonMain*() =
   suite "persist block json tests":
     jsonTest("PersistBlockTests", testFixture)
-

tests/test_transaction_json.nim

@@ -1,7 +1,8 @@
 import
   unittest2, json, os, tables, strformat, strutils,
   eth/[common, rlp],
-  ./test_helpers, ../nimbus/[transaction, utils, errors]
+  ./test_helpers, ./test_allowed_to_fail,
+  ../nimbus/[transaction, utils, errors]
 
 const
   FIXTURE_FORK_SKIPS = ["_info", "rlp", "Constantinople"]
@@ -10,7 +11,7 @@ proc testFixture(node: JsonNode, testStatusIMPL: var TestStatus)
 
 proc transactionJsonMain*() =
   suite "Transactions tests":
-    jsonTest("TransactionTests", testFixture)
+    jsonTest("TransactionTests", testFixture, skipTxTests)
 
 when isMainModule:
   transactionJsonMain()

tests/test_vm_json.nim

@@ -8,7 +8,7 @@
 import
   unittest2, strformat, strutils, tables, json, os, times,
   stew/byteutils, stew/ranges/typedranges, eth/[rlp, common], eth/trie/db,
-  ./test_helpers, ../nimbus/vm/interpreter,
+  ./test_helpers, ./test_allowed_to_fail, ../nimbus/vm/interpreter,
   ../nimbus/[constants, vm_state, vm_types, utils],
   ../nimbus/db/[db_chain, state_db]
 
@@ -16,7 +16,7 @@ proc testFixture(fixtures: JsonNode, testStatusIMPL: var TestStatus)
 
 proc vmJsonMain*() =
   suite "vm json tests":
-    jsonTest("VMTests", testFixture)
+    jsonTest("VMTests", testFixture, skipVMTests)
 
 proc testFixture(fixtures: JsonNode, testStatusIMPL: var TestStatus) =
   var fixture: JsonNode