status-im
/
nimbus-eth1
mirror of https://github.com/status-im/nimbus-eth1.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #423 from status-im/tester_skip_table 

fixes #421, each test suite have their own skip table
This commit is contained in:
andri lim 2019-12-06 21:37:41 +07:00 committed by GitHub
parent 1e54e2537b 3683ce2cbc
commit a70c9d5e10
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 218 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
GeneralStateTests.md
View File

@ -387,7 +387,7 @@ OK: 3/3 Fail: 0/3 Skip: 0/3
+ CreateMessageRevertedOOGInInit.json OK
+ RevertDepthCreate2OOG.json OK
+ RevertDepthCreateAddressCollision.json OK
RevertInCreateInInitCreate2.json Skip
+ RevertInCreateInInitCreate2.json OK
+ RevertOpcodeCreate.json OK
+ RevertOpcodeInCreateReturnsCreate2.json OK
+ call_outsize_then_create2_successful_then_returndatasize.json OK
@ -413,7 +413,7 @@ OK: 3/3 Fail: 0/3 Skip: 0/3
+ returndatacopy_following_successful_create.json OK
+ returndatasize_following_successful_create.json OK
```
OK: 41/44 Fail: 0/44 Skip: 3/44
OK: 42/44 Fail: 0/44 Skip: 2/44
## stCreateTest
```diff
+ CREATE_AcreateB_BSuicide_BStore.json OK
@ -1686,7 +1686,7 @@ OK: 38/38 Fail: 0/38 Skip: 0/38
+ RevertDepthCreateAddressCollision.json OK
+ RevertDepthCreateOOG.json OK
+ RevertInCallCode.json OK
RevertInCreateInInit.json Skip
+ RevertInCreateInInit.json OK
+ RevertInDelegateCall.json OK
+ RevertInStaticCall.json OK
+ RevertOnEmptyStack.json OK
@ -1722,10 +1722,10 @@ OK: 38/38 Fail: 0/38 Skip: 0/38
+ TouchToEmptyAccountRevert2.json OK
+ TouchToEmptyAccountRevert3.json OK
```
OK: 40/46 Fail: 0/46 Skip: 6/46
OK: 41/46 Fail: 0/46 Skip: 5/46
## stSStoreTest
```diff
InitCollision.json Skip
+ InitCollision.json OK
+ InitCollisionNonZeroNonce.json OK
+ SstoreCallToSelfSubRefundBelowZero.json OK
+ sstore_0to0.json OK
@ -1755,7 +1755,7 @@ OK: 40/46 Fail: 0/46 Skip: 6/46
+ sstore_combinations_initial1.json OK
+ sstore_combinations_initial2.json OK
```
OK: 28/29 Fail: 0/29 Skip: 1/29
OK: 29/29 Fail: 0/29 Skip: 0/29
## stShift
```diff
+ sar00.json OK
@ -2645,4 +2645,4 @@ OK: 133/133 Fail: 0/133 Skip: 0/133
OK: 130/130 Fail: 0/130 Skip: 0/130
---TOTAL---
OK: 2340/2447 Fail: 0/2447 Skip: 107/2447
OK: 2343/2447 Fail: 0/2447 Skip: 104/2447

6
PrecompileTests.md
View File

@ -6,12 +6,12 @@ PrecompileTests
+ bn256mul.json OK
+ ecrecover.json OK
+ identity.json OK
modexp.json Skip
+ modexp.json OK
+ pairing.json OK
+ ripemd160.json OK
+ sha256.json OK
```
OK: 7/8 Fail: 0/8 Skip: 1/8
OK: 8/8 Fail: 0/8 Skip: 0/8
---TOTAL---
OK: 7/8 Fail: 0/8 Skip: 1/8
OK: 8/8 Fail: 0/8 Skip: 0/8

188
tests/test_allowed_to_fail.nim Normal file
View File

@ -0,0 +1,188 @@
# Nimbus
# Copyright (c) 2018 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed except according to those terms.
# XXX: when all but a relative few dozen, say, GeneralStateTests run, remove this,
# but for now, this enables some CI use before that to prevent regressions. In the
# separate file here because it would otherwise just distract. Could use all sorts
# of O(1) or O(log n) lookup structures, or be more careful to only initialize the
# table once, but notion's that it should shrink reasonable quickly and disappear,
# being mostly used for short-term regression prevention.
import strutils
func slowGSTTests(folder: string, name: string): bool =
result = folder == "stQuadraticComplexityTest" or
name in @["randomStatetest352.json", "randomStatetest1.json",
"randomStatetest32.json", "randomStatetest347.json",
"randomStatetest393.json", "randomStatetest626.json",
"CALLCODE_Bounds.json", "DELEGATECALL_Bounds3.json",
"CALLCODE_Bounds4.json", "CALL_Bounds.json",
"DELEGATECALL_Bounds2.json", "CALL_Bounds3.json",
"CALLCODE_Bounds2.json", "CALLCODE_Bounds3.json",
"DELEGATECALL_Bounds.json", "CALL_Bounds2a.json",
"CALL_Bounds2.json",
"CallToNameRegistratorMemOOGAndInsufficientBalance.json",
"CallToNameRegistratorTooMuchMemory0.json",
# all these tests below actually pass
# but they are very slow
# constantinople slow tests
"Create2Recursive.json",
# byzantium slow tests
"LoopCallsDepthThenRevert3.json",
"LoopCallsDepthThenRevert2.json",
"LoopCallsDepthThenRevert.json",
"static_Call50000.json",
"static_Call50000_ecrec.json",
"static_Call50000_identity.json",
"static_Call50000_identity2.json",
"static_Call50000_rip160.json",
"static_Call50000_sha256.json",
"LoopCallsThenRevert.json",
"LoopDelegateCallsDepthThenRevert.json",
"recursiveCreateReturnValue.json",
"static_Call1024PreCalls2.json",
"Callcode1024BalanceTooLow.json",
"static_Call1024BalanceTooLow.json",
"static_Call1024BalanceTooLow2.json",
"static_Call1024OOG.json",
"static_Call1024PreCalls3.json",
"static_Call1024PreCalls.json",
"static_Call1MB1024Calldepth.json",
# Homestead recursives
"ContractCreationSpam.json",
"Call1024OOG.json",
"Call1024PreCalls.json",
"CallRecursiveBombPreCall.json",
"Delegatecall1024.json",
"Delegatecall1024OOG.json",
"JUMPDEST_Attack.json",
"JUMPDEST_AttackwithJump.json",
"ABAcalls1.json",
"ABAcalls2.json",
"CallRecursiveBomb0.json",
"CallRecursiveBomb0_OOG_atMaxCallDepth.json",
"CallRecursiveBomb1.json",
"CallRecursiveBomb2.json",
"CallRecursiveBombLog.json",
"CallRecursiveBombLog2.json",
"Call1024BalanceTooLow.json",
# Frontier recursives
"Callcode1024OOG.json",
"callcallcodecall_ABCB_RECURSIVE.json",
"callcallcodecallcode_ABCB_RECURSIVE.json",
"callcodecallcall_ABCB_RECURSIVE.json",
"callcodecallcallcode_ABCB_RECURSIVE.json",
"callcodecallcodecall_ABCB_RECURSIVE.json",
"callcodecallcodecallcode_ABCB_RECURSIVE.json",
"callcallcallcode_ABCB_RECURSIVE.json"
]
func failIn32Bits(folder, name: string): bool =
return name in @[
# crash with OOM
"static_Return50000_2.json",
"randomStatetest185.json",
"randomStatetest159.json",
"randomStatetest48.json",
# OOM in AppVeyor, not on my machine
"randomStatetest36.json"
]
func allowedFailingGeneralStateTest(folder, name: string): bool =
let allowedFailingGeneralStateTests = @[
# conflicts between native int and big int.
# gasFee calculation in modexp precompiled
# contracts
"modexp.json",
# perhaps a design flaw with create/create2 opcode.
# a conflict between balance checker and
# static call context checker
"create2noCash.json",
# Failure once spotted on Travis CI Linux AMD64:
# "out of memorysubtest no: 7 failed"
# "randomStatetest159.json",
]
result = name in allowedFailingGeneralStateTests
func allowedFailInCurrentBuild(folder, name: string): bool =
when sizeof(int) == 4:
if failIn32Bits(folder, name):
return true
return allowedFailingGeneralStateTest(folder, name)
func skipGSTTests*(folder: string, name: string): bool =
# we skip tests that are slow or expected to fail for now
if slowGSTTests(folder, name):
return true
result = allowedFailInCurrentBuild(folder, name)
func skipNewGSTTests*(folder: string, name: string): bool =
# share the same slow and failing tests
if skipGSTTests(folder, name):
return true
result = name in @[
# skip slow tests
"CALLBlake2f_MaxRounds.json",
# py-evm claims these tests are incorrect
# nimbus also agree
"RevertInCreateInInit.json",
"RevertInCreateInInitCreate2.json",
"InitCollision.json"
]
func skipVMTests*(folder: string, name: string): bool =
when sizeof(int) == 4:
if name == "sha3_bigSize.json":
return true
result = (folder == "vmPerformance" and "loop" in name)
func skipBCTests*(folder: string, name: string): bool =
let allowedFailingBCTests = @[
# BlockChain slow tests
"SuicideIssue.json",
# BC OOM tests
"randomStatetest94.json", # pre istanbul
# pre istanbul failing
"SuicidesMixingCoinbase.json",
"suicideCoinbase.json",
"suicideCoinbaseState.json"
]
result = name in allowedFailingBCTests
func skipNewBCTests*(folder: string, name: string): bool =
let allowedFailingBCTests = @[
# Istanbul bc tests
# py-evm claims these tests are incorrect
# nimbus also agree
"RevertInCreateInInit.json",
"RevertInCreateInInitCreate2.json",
"InitCollision.json",
# BC OOM
"static_Return50000_2.json" # istanbul
]
result = name in allowedFailingBCTests
func skipTxTests*(folder: string, name: string): bool =
# from test_transaction_json
when sizeof(int) == 4:
result = name == "RLPHeaderSizeOverflowInt32.json"
else:
false

7
tests/test_blockchain_json.nim
View File

@ -10,7 +10,8 @@ import
options,
eth/[common, rlp, bloom], eth/trie/[db, trie_defs],
ethash, stew/endians2, nimcrypto,
./test_helpers, ../premix/parser, test_config,
./test_helpers, , ./test_allowed_to_fail,
../premix/parser, test_config,
../nimbus/vm/interpreter/vm_forks,
../nimbus/[vm_state, utils, vm_types, errors, transaction, constants],
../nimbus/db/[db_chain, state_db],
@ -668,9 +669,9 @@ proc main() =
if paramCount() == 0:
# run all test fixtures
suite "block chain json tests":
jsonTest("BlockchainTests", testFixture)
jsonTest("BlockchainTests", testFixture, skipBCTests)
suite "new block chain json tests":
jsonTest("newBlockchainTests", testFixture)
jsonTest("newBlockchainTests", testFixture, skipNewBCTests)
else:
# execute single test in debug mode
let config = getConfiguration()

9
tests/test_generalstate_json.nim
View File

@ -9,7 +9,8 @@ import
unittest2, strformat, strutils, tables, json, times, os,
stew/ranges/typedranges, nimcrypto, options,
eth/[rlp, common], eth/trie/[db, trie_defs], chronicles,
./test_helpers, ../nimbus/p2p/executor, test_config,
./test_helpers, ./test_allowed_to_fail,
../nimbus/p2p/executor, test_config,
../nimbus/transaction,
../nimbus/[vm_state, vm_types, vm_state_transactions, utils],
../nimbus/vm/interpreter,
@ -178,10 +179,10 @@ proc testFixture(fixtures: JsonNode, testStatusIMPL: var TestStatus,
proc generalStateJsonMain*(debugMode = false) =
if paramCount() == 0 or not debugMode:
# run all test fixtures
suite "generalstate json tests":
jsonTest("GeneralStateTests", testFixture)
#suite "generalstate json tests":
#jsonTest("GeneralStateTests", testFixture, skipGSTTests)
suite "new generalstate json tests":
jsonTest("newGeneralStateTests", testFixture)
jsonTest("newGeneralStateTests", testFixture, skipNewGSTTests)
else:
# execute single test in debug mode
let config = getConfiguration()

117
tests/test_helpers.nim
View File

@ -10,8 +10,7 @@ import
stew/byteutils, stew/ranges/typedranges, net, eth/[common, keys, rlp, p2p], unittest2,
../nimbus/[vm_state, config, transaction, utils, errors],
../nimbus/db/[db_chain, state_db],
../nimbus/vm/interpreter/vm_forks,
../tests/test_generalstate_failing
../nimbus/vm/interpreter/vm_forks
func revmap(x: Table[Fork, string]): Table[string, Fork] =
result = initTable[string, Fork]()
@ -44,112 +43,7 @@ const
type
Status* {.pure.} = enum OK, Fail, Skip
func slowTest*(folder: string, name: string): bool =
result =
(folder == "vmPerformance" and "loop" in name) or
folder == "stQuadraticComplexityTest" or
name in @["randomStatetest352.json", "randomStatetest1.json",
"randomStatetest32.json", "randomStatetest347.json",
"randomStatetest393.json", "randomStatetest626.json",
"CALLCODE_Bounds.json", "DELEGATECALL_Bounds3.json",
"CALLCODE_Bounds4.json", "CALL_Bounds.json",
"DELEGATECALL_Bounds2.json", "CALL_Bounds3.json",
"CALLCODE_Bounds2.json", "CALLCODE_Bounds3.json",
"DELEGATECALL_Bounds.json", "CALL_Bounds2a.json",
"CALL_Bounds2.json",
"CallToNameRegistratorMemOOGAndInsufficientBalance.json",
"CallToNameRegistratorTooMuchMemory0.json",
# all these tests below actually pass
# but they are very slow
# constantinople slow tests
"Create2Recursive.json",
# byzantium slow tests
"LoopCallsDepthThenRevert3.json",
"LoopCallsDepthThenRevert2.json",
"LoopCallsDepthThenRevert.json",
"static_Call50000.json",
"static_Call50000_ecrec.json",
"static_Call50000_identity.json",
"static_Call50000_identity2.json",
"static_Call50000_rip160.json",
"static_Call50000_sha256.json",
"LoopCallsThenRevert.json",
"LoopDelegateCallsDepthThenRevert.json",
"recursiveCreateReturnValue.json",
"static_Call1024PreCalls2.json",
"Callcode1024BalanceTooLow.json",
"static_Call1024BalanceTooLow.json",
"static_Call1024BalanceTooLow2.json",
"static_Call1024OOG.json",
"static_Call1024PreCalls3.json",
"static_Call1024PreCalls.json",
"static_Call1MB1024Calldepth.json",
# Homestead recursives
"ContractCreationSpam.json",
"Call1024OOG.json",
"Call1024PreCalls.json",
"CallRecursiveBombPreCall.json",
"Delegatecall1024.json",
"Delegatecall1024OOG.json",
"JUMPDEST_Attack.json",
"JUMPDEST_AttackwithJump.json",
"ABAcalls1.json",
"ABAcalls2.json",
"CallRecursiveBomb0.json",
"CallRecursiveBomb0_OOG_atMaxCallDepth.json",
"CallRecursiveBomb1.json",
"CallRecursiveBomb2.json",
"CallRecursiveBombLog.json",
"CallRecursiveBombLog2.json",
"Call1024BalanceTooLow.json",
# Frontier recursives
"Callcode1024OOG.json",
"callcallcodecall_ABCB_RECURSIVE.json",
"callcallcodecallcode_ABCB_RECURSIVE.json",
"callcodecallcall_ABCB_RECURSIVE.json",
"callcodecallcallcode_ABCB_RECURSIVE.json",
"callcodecallcodecall_ABCB_RECURSIVE.json",
"callcodecallcodecallcode_ABCB_RECURSIVE.json",
"callcallcallcode_ABCB_RECURSIVE.json",
# BlockChain slow tests
"SuicideIssue.json",
"CALLBlake2f_MaxRounds.json"
]
func failIn32Bits(folder, name: string): bool =
return name in @[
"sha3_bigSize.json", # from vm_test
# crash with OOM
"static_Return50000_2.json",
"randomStatetest185.json",
"randomStatetest159.json",
"randomStatetest48.json",
# OOM in AppVeyor, not on my machine
"randomStatetest36.json",
# from test_transaction_json
"RLPHeaderSizeOverflowInt32.json"
]
func allowedFailInCurrentBuild(folder, name: string): bool =
when sizeof(int) == 4:
if failIn32Bits(folder, name):
return true
return allowedFailingGeneralStateTest(folder, name)
func validTest*(folder: string, name: string): bool =
# we skip tests that are slow or expected to fail for now
result =
not slowTest(folder, name) and
not allowedFailInCurrentBuild(folder, name)
func skipNothing*(folder: string, name: string): bool = false
proc lacksSupportedForks*(fixtures: JsonNode): bool =
# XXX: Until Nimbus supports all forks, some of the GeneralStateTests won't work.
@ -171,7 +65,7 @@ proc lacksSupportedForks*(fixtures: JsonNode): bool =
var status = initOrderedTable[string, OrderedTable[string, Status]]()
macro jsonTest*(s: static[string], handler: untyped): untyped =
macro jsonTest*(s: static[string], handler: untyped, skipTest: untyped = skipNothing): untyped =
let
testStatusIMPL = ident("testStatusIMPL")
testName = ident("testName")
@ -191,8 +85,9 @@ macro jsonTest*(s: static[string], handler: untyped): untyped =
if not status.hasKey(last):
status[last] = initOrderedTable[string, Status]()
status[last][name] = Status.Skip
if last.validTest(name):
filenames.add(filename)
if `skipTest`(last, name):
continue
filenames.add(filename)
for fname in filenames:
test fname:
{.gcsafe.}:

1
tests/test_persistblock_json.nim
View File

@ -41,4 +41,3 @@ proc testFixture(node: JsonNode, testStatusIMPL: var TestStatus) =
proc persistBlockJsonMain*() =
suite "persist block json tests":
jsonTest("PersistBlockTests", testFixture)

5
tests/test_transaction_json.nim
View File

@ -1,7 +1,8 @@
import
unittest2, json, os, tables, strformat, strutils,
eth/[common, rlp],
./test_helpers, ../nimbus/[transaction, utils, errors]
./test_helpers, ./test_allowed_to_fail,
../nimbus/[transaction, utils, errors]
const
FIXTURE_FORK_SKIPS = ["_info", "rlp", "Constantinople"]
@ -10,7 +11,7 @@ proc testFixture(node: JsonNode, testStatusIMPL: var TestStatus)
proc transactionJsonMain*() =
suite "Transactions tests":
jsonTest("TransactionTests", testFixture)
jsonTest("TransactionTests", testFixture, skipTxTests)
when isMainModule:
transactionJsonMain()

4
tests/test_vm_json.nim
View File

@ -8,7 +8,7 @@
import
unittest2, strformat, strutils, tables, json, os, times,
stew/byteutils, stew/ranges/typedranges, eth/[rlp, common], eth/trie/db,
./test_helpers, ../nimbus/vm/interpreter,
./test_helpers, ./test_allowed_to_fail, ../nimbus/vm/interpreter,
../nimbus/[constants, vm_state, vm_types, utils],
../nimbus/db/[db_chain, state_db]
@ -16,7 +16,7 @@ proc testFixture(fixtures: JsonNode, testStatusIMPL: var TestStatus)
proc vmJsonMain*() =
suite "vm json tests":
jsonTest("VMTests", testFixture)
jsonTest("VMTests", testFixture, skipVMTests)
proc testFixture(fixtures: JsonNode, testStatusIMPL: var TestStatus) =
var fixture: JsonNode