Add --skip option to CLI

Disassociates hardcoded tests to skip when running via CLI. Tests still
skipped when running `go test`

cmd/ethtest/main.go

@@ -28,6 +28,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/codegangsta/cli"
 	"github.com/ethereum/go-ethereum/logger/glog"
@@ -40,6 +41,7 @@ var (
 	defaultTest     = "all"
 	defaultDir      = "."
 	allTests        = []string{"BlockTests", "StateTests", "TransactionTests", "VMTests"}
+	skipTests       = []string{}
 
 	TestFlag = cli.StringFlag{
 		Name:  "test",
@@ -60,6 +62,10 @@ var (
 		Name:  "stdin",
 		Usage: "Accept input from stdin instead of reading from file",
 	}
+	SkipTestsFlag = cli.StringFlag{
+		Name:  "skip",
+		Usage: "Tests names to skip",
+	}
 )
 
 func runTestWithReader(test string, r io.Reader) error {
@@ -67,13 +73,13 @@ func runTestWithReader(test string, r io.Reader) error {
 	var err error
 	switch test {
 	case "bt", "BlockTest", "BlockTests", "BlockChainTest":
-		err = tests.RunBlockTestWithReader(r)
+		err = tests.RunBlockTestWithReader(r, skipTests)
 	case "st", "state", "StateTest", "StateTests":
-		err = tests.RunStateTestWithReader(r)
+		err = tests.RunStateTestWithReader(r, skipTests)
 	case "tx", "TransactionTest", "TransactionTests":
-		err = tests.RunTransactionTestsWithReader(r)
+		err = tests.RunTransactionTestsWithReader(r, skipTests)
 	case "vm", "VMTest", "VMTests":
-		err = tests.RunVmTestWithReader(r)
+		err = tests.RunVmTestWithReader(r, skipTests)
 	default:
 		err = fmt.Errorf("Invalid test type specified: %v", test)
 	}
@@ -174,6 +180,7 @@ func setupApp(c *cli.Context) {
 	flagFile := c.GlobalString(FileFlag.Name)
 	continueOnError = c.GlobalBool(ContinueOnErrorFlag.Name)
 	useStdIn := c.GlobalBool(ReadStdInFlag.Name)
+	skipTests = strings.Split(c.GlobalString(SkipTestsFlag.Name), " ")
 
 	if !useStdIn {
 		runSuite(flagTest, flagFile)
@@ -200,6 +207,7 @@ func main() {
 		FileFlag,
 		ContinueOnErrorFlag,
 		ReadStdInFlag,
+		SkipTestsFlag,
 	}
 
 	if err := app.Run(os.Args); err != nil {

tests/block_test.go

@@ -6,67 +6,67 @@ import (
 )
 
 func TestBcValidBlockTests(t *testing.T) {
-	err := RunBlockTest(filepath.Join(blockTestDir, "bcValidBlockTest.json"))
+	err := RunBlockTest(filepath.Join(blockTestDir, "bcValidBlockTest.json"), BlockSkipTests)
 	if err != nil {
 		t.Fatal(err)
 	}
 }
 
 func TestBcUncleTests(t *testing.T) {
-	err := RunBlockTest(filepath.Join(blockTestDir, "bcUncleTest.json"))
+	err := RunBlockTest(filepath.Join(blockTestDir, "bcUncleTest.json"), BlockSkipTests)
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = RunBlockTest(filepath.Join(blockTestDir, "bcBruncleTest.json"))
+	err = RunBlockTest(filepath.Join(blockTestDir, "bcBruncleTest.json"), BlockSkipTests)
 	if err != nil {
 		t.Fatal(err)
 	}
 }
 
 func TestBcUncleHeaderValidityTests(t *testing.T) {
-	err := RunBlockTest(filepath.Join(blockTestDir, "bcUncleHeaderValiditiy.json"))
+	err := RunBlockTest(filepath.Join(blockTestDir, "bcUncleHeaderValiditiy.json"), BlockSkipTests)
 	if err != nil {
 		t.Fatal(err)
 	}
 }
 
 func TestBcInvalidHeaderTests(t *testing.T) {
-	err := RunBlockTest(filepath.Join(blockTestDir, "bcInvalidHeaderTest.json"))
+	err := RunBlockTest(filepath.Join(blockTestDir, "bcInvalidHeaderTest.json"), BlockSkipTests)
 	if err != nil {
 		t.Fatal(err)
 	}
 }
 
 func TestBcInvalidRLPTests(t *testing.T) {
-	err := RunBlockTest(filepath.Join(blockTestDir, "bcInvalidRLPTest.json"))
+	err := RunBlockTest(filepath.Join(blockTestDir, "bcInvalidRLPTest.json"), BlockSkipTests)
 	if err != nil {
 		t.Fatal(err)
 	}
 }
 
 func TestBcRPCAPITests(t *testing.T) {
-	err := RunBlockTest(filepath.Join(blockTestDir, "bcRPC_API_Test.json"))
+	err := RunBlockTest(filepath.Join(blockTestDir, "bcRPC_API_Test.json"), BlockSkipTests)
 	if err != nil {
 		t.Fatal(err)
 	}
 }
 
 func TestBcForkBlockTests(t *testing.T) {
-	err := RunBlockTest(filepath.Join(blockTestDir, "bcForkBlockTest.json"))
+	err := RunBlockTest(filepath.Join(blockTestDir, "bcForkBlockTest.json"), BlockSkipTests)
 	if err != nil {
 		t.Fatal(err)
 	}
 }
 
 func TestBcTotalDifficulty(t *testing.T) {
-	err := RunBlockTest(filepath.Join(blockTestDir, "bcTotalDifficultyTest.json"))
+	err := RunBlockTest(filepath.Join(blockTestDir, "bcTotalDifficultyTest.json"), BlockSkipTests)
 	if err != nil {
 		t.Fatal(err)
 	}
 }
 
 func TestBcWallet(t *testing.T) {
-	err := RunBlockTest(filepath.Join(blockTestDir, "bcWalletTest.json"))
+	err := RunBlockTest(filepath.Join(blockTestDir, "bcWalletTest.json"), BlockSkipTests)
 	if err != nil {
 		t.Fatal(err)
 	}

tests/block_test_util.go

@@ -86,7 +86,7 @@ type btTransaction struct {
 	Value    string
 }
 
-func RunBlockTestWithReader(r io.Reader) error {
+func RunBlockTestWithReader(r io.Reader, skipTests []string) error {
 	btjs := make(map[string]*btJSON)
 	if err := readJson(r, &btjs); err != nil {
 		return err
@@ -97,13 +97,13 @@ func RunBlockTestWithReader(r io.Reader) error {
 		return err
 	}
 
-	if err := runBlockTests(bt); err != nil {
+	if err := runBlockTests(bt, skipTests); err != nil {
 		return err
 	}
 	return nil
 }
 
-func RunBlockTest(file string) error {
+func RunBlockTest(file string, skipTests []string) error {
 	btjs := make(map[string]*btJSON)
 	if err := readJsonFile(file, &btjs); err != nil {
 		return err
@@ -113,15 +113,15 @@ func RunBlockTest(file string) error {
 	if err != nil {
 		return err
 	}
-	if err := runBlockTests(bt); err != nil {
+	if err := runBlockTests(bt, skipTests); err != nil {
 		return err
 	}
 	return nil
 }
 
-func runBlockTests(bt map[string]*BlockTest) error {
+func runBlockTests(bt map[string]*BlockTest, skipTests []string) error {
-	skipTest := make(map[string]bool, len(BlockSkipTests))
+	skipTest := make(map[string]bool, len(skipTests))
-	for _, name := range BlockSkipTests {
+	for _, name := range skipTests {
 		skipTest[name] = true
 	}
 

tests/state_test.go

@@ -8,84 +8,84 @@ import (
 
 func TestStateSystemOperations(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stSystemOperationsTest.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestStateExample(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stExample.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestStatePreCompiledContracts(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stPreCompiledContracts.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestStateRecursiveCreate(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stRecursiveCreate.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestStateSpecial(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stSpecialTest.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestStateRefund(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stRefundTest.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestStateBlockHash(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stBlockHashTest.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestStateInitCode(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stInitCodeTest.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestStateLog(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stLogTests.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestStateTransaction(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stTransactionTest.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestCallCreateCallCode(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stCallCreateCallCodeTest.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestMemory(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stMemoryTest.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
@@ -95,7 +95,7 @@ func TestMemoryStress(t *testing.T) {
 		t.Skip()
 	}
 	fn := filepath.Join(stateTestDir, "stMemoryStressTest.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
@@ -105,21 +105,21 @@ func TestQuadraticComplexity(t *testing.T) {
 		t.Skip()
 	}
 	fn := filepath.Join(stateTestDir, "stQuadraticComplexityTest.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestSolidity(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stSolidityTest.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestWallet(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stWalletTest.json")
-	if err := RunStateTest(fn); err != nil {
+	if err := RunStateTest(fn, StateSkipTests); err != nil {
 		t.Error(err)
 	}
 }
@@ -127,7 +127,7 @@ func TestWallet(t *testing.T) {
 func TestStateTestsRandom(t *testing.T) {
 	fns, _ := filepath.Glob("./files/StateTests/RandomTests/*")
 	for _, fn := range fns {
-		if err := RunStateTest(fn); err != nil {
+		if err := RunStateTest(fn, StateSkipTests); err != nil {
 			t.Error(err)
 		}
 	}

tests/state_test_util.go

@@ -16,26 +16,26 @@ import (
 	"github.com/ethereum/go-ethereum/logger/glog"
 )
 
-func RunStateTestWithReader(r io.Reader) error {
+func RunStateTestWithReader(r io.Reader, skipTests []string) error {
 	tests := make(map[string]VmTest)
 	if err := readJson(r, &tests); err != nil {
 		return err
 	}
 
-	if err := runStateTests(tests); err != nil {
+	if err := runStateTests(tests, skipTests); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func RunStateTest(p string) error {
+func RunStateTest(p string, skipTests []string) error {
 	tests := make(map[string]VmTest)
 	if err := readJsonFile(p, &tests); err != nil {
 		return err
 	}
 
-	if err := runStateTests(tests); err != nil {
+	if err := runStateTests(tests, skipTests); err != nil {
 		return err
 	}
 
@@ -43,9 +43,9 @@ func RunStateTest(p string) error {
 
 }
 
-func runStateTests(tests map[string]VmTest) error {
+func runStateTests(tests map[string]VmTest, skipTests []string) error {
-	skipTest := make(map[string]bool, len(StateSkipTests))
+	skipTest := make(map[string]bool, len(skipTests))
-	for _, name := range StateSkipTests {
+	for _, name := range skipTests {
 		skipTest[name] = true
 	}
 

tests/transaction_test.go

@@ -6,21 +6,21 @@ import (
 )
 
 func TestTransactions(t *testing.T) {
-	err := RunTransactionTests(filepath.Join(transactionTestDir, "ttTransactionTest.json"))
+	err := RunTransactionTests(filepath.Join(transactionTestDir, "ttTransactionTest.json"), TransSkipTests)
 	if err != nil {
 		t.Fatal(err)
 	}
 }
 
 func TestWrongRLPTransactions(t *testing.T) {
-	err := RunTransactionTests(filepath.Join(transactionTestDir, "ttWrongRLPTransaction.json"))
+	err := RunTransactionTests(filepath.Join(transactionTestDir, "ttWrongRLPTransaction.json"), TransSkipTests)
 	if err != nil {
 		t.Fatal(err)
 	}
 }
 
 func Test10MBtx(t *testing.T) {
-	err := RunTransactionTests(filepath.Join(transactionTestDir, "tt10mbDataField.json"))
+	err := RunTransactionTests(filepath.Join(transactionTestDir, "tt10mbDataField.json"), TransSkipTests)
 	if err != nil {
 		t.Fatal(err)
 	}

tests/transaction_test_util.go

@@ -32,9 +32,9 @@ type TransactionTest struct {
 	Transaction TtTransaction
 }
 
-func RunTransactionTestsWithReader(r io.Reader) error {
+func RunTransactionTestsWithReader(r io.Reader, skipTests []string) error {
-	skipTest := make(map[string]bool, len(TransSkipTests))
+	skipTest := make(map[string]bool, len(skipTests))
-	for _, name := range TransSkipTests {
+	for _, name := range skipTests {
 		skipTest[name] = true
 	}
 
@@ -59,18 +59,25 @@ func RunTransactionTestsWithReader(r io.Reader) error {
 	return nil
 }
 
-func RunTransactionTests(file string) error {
+func RunTransactionTests(file string, skipTests []string) error {
-	skipTest := make(map[string]bool, len(TransSkipTests))
+	tests := make(map[string]TransactionTest)
-	for _, name := range TransSkipTests {
+	if err := readJsonFile(file, &tests); err != nil {
-		skipTest[name] = true
-	}
-
-	bt := make(map[string]TransactionTest)
-	if err := readJsonFile(file, &bt); err != nil {
 		return err
 	}
 
-	for name, test := range bt {
+	if err := runTransactionTests(tests, skipTests); err != nil {
+		return err
+	}
+	return nil
+}
+
+func runTransactionTests(tests map[string]TransactionTest, skipTests []string) error {
+	skipTest := make(map[string]bool, len(skipTests))
+	for _, name := range skipTests {
+		skipTest[name] = true
+	}
+
+	for name, test := range tests {
 		// if the test should be skipped, return
 		if skipTest[name] {
 			glog.Infoln("Skipping transaction test", name)

tests/vm_test.go

@@ -8,91 +8,91 @@ import (
 // I've created a new function for each tests so it's easier to identify where the problem lies if any of them fail.
 func TestVMArithmetic(t *testing.T) {
 	fn := filepath.Join(vmTestDir, "vmArithmeticTest.json")
-	if err := RunVmTest(fn); err != nil {
+	if err := RunVmTest(fn, VmSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestBitwiseLogicOperation(t *testing.T) {
 	fn := filepath.Join(vmTestDir, "vmBitwiseLogicOperationTest.json")
-	if err := RunVmTest(fn); err != nil {
+	if err := RunVmTest(fn, VmSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestBlockInfo(t *testing.T) {
 	fn := filepath.Join(vmTestDir, "vmBlockInfoTest.json")
-	if err := RunVmTest(fn); err != nil {
+	if err := RunVmTest(fn, VmSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestEnvironmentalInfo(t *testing.T) {
 	fn := filepath.Join(vmTestDir, "vmEnvironmentalInfoTest.json")
-	if err := RunVmTest(fn); err != nil {
+	if err := RunVmTest(fn, VmSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestFlowOperation(t *testing.T) {
 	fn := filepath.Join(vmTestDir, "vmIOandFlowOperationsTest.json")
-	if err := RunVmTest(fn); err != nil {
+	if err := RunVmTest(fn, VmSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestLogTest(t *testing.T) {
 	fn := filepath.Join(vmTestDir, "vmLogTest.json")
-	if err := RunVmTest(fn); err != nil {
+	if err := RunVmTest(fn, VmSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestPerformance(t *testing.T) {
 	fn := filepath.Join(vmTestDir, "vmPerformanceTest.json")
-	if err := RunVmTest(fn); err != nil {
+	if err := RunVmTest(fn, VmSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestPushDupSwap(t *testing.T) {
 	fn := filepath.Join(vmTestDir, "vmPushDupSwapTest.json")
-	if err := RunVmTest(fn); err != nil {
+	if err := RunVmTest(fn, VmSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestVMSha3(t *testing.T) {
 	fn := filepath.Join(vmTestDir, "vmSha3Test.json")
-	if err := RunVmTest(fn); err != nil {
+	if err := RunVmTest(fn, VmSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestVm(t *testing.T) {
 	fn := filepath.Join(vmTestDir, "vmtests.json")
-	if err := RunVmTest(fn); err != nil {
+	if err := RunVmTest(fn, VmSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestVmLog(t *testing.T) {
 	fn := filepath.Join(vmTestDir, "vmLogTest.json")
-	if err := RunVmTest(fn); err != nil {
+	if err := RunVmTest(fn, VmSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestInputLimits(t *testing.T) {
 	fn := filepath.Join(vmTestDir, "vmInputLimits.json")
-	if err := RunVmTest(fn); err != nil {
+	if err := RunVmTest(fn, VmSkipTests); err != nil {
 		t.Error(err)
 	}
 }
 
 func TestInputLimitsLight(t *testing.T) {
 	fn := filepath.Join(vmTestDir, "vmInputLimitsLight.json")
-	if err := RunVmTest(fn); err != nil {
+	if err := RunVmTest(fn, VmSkipTests); err != nil {
 		t.Error(err)
 	}
 }
@@ -100,7 +100,7 @@ func TestInputLimitsLight(t *testing.T) {
 func TestVMRandom(t *testing.T) {
 	fns, _ := filepath.Glob(filepath.Join(baseDir, "RandomTests", "*"))
 	for _, fn := range fns {
-		if err := RunVmTest(fn); err != nil {
+		if err := RunVmTest(fn, VmSkipTests); err != nil {
 			t.Error(err)
 		}
 	}

tests/vm_test_util.go

@@ -14,7 +14,7 @@ import (
 	"github.com/ethereum/go-ethereum/logger/glog"
 )
 
-func RunVmTestWithReader(r io.Reader) error {
+func RunVmTestWithReader(r io.Reader, skipTests []string) error {
 	tests := make(map[string]VmTest)
 	err := readJson(r, &tests)
 	if err != nil {
@@ -25,14 +25,14 @@ func RunVmTestWithReader(r io.Reader) error {
 		return err
 	}
 
-	if err := runVmTests(tests); err != nil {
+	if err := runVmTests(tests, skipTests); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func RunVmTest(p string) error {
+func RunVmTest(p string, skipTests []string) error {
 
 	tests := make(map[string]VmTest)
 	err := readJsonFile(p, &tests)
@@ -40,16 +40,16 @@ func RunVmTest(p string) error {
 		return err
 	}
 
-	if err := runVmTests(tests); err != nil {
+	if err := runVmTests(tests, skipTests); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func runVmTests(tests map[string]VmTest) error {
+func runVmTests(tests map[string]VmTest, skipTests []string) error {
-	skipTest := make(map[string]bool, len(VmSkipTests))
+	skipTest := make(map[string]bool, len(skipTests))
-	for _, name := range VmSkipTests {
+	for _, name := range skipTests {
 		skipTest[name] = true
 	}