Merge pull request #232 from faizkhan00/freezehash
Add per-commit testing
This commit is contained in:
commit
4d55f020dd
|
@ -31,6 +31,13 @@ public class JSONReader {
|
||||||
return json.isEmpty() ? getFromLocal(filename) : json;
|
return json.isEmpty() ? getFromLocal(filename) : json;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static String loadJSONFromCommit(String filename, String shacommit) {
|
||||||
|
String json = "";
|
||||||
|
if (!SystemProperties.CONFIG.vmTestLoadLocal())
|
||||||
|
json = getFromUrl("https://raw.githubusercontent.com/ethereum/tests/" + shacommit + "/" + filename);
|
||||||
|
return json.isEmpty() ? getFromLocal(filename) : json;
|
||||||
|
}
|
||||||
|
|
||||||
public static String getFromLocal(String filename) {
|
public static String getFromLocal(String filename) {
|
||||||
System.out.println("Loading local file: " + filename);
|
System.out.println("Loading local file: " + filename);
|
||||||
try {
|
try {
|
||||||
|
|
|
@ -15,10 +15,13 @@ import java.util.Set;
|
||||||
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
|
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
|
||||||
public class GitHubStateTest {
|
public class GitHubStateTest {
|
||||||
|
|
||||||
|
//SHACOMMIT of tested commit, ethereum/tests.git
|
||||||
|
public String shacommit = "cfb120d1793dbb14404d9991f67cfb16bf573887";
|
||||||
|
|
||||||
@Ignore
|
@Ignore
|
||||||
@Test // this method is mostly for hands-on convenient testing
|
@Test // this method is mostly for hands-on convenient testing
|
||||||
public void stSingleTest() throws ParseException {
|
public void stSingleTest() throws ParseException {
|
||||||
String json = JSONReader.loadJSON("StateTests/stSystemOperationsTest.json");
|
String json = JSONReader.loadJSONFromCommit("StateTests/stSystemOperationsTest.json", shacommit);
|
||||||
GitHubJSONTestSuite.runGitHubJsonStateTest(json, "CallRecursiveBombLog2");
|
GitHubJSONTestSuite.runGitHubJsonStateTest(json, "CallRecursiveBombLog2");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -28,14 +31,14 @@ public class GitHubStateTest {
|
||||||
Set<String> excluded = new HashSet<>();
|
Set<String> excluded = new HashSet<>();
|
||||||
excluded.add("CallRipemd160_5");
|
excluded.add("CallRipemd160_5");
|
||||||
excluded.add("CallSha256_5");
|
excluded.add("CallSha256_5");
|
||||||
String json = JSONReader.loadJSON("StateTests/stPreCompiledContracts.json");
|
String json = JSONReader.loadJSONFromCommit("StateTests/stPreCompiledContracts.json", shacommit);
|
||||||
GitHubJSONTestSuite.runGitHubJsonStateTest(json, excluded);
|
GitHubJSONTestSuite.runGitHubJsonStateTest(json, excluded);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void stExample() throws ParseException { // [V]
|
public void stExample() throws ParseException { // [V]
|
||||||
|
|
||||||
String json = JSONReader.loadJSON("StateTests/stExample.json");
|
String json = JSONReader.loadJSONFromCommit("StateTests/stExample.json", shacommit);
|
||||||
GitHubJSONTestSuite.runGitHubJsonStateTest(json);
|
GitHubJSONTestSuite.runGitHubJsonStateTest(json);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -43,14 +46,14 @@ public class GitHubStateTest {
|
||||||
public void stInitCodeTest() throws ParseException { // [V]
|
public void stInitCodeTest() throws ParseException { // [V]
|
||||||
|
|
||||||
Set<String> excluded = new HashSet<>();
|
Set<String> excluded = new HashSet<>();
|
||||||
String json = JSONReader.loadJSON("StateTests/stInitCodeTest.json");
|
String json = JSONReader.loadJSONFromCommit("StateTests/stInitCodeTest.json", shacommit);
|
||||||
GitHubJSONTestSuite.runGitHubJsonStateTest(json, excluded);
|
GitHubJSONTestSuite.runGitHubJsonStateTest(json, excluded);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void stLogTests() throws ParseException { // [V]
|
public void stLogTests() throws ParseException { // [V]
|
||||||
|
|
||||||
String json = JSONReader.loadJSON("StateTests/stLogTests.json");
|
String json = JSONReader.loadJSONFromCommit("StateTests/stLogTests.json", shacommit);
|
||||||
GitHubJSONTestSuite.runGitHubJsonStateTest(json);
|
GitHubJSONTestSuite.runGitHubJsonStateTest(json);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -59,14 +62,14 @@ public class GitHubStateTest {
|
||||||
Set<String> excluded = new HashSet<>();
|
Set<String> excluded = new HashSet<>();
|
||||||
excluded.add("CallRipemd160_5");
|
excluded.add("CallRipemd160_5");
|
||||||
excluded.add("CallSha256_5");
|
excluded.add("CallSha256_5");
|
||||||
String json = JSONReader.loadJSON("StateTests/stPreCompiledContracts.json");
|
String json = JSONReader.loadJSONFromCommit("StateTests/stPreCompiledContracts.json", shacommit);
|
||||||
GitHubJSONTestSuite.runGitHubJsonStateTest(json, excluded);
|
GitHubJSONTestSuite.runGitHubJsonStateTest(json, excluded);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void stRecursiveCreate() throws ParseException { // [V]
|
public void stRecursiveCreate() throws ParseException { // [V]
|
||||||
|
|
||||||
String json = JSONReader.loadJSON("StateTests/stRecursiveCreate.json");
|
String json = JSONReader.loadJSONFromCommit("StateTests/stRecursiveCreate.json", shacommit);
|
||||||
GitHubJSONTestSuite.runGitHubJsonStateTest(json);
|
GitHubJSONTestSuite.runGitHubJsonStateTest(json);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -75,21 +78,21 @@ public class GitHubStateTest {
|
||||||
Set<String> excluded = new HashSet<>();
|
Set<String> excluded = new HashSet<>();
|
||||||
excluded.add("refund_CallA");
|
excluded.add("refund_CallA");
|
||||||
excluded.add("refund_CallA2");
|
excluded.add("refund_CallA2");
|
||||||
String json = JSONReader.loadJSON("StateTests/stRefundTest.json");
|
String json = JSONReader.loadJSONFromCommit("StateTests/stRefundTest.json", shacommit);
|
||||||
GitHubJSONTestSuite.runGitHubJsonStateTest(json, excluded);
|
GitHubJSONTestSuite.runGitHubJsonStateTest(json, excluded);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void stSpecialTest() throws ParseException { // [V]
|
public void stSpecialTest() throws ParseException { // [V]
|
||||||
|
|
||||||
String json = JSONReader.loadJSON("StateTests/stSpecialTest.json");
|
String json = JSONReader.loadJSONFromCommit("StateTests/stSpecialTest.json", shacommit);
|
||||||
GitHubJSONTestSuite.runGitHubJsonStateTest(json);
|
GitHubJSONTestSuite.runGitHubJsonStateTest(json);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void stBlockHashTest() throws ParseException {
|
public void stBlockHashTest() throws ParseException {
|
||||||
|
|
||||||
String json = JSONReader.loadJSON("StateTests/stBlockHashTest.json");
|
String json = JSONReader.loadJSONFromCommit("StateTests/stBlockHashTest.json", shacommit);
|
||||||
GitHubJSONTestSuite.runGitHubJsonStateTest(json);
|
GitHubJSONTestSuite.runGitHubJsonStateTest(json);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -98,7 +101,7 @@ public class GitHubStateTest {
|
||||||
public void stSystemOperationsTest() throws ParseException {
|
public void stSystemOperationsTest() throws ParseException {
|
||||||
|
|
||||||
Set<String> excluded = new HashSet<>();
|
Set<String> excluded = new HashSet<>();
|
||||||
String json = JSONReader.loadJSON("StateTests/stSystemOperationsTest.json");
|
String json = JSONReader.loadJSONFromCommit("StateTests/stSystemOperationsTest.json", shacommit);
|
||||||
GitHubJSONTestSuite.runGitHubJsonStateTest(json, excluded);
|
GitHubJSONTestSuite.runGitHubJsonStateTest(json, excluded);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -114,7 +117,7 @@ public class GitHubStateTest {
|
||||||
excluded.add("SuicidesAndInternlCallSuicides");
|
excluded.add("SuicidesAndInternlCallSuicides");
|
||||||
excluded.add("SuicidesMixingCoinbase");
|
excluded.add("SuicidesMixingCoinbase");
|
||||||
excluded.add("CreateTransactionReverted");
|
excluded.add("CreateTransactionReverted");
|
||||||
String json = JSONReader.loadJSON("StateTests/stTransactionTest.json");
|
String json = JSONReader.loadJSONFromCommit("StateTests/stTransactionTest.json", shacommit);
|
||||||
GitHubJSONTestSuite.runGitHubJsonStateTest(json, excluded);
|
GitHubJSONTestSuite.runGitHubJsonStateTest(json, excluded);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -14,17 +14,16 @@ import java.util.Set;
|
||||||
|
|
||||||
import static org.ethereum.jsontestsuite.JSONReader.getFileNamesForTreeSha;
|
import static org.ethereum.jsontestsuite.JSONReader.getFileNamesForTreeSha;
|
||||||
|
|
||||||
|
|
||||||
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
|
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
|
||||||
public class GitHubVMTest {
|
public class GitHubVMTest {
|
||||||
|
|
||||||
//SHACOMMIT of VMTESTS TREE (not main tree)
|
//SHACOMMIT of tested commit, ethereum/tests.git
|
||||||
public String shacommit = "a713843af6e6274915bdbbc03d62dc5a0007548b";
|
public String shacommit = "eecee75336681dc8c0b7a2423997178eb2101f4e";
|
||||||
public List<String> vmTestFiles = getFileNamesForTreeSha(shacommit);
|
//public List<String> vmTestFiles = getFileNamesForTreeSha(shacommit);
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void runSingle() throws ParseException {
|
public void runSingle() throws ParseException {
|
||||||
String json = JSONReader.loadJSON("VMTests/vmEnvironmentalInfoTest.json");
|
String json = JSONReader.loadJSONFromCommit("VMTests/vmEnvironmentalInfoTest.json", shacommit);
|
||||||
GitHubJSONTestSuite.runGitHubJsonVMTest(json, "extcodecopy0AddressTooBigRight");
|
GitHubJSONTestSuite.runGitHubJsonVMTest(json, "extcodecopy0AddressTooBigRight");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -33,14 +32,15 @@ public class GitHubVMTest {
|
||||||
Set<String> excluded = new HashSet<>();
|
Set<String> excluded = new HashSet<>();
|
||||||
excluded.add("addmod1_overflowDiff");
|
excluded.add("addmod1_overflowDiff");
|
||||||
excluded.add("addmod1_overflow3");
|
excluded.add("addmod1_overflow3");
|
||||||
String json = JSONReader.getTestBlobForTreeSha(shacommit, "vmArithmeticTest.json");
|
String json = JSONReader.loadJSONFromCommit("VMTests/vmArithmeticTest.json", shacommit);
|
||||||
|
//String json = JSONReader.getTestBlobForTreeSha(shacommit, "vmArithmeticTest.json");
|
||||||
GitHubJSONTestSuite.runGitHubJsonVMTest(json, excluded);
|
GitHubJSONTestSuite.runGitHubJsonVMTest(json, excluded);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test // testing full suite
|
@Test // testing full suite
|
||||||
public void testBitwiseLogicOperationFromGitHub() throws ParseException {
|
public void testBitwiseLogicOperationFromGitHub() throws ParseException {
|
||||||
Set<String> excluded = new HashSet<>();
|
Set<String> excluded = new HashSet<>();
|
||||||
String json = JSONReader.loadJSON("VMTests/vmBitwiseLogicOperationTest.json");
|
String json = JSONReader.loadJSONFromCommit("VMTests/vmBitwiseLogicOperationTest.json", shacommit);
|
||||||
GitHubJSONTestSuite.runGitHubJsonVMTest(json, excluded);
|
GitHubJSONTestSuite.runGitHubJsonVMTest(json, excluded);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -48,7 +48,7 @@ public class GitHubVMTest {
|
||||||
@Test // testing full suite
|
@Test // testing full suite
|
||||||
public void testBlockInfoFromGitHub() throws ParseException {
|
public void testBlockInfoFromGitHub() throws ParseException {
|
||||||
Set<String> excluded = new HashSet<>();
|
Set<String> excluded = new HashSet<>();
|
||||||
String json = JSONReader.loadJSON("VMTests/vmBlockInfoTest.json");
|
String json = JSONReader.loadJSONFromCommit("VMTests/vmBlockInfoTest.json", shacommit);
|
||||||
GitHubJSONTestSuite.runGitHubJsonVMTest(json, excluded);
|
GitHubJSONTestSuite.runGitHubJsonVMTest(json, excluded);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -57,7 +57,7 @@ public class GitHubVMTest {
|
||||||
public void testEnvironmentalInfoFromGitHub() throws ParseException {
|
public void testEnvironmentalInfoFromGitHub() throws ParseException {
|
||||||
Set<String> excluded = new HashSet<>();
|
Set<String> excluded = new HashSet<>();
|
||||||
excluded.add("env1"); //Bug in test runner- this passes if VM logging is on "ALL"
|
excluded.add("env1"); //Bug in test runner- this passes if VM logging is on "ALL"
|
||||||
String json = JSONReader.loadJSON("VMTests/vmEnvironmentalInfoTest.json");
|
String json = JSONReader.loadJSONFromCommit("VMTests/vmEnvironmentalInfoTest.json", shacommit);
|
||||||
GitHubJSONTestSuite.runGitHubJsonVMTest(json, excluded);
|
GitHubJSONTestSuite.runGitHubJsonVMTest(json, excluded);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -65,7 +65,7 @@ public class GitHubVMTest {
|
||||||
@Test // testing full suite
|
@Test // testing full suite
|
||||||
public void testIOandFlowOperationsFromGitHub() throws ParseException {
|
public void testIOandFlowOperationsFromGitHub() throws ParseException {
|
||||||
Set<String> excluded = new HashSet<>();
|
Set<String> excluded = new HashSet<>();
|
||||||
String json = JSONReader.loadJSON("VMTests/vmIOandFlowOperationsTest.json");
|
String json = JSONReader.loadJSONFromCommit("VMTests/vmIOandFlowOperationsTest.json", shacommit);
|
||||||
GitHubJSONTestSuite.runGitHubJsonVMTest(json, excluded);
|
GitHubJSONTestSuite.runGitHubJsonVMTest(json, excluded);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -73,44 +73,43 @@ public class GitHubVMTest {
|
||||||
@Test // testing random
|
@Test // testing random
|
||||||
public void testvmInputLimitsTest1FromGitHub() throws ParseException {
|
public void testvmInputLimitsTest1FromGitHub() throws ParseException {
|
||||||
Set<String> excluded = new HashSet<>();
|
Set<String> excluded = new HashSet<>();
|
||||||
String json = JSONReader.loadJSON("VMTests/vmInputLimitsTest1.json");
|
String json = JSONReader.loadJSONFromCommit("VMTests/vmInputLimitsTest1.json", shacommit);
|
||||||
GitHubJSONTestSuite.runGitHubJsonVMTest(json, excluded);
|
GitHubJSONTestSuite.runGitHubJsonVMTest(json, excluded);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test // testing full suite
|
@Test // testing full suite
|
||||||
public void testVMLogGitHub() throws ParseException {
|
public void testVMLogGitHub() throws ParseException {
|
||||||
Set<String> excluded = new HashSet<>();
|
Set<String> excluded = new HashSet<>();
|
||||||
String json = JSONReader.loadJSON("VMTests/vmLogTest.json");
|
String json = JSONReader.loadJSONFromCommit("VMTests/vmLogTest.json", shacommit);
|
||||||
GitHubJSONTestSuite.runGitHubJsonVMTest(json, excluded);
|
GitHubJSONTestSuite.runGitHubJsonVMTest(json, excluded);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test // testing full suite
|
@Test // testing full suite
|
||||||
public void testPushDupSwapFromGitHub() throws ParseException {
|
public void testPushDupSwapFromGitHub() throws ParseException {
|
||||||
Set<String> excluded = new HashSet<>();
|
Set<String> excluded = new HashSet<>();
|
||||||
String json = JSONReader.loadJSON("VMTests/vmPushDupSwapTest.json");
|
String json = JSONReader.loadJSONFromCommit("VMTests/vmPushDupSwapTest.json", shacommit);
|
||||||
GitHubJSONTestSuite.runGitHubJsonVMTest(json, excluded);
|
GitHubJSONTestSuite.runGitHubJsonVMTest(json, excluded);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test // testing full suite
|
@Test // testing full suite
|
||||||
public void testShaFromGitHub() throws ParseException {
|
public void testShaFromGitHub() throws ParseException {
|
||||||
Set<String> excluded = new HashSet<>();
|
Set<String> excluded = new HashSet<>();
|
||||||
String json = JSONReader.loadJSON("VMTests/vmSha3Test.json");
|
String json = JSONReader.loadJSONFromCommit("VMTests/vmSha3Test.json", shacommit);
|
||||||
GitHubJSONTestSuite.runGitHubJsonVMTest(json, excluded);
|
GitHubJSONTestSuite.runGitHubJsonVMTest(json, excluded);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test // testing full suite
|
@Test // testing full suite
|
||||||
public void testvmSystemOperationsTestGitHub() throws ParseException {
|
public void testvmSystemOperationsTestGitHub() throws ParseException {
|
||||||
Set<String> excluded = new HashSet<>();
|
Set<String> excluded = new HashSet<>();
|
||||||
String json = JSONReader.loadJSON("VMTests/vmSystemOperationsTest.json");
|
String json = JSONReader.loadJSONFromCommit("VMTests/vmSystemOperationsTest.json", shacommit);
|
||||||
GitHubJSONTestSuite.runGitHubJsonVMTest(json, excluded);
|
GitHubJSONTestSuite.runGitHubJsonVMTest(json, excluded);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Test // testing full suite
|
@Test // testing full suite
|
||||||
public void testVMGitHub() throws ParseException {
|
public void testVMGitHub() throws ParseException {
|
||||||
|
|
||||||
Set<String> excluded = new HashSet<>();
|
Set<String> excluded = new HashSet<>();
|
||||||
String json = JSONReader.loadJSON("VMTests/vmtests.json");
|
String json = JSONReader.loadJSONFromCommit("VMTests/vmtests.json", shacommit);
|
||||||
GitHubJSONTestSuite.runGitHubJsonVMTest(json, excluded);
|
GitHubJSONTestSuite.runGitHubJsonVMTest(json, excluded);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -121,7 +120,7 @@ public class GitHubVMTest {
|
||||||
List<String> fileNames = getFileNamesForTreeSha(sha);
|
List<String> fileNames = getFileNamesForTreeSha(sha);
|
||||||
List<String> excludedFiles =
|
List<String> excludedFiles =
|
||||||
Arrays.asList(
|
Arrays.asList(
|
||||||
"201501150842LARGE_DATA_IN_CALLCREATE_GOjson" //Badly named file
|
"201501150842LARGE_DATA_IN_CALLCREATE_GOjson" //Badly named file
|
||||||
);
|
);
|
||||||
|
|
||||||
for (String fileName : fileNames) {
|
for (String fileName : fileNames) {
|
||||||
|
|
Loading…
Reference in New Issue