Fix VMTest

+ Design special type repository that acts according the
 rules of VMTesting means save any account been approached
 even if didn't change.

 + Automatically fetch and run all tests in github
 tree {ethereum_git_tests_repo}/VMTests/RandomTests

 + Fix all tests in VMTests/vmEnvironmentalInfoTest

[Delivers #88044620]  [Delivers #87875292]

ethereumj-core/src/main/java/org/ethereum/db/BlockStoreDummy.java

@@ -10,7 +10,7 @@ import java.util.List;
 
 /**
  * @author Roman Mandeleil
- * @since 08.01.2015
+ * @since 10.02.2015
  */
 public class BlockStoreDummy implements BlockStore {
 

ethereumj-core/src/main/java/org/ethereum/db/RepositoryDummy.java

@@ -23,12 +23,15 @@ import static org.ethereum.util.ByteUtil.wrap;
  * @author Roman Mandeleil
  * @since 17.11.2014
  */
-public class RepositoryDummy implements Repository {
+public class RepositoryDummy extends RepositoryImpl {
 
     private static final Logger logger = LoggerFactory.getLogger("repository");
     private Map<ByteArrayWrapper, AccountState> worldState = new HashMap<>();
     private Map<ByteArrayWrapper, ContractDetails> detailsDB = new HashMap<>();
 
+    public RepositoryDummy() {
+        super(false);
+    }
 
     @Override
     public void reset() {

ethereumj-core/src/main/java/org/ethereum/db/RepositoryImpl.java

@@ -29,6 +29,7 @@ import java.io.IOException;
 import java.math.BigInteger;
 
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
@@ -54,17 +55,20 @@ public class RepositoryImpl implements Repository {
 
     KeyValueDataSource detailsDS = null;
     KeyValueDataSource stateDS = null;
-    
+
     public RepositoryImpl() {
         this(DETAILS_DB, STATE_DB);
     }
 
+    public RepositoryImpl(boolean createDb){
+    }
+
     public RepositoryImpl(KeyValueDataSource detailsDS, KeyValueDataSource stateDS) {
 
         detailsDS.setName(DETAILS_DB);
         detailsDS.init();
         this.detailsDS = detailsDS;
-        
+
         stateDS.setName(STATE_DB);
         stateDS.init();
         this.stateDS = stateDS;
@@ -73,7 +77,7 @@ public class RepositoryImpl implements Repository {
         stateDB = new DatabaseImpl(stateDS);
         worldState = new TrieImpl(stateDB.getDb());
     }
-    
+
     public RepositoryImpl(String detailsDbName, String stateDbName) {
         detailsDB = new DatabaseImpl(detailsDbName);
         stateDB = new DatabaseImpl(stateDbName);
@@ -87,7 +91,7 @@ public class RepositoryImpl implements Repository {
 
         detailsDS.init();
         detailsDB = new DatabaseImpl(detailsDS);
-        
+
         stateDS.init();
         stateDB = new DatabaseImpl(stateDS);
         worldState = new TrieImpl(stateDB.getDb());
@@ -461,6 +465,12 @@ public class RepositoryImpl implements Repository {
         cacheDetails.put(wrap(addr), details);
     }
 
+    public Set<ByteArrayWrapper> getFullAddressSet() {
+        Set<ByteArrayWrapper> setKeys = new HashSet<>(detailsDB.dumpKeys());
+        return setKeys;
+    }
+
+
     @Override
     public byte[] getRoot() {
         return worldState.getRootHash();

ethereumj-core/src/main/java/org/ethereum/db/RepositoryVMTestDummy.java

@@ -0,0 +1,305 @@
+package org.ethereum.db;
+
+import org.ethereum.core.AccountState;
+import org.ethereum.core.Block;
+import org.ethereum.facade.Repository;
+import org.ethereum.vm.DataWord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.spongycastle.util.encoders.Hex;
+
+import java.math.BigInteger;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import static org.ethereum.crypto.SHA3Helper.sha3;
+import static org.ethereum.util.ByteUtil.wrap;
+
+/**
+ * @author Roman Mandeleil
+ * @since 10.02.2015
+ */
+public class RepositoryVMTestDummy extends RepositoryImpl{
+
+    private static final Logger logger = LoggerFactory.getLogger("repository");
+    private Map<ByteArrayWrapper, AccountState> worldState = new HashMap<>();
+    private Map<ByteArrayWrapper, ContractDetails> detailsDB = new HashMap<>();
+
+    public RepositoryVMTestDummy() {
+        super(false);
+    }
+
+    @Override
+    public void reset() {
+
+        worldState.clear();
+        detailsDB.clear();
+    }
+
+    @Override
+    public void close() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean isClosed() {
+        throw new UnsupportedOperationException();
+    }
+
+
+    @Override
+    public void updateBatch(HashMap<ByteArrayWrapper, AccountState> stateCache, HashMap<ByteArrayWrapper,
+            ContractDetails> detailsCache) {
+
+        for (ByteArrayWrapper hash : stateCache.keySet()) {
+
+            AccountState accountState = stateCache.get(hash);
+            ContractDetails contractDetails = detailsCache.get(hash);
+
+            if (accountState.isDeleted()) {
+                worldState.remove(hash);
+                detailsDB.remove(hash);
+
+                logger.debug("delete: [{}]",
+                        Hex.toHexString(hash.getData()));
+
+            } else {
+
+                if (accountState.isDirty() || contractDetails.isDirty()) {
+                    detailsDB.put(hash, contractDetails);
+                    accountState.setStateRoot(contractDetails.getStorageHash());
+                    accountState.setCodeHash(sha3(contractDetails.getCode()));
+                    worldState.put(hash, accountState);
+                    if (logger.isDebugEnabled()) {
+                        logger.debug("update: [{}],nonce: [{}] balance: [{}] \n [{}]",
+                                Hex.toHexString(hash.getData()),
+                                accountState.getNonce(),
+                                accountState.getBalance(),
+                                contractDetails.getStorage());
+                    }
+
+                }
+
+            }
+        }
+
+        stateCache.clear();
+        detailsCache.clear();
+
+    }
+
+
+    @Override
+    public void flush() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void rollback() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void commit() {
+        throw new UnsupportedOperationException();
+    }
+
+
+    @Override
+    public void syncToRoot(byte[] root) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public Repository startTracking() {
+        return new RepositoryTrack(this);
+    }
+
+    @Override
+    public void dumpState(Block block, long gasUsed, int txNumber, byte[] txHash) {
+
+    }
+
+    @Override
+    public Set<byte[]> getAccountsKeys() {
+        return null;
+    }
+
+    public Set<ByteArrayWrapper> getFullAddressSet() {
+        return worldState.keySet();
+    }
+
+
+    @Override
+    public BigInteger addBalance(byte[] addr, BigInteger value) {
+        AccountState account = getAccountState(addr);
+
+        if (account == null)
+            account = createAccount(addr);
+
+        BigInteger result = account.addToBalance(value);
+        worldState.put(wrap(addr), account);
+
+        return result;
+    }
+
+    @Override
+    public BigInteger getBalance(byte[] addr) {
+        AccountState account = getAccountState(addr);
+
+        if (account== null){
+            account = createAccount(addr);
+        }
+
+        return account.getBalance();
+    }
+
+    @Override
+    public DataWord getStorageValue(byte[] addr, DataWord key) {
+        ContractDetails details = getContractDetails(addr);
+
+        if (details == null){
+            createAccount(addr);
+            details = getContractDetails(addr);
+        }
+
+        return details.get(key);
+    }
+
+    @Override
+    public void addStorageRow(byte[] addr, DataWord key, DataWord value) {
+        ContractDetails details = getContractDetails(addr);
+
+        if (details == null) {
+            createAccount(addr);
+            details = getContractDetails(addr);
+        }
+
+        details.put(key, value);
+        detailsDB.put(wrap(addr), details);
+    }
+
+    @Override
+    public byte[] getCode(byte[] addr) {
+        ContractDetails details = getContractDetails(addr);
+
+        if (details == null){
+            createAccount(addr);
+            details = getContractDetails(addr);
+        }
+
+        return details.getCode();
+    }
+
+
+    @Override
+    public void saveCode(byte[] addr, byte[] code) {
+        ContractDetails details = getContractDetails(addr);
+
+        if (details == null) {
+            createAccount(addr);
+            details = getContractDetails(addr);
+        }
+
+        details.setCode(code);
+        detailsDB.put(wrap(addr), details);
+    }
+
+    @Override
+    public BigInteger getNonce(byte[] addr) {
+        AccountState account = getAccountState(addr);
+
+        if (account == null)
+            account = createAccount(addr);
+
+        return account.getNonce();
+    }
+
+    @Override
+    public BigInteger increaseNonce(byte[] addr) {
+        AccountState account = getAccountState(addr);
+
+        if (account == null)
+            account = createAccount(addr);
+
+        account.incrementNonce();
+        worldState.put(wrap(addr), account);
+
+        return account.getNonce();
+    }
+
+    public BigInteger setNonce(byte[] addr, BigInteger nonce) {
+
+        AccountState account = getAccountState(addr);
+
+        if (account == null)
+            account = createAccount(addr);
+
+        account.setNonce(nonce);
+        worldState.put(wrap(addr), account);
+
+        return account.getNonce();
+    }
+
+
+    @Override
+    public void delete(byte[] addr) {
+        worldState.remove(wrap(addr));
+        detailsDB.remove(wrap(addr));
+    }
+
+    @Override
+    public ContractDetails getContractDetails(byte[] addr) {
+
+        return detailsDB.get(wrap(addr));
+    }
+
+
+    @Override
+    public AccountState getAccountState(byte[] addr) {
+        return worldState.get(wrap(addr));
+    }
+
+    @Override
+    public AccountState createAccount(byte[] addr) {
+        AccountState accountState = new AccountState();
+        worldState.put(wrap(addr), accountState);
+
+        ContractDetails contractDetails = new ContractDetails();
+        detailsDB.put(wrap(addr), contractDetails);
+
+        return accountState;
+    }
+
+
+    @Override
+    public boolean isExist(byte[] addr) {
+        return getAccountState(addr) != null;
+    }
+
+    @Override
+    public byte[] getRoot() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void loadAccount(byte[] addr, HashMap<ByteArrayWrapper, AccountState> cacheAccounts, HashMap<ByteArrayWrapper, ContractDetails> cacheDetails) {
+
+        AccountState account = getAccountState(addr);
+        ContractDetails details = getContractDetails(addr);
+
+        if (account == null)
+            account = new AccountState();
+        else
+            account = account.clone();
+
+        if (details == null)
+            details = new ContractDetails();
+        else
+            details = details.clone();
+
+        cacheAccounts.put(wrap(addr), account);
+        cacheDetails.put(wrap(addr), details);
+    }
+}

ethereumj-core/src/main/java/org/ethereum/jsontestsuite/JSONReader.java

@@ -1,6 +1,10 @@
 package org.ethereum.jsontestsuite;
 
 import org.ethereum.config.SystemProperties;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -12,6 +16,9 @@ import java.net.HttpURLConnection;
 import java.net.URL;
 
 import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
 
 public class JSONReader {
 
@@ -63,4 +70,26 @@ public class JSONReader {
         }
         return result;
     }
+
+    public static List<String> getFileNamesForTreeSha(String sha){
+
+        String result = getFromUrl("https://api.github.com/repos/ethereum/tests/git/trees/" + sha);
+
+        JSONParser parser = new JSONParser();
+        JSONObject testSuiteObj = null;
+
+        List<String> fileNames = new ArrayList<String>();
+        try {
+            testSuiteObj = (JSONObject) parser.parse(result);
+            JSONArray tree = (JSONArray)testSuiteObj.get("tree");
+
+            for (Object oEntry : tree) {
+                JSONObject entry = (JSONObject) oEntry;
+                String testName = (String) entry.get("path");
+                fileNames.add(testName);
+            }
+        } catch (ParseException e) {e.printStackTrace();}
+
+        return fileNames;
+    }
 }

ethereumj-core/src/main/java/org/ethereum/jsontestsuite/TestRunner.java

@@ -2,10 +2,7 @@ package org.ethereum.jsontestsuite;
 
 import org.ethereum.core.BlockchainImpl;
 import org.ethereum.core.TransactionExecutor;
-import org.ethereum.db.BlockStoreDummy;
-import org.ethereum.db.ByteArrayWrapper;
-import org.ethereum.db.ContractDetails;
-import org.ethereum.db.RepositoryDummy;
+import org.ethereum.db.*;
 import org.ethereum.facade.Repository;
 import org.ethereum.util.ByteUtil;
 import org.ethereum.vm.DataWord;
@@ -67,7 +64,8 @@ public class TestRunner {
         logger.info("***\n");
 
         logger.info("--------- PRE ---------");
-        RepositoryDummy repository = loadRepository(testCase.getPre());
+        RepositoryImpl repository = loadRepository(new RepositoryDummy(),  testCase.getPre());
+
 
         logger.info("loaded repository");
 
@@ -150,7 +148,7 @@ public class TestRunner {
 
 
         logger.info("--------- PRE ---------");
-        RepositoryDummy repository = loadRepository(testCase.getPre());
+        RepositoryImpl repository = loadRepository(new RepositoryVMTestDummy(),  testCase.getPre());
 
         try {
 
@@ -519,11 +517,9 @@ public class TestRunner {
         return transaction;
     }
 
-    public RepositoryDummy loadRepository(Map<ByteArrayWrapper, AccountState> pre) {
+    public RepositoryImpl loadRepository(RepositoryImpl track, Map<ByteArrayWrapper, AccountState> pre) {
 
 
-        RepositoryDummy track = new RepositoryDummy();
-
             /* 1. Store pre-exist accounts - Pre */
         for (ByteArrayWrapper key : pre.keySet()) {
 

ethereumj-core/src/test/java/test/ethereum/jsontestsuite/GitHubJSONTestSuite.java

@@ -19,10 +19,7 @@ import org.junit.runners.Suite.SuiteClasses;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
 
 /**
  * Test file specific for tests maintained in the GitHub repository
@@ -68,6 +65,10 @@ public class GitHubJSONTestSuite {
         }
     }
 
+    protected static void runGitHubJsonVMTest(String json) throws ParseException {
+        Set<String> excluded = new HashSet<>();
+        runGitHubJsonVMTest(json, excluded);
+    }
 
 
     protected static void runGitHubJsonVMTest(String json, Set<String> excluded) throws ParseException {

ethereumj-core/src/test/java/test/ethereum/jsontestsuite/GitHubVMTest.java

@@ -1,17 +1,19 @@
 package test.ethereum.jsontestsuite;
 
 import org.ethereum.jsontestsuite.JSONReader;
-
 import org.json.simple.parser.ParseException;
-
 import org.junit.FixMethodOrder;
 import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runners.MethodSorters;
 
+import java.util.Arrays;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 
+import static org.ethereum.jsontestsuite.JSONReader.getFileNamesForTreeSha;
+
 
 @FixMethodOrder(MethodSorters.NAME_ASCENDING)
 public class GitHubVMTest {
@@ -19,8 +21,8 @@ public class GitHubVMTest {
 
     @Test
     public void runSingle() throws ParseException {
-        String json = JSONReader.loadJSON("VMTests/vmBlockInfoTest.json");
-        GitHubJSONTestSuite.runGitHubJsonVMTest(json, "blockhash257Block");
+        String json = JSONReader.loadJSON("VMTests/vmEnvironmentalInfoTest.json");
+        GitHubJSONTestSuite.runGitHubJsonVMTest(json, "extcodecopy0AddressTooBigRight");
     }
 
 
@@ -43,7 +45,6 @@ public class GitHubVMTest {
     @Test // testing full suite
     public void testBlockInfoFromGitHub() throws ParseException {
         Set<String> excluded = new HashSet<>();
-
         String json = JSONReader.loadJSON("VMTests/vmBlockInfoTest.json");
         GitHubJSONTestSuite.runGitHubJsonVMTest(json, excluded);
     }
@@ -52,11 +53,6 @@ public class GitHubVMTest {
     @Test // testing full suite
     public void testEnvironmentalInfoFromGitHub() throws ParseException {
         Set<String> excluded = new HashSet<>();
-        excluded.add("ExtCodeSizeAddressInputTooBigRightMyAddress");
-        excluded.add("balanceAddressInputTooBigRightMyAddress");
-        excluded.add("balanceAddressInputTooBig");
-        excluded.add("extcodecopy0AddressTooBigRight");
-
         String json = JSONReader.loadJSON("VMTests/vmEnvironmentalInfoTest.json");
         GitHubJSONTestSuite.runGitHubJsonVMTest(json, excluded);
     }
@@ -114,4 +110,25 @@ public class GitHubVMTest {
         GitHubJSONTestSuite.runGitHubJsonVMTest(json, excluded);
     }
 
+    @Test // testing full suite
+    public void testRandomVMGitHub() throws ParseException {
+
+        String sha = "60b921af8bf7bbe38565f8543e52a54e5f465ae8";
+        List<String> fileNames = getFileNamesForTreeSha(sha);
+        List<String> excludedFiles =
+                Arrays.asList(
+                        ""
+                );
+
+        for (String fileName : fileNames) {
+
+            if (excludedFiles.contains(fileName)) continue;
+            System.out.println("Running: " + fileName);
+            String json = JSONReader.loadJSON("VMTests//RandomTests/" + fileName);
+            GitHubJSONTestSuite.runGitHubJsonVMTest(json);
+        }
+
+    }
+
+
 }