Merged latest develop changes.

ethereumj-core-android/src/main/java/org/ethereum/android/datasource/MapDBDataSource.java

@@ -16,6 +16,8 @@ import static org.ethereum.util.ByteUtil.wrap;
 
 public class MapDBDataSource implements KeyValueDataSource {
     
+    private static final int BATCH_SIZE = 1024 * 1000 * 10;
+    
     private DB db;
     private HTreeMap<ByteArrayWrapper, byte[]> map;
     private String name;
@@ -30,8 +32,9 @@ public class MapDBDataSource implements KeyValueDataSource {
         db = DBMaker.newFileDB(new File(dbLocation, name))
                 .asyncWriteEnable()
                 .mmapFileEnableIfSupported()
-                .compressionEnable()
+//                .compressionEnable()
-                .cacheSize(512)
+                .cacheDisable()
+//                .asyncWriteFlushDelay(1000)
                 .closeOnJvmShutdown()
                 .make();
 
@@ -77,9 +80,17 @@ public class MapDBDataSource implements KeyValueDataSource {
 
     @Override
     public void updateBatch(Map<byte[], byte[]> rows) {
+        int savedSize = 0;
         try {
             for (byte[] key : rows.keySet()) {
-                map.put(wrap(key), rows.get(key));
+                byte[] value = rows.get(key);
+                savedSize += value.length;
+                
+                map.put(wrap(key), value);
+                if (savedSize > BATCH_SIZE) {
+                    db.commit();
+                    savedSize = 0;
+                }
             }
         } finally {
             db.commit();

ethereumj-core/build.gradle

@@ -130,9 +130,8 @@ dependencies {
     compile('io.netty:netty-all:4.0.28.Final')
     compile "com.madgag.spongycastle:core:${scastleVersion}" // for SHA3 and SECP256K1
     compile "com.madgag.spongycastle:prov:${scastleVersion}" // for SHA3 and SECP256K1
-    compile "org.iq80.leveldb:leveldb:${leveldbVersion}"
 
-    compile "org.fusesource.leveldbjni:leveldbjni:1.8"
+    compile "org.iq80.leveldb:leveldb:${leveldbVersion}"
 
     compile "com.cedarsoftware:java-util:1.8.0" // for deep equals
     compile "org.antlr:antlr4-runtime:4.5" // for serpent compilation

ethereumj-core/src/main/java/org/ethereum/datasource/mapdb/MapDBDataSource.java

@@ -17,6 +17,8 @@ import static org.ethereum.util.ByteUtil.wrap;
 
 public class MapDBDataSource implements KeyValueDataSource {
 
+    private static final int BATCH_SIZE = 1024 * 1000 * 10;
+    
     private DB db;
     private HTreeMap<ByteArrayWrapper, byte[]> map;
     private String name;
@@ -31,8 +33,9 @@ public class MapDBDataSource implements KeyValueDataSource {
         db = DBMaker.newFileDB(new File(dbLocation, name))
                 .asyncWriteEnable()
                 .mmapFileEnableIfSupported()
-                .compressionEnable()
+//                .compressionEnable()
-                .cacheSize(512)
+                .cacheDisable()
+//                .asyncWriteFlushDelay(1000)
                 .closeOnJvmShutdown()
                 .make();
 
@@ -78,9 +81,17 @@ public class MapDBDataSource implements KeyValueDataSource {
 
     @Override
     public void updateBatch(Map<byte[], byte[]> rows) {
+        int savedSize = 0;
         try {
             for (byte[] key : rows.keySet()) {
-                map.put(wrap(key), rows.get(key));
+                byte[] value = rows.get(key);
+                savedSize += value.length;
+                
+                map.put(wrap(key), value);
+                if (savedSize > BATCH_SIZE) {
+                    db.commit();
+                    savedSize = 0;
+                }
             }
         } finally {
             db.commit();

ethereumj-core/src/main/java/org/ethereum/db/ContractDetails.java

@@ -1,12 +1,12 @@
 package org.ethereum.db;
 
-import org.ethereum.db.ContractDetailsImpl;
 import org.ethereum.vm.DataWord;
 
 import java.util.List;
 import java.util.Map;
 
 public interface ContractDetails {
+
     void put(DataWord key, DataWord value);
 
     DataWord get(DataWord key);

ethereumj-core/src/main/java/org/ethereum/db/DetailsDataStore.java

@@ -9,7 +9,9 @@ import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 
+import static java.lang.String.format;
 import static org.ethereum.util.ByteUtil.wrap;
+import static org.spongycastle.util.encoders.Hex.decode;
 
 public class DetailsDataStore {
 
@@ -19,27 +21,28 @@ public class DetailsDataStore {
     private HashMap<ByteArrayWrapper, ContractDetails> cache = new HashMap<>();
     private Set<ByteArrayWrapper> removes = new HashSet<>();
 
-    public void setDB(DatabaseImpl db){
+    public void setDB(DatabaseImpl db) {
         this.db = db;
     }
 
-    public ContractDetails get(byte[] key){
+    public ContractDetails get(byte[] key) {
 
-        ContractDetails details = cache.get(wrap(key));
+        ByteArrayWrapper wrappedKey = wrap(key);
+        ContractDetails details = cache.get(wrappedKey);
 
-        if (details == null){
+        if (details == null) {
 
-            if ( removes.contains(wrap(key)))  return null;
+            if (removes.contains(wrappedKey)) return null;
 
             byte[] data = db.get(key);
             if (data == null) return null;
 
             details = new ContractDetailsImpl(data);
-            cache.put( wrap(key), details);
+            cache.put(wrappedKey, details);
 
-            float out = ((float)data.length) / 1048576;
+            float out = ((float) data.length) / 1048576;
             if (out > 10) {
-                String sizeFmt = String.format("%02.2f", out);
+                String sizeFmt = format("%02.2f", out);
                 System.out.println("loaded: key: " + Hex.toHexString(key) + " size: " + sizeFmt + "MB");
             }
         }
@@ -47,60 +50,65 @@ public class DetailsDataStore {
         return details;
     }
 
-    public void update(byte[] key, ContractDetails contractDetails){
+    public void update(byte[] key, ContractDetails contractDetails) {
-        cache.put(wrap(key), contractDetails);
+        ByteArrayWrapper wrappedKey = wrap(key);
-
+        cache.put(wrappedKey, contractDetails);
-        if (removes.contains(wrap(key)))
+        removes.remove(wrappedKey);
-            removes.remove(wrap(key));
     }
 
-    public void remove(byte[] key){
+    public void remove(byte[] key) {
-        cache.remove(wrap(key));
+        ByteArrayWrapper wrappedKey = wrap(key);
-        removes.add(wrap(key));
+        cache.remove(wrappedKey);
+        removes.add(wrappedKey);
     }
 
-    public void flush(){
+    public void flush() {
+        long keys = cache.size();
 
-        long t = System.nanoTime();
+        ByteArrayWrapper largeDetailsKey = wrap(decode("b61662398570293e4f0d25525e2b3002b7fe0836"));
+        ContractDetails largeDetails = cache.get(largeDetailsKey);
+
+        long start = System.nanoTime();
+        long totalSize = flushInternal();
+        long finish = System.nanoTime();
+
+        if (largeDetails != null) cache.put(largeDetailsKey, largeDetails);
+
+        float flushSize = (float) totalSize / 1_048_576;
+        float flushTime = (float) (finish - start) / 1_000_000;
+        gLogger.info(format("Flush details in: %02.2f ms, %d keys, %02.2fMB", flushTime, keys, flushSize));
+    }
+
+    private long flushInternal() {
+        long totalSize = 0;
 
         Map<byte[], byte[]> batch = new HashMap<>();
-        long totalSize = 0;
+        for (Map.Entry<ByteArrayWrapper, ContractDetails> entry : cache.entrySet()) {
-        for (ByteArrayWrapper key : cache.keySet()){
+            byte[] key = entry.getKey().getData();
-            ContractDetails contractDetails = cache.get(key);
+            byte[] value = entry.getValue().getEncoded();
-            byte[] value = contractDetails.getEncoded();
+
-            db.put(key.getData(), value);
+            batch.put(key, value);
-            batch.put(key.getData(), value);
             totalSize += value.length;
         }
 
         db.getDb().updateBatch(batch);
 
-        for (ByteArrayWrapper key : removes){
+        for (ByteArrayWrapper key : removes) {
             db.delete(key.getData());
         }
 
-        long keys = cache.size();
-
-        byte[] aKey = Hex.decode("b61662398570293e4f0d25525e2b3002b7fe0836");
-        ContractDetails aDetails = cache.get(wrap(aKey));
-
         cache.clear();
         removes.clear();
 
-        if (aDetails != null) cache.put(wrap(aKey), aDetails);
+        return totalSize;
-
-        long t_ = System.nanoTime();
-        String sizeFmt = String.format("%02.2f", ((float)totalSize) / 1048576);
-        gLogger.info("Flush details in: {} ms, {} keys, {}MB",
-                ((float)(t_ - t) / 1_000_000), keys, sizeFmt);
     }
 
 
-    public Set<ByteArrayWrapper> keys(){
+    public Set<ByteArrayWrapper> keys() {
-
         Set<ByteArrayWrapper> keys = new HashSet<>();
         keys.addAll(cache.keySet());
         keys.addAll(db.dumpKeys());
+        
         return keys;
     }
 }

ethereumj-core/src/main/java/org/ethereum/facade/EthereumImpl.java

@@ -192,7 +192,7 @@ public class EthereumImpl implements Ethereum {
         PeerClient peer = worldManager.getActivePeer();
         if (peer == null) {
 
-            peer = peerClientProvider.get();
+            peer = new PeerClient();
             worldManager.setActivePeer(peer);
         }
         return peer;

ethereumj-core/src/main/java/org/ethereum/trie/Cache.java

@@ -1,6 +1,5 @@
 package org.ethereum.trie;
 
-import org.ethereum.crypto.HashUtil;
 import org.ethereum.datasource.KeyValueDataSource;
 import org.ethereum.db.ByteArrayWrapper;
 import org.ethereum.util.Value;
@@ -12,7 +11,9 @@ import java.util.Iterator;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
+import static java.lang.String.format;
 import static org.ethereum.util.ByteUtil.wrap;
+import static org.ethereum.util.Value.fromRlpEncoded;
 
 /**
  * @author Nick Savers
@@ -49,53 +50,44 @@ public class Cache {
     }
 
     public Value get(byte[] key) {
-        ByteArrayWrapper keyObj = new ByteArrayWrapper(key);
+        ByteArrayWrapper wrappedKey = wrap(key);
         // First check if the key is the cache
-        if (this.nodes.get(keyObj) != null) {
+        Node node = this.nodes.get(wrappedKey);
-            return this.nodes.get(keyObj).getValue();
+        if (node == null) {
+            byte[] data = this.dataSource.get(key);
+            node = new Node(fromRlpEncoded(data), false);
+            
+            this.nodes.put(wrappedKey, node);
         }
 
-        // Get the key of the database instead and cache it
+        return node.getValue();
-        byte[] data = this.dataSource.get(key);
-        Value value = Value.fromRlpEncoded(data);
-        // Create caching node
-        this.nodes.put(keyObj, new Node(value, false));
-
-        return value;
     }
 
     public void delete(byte[] key) {
-        ByteArrayWrapper keyObj = new ByteArrayWrapper(key);
+        this.nodes.remove(wrap(key));
-        this.nodes.remove(keyObj);
 
         if (dataSource == null) return;
         this.dataSource.delete(key);
     }
 
     public void commit() {
-
-        long t = System.nanoTime();
-        if (dataSource == null) return;
-
         // Don't try to commit if it isn't dirty
-        if (!this.isDirty) {
+        if ((dataSource == null) || !this.isDirty) return;
-            return;
-        }
 
+        long start = System.nanoTime();
 
-        long size = 0;
+        long totalSize = 0;
-        long keys = 0;
         Map<byte[], byte[]> batch = new HashMap<>();
         for (ByteArrayWrapper key : this.nodes.keySet()) {
             Node node = this.nodes.get(key);
+            
             if (node.isDirty()) {
+                node.setDirty(false);
                 
                 byte[] value = node.getValue().encode();
                 batch.put(key.getData(), value);
-                node.setDirty(false);
 
-                size += value.length;
+                totalSize += value.length;
-                keys += 1;
             }
         }
 
@@ -103,11 +95,11 @@ public class Cache {
         this.isDirty = false;
         this.nodes.clear();
 
-        long t_ = System.nanoTime();
+        long finish = System.nanoTime();
-        String sizeFmt = String.format("%02.2f", ((float)size) / 1048576);
-        gLogger.info("Flush state in: {} ms, {} nodes, {}MB",
-                ((float)(t_ - t) / 1_000_000), keys, sizeFmt);
         
+        float flushSize = (float) totalSize / 1048576;
+        float flushTime = (float) (finish - start) / 1_000_000;
+        gLogger.info(format("Flush state in: %02.2f ms, %d nodes, %02.2fMB", flushTime, batch.size(), flushSize));
     }
 
     public void undo() {

ethereumj-core/src/main/java/org/ethereum/trie/TrieImpl.java

@@ -1,6 +1,5 @@
 package org.ethereum.trie;
 
-import org.ethereum.crypto.HashUtil;
 import org.ethereum.datasource.KeyValueDataSource;
 import org.ethereum.db.ByteArrayWrapper;
 import org.ethereum.util.RLP;