Connect KeyValueDataSource abstraction to the rest of the application

Repository , Trie implementation and Cache now going to the data through KeyValueDatasource

ethereumj-core/src/main/java/org/ethereum/datasource/KeyValueDataSource.java

@@ -18,8 +18,7 @@ public interface KeyValueDataSource {
     public void put(byte[] key, byte[] value);
     public void delete(byte[] key);
     public Set<byte[]> keys();
-    public void setBatch( Map<byte[], byte[]> rows);
-    
-    
+    public void updateBatch( Map<byte[], byte[]> rows);
 
+    public void close();
 }

ethereumj-core/src/main/java/org/ethereum/datasource/LevelDbDataSource.java

@@ -1,12 +1,24 @@
 package org.ethereum.datasource;
 
+import org.ethereum.config.SystemProperties;
+import org.ethereum.db.ByteArrayWrapper;
 import org.ethereum.db.Database;
 import org.ethereum.db.DatabaseImpl;
-import org.iq80.leveldb.DB;
+import org.ethereum.trie.Node;
+import org.iq80.leveldb.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.spongycastle.util.encoders.Hex;
 
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 
+import static org.iq80.leveldb.impl.Iq80DBFactory.factory;
+
 /**
  *
  * @author: Roman Mandeleil
@@ -15,16 +27,50 @@ import java.util.Set;
 
 public class LevelDbDataSource implements KeyValueDataSource{
 
+    private static final Logger logger = LoggerFactory.getLogger("db");
+
     String name;
-    Database db;
+    private DB db;
     
     @Override
     public void init() {
         
         if (name == null) throw new NullPointerException("no name set to the db");
-        db = new DatabaseImpl(name);
+        
+        Options options = new Options();
+        options.createIfMissing(true);
+        options.compressionType(CompressionType.NONE);
+        try {
+            logger.debug("Opening database");
+            File dbLocation = new File(System.getProperty("user.dir") + "/" +
+                    SystemProperties.CONFIG.databaseDir() + "/");
+            File fileLocation = new File(dbLocation, name);
+
+            if (SystemProperties.CONFIG.databaseReset()) {
+                destroyDB(fileLocation);
+            }
+
+            logger.debug("Initializing new or existing database: '{}'", name);
+            db = factory.open(fileLocation, options);
+
+        } catch (IOException ioe) {
+            logger.error(ioe.getMessage(), ioe);
+            throw new RuntimeException("Can't initialize database");
+        }
     }
 
+
+    public void destroyDB(File fileLocation) {
+        logger.debug("Destroying existing database");
+        Options options = new Options();
+        try {
+            factory.destroy(fileLocation, options);
+        } catch (IOException e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+
     @Override
     public void setName(String name) {
         this.name = name;
@@ -47,14 +93,35 @@ public class LevelDbDataSource implements KeyValueDataSource{
 
     @Override
     public Set<byte[]> keys() {
-        
-        // todo: re-modelling DataBase for that
-        throw new UnsupportedOperationException();
+
+        DBIterator dbIterator = db.iterator();
+        Set<byte[]> keys = new HashSet<>();
+        while (dbIterator.hasNext()){
+            
+            Map.Entry<byte[], byte[]> entry = dbIterator.next();
+            keys.add(entry.getKey());
+        }
+        return keys;
     }
 
     @Override
-    public void setBatch(Map<byte[], byte[]> rows) {
-        // todo: re-modelling DataBase for that
-        throw new UnsupportedOperationException();
+    public void updateBatch(Map<byte[], byte[]> rows) {
+
+        WriteBatch batch = db.createWriteBatch();
+
+        for (Map.Entry<byte[], byte[]> row : rows.entrySet())
+                batch.put(row.getKey(), row.getValue());
+
+        db.write(batch);
+    }
+
+    @Override
+    public void close() {
+        try {
+            logger.info("Close db: {}", name);
+            db.close();
+        } catch (IOException e) {
+            logger.error("Failed to find the db file on the close: {} ", name);
+        }
     }
 }

ethereumj-core/src/main/java/org/ethereum/datasource/RedisDataSource.java

@@ -60,7 +60,7 @@ public class RedisDataSource implements KeyValueDataSource{
     }
 
     @Override
-    public void setBatch(Map<byte[], byte[]> rows) {
+    public void updateBatch(Map<byte[], byte[]> rows) {
         jedis.select(index);
         Pipeline pipeline = jedis.pipelined();
 
@@ -75,6 +75,10 @@ public class RedisDataSource implements KeyValueDataSource{
         pipeline.sync();
     }
 
+    @Override
+    public void close() {
+        jedis.close();
+    }
 
     private static Integer nameToIndex(String name) {
         Integer index = DBNameScheme.get(name);

ethereumj-core/src/main/java/org/ethereum/db/DatabaseImpl.java

@@ -1,26 +1,16 @@
 package org.ethereum.db;
 
-import org.ethereum.config.SystemProperties;
-
-import org.iq80.leveldb.CompressionType;
-import org.iq80.leveldb.DB;
-import org.iq80.leveldb.DBIterator;
-import org.iq80.leveldb.Options;
-
+import org.ethereum.datasource.KeyValueDataSource;
+import org.ethereum.datasource.LevelDbDataSource;
+import org.ethereum.util.ByteUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-
 import org.spongycastle.util.encoders.Hex;
 
-import java.io.File;
-import java.io.IOException;
-
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
-import static org.iq80.leveldb.impl.Iq80DBFactory.factory;
-
 /**
  * Generic interface for Ethereum database
  *
@@ -32,63 +22,20 @@ import static org.iq80.leveldb.impl.Iq80DBFactory.factory;
 public class DatabaseImpl implements Database {
 
     private static final Logger logger = LoggerFactory.getLogger("db");
-    private DB db;
     private String name;
+    private KeyValueDataSource dataSource;
 
     public DatabaseImpl(String name) {
         // Initialize Database
-        this.name = name;
-        Options options = new Options();
-        options.createIfMissing(true);
-        options.compressionType(CompressionType.NONE);
-        try {
-            logger.debug("Opening database");
-            File dbLocation = new File(System.getProperty("user.dir") + "/" +
-                    SystemProperties.CONFIG.databaseDir() + "/");
-            File fileLocation = new File(dbLocation, name);
-
-            if (SystemProperties.CONFIG.databaseReset()) {
-                destroyDB(fileLocation);
-            }
-
-            logger.debug("Initializing new or existing database: '{}'", name);
-            db = factory.open(fileLocation, options);
-//          logger.debug("Showing database stats");
-//          String stats = DATABASE.getProperty("leveldb.stats");
-//          logger.debug(stats);
-
-            if (logger.isTraceEnabled()) {
-
-                logger.trace("Dump for: {}", fileLocation.toString());
-                DBIterator iter = db.iterator();
-
-                while (iter.hasNext()) {
-                    byte[] key = iter.peekNext().getKey();
-                    byte[] value = iter.peekNext().getValue();
-
-                    logger.trace("key={}, value={}", Hex.toHexString(key), Hex.toHexString(value));
-                    iter.next();
-                }
-            }
-        } catch (IOException ioe) {
-            logger.error(ioe.getMessage(), ioe);
-            throw new RuntimeException("Can't initialize database");
-        }
+        dataSource  = new LevelDbDataSource();
+        dataSource.setName(name);
+        dataSource.init();
     }
 
-    public void destroyDB(File fileLocation) {
-        logger.debug("Destroying existing database");
-        Options options = new Options();
-        try {
-            factory.destroy(fileLocation, options);
-        } catch (IOException e) {
-            logger.error(e.getMessage(), e);
-        }
-    }
 
     @Override
     public byte[] get(byte[] key) {
-        return db.get(key);
+        return dataSource.get(key);
     }
 
     @Override
@@ -98,7 +45,7 @@ public class DatabaseImpl implements Database {
             logger.debug("put: key: [{}], value: [{}]",
                     Hex.toHexString(key),
                     Hex.toHexString(value));
-        db.put(key, value);
+        dataSource.put(key, value);
     }
 
     @Override
@@ -106,34 +53,24 @@ public class DatabaseImpl implements Database {
         if (logger.isDebugEnabled())
             logger.debug("delete: key: [{}]");
 
-        db.delete(key);
+        dataSource.delete(key);
     }
 
-    public DBIterator iterator() {
-        return db.iterator();
-    }
-
-    public DB getDb() {
-        return this.db;
+    public KeyValueDataSource getDb() {
+        return this.dataSource;
     }
 
     @Override
     public void close() {
-        try {
-            logger.info("Close db: {}", name);
-            db.close();
-        } catch (IOException e) {
-            logger.error("Failed to find the db file on the close: {} ", name);
-        }
+        dataSource.close();
     }
 
     public List<ByteArrayWrapper> dumpKeys() {
-        DBIterator iterator = getDb().iterator();
+        
         ArrayList<ByteArrayWrapper> keys = new ArrayList<>();
 
-        while (iterator.hasNext()) {
-            ByteArrayWrapper key = new ByteArrayWrapper(iterator.next().getKey());
-            keys.add(key);
+        for (byte[] key : dataSource.keys()) {
+            keys.add(ByteUtil.wrap(key));
         }
         Collections.sort(keys);
         return keys;

ethereumj-core/src/main/java/org/ethereum/db/RepositoryDummy.java

@@ -124,7 +124,7 @@ public class RepositoryDummy implements Repository {
     }
 
     @Override
-    public DBIterator getAccountsIterator() {
+    public Set<byte[]> getAccountsKeys() {
         return null;
     }
 

ethereumj-core/src/main/java/org/ethereum/db/RepositoryImpl.java

@@ -1,5 +1,7 @@
 package org.ethereum.db;
 
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import org.ethereum.core.AccountState;
 import org.ethereum.core.Block;
 import org.ethereum.facade.Repository;
@@ -8,17 +10,9 @@ import org.ethereum.json.JSONHelper;
 import org.ethereum.trie.Trie;
 import org.ethereum.trie.TrieImpl;
 import org.ethereum.vm.DataWord;
-
-import com.fasterxml.jackson.databind.node.JsonNodeFactory;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
-import org.iq80.leveldb.DBIterator;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-
 import org.spongycastle.util.encoders.Hex;
-
 import org.springframework.stereotype.Component;
 import org.springframework.util.FileSystemUtils;
 
@@ -26,11 +20,10 @@ import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
-
 import java.math.BigInteger;
-
 import java.util.HashMap;
 import java.util.List;
+import java.util.Set;
 
 import static org.ethereum.config.SystemProperties.CONFIG;
 import static org.ethereum.crypto.SHA3Helper.sha3;
@@ -253,8 +246,8 @@ public class RepositoryImpl implements Repository {
 
 
     @Override
-    public DBIterator getAccountsIterator() {
-        return detailsDB.iterator();
+    public Set<byte[]> getAccountsKeys() {
+        return detailsDB.getDb().keys();
     }
 
     @Override

ethereumj-core/src/main/java/org/ethereum/db/RepositoryTrack.java

@@ -4,16 +4,11 @@ import org.ethereum.core.AccountState;
 import org.ethereum.core.Block;
 import org.ethereum.facade.Repository;
 import org.ethereum.vm.DataWord;
-
-import org.iq80.leveldb.DBIterator;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-
 import org.spongycastle.util.encoders.Hex;
 
 import java.math.BigInteger;
-
 import java.util.HashMap;
 import java.util.Set;
 
@@ -207,10 +202,11 @@ public class RepositoryTrack implements Repository {
 
 
     @Override
-    public DBIterator getAccountsIterator() {
+    public Set<byte[]> getAccountsKeys() {
         throw new UnsupportedOperationException();
     }
 
+
     public Set<ByteArrayWrapper> getFullAddressSet() {
         return cacheAccounts.keySet();
     }

ethereumj-core/src/main/java/org/ethereum/facade/Repository.java

@@ -11,6 +11,8 @@ import org.iq80.leveldb.DBIterator;
 import java.math.BigInteger;
 
 import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
 
 /**
  * @author Roman Mandeleil
@@ -127,11 +129,9 @@ public interface Repository {
     public BigInteger addBalance(byte[] addr, BigInteger value);
 
     /**
-     * Returns an iterator over the accounts in this database in proper sequence
-     *
-     * @return an iterator over the accounts in this database in proper sequence
+     * @return Returns set of all the account addresses
      */
-    public DBIterator getAccountsIterator();
+    public Set<byte[]> getAccountsKeys();
 
 
     /**

ethereumj-core/src/main/java/org/ethereum/trie/Cache.java

@@ -1,12 +1,14 @@
 package org.ethereum.trie;
 
 import org.ethereum.crypto.HashUtil;
+import org.ethereum.datasource.KeyValueDataSource;
 import org.ethereum.db.ByteArrayWrapper;
 import org.ethereum.util.Value;
 
 import org.iq80.leveldb.DB;
 import org.iq80.leveldb.WriteBatch;
 
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -17,12 +19,12 @@ import java.util.concurrent.ConcurrentHashMap;
  */
 public class Cache {
 
-    private final DB db;
+    private final KeyValueDataSource dataSource;
     private Map<ByteArrayWrapper, Node> nodes = new ConcurrentHashMap<>();
     private boolean isDirty;
 
-    public Cache(DB db) {
-        this.db = db;
+    public Cache(KeyValueDataSource dataSource) {
+        this.dataSource = dataSource;
     }
 
     /**
@@ -51,7 +53,7 @@ public class Cache {
         }
 
         // Get the key of the database instead and cache it
-        byte[] data = this.db.get(key);
+        byte[] data = this.dataSource.get(key);
         Value value = Value.fromRlpEncoded(data);
         // Create caching node
         this.nodes.put(keyObj, new Node(value, false));
@@ -63,21 +65,21 @@ public class Cache {
         ByteArrayWrapper keyObj = new ByteArrayWrapper(key);
         this.nodes.remove(keyObj);
 
-        if (db == null) return;
-        this.db.delete(key);
+        if (dataSource == null) return;
+        this.dataSource.delete(key);
     }
 
     public void commit() {
 
-        if (db == null) return;
+        if (dataSource == null) return;
 
         // Don't try to commit if it isn't dirty
         if (!this.isDirty) {
             return;
         }
 
-        WriteBatch batch = db.createWriteBatch();
 
+        Map<byte[], byte[]> batch = new HashMap<>();
         for (ByteArrayWrapper key : this.nodes.keySet()) {
             Node node = this.nodes.get(key);
             if (node.isDirty()) {
@@ -86,7 +88,7 @@ public class Cache {
             }
         }
 
-        db.write(batch);
+        dataSource.updateBatch(batch);
         this.isDirty = false;
     }
 
@@ -112,8 +114,8 @@ public class Cache {
         return nodes;
     }
 
-    public DB getDb() {
-        return db;
+    public KeyValueDataSource getDb() {
+        return dataSource;
     }
 
     public String cacheDump() {

ethereumj-core/src/main/java/org/ethereum/trie/TrieImpl.java

@@ -1,6 +1,7 @@
 package org.ethereum.trie;
 
 import org.ethereum.crypto.HashUtil;
+import org.ethereum.datasource.KeyValueDataSource;
 import org.ethereum.db.ByteArrayWrapper;
 import org.ethereum.util.Value;
 
@@ -54,11 +55,11 @@ public class TrieImpl implements Trie {
     private Object root;
     private Cache cache;
 
-    public TrieImpl(DB db) {
+    public TrieImpl(KeyValueDataSource db) {
         this(db, "");
     }
 
-    public TrieImpl(DB db, Object root) {
+    public TrieImpl(KeyValueDataSource db, Object root) {
         this.cache = new Cache(db);
         this.root = root;
         this.prevRoot = root;

ethereumj-core/src/test/java/test/ethereum/db/MockDB.java

@@ -1,52 +1,23 @@
 package test.ethereum.db;
 
+import org.ethereum.datasource.KeyValueDataSource;
 import org.ethereum.db.ByteArrayWrapper;
-
-import org.iq80.leveldb.DB;
 import org.iq80.leveldb.DBException;
-import org.iq80.leveldb.DBIterator;
-import org.iq80.leveldb.Range;
-import org.iq80.leveldb.ReadOptions;
-import org.iq80.leveldb.Snapshot;
-import org.iq80.leveldb.WriteBatch;
-import org.iq80.leveldb.WriteOptions;
-
-import java.io.IOException;
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Set;
 
-public class MockDB implements DB {
+public class MockDB implements KeyValueDataSource {
 
     Map<ByteArrayWrapper, byte[]> storage = new HashMap<>();
 
-    @Override
-    public void close() throws IOException {
-        storage.clear();
-    }
-
-    @Override
-    public void compactRange(byte[] arg0, byte[] arg1) throws DBException {
-        // TODO Auto-generated method stub
-    }
-
-    @Override
-    public WriteBatch createWriteBatch() {
-        // TODO Auto-generated method stub
-        return null;
-    }
 
     @Override
     public void delete(byte[] arg0) throws DBException {
         storage.remove(arg0);
     }
 
-    @Override
-    public Snapshot delete(byte[] arg0, WriteOptions arg1)
-            throws DBException {
-        // TODO Auto-generated method stub
-        return null;
-    }
 
     @Override
     public byte[] get(byte[] arg0) throws DBException {
@@ -54,77 +25,13 @@ public class MockDB implements DB {
         return storage.get(new ByteArrayWrapper(arg0));
     }
 
-    @Override
-    public byte[] get(byte[] arg0, ReadOptions arg1) throws DBException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public long[] getApproximateSizes(Range... arg0) {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public String getProperty(String arg0) {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Snapshot getSnapshot() {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public DBIterator iterator() {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public DBIterator iterator(ReadOptions arg0) {
-        // TODO Auto-generated method stub
-        return null;
-    }
 
     @Override
     public void put(byte[] key, byte[] value) throws DBException {
 
         storage.put(new ByteArrayWrapper(key), value);
     }
-
-    @Override
-    public Snapshot put(byte[] arg0, byte[] arg1, WriteOptions arg2)
-            throws DBException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public void resumeCompactions() {
-        // TODO Auto-generated method stub
-    }
-
-    @Override
-    public void suspendCompactions() throws InterruptedException {
-        // TODO Auto-generated method stub
-    }
-
-    @Override
-    public void write(WriteBatch arg0) throws DBException {
-        // TODO Auto-generated method stub
-    }
-
-    @Override
-    public Snapshot write(WriteBatch arg0, WriteOptions arg1)
-            throws DBException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
+    
     /**
      * Returns the number of items added to this Mock DB
      *
@@ -133,4 +40,29 @@ public class MockDB implements DB {
     public int getAddedItems() {
         return storage.size();
     }
+
+    @Override
+    public void init() {
+
+    }
+
+    @Override
+    public void setName(String name) {
+
+    }
+
+    @Override
+    public Set<byte[]> keys() {
+        return null;
+    }
+
+    @Override
+    public void updateBatch(Map<byte[], byte[]> rows) {
+
+    }
+
+    @Override
+    public void close() {
+
+    }
 }

ethereumj-studio/src/main/java/org/ethereum/gui/AccountsListWindow.java

@@ -4,19 +4,15 @@ import org.ethereum.core.AccountState;
 import org.ethereum.core.Denomination;
 import org.ethereum.crypto.HashUtil;
 import org.ethereum.facade.Repository;
-
-import org.iq80.leveldb.DBIterator;
-
 import org.spongycastle.util.Arrays;
 import org.spongycastle.util.encoders.Hex;
 
-import java.awt.*;
-
-import java.util.ArrayList;
-import java.util.List;
-
 import javax.swing.*;
 import javax.swing.table.AbstractTableModel;
+import java.awt.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
 
 public class AccountsListWindow extends JFrame {
 
@@ -54,11 +50,10 @@ public class AccountsListWindow extends JFrame {
             @Override
             public void run() {
                 Repository repository = UIEthereumManager.ethereum.getRepository();
-                DBIterator i = repository.getAccountsIterator();
-                while (i.hasNext()) {
+                Set<byte[]> keys = repository.getAccountsKeys();
+                for (byte[] key : keys){
                     DataClass dc = new DataClass();
-                    dc.address = i.next().getKey();
-
+                    dc.address = key;
                     AccountState state = repository.getAccountState(dc.address);
                     dc.accountState = state;