status-im
/
ethereumj-personal
mirror of https://github.com/status-im/ethereumj-personal.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merged latest develop changes.

This commit is contained in:
Adrian Tiberius 2015-06-25 00:55:07 +02:00
parent bc618bacc1
commit 16d6c57c20
8 changed files with 102 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
ethereumj-core-android/src/main/java/org/ethereum/android/datasource/MapDBDataSource.java
View File

@ -16,6 +16,8 @@ import static org.ethereum.util.ByteUtil.wrap;
public class MapDBDataSource implements KeyValueDataSource {
private static final int BATCH_SIZE = 1024 * 1000 * 10;
private DB db;
private HTreeMap<ByteArrayWrapper, byte[]> map;
private String name;
@ -30,8 +32,9 @@ public class MapDBDataSource implements KeyValueDataSource {
db = DBMaker.newFileDB(new File(dbLocation, name))
.asyncWriteEnable()
.mmapFileEnableIfSupported()
.compressionEnable()
.cacheSize(512)
// .compressionEnable()
.cacheDisable()
// .asyncWriteFlushDelay(1000)
.closeOnJvmShutdown()
.make();
@ -70,16 +73,24 @@ public class MapDBDataSource implements KeyValueDataSource {
public Set<byte[]> keys() {
HashSet<byte[]> result = new HashSet<>();
for (ByteArrayWrapper key : map.keySet()) {
result.add(key.getData());
result.add(key.getData());
}
return result;
}
@Override
public void updateBatch(Map<byte[], byte[]> rows) {
int savedSize = 0;
try {
for (byte[] key : rows.keySet()) {
map.put(wrap(key), rows.get(key));
byte[] value = rows.get(key);
savedSize += value.length;
map.put(wrap(key), value);
if (savedSize > BATCH_SIZE) {
db.commit();
savedSize = 0;
}
}
} finally {
db.commit();

3
ethereumj-core/build.gradle
View File

@ -130,9 +130,8 @@ dependencies {
compile('io.netty:netty-all:4.0.28.Final')
compile "com.madgag.spongycastle:core:${scastleVersion}" // for SHA3 and SECP256K1
compile "com.madgag.spongycastle:prov:${scastleVersion}" // for SHA3 and SECP256K1
compile "org.iq80.leveldb:leveldb:${leveldbVersion}"
compile "org.fusesource.leveldbjni:leveldbjni:1.8"
compile "org.iq80.leveldb:leveldb:${leveldbVersion}"
compile "com.cedarsoftware:java-util:1.8.0" // for deep equals
compile "org.antlr:antlr4-runtime:4.5" // for serpent compilation

19
ethereumj-core/src/main/java/org/ethereum/datasource/mapdb/MapDBDataSource.java
View File

@ -16,6 +16,8 @@ import static java.lang.System.getProperty;
import static org.ethereum.util.ByteUtil.wrap;
public class MapDBDataSource implements KeyValueDataSource {
private static final int BATCH_SIZE = 1024 * 1000 * 10;
private DB db;
private HTreeMap<ByteArrayWrapper, byte[]> map;
@ -31,8 +33,9 @@ public class MapDBDataSource implements KeyValueDataSource {
db = DBMaker.newFileDB(new File(dbLocation, name))
.asyncWriteEnable()
.mmapFileEnableIfSupported()
.compressionEnable()
.cacheSize(512)
// .compressionEnable()
.cacheDisable()
// .asyncWriteFlushDelay(1000)
.closeOnJvmShutdown()
.make();
@ -71,16 +74,24 @@ public class MapDBDataSource implements KeyValueDataSource {
public Set<byte[]> keys() {
HashSet<byte[]> result = new HashSet<>();
for (ByteArrayWrapper key : map.keySet()) {
result.add(key.getData());
result.add(key.getData());
}
return result;
}
@Override
public void updateBatch(Map<byte[], byte[]> rows) {
int savedSize = 0;
try {
for (byte[] key : rows.keySet()) {
map.put(wrap(key), rows.get(key));
byte[] value = rows.get(key);
savedSize += value.length;
map.put(wrap(key), value);
if (savedSize > BATCH_SIZE) {
db.commit();
savedSize = 0;
}
}
} finally {
db.commit();

2
ethereumj-core/src/main/java/org/ethereum/db/ContractDetails.java
View File

@ -1,12 +1,12 @@
package org.ethereum.db;
import org.ethereum.db.ContractDetailsImpl;
import org.ethereum.vm.DataWord;
import java.util.List;
import java.util.Map;
public interface ContractDetails {
void put(DataWord key, DataWord value);
DataWord get(DataWord key);

84
ethereumj-core/src/main/java/org/ethereum/db/DetailsDataStore.java
View File

@ -9,7 +9,9 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import static java.lang.String.format;
import static org.ethereum.util.ByteUtil.wrap;
import static org.spongycastle.util.encoders.Hex.decode;
public class DetailsDataStore {
@ -19,27 +21,28 @@ public class DetailsDataStore {
private HashMap<ByteArrayWrapper, ContractDetails> cache = new HashMap<>();
private Set<ByteArrayWrapper> removes = new HashSet<>();
public void setDB(DatabaseImpl db){
public void setDB(DatabaseImpl db) {
this.db = db;
}
public ContractDetails get(byte[] key){
public ContractDetails get(byte[] key) {
ContractDetails details = cache.get(wrap(key));
ByteArrayWrapper wrappedKey = wrap(key);
ContractDetails details = cache.get(wrappedKey);
if (details == null){
if (details == null) {
if ( removes.contains(wrap(key))) return null;
if (removes.contains(wrappedKey)) return null;
byte[] data = db.get(key);
if (data == null) return null;
details = new ContractDetailsImpl(data);
cache.put( wrap(key), details);
cache.put(wrappedKey, details);
float out = ((float)data.length) / 1048576;
float out = ((float) data.length) / 1048576;
if (out > 10) {
String sizeFmt = String.format("%02.2f", out);
String sizeFmt = format("%02.2f", out);
System.out.println("loaded: key: " + Hex.toHexString(key) + " size: " + sizeFmt + "MB");
}
}
@ -47,60 +50,65 @@ public class DetailsDataStore {
return details;
}
public void update(byte[] key, ContractDetails contractDetails){
cache.put(wrap(key), contractDetails);
if (removes.contains(wrap(key)))
removes.remove(wrap(key));
public void update(byte[] key, ContractDetails contractDetails) {
ByteArrayWrapper wrappedKey = wrap(key);
cache.put(wrappedKey, contractDetails);
removes.remove(wrappedKey);
}
public void remove(byte[] key){
cache.remove(wrap(key));
removes.add(wrap(key));
public void remove(byte[] key) {
ByteArrayWrapper wrappedKey = wrap(key);
cache.remove(wrappedKey);
removes.add(wrappedKey);
}
public void flush(){
public void flush() {
long keys = cache.size();
long t = System.nanoTime();
ByteArrayWrapper largeDetailsKey = wrap(decode("b61662398570293e4f0d25525e2b3002b7fe0836"));
ContractDetails largeDetails = cache.get(largeDetailsKey);
long start = System.nanoTime();
long totalSize = flushInternal();
long finish = System.nanoTime();
if (largeDetails != null) cache.put(largeDetailsKey, largeDetails);
float flushSize = (float) totalSize / 1_048_576;
float flushTime = (float) (finish - start) / 1_000_000;
gLogger.info(format("Flush details in: %02.2f ms, %d keys, %02.2fMB", flushTime, keys, flushSize));
}
private long flushInternal() {
long totalSize = 0;
Map<byte[], byte[]> batch = new HashMap<>();
long totalSize = 0;
for (ByteArrayWrapper key : cache.keySet()){
ContractDetails contractDetails = cache.get(key);
byte[] value = contractDetails.getEncoded();
db.put(key.getData(), value);
batch.put(key.getData(), value);
for (Map.Entry<ByteArrayWrapper, ContractDetails> entry : cache.entrySet()) {
byte[] key = entry.getKey().getData();
byte[] value = entry.getValue().getEncoded();
batch.put(key, value);
totalSize += value.length;
}
db.getDb().updateBatch(batch);
for (ByteArrayWrapper key : removes){
for (ByteArrayWrapper key : removes) {
db.delete(key.getData());
}
long keys = cache.size();
byte[] aKey = Hex.decode("b61662398570293e4f0d25525e2b3002b7fe0836");
ContractDetails aDetails = cache.get(wrap(aKey));
cache.clear();
removes.clear();
if (aDetails != null) cache.put(wrap(aKey), aDetails);
long t_ = System.nanoTime();
String sizeFmt = String.format("%02.2f", ((float)totalSize) / 1048576);
gLogger.info("Flush details in: {} ms, {} keys, {}MB",
((float)(t_ - t) / 1_000_000), keys, sizeFmt);
return totalSize;
}
public Set<ByteArrayWrapper> keys(){
public Set<ByteArrayWrapper> keys() {
Set<ByteArrayWrapper> keys = new HashSet<>();
keys.addAll(cache.keySet());
keys.addAll(db.dumpKeys());
return keys;
}
}

2
ethereumj-core/src/main/java/org/ethereum/facade/EthereumImpl.java
View File

@ -192,7 +192,7 @@ public class EthereumImpl implements Ethereum {
PeerClient peer = worldManager.getActivePeer();
if (peer == null) {
peer = peerClientProvider.get();
peer = new PeerClient();
worldManager.setActivePeer(peer);
}
return peer;

54
ethereumj-core/src/main/java/org/ethereum/trie/Cache.java
View File

@ -1,6 +1,5 @@
package org.ethereum.trie;
import org.ethereum.crypto.HashUtil;
import org.ethereum.datasource.KeyValueDataSource;
import org.ethereum.db.ByteArrayWrapper;
import org.ethereum.util.Value;
@ -12,7 +11,9 @@ import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import static java.lang.String.format;
import static org.ethereum.util.ByteUtil.wrap;
import static org.ethereum.util.Value.fromRlpEncoded;
/**
* @author Nick Savers
@ -49,53 +50,44 @@ public class Cache {
}
public Value get(byte[] key) {
ByteArrayWrapper keyObj = new ByteArrayWrapper(key);
ByteArrayWrapper wrappedKey = wrap(key);
// First check if the key is the cache
if (this.nodes.get(keyObj) != null) {
return this.nodes.get(keyObj).getValue();
Node node = this.nodes.get(wrappedKey);
if (node == null) {
byte[] data = this.dataSource.get(key);
node = new Node(fromRlpEncoded(data), false);
this.nodes.put(wrappedKey, node);
}
// Get the key of the database instead and cache it
byte[] data = this.dataSource.get(key);
Value value = Value.fromRlpEncoded(data);
// Create caching node
this.nodes.put(keyObj, new Node(value, false));
return value;
return node.getValue();
}
public void delete(byte[] key) {
ByteArrayWrapper keyObj = new ByteArrayWrapper(key);
this.nodes.remove(keyObj);
this.nodes.remove(wrap(key));
if (dataSource == null) return;
this.dataSource.delete(key);
}
public void commit() {
long t = System.nanoTime();
if (dataSource == null) return;
// Don't try to commit if it isn't dirty
if (!this.isDirty) {
return;
}
if ((dataSource == null) || !this.isDirty) return;
long start = System.nanoTime();
long size = 0;
long keys = 0;
long totalSize = 0;
Map<byte[], byte[]> batch = new HashMap<>();
for (ByteArrayWrapper key : this.nodes.keySet()) {
Node node = this.nodes.get(key);
if (node.isDirty()) {
node.setDirty(false);
byte[] value = node.getValue().encode();
batch.put(key.getData(), value);
node.setDirty(false);
size += value.length;
keys += 1;
totalSize += value.length;
}
}
@ -103,11 +95,11 @@ public class Cache {
this.isDirty = false;
this.nodes.clear();
long t_ = System.nanoTime();
String sizeFmt = String.format("%02.2f", ((float)size) / 1048576);
gLogger.info("Flush state in: {} ms, {} nodes, {}MB",
((float)(t_ - t) / 1_000_000), keys, sizeFmt);
long finish = System.nanoTime();
float flushSize = (float) totalSize / 1048576;
float flushTime = (float) (finish - start) / 1_000_000;
gLogger.info(format("Flush state in: %02.2f ms, %d nodes, %02.2fMB", flushTime, batch.size(), flushSize));
}
public void undo() {

1
ethereumj-core/src/main/java/org/ethereum/trie/TrieImpl.java
View File

@ -1,6 +1,5 @@
package org.ethereum.trie;
import org.ethereum.crypto.HashUtil;
import org.ethereum.datasource.KeyValueDataSource;
import org.ethereum.db.ByteArrayWrapper;
import org.ethereum.util.RLP;