From 43b956a1061499b6bcd77d3df51e7fac2bbf4e4c Mon Sep 17 00:00:00 2001 From: eugene-shevchenko Date: Thu, 26 Mar 2015 10:25:49 +0200 Subject: [PATCH 1/8] Pending transaction save in external datasource --- .../org/ethereum/config/SystemProperties.java | 12 +- .../org/ethereum/core/BlockchainImpl.java | 6 +- .../datasource/redis/RedisConnection.java | 17 ++ .../datasource/redis/RedisConnectionImpl.java | 104 ++++++++++++ .../redis/RedisKeyValueDataSource.java | 91 +++++++++++ .../ethereum/datasource/redis/RedisMap.java | 152 ++++++++++++++++++ .../datasource/redis/RedisSerializer.java | 10 ++ .../ethereum/datasource/redis/RedisSet.java | 120 ++++++++++++++ .../datasource/redis/RedisStorage.java | 114 +++++++++++++ .../datasource/redis/Serializers.java | 117 ++++++++++++++ .../org/ethereum/facade/DefaultConfig.java | 22 ++- .../org/ethereum/facade/RemoteConfig.java | 21 ++- .../java/org/ethereum/util/Functional.java | 63 ++++++++ .../datasource/RedisConnectionTest.java | 109 +++++++++++++ 14 files changed, 953 insertions(+), 5 deletions(-) create mode 100644 ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisConnection.java create mode 100644 ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisConnectionImpl.java create mode 100644 ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisKeyValueDataSource.java create mode 100644 ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisMap.java create mode 100644 ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisSerializer.java create mode 100644 ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisSet.java create mode 100644 ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisStorage.java create mode 100644 ethereumj-core/src/main/java/org/ethereum/datasource/redis/Serializers.java create mode 100644 ethereumj-core/src/main/java/org/ethereum/util/Functional.java create mode 100644 ethereumj-core/src/test/java/test/ethereum/datasource/RedisConnectionTest.java diff --git a/ethereumj-core/src/main/java/org/ethereum/config/SystemProperties.java b/ethereumj-core/src/main/java/org/ethereum/config/SystemProperties.java index 5ba2573a..9367b85f 100644 --- a/ethereumj-core/src/main/java/org/ethereum/config/SystemProperties.java +++ b/ethereumj-core/src/main/java/org/ethereum/config/SystemProperties.java @@ -50,6 +50,7 @@ public class SystemProperties { private final static String DEFAULT_VM_TRACE_DIR = "dmp"; private final static int DEFAULT_PEER_LISTEN_PORT = 30303; private final static String DEFAULT_KEY_VALUE_DATA_SOURCE = "leveldb"; + private final static boolean DEFAULT_REDIS_ENABLED = true; /* Testing */ @@ -239,7 +240,11 @@ public class SystemProperties { } public boolean vmTrace() { - return Boolean.parseBoolean(prop.getProperty("vm.structured.trace", String.valueOf(DEFAULT_VM_TRACE))); + return boolProperty("vm.structured.trace", DEFAULT_VM_TRACE); + } + + private boolean boolProperty(String key, Boolean defaultValue) { + return Boolean.parseBoolean(prop.getProperty(key, String.valueOf(defaultValue))); } public String vmTraceDir() { @@ -250,10 +255,13 @@ public class SystemProperties { return Integer.parseInt(prop.getProperty("peer.listen.port", String.valueOf(DEFAULT_PEER_LISTEN_PORT))); } - public String getKeyValueDataSource() { return prop.getProperty("keyvalue.datasource", DEFAULT_KEY_VALUE_DATA_SOURCE); } + + public boolean isRedisEnabled() { + return boolProperty("redis.enabled", DEFAULT_REDIS_ENABLED); + } public void setListenPort(Integer port) { prop.setProperty("peer.listen.port", port.toString()); diff --git a/ethereumj-core/src/main/java/org/ethereum/core/BlockchainImpl.java b/ethereumj-core/src/main/java/org/ethereum/core/BlockchainImpl.java index 250d0174..e8d1466f 100644 --- a/ethereumj-core/src/main/java/org/ethereum/core/BlockchainImpl.java +++ b/ethereumj-core/src/main/java/org/ethereum/core/BlockchainImpl.java @@ -16,9 +16,11 @@ import org.slf4j.LoggerFactory; import org.spongycastle.util.encoders.Hex; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; import org.springframework.util.FileSystemUtils; +import javax.annotation.Resource; import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; @@ -72,7 +74,9 @@ public class BlockchainImpl implements Blockchain { // to avoid using minGasPrice=0 from Genesis for the wallet private static final long INITIAL_MIN_GAS_PRICE = 10 * SZABO.longValue(); - private final Set pendingTransactions = Collections.synchronizedSet(new HashSet()); + @Resource + @Qualifier("pendingTransactions") + private Set pendingTransactions; @Autowired private Repository repository; diff --git a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisConnection.java b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisConnection.java new file mode 100644 index 00000000..e7085c92 --- /dev/null +++ b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisConnection.java @@ -0,0 +1,17 @@ +package org.ethereum.datasource.redis; + +import org.ethereum.core.Transaction; +import org.ethereum.datasource.KeyValueDataSource; + +import java.util.Set; + +public interface RedisConnection { + + boolean isAvailable(); + + Set createSetFor(Class clazz, String name); + + Set createTransactionSet(String name); + + KeyValueDataSource createKeyValueDataSource(String name); +} diff --git a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisConnectionImpl.java b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisConnectionImpl.java new file mode 100644 index 00000000..346d8fdd --- /dev/null +++ b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisConnectionImpl.java @@ -0,0 +1,104 @@ +package org.ethereum.datasource.redis; + +import org.ethereum.config.SystemProperties; +import org.ethereum.core.Transaction; +import org.ethereum.datasource.KeyValueDataSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.JedisPoolConfig; +import redis.clients.jedis.Protocol; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.net.URI; +import java.util.Set; + +import static org.springframework.util.StringUtils.isEmpty; + +@Component +public class RedisConnectionImpl implements RedisConnection { + + private static final Logger logger = LoggerFactory.getLogger("db"); + + private JedisPool jedisPool; + + @PostConstruct + public void init() { + if (!SystemProperties.CONFIG.isRedisEnabled()) return; + + String redisCloudUrl = System.getenv("REDISCLOUD_URL"); + if (isEmpty(redisCloudUrl)) { + logger.info("Cannot connect to Redis. 'REDISCLOUD_URL' environment variable is not defined."); + return; + } + + logger.info("Redis pool creating: " + redisCloudUrl); + try { + jedisPool = createJedisPool(new URI(redisCloudUrl)); + } catch (Exception e) { + logger.warn("Cannot connect to Redis cloud: ", e); + } finally { + logger.info(isAvailable() ? "Redis cloud connected successfully." : "Redis cloud connection failed."); + } + } + + private static JedisPool createJedisPool(URI redisUri) { + String userInfo = redisUri.getUserInfo(); + if (StringUtils.hasText(userInfo)) { + return new JedisPool(new JedisPoolConfig(), + redisUri.getHost(), + redisUri.getPort(), + Protocol.DEFAULT_TIMEOUT, + userInfo.split(":", 2)[1]); + } + + return new JedisPool(new JedisPoolConfig(), + redisUri.getHost(), + redisUri.getPort(), + Protocol.DEFAULT_TIMEOUT); + } + + @PreDestroy + public void destroy() { + if (jedisPool != null) { + jedisPool.destroy(); + } + } + + @Override + public boolean isAvailable() { + boolean available = jedisPool != null; + if (available) { + Jedis jedis = jedisPool.getResource(); + try { + available = jedis.isConnected(); + } catch (Throwable t) { + logger.warn("Connection testing failed: ", t); + available = false; + } finally { + jedisPool.returnResource(jedis); + } + } + return available; + } + + @Override + public Set createSetFor(Class clazz, String name) { + return new RedisSet(name, jedisPool, Serializers.forClass(clazz)); + } + + @Override + public Set createTransactionSet(String name) { + return createSetFor(Transaction.class, name); + } + + @Override + public KeyValueDataSource createKeyValueDataSource(String name) { + return new RedisKeyValueDataSource(name, jedisPool, null); + + } +} diff --git a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisKeyValueDataSource.java b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisKeyValueDataSource.java new file mode 100644 index 00000000..42d556c5 --- /dev/null +++ b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisKeyValueDataSource.java @@ -0,0 +1,91 @@ +package org.ethereum.datasource.redis; + +import org.ethereum.datasource.KeyValueDataSource; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.Pipeline; + +import java.util.Map; +import java.util.Set; + +import static org.ethereum.config.SystemProperties.CONFIG; +import static org.ethereum.util.Functional.*; + +public class RedisKeyValueDataSource extends RedisStorage implements KeyValueDataSource { + + RedisKeyValueDataSource(String namespace, JedisPool pool, RedisSerializer serializer) { + super(namespace, pool, serializer); + } + + @Override + public void init() { + if (CONFIG.databaseReset()) { + pooled(new Consumer() { + @Override + public void accept(Jedis jedis) { + jedis.flushAll(); + } + }); + } + } + + @Override + public void setName(String name) { + setNamespace(name); + } + + @Override + public byte[] get(final byte[] key) { + return pooledWithResult(new Function() { + @Override + public byte[] apply(Jedis jedis) { + return jedis.get(getNamespace()); + } + }); + } + + @Override + public void put(final byte[] key, final byte[] value) { + pooled(new Consumer() { + @Override + public void accept(Jedis jedis) { + jedis.set(formatKey(key), value); + } + }); + } + + @Override + public void delete(final byte[] key) { + pooled(new Consumer() { + @Override + public void accept(Jedis jedis) { + jedis.del(formatKey(key)); + } + }); + } + + @Override + public Set keys() { + return pooledWithResult(new Function>() { + @Override + public Set apply(Jedis jedis) { + return jedis.keys(formatKey("*".getBytes())); + } + }); + } + + @Override + public void updateBatch(final Map rows) { + doInPipeline(rows.entrySet(), new BiConsumer, Pipeline>() { + @Override + public void accept(Map.Entry entry, Pipeline pipeline) { + pipeline.set(formatKey(entry.getKey()), entry.getValue()); + } + }); + } + + @Override + public void close() { + + } +} diff --git a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisMap.java b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisMap.java new file mode 100644 index 00000000..679022c2 --- /dev/null +++ b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisMap.java @@ -0,0 +1,152 @@ +package org.ethereum.datasource.redis; + +import org.apache.commons.collections4.Transformer; +import org.apache.commons.collections4.keyvalue.AbstractMapEntry; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; + +import java.util.Collection; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import static org.apache.commons.collections4.CollectionUtils.collect; +import static org.ethereum.util.Functional.Consumer; +import static org.ethereum.util.Functional.Function; + +public class RedisMap extends RedisStorage implements Map { + + private final RedisSerializer keySerializer; + + RedisMap(String namespace, JedisPool pool, RedisSerializer keySerializer, RedisSerializer valueSerializer) { + super(namespace, pool, valueSerializer); + this.keySerializer = keySerializer; + } + + @Override + public int size() { + return pooledWithResult(new Function() { + @Override + public Integer apply(Jedis jedis) { + return jedis.hlen(getNamespace()).intValue(); + } + }); + } + + @Override + public boolean isEmpty() { + return size() == 0; + } + + @Override + public boolean containsKey(final Object key) { + return pooledWithResult(new Function() { + @Override + public Boolean apply(Jedis jedis) { + return jedis.hexists(getNamespace(), keySerializer.serialize((K) key)); + } + }); + } + + @Override + public boolean containsValue(Object value) { + return false; + } + + @Override + public V get(final Object key) { + return pooledWithResult(new Function() { + @Override + public V apply(Jedis jedis) { + byte[] value = jedis.hget(getNamespace(), keySerializer.serialize((K) key)); + return deserialize(value); + } + }); + } + + @Override + public V put(K key, V value) { +/* + return pooledWithResult(new Function() { + @Override + public V apply(Jedis jedis) { + return jedis.hset(getNamespace(), keySerializer.serialize(key), serialize(value)); + } + }); +*/ + return null; + } + + @Override + public V remove(Object key) { + return null; + } + + @Override + public void putAll(Map m) { + + } + + @Override + public void clear() { + pooled(new Consumer() { + @Override + public void accept(Jedis jedis) { + jedis.del(getNamespace()); + } + }); + } + + @Override + public Set keySet() { + return pooledWithResult(new Function>() { + @Override + public Set apply(Jedis jedis) { + Set result = new HashSet(); + collect(jedis.hkeys(getNamespace()), new Transformer() { + @Override + public K transform(byte[] input) { + return keySerializer.deserialize(input); + } + }, result); + return result; + } + }); + } + + @Override + public Collection values() { + return pooledWithResult(new Function>() { + @Override + public Collection apply(Jedis jedis) { + return deserialize(jedis.hvals(getNamespace())); + } + }); + } + + @Override + public Set> entrySet() { + return pooledWithResult(new Function>>() { + @Override + public Set> apply(Jedis jedis) { + Set> result = new HashSet>(); + collect(jedis.hgetAll(getNamespace()).entrySet(), new Transformer, Entry>() { + @Override + public Entry transform(Entry input) { + K key = keySerializer.deserialize(input.getKey()); + V value = deserialize(input.getValue()); + return new RedisMapEntry(key, value); + } + }, result); + return result; + } + }); + } + + private class RedisMapEntry extends AbstractMapEntry { + + RedisMapEntry(K key, V value) { + super(key, value); + } + } +} diff --git a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisSerializer.java b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisSerializer.java new file mode 100644 index 00000000..b3c3d979 --- /dev/null +++ b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisSerializer.java @@ -0,0 +1,10 @@ +package org.ethereum.datasource.redis; + +public interface RedisSerializer { + + boolean canSerialize(Object o); + + byte[] serialize(T t); + + T deserialize(byte[] bytes); +} \ No newline at end of file diff --git a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisSet.java b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisSet.java new file mode 100644 index 00000000..ba54a367 --- /dev/null +++ b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisSet.java @@ -0,0 +1,120 @@ +package org.ethereum.datasource.redis; + +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; +import java.util.Set; + +import static org.ethereum.util.Functional.Consumer; +import static org.ethereum.util.Functional.Function; + +public class RedisSet extends RedisStorage implements Set { + + RedisSet(String namespace, JedisPool pool, RedisSerializer serializer) { + super(namespace, pool, serializer); + } + + @Override + public int size() { + return pooledWithResult(new Function() { + @Override + public Integer apply(Jedis jedis) { + return jedis.scard(getNamespace()).intValue(); + } + }); + } + + @Override + public boolean isEmpty() { + return size() == 0; + } + + @Override + public boolean contains(final Object o) { + return pooledWithResult(new Function() { + @Override + public Boolean apply(Jedis jedis) { + return jedis.sismember(getNamespace(), serialize(o)); + } + }); + } + + @Override + public Iterator iterator() { + Set members = pooledWithResult(new Function>() { + @Override + public Set apply(Jedis jedis) { + return jedis.smembers(getNamespace()); + } + }); + return deserialize(members).iterator(); + } + + @Override + public Object[] toArray() { + throw new UnsupportedOperationException(); + } + + @Override + public T1[] toArray(T1[] a) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean add(T t) { + return addAll(Arrays.asList(t)); + } + + @Override + public boolean remove(Object o) { + return removeAll(Arrays.asList(o)); + } + + @Override + public boolean containsAll(Collection c) { + return false; + } + + @Override + public boolean addAll(final Collection c) { + return pooledWithResult(new Function() { + @Override + public Boolean apply(Jedis jedis) { + return jedis.sadd(getNamespace(), serialize(c)) == 1; + } + }); + } + + @Override + public boolean retainAll(final Collection c) { + return pooledWithResult(new Function() { + @Override + public Boolean apply(Jedis jedis) { + return jedis.sinterstore(getNamespace(), serialize(c)) == c.size(); + } + }); + } + + @Override + public boolean removeAll(final Collection c) { + return pooledWithResult(new Function() { + @Override + public Boolean apply(Jedis jedis) { + return jedis.srem(getNamespace(), serialize(c)) == 1; + } + }); + } + + @Override + public void clear() { + pooled(new Consumer() { + @Override + public void accept(Jedis jedis) { + jedis.del(getNamespace()); + } + }); + } +} diff --git a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisStorage.java b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisStorage.java new file mode 100644 index 00000000..a20de52f --- /dev/null +++ b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisStorage.java @@ -0,0 +1,114 @@ +package org.ethereum.datasource.redis; + +import org.apache.commons.collections4.Transformer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.Pipeline; + +import java.util.Collection; + +import static org.apache.commons.collections4.CollectionUtils.collect; +import static org.ethereum.util.Functional.*; + +public abstract class RedisStorage { + + private static final Logger logger = LoggerFactory.getLogger("db"); + + private String namespace; + private final JedisPool pool; + private final RedisSerializer serializer; + + RedisStorage(String namespace, JedisPool pool, RedisSerializer serializer) { + this.pool = pool; + this.serializer = serializer; + + setNamespace(namespace); + } + + protected byte[] getNamespace() { + return namespace.getBytes(); + } + + protected void setNamespace(String namespace) { + this.namespace = namespace + ":"; + } + + protected byte[] formatKey(byte[] key) { + byte[] prefix = getNamespace(); + + int length = prefix.length + key.length; + byte[] result = new byte[length]; + System.arraycopy(prefix, 0, result, 0, prefix.length); + System.arraycopy(key, 0, result, prefix.length, key.length); + return result; + } + + protected byte[] serialize(Object o) { + if (serializer.canSerialize(o)) { + return serializer.serialize((T) o); + } + + logger.warn("Cannot serialize '%s'.", o.getClass()); + return new byte[0]; + } + + protected byte[][] serialize(Collection collection) { + return collect(collection, new Transformer() { + @Override + public byte[] transform(Object input) { + return serialize(input); + } + }).toArray(new byte[][]{}); + } + + protected T deserialize(byte[] bytes) { + return serializer.deserialize(bytes); + } + + + protected Collection deserialize(Collection bytesCollection) { + return collect(bytesCollection, new Transformer() { + @Override + public T transform(byte[] input) { + return deserialize(input); + } + }); + } + + protected

void doInPipeline(final Collection

collection, final BiConsumer consumer) { + pooled(new Consumer() { + @Override + public void accept(Jedis jedis) { + Pipeline pipeline = jedis.pipelined(); + try { + for (P el : collection) { + consumer.accept(el, pipeline); + } + } finally { + pipeline.sync(); + } + } + }); + } + + protected void pooled(final Consumer consumer) { + pooledWithResult(new Function() { + @Override + public Object apply(Jedis jedis) { + consumer.accept(jedis); + return null; + } + }); + } + + protected R pooledWithResult(Function function) { + Jedis jedis = pool.getResource(); + try { + return function.apply(jedis); + } finally { + pool.returnResource(jedis); + } + } +} diff --git a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/Serializers.java b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/Serializers.java new file mode 100644 index 00000000..27b67cdb --- /dev/null +++ b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/Serializers.java @@ -0,0 +1,117 @@ +package org.ethereum.datasource.redis; + +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.type.TypeFactory; +import org.ethereum.core.Transaction; +import org.ethereum.core.TransactionReceipt; + +import java.util.HashSet; +import java.util.Set; + +public final class Serializers { + + private static abstract class BaseRedisSerializer implements RedisSerializer { + + public abstract boolean supports(Class aClass); + + @Override + public boolean canSerialize(Object o) { + return (o != null) && supports(o.getClass()); + } + } + + private static class JacksonJsonRedisSerializer implements RedisSerializer { + + private final ObjectMapper objectMapper = new ObjectMapper(); + private final JavaType javaType; + + JacksonJsonRedisSerializer(Class clazz) { + this.javaType = TypeFactory.defaultInstance().constructType(clazz); + } + + @Override + public T deserialize(byte[] bytes) { + if (isEmpty(bytes)) return null; + + try { + return (T) this.objectMapper.readValue(bytes, 0, bytes.length, javaType); + } catch (Exception ex) { + throw new RuntimeException("Cannot read JSON: " + ex.getMessage(), ex); + } + } + + @Override + public byte[] serialize(Object t) { + if (t == null) return EMPTY_ARRAY; + + try { + return this.objectMapper.writeValueAsBytes(t); + } catch (Exception ex) { + throw new RuntimeException("Cannot write JSON: " + ex.getMessage(), ex); + } + } + + @Override + public boolean canSerialize(Object o) { + return (o != null) && javaType.hasRawClass(o.getClass()); + } + } + + private static class TransactionReceiptSerializer extends BaseRedisSerializer { + + @Override + public boolean supports(Class aClass) { + return TransactionReceipt.class.isAssignableFrom(aClass); + } + + + @Override + public byte[] serialize(TransactionReceipt transactionReceipt) { + return transactionReceipt.getEncoded(); + } + + @Override + public TransactionReceipt deserialize(byte[] bytes) { + return new TransactionReceipt(bytes); + } + } + + private static class TransactionSerializer extends BaseRedisSerializer { + + @Override + public boolean supports(Class aClass) { + return Transaction.class.isAssignableFrom(aClass); + } + + @Override + public byte[] serialize(Transaction transaction) { + return transaction.getEncoded(); + } + + @Override + public Transaction deserialize(byte[] bytes) { + return new Transaction(bytes); + } + } + + + private static final byte[] EMPTY_ARRAY = new byte[0]; + private static final Set SERIALIZERS = new HashSet() {{ + add(new TransactionSerializer()); + add(new TransactionReceiptSerializer()); + }}; + + public static RedisSerializer forClass(Class clazz) { + for (BaseRedisSerializer serializer : SERIALIZERS) { + if (serializer.supports(clazz)) { + return serializer; + } + } + return new JacksonJsonRedisSerializer(clazz); + } + + private static boolean isEmpty(byte[] data) { + return (data == null || data.length == 0); + } +} diff --git a/ethereumj-core/src/main/java/org/ethereum/facade/DefaultConfig.java b/ethereumj-core/src/main/java/org/ethereum/facade/DefaultConfig.java index e79120f3..6f5cf80a 100644 --- a/ethereumj-core/src/main/java/org/ethereum/facade/DefaultConfig.java +++ b/ethereumj-core/src/main/java/org/ethereum/facade/DefaultConfig.java @@ -1,13 +1,15 @@ package org.ethereum.facade; import org.ethereum.config.SystemProperties; +import org.ethereum.core.Transaction; import org.ethereum.datasource.KeyValueDataSource; import org.ethereum.datasource.LevelDbDataSource; import org.ethereum.datasource.RedisDataSource; +import org.ethereum.datasource.redis.RedisConnection; +import org.ethereum.datasource.redis.RedisConnectionImpl; import org.ethereum.db.BlockStore; import org.ethereum.db.BlockStoreImpl; import org.ethereum.db.RepositoryImpl; -import org.hibernate.Session; import org.hibernate.SessionFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,7 +26,10 @@ import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; import java.sql.SQLException; +import java.util.Collections; +import java.util.HashSet; import java.util.Properties; +import java.util.Set; import static org.ethereum.config.SystemProperties.CONFIG; @@ -59,6 +64,21 @@ public class DefaultConfig { return new LevelDbDataSource(); } + @Bean + public RedisConnection redisConnection() { + RedisConnectionImpl connection = new RedisConnectionImpl(); + connection.init(); + return connection; + } + + @Bean + public Set pendingTransactions() { + RedisConnection connection = redisConnection(); + return connection.isAvailable() + ? connection.createTransactionSet("pendingTransactions") + : Collections.synchronizedSet(new HashSet()); + } + @Bean @Transactional(propagation = Propagation.SUPPORTS) public BlockStore blockStore(SessionFactory sessionFactory){ diff --git a/ethereumj-core/src/main/java/org/ethereum/facade/RemoteConfig.java b/ethereumj-core/src/main/java/org/ethereum/facade/RemoteConfig.java index b44dd8fe..8fec9be1 100644 --- a/ethereumj-core/src/main/java/org/ethereum/facade/RemoteConfig.java +++ b/ethereumj-core/src/main/java/org/ethereum/facade/RemoteConfig.java @@ -1,11 +1,13 @@ package org.ethereum.facade; import org.ethereum.config.SystemProperties; +import org.ethereum.core.Transaction; import org.ethereum.datasource.KeyValueDataSource; import org.ethereum.datasource.LevelDbDataSource; import org.ethereum.datasource.RedisDataSource; +import org.ethereum.datasource.redis.RedisConnection; +import org.ethereum.datasource.redis.RedisConnectionImpl; import org.ethereum.db.BlockStore; -import org.ethereum.db.BlockStoreImpl; import org.ethereum.db.InMemoryBlockStore; import org.ethereum.db.RepositoryImpl; import org.hibernate.SessionFactory; @@ -24,7 +26,10 @@ import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; import java.sql.SQLException; +import java.util.Collections; +import java.util.HashSet; import java.util.Properties; +import java.util.Set; import static org.ethereum.config.SystemProperties.CONFIG; @@ -62,6 +67,20 @@ public class RemoteConfig { return new LevelDbDataSource(); } + @Bean + public RedisConnection redisConnection() { + RedisConnectionImpl connection = new RedisConnectionImpl(); + connection.init(); + return connection; + } + + @Bean + public Set pendingTransactions() { + RedisConnection connection = redisConnection(); + return connection.isAvailable() + ? connection.createTransactionSet("pendingTransactions") + : Collections.synchronizedSet(new HashSet()); + } @Bean @Transactional(propagation = Propagation.SUPPORTS) diff --git a/ethereumj-core/src/main/java/org/ethereum/util/Functional.java b/ethereumj-core/src/main/java/org/ethereum/util/Functional.java new file mode 100644 index 00000000..9f188d3c --- /dev/null +++ b/ethereumj-core/src/main/java/org/ethereum/util/Functional.java @@ -0,0 +1,63 @@ +package org.ethereum.util; + + +public interface Functional { + + /** + * Represents an operation that accepts a single input argument and returns no + * result. Unlike most other functional interfaces, {@code Consumer} is expected + * to operate via side-effects. + * + * @param the type of the input to the operation + */ + public static interface Consumer { + + /** + * Performs this operation on the given argument. + * + * @param t the input argument + */ + void accept(T t); + } + + /** + * Represents an operation that accepts two input arguments and returns no + * result. This is the two-arity specialization of {@link java.util.function.Consumer}. + * Unlike most other functional interfaces, {@code BiConsumer} is expected + * to operate via side-effects. + * + * @param the type of the first argument to the operation + * @param the type of the second argument to the operation + * + * @see org.ethereum.util.Functional.Consumer + */ + public interface BiConsumer { + + /** + * Performs this operation on the given arguments. + * + * @param t the first input argument + * @param u the second input argument + */ + void accept(T t, U u); + } + + + /** + * Represents a function that accepts one argument and produces a result. + * + * @param the type of the input to the function + * @param the type of the result of the function + */ + public static interface Function { + + /** + * Applies this function to the given argument. + * + * @param t the function argument + * @return the function result + */ + R apply(T t); + } + +} diff --git a/ethereumj-core/src/test/java/test/ethereum/datasource/RedisConnectionTest.java b/ethereumj-core/src/test/java/test/ethereum/datasource/RedisConnectionTest.java new file mode 100644 index 00000000..a1ab18b1 --- /dev/null +++ b/ethereumj-core/src/test/java/test/ethereum/datasource/RedisConnectionTest.java @@ -0,0 +1,109 @@ +package test.ethereum.datasource; + +import org.ethereum.core.Transaction; +import org.ethereum.crypto.ECKey; +import org.ethereum.crypto.HashUtil; +import org.ethereum.datasource.redis.RedisConnection; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.spongycastle.util.encoders.Hex; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.test.context.support.AnnotationConfigContextLoader; +import test.ethereum.TestContext; + +import java.math.BigInteger; +import java.util.Set; + +import static org.junit.Assert.*; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(loader = AnnotationConfigContextLoader.class) +public class RedisConnectionTest { + + @Configuration + @ComponentScan(basePackages = "org.ethereum") + static class ContextConfiguration extends TestContext { } + + @Autowired + private RedisConnection redisConnection; + + @Test + public void test() { + if (!redisConnection.isAvailable()) return; + + Set pojos = redisConnection.createSetFor(Pojo.class, "pojos"); + + Pojo pojo = Pojo.create(1L, "test"); + pojos.add(pojo); + assertTrue(pojos.contains(pojo)); + assertEquals(1, pojos.size()); + Pojo next = pojos.iterator().next(); + assertNotNull(next); + assertEquals(pojo.getId(), next.getId()); + assertEquals(pojo.getName(), next.getName()); + assertTrue(pojos.remove(pojo)); + assertTrue(pojos.isEmpty()); + } + + @Test + public void transactionStorageTest() { + if (!redisConnection.isAvailable()) return; + + String namespace = "txnNamespace"; + Set transactions = redisConnection.createTransactionSet(namespace); + transactions.add(createTransaction("09184e72a000", "4255", "1000000000000000000000", "cat")); + transactions.add(createTransaction("09184e72a000", "4255", "1000000000000000000000", "dog")); + transactions.add(createTransaction("09184e72a000", "4255", "1000000000000000000000", "rabbit")); + + Set transactions1 = redisConnection.createTransactionSet(namespace); + transactions1.add(createTransaction("09184e72a000", "4255", "1000000000000000000000", "duck")); + transactions1.add(createTransaction("09184e72a000", "4255", "1000000000000000000000", "chicken")); + transactions1.add(createTransaction("09184e72a000", "4255", "1000000000000000000000", "cow")); + + assertEquals(6, transactions1.size()); + transactions.clear(); + assertTrue(transactions1.isEmpty()); + } + + private static class Pojo { + private long id; + private String name; + + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public static Pojo create(long id, String name) { + Pojo result = new Pojo(); + result.setId(id); + result.setName(name); + return result; + } + } + + public static Transaction createTransaction(String gasPrice, String gas, String val, String secret) { + + ECKey ecKey = ECKey.fromPrivate(HashUtil.sha3(secret.getBytes())); + + // Tn (nonce); Tp(pgas); Tg(gaslimi); Tt(value); Tv(value); Ti(sender); Tw; Tr; Ts + return new Transaction(null, Hex.decode(gasPrice), Hex.decode(gas), ecKey.getAddress(), + new BigInteger(val).toByteArray(), + null); + } +} \ No newline at end of file From 40e1019d5bfcc8d1827fdd1cb568372d9ceba27c Mon Sep 17 00:00:00 2001 From: eugene-shevchenko Date: Thu, 26 Mar 2015 23:32:40 +0200 Subject: [PATCH 2/8] Pending Transaction - save in external datasource --- .../datasource/KeyValueDataSource.java | 16 +-- .../datasource/LevelDbDataSource.java | 3 +- .../ethereum/datasource/RedisDataSource.java | 104 --------------- .../org/ethereum/datasource/RedisPool.java | 61 --------- .../datasource/redis/RedisConnection.java | 4 +- .../datasource/redis/RedisConnectionImpl.java | 7 +- .../datasource/redis/RedisDataSource.java | 80 +++++++++++ .../redis/RedisKeyValueDataSource.java | 91 ------------- .../ethereum/datasource/redis/RedisMap.java | 70 ++++++---- .../ethereum/datasource/redis/RedisSet.java | 24 ++-- .../datasource/redis/RedisStorage.java | 36 ++--- .../datasource/redis/Serializers.java | 76 +++++++++-- .../java/org/ethereum/db/DatabaseImpl.java | 6 +- .../org/ethereum/facade/CommonConfig.java | 123 +++++++++++++++++ .../org/ethereum/facade/DefaultConfig.java | 124 +---------------- .../org/ethereum/facade/EthereumImpl.java | 7 +- .../org/ethereum/facade/RemoteConfig.java | 126 +----------------- .../datasource/AbstractRedisTest.java | 24 ++++ .../datasource/RedisConnectionTest.java | 20 +-- .../datasource/RedisDataSourceTest.java | 61 ++++----- .../test/java/test/ethereum/db/MockDB.java | 6 +- 21 files changed, 423 insertions(+), 646 deletions(-) delete mode 100644 ethereumj-core/src/main/java/org/ethereum/datasource/RedisDataSource.java delete mode 100644 ethereumj-core/src/main/java/org/ethereum/datasource/RedisPool.java create mode 100644 ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisDataSource.java delete mode 100644 ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisKeyValueDataSource.java create mode 100644 ethereumj-core/src/main/java/org/ethereum/facade/CommonConfig.java create mode 100644 ethereumj-core/src/test/java/test/ethereum/datasource/AbstractRedisTest.java diff --git a/ethereumj-core/src/main/java/org/ethereum/datasource/KeyValueDataSource.java b/ethereumj-core/src/main/java/org/ethereum/datasource/KeyValueDataSource.java index ad05fc27..eb09f7d2 100644 --- a/ethereumj-core/src/main/java/org/ethereum/datasource/KeyValueDataSource.java +++ b/ethereumj-core/src/main/java/org/ethereum/datasource/KeyValueDataSource.java @@ -9,19 +9,19 @@ import java.util.Set; */ public interface KeyValueDataSource { - public void init(); + void init(); - public void setName(String name); + void setName(String name); - public byte[] get(byte[] key); + byte[] get(byte[] key); - public void put(byte[] key, byte[] value); + byte[] put(byte[] key, byte[] value); - public void delete(byte[] key); + void delete(byte[] key); - public Set keys(); + Set keys(); - public void updateBatch(Map rows); + void updateBatch(Map rows); - public void close(); + void close(); } diff --git a/ethereumj-core/src/main/java/org/ethereum/datasource/LevelDbDataSource.java b/ethereumj-core/src/main/java/org/ethereum/datasource/LevelDbDataSource.java index ed6e7864..d18e2426 100644 --- a/ethereumj-core/src/main/java/org/ethereum/datasource/LevelDbDataSource.java +++ b/ethereumj-core/src/main/java/org/ethereum/datasource/LevelDbDataSource.java @@ -88,8 +88,9 @@ public class LevelDbDataSource implements KeyValueDataSource { } @Override - public void put(byte[] key, byte[] value) { + public byte[] put(byte[] key, byte[] value) { db.put(key, value); + return value; } @Override diff --git a/ethereumj-core/src/main/java/org/ethereum/datasource/RedisDataSource.java b/ethereumj-core/src/main/java/org/ethereum/datasource/RedisDataSource.java deleted file mode 100644 index 416b0a45..00000000 --- a/ethereumj-core/src/main/java/org/ethereum/datasource/RedisDataSource.java +++ /dev/null @@ -1,104 +0,0 @@ -package org.ethereum.datasource; - -import redis.clients.jedis.Jedis; -import redis.clients.jedis.Pipeline; - -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.ethereum.config.SystemProperties.CONFIG; - -/** - * - * @author: Roman Mandeleil - * Created on: 18/01/2015 21:48 - */ - -public class RedisDataSource implements KeyValueDataSource{ - - String name; - int index; - - Jedis jedis; - - public RedisDataSource() { - } - - public RedisDataSource(String name) { - this.name = name; - } - - @Override - public void init() { - - if (name == null) throw new NullPointerException("no name set to the db"); - - this.jedis = RedisPool.getResource(name); - if (jedis == null) this.jedis = new Jedis("localhost"); - - if (CONFIG.databaseReset()) - this.jedis.flushAll(); - } - - @Override - public void setName(String name) { - this.name = name; - index = nameToIndex(name); - } - - @Override - public byte[] get(byte[] key) { - return jedis.get(key); - } - - @Override - public void put(byte[] key, byte[] value) { - jedis.set(key, value); - } - - @Override - public void delete(byte[] key) { - jedis.del(key); - } - - @Override - public Set keys() { - return jedis.keys("*".getBytes()); - } - - @Override - public void updateBatch(Map rows) { - Pipeline pipeline = jedis.pipelined(); - - Iterator> iterator = rows.entrySet().iterator(); - while(iterator.hasNext()){ - - Map.Entry row = iterator.next(); - byte[] key = row.getKey(); - byte[] val = row.getValue(); - pipeline.set(key, val); - } - pipeline.sync(); - } - - @Override - public void close() { - jedis.close(); - } - - private static Integer nameToIndex(String name) { - Integer index = DBNameScheme.get(name); - if (index == null) { - index = indexCounter.getAndIncrement(); - DBNameScheme.put(name, index); - indexCounter.intValue(); - } - return index; - } - - private static Map DBNameScheme = new HashMap<>(); - private static AtomicInteger indexCounter = new AtomicInteger(1); -} diff --git a/ethereumj-core/src/main/java/org/ethereum/datasource/RedisPool.java b/ethereumj-core/src/main/java/org/ethereum/datasource/RedisPool.java deleted file mode 100644 index f1e8d120..00000000 --- a/ethereumj-core/src/main/java/org/ethereum/datasource/RedisPool.java +++ /dev/null @@ -1,61 +0,0 @@ -package org.ethereum.datasource; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.JedisPool; -import redis.clients.jedis.JedisPoolConfig; -import redis.clients.jedis.Protocol; - -import java.net.URI; -import java.net.URISyntaxException; - -/** - * - * @author: Roman Mandeleil - * Created on: 20/01/2015 13:27 - */ - -public class RedisPool { - - private static final Logger logger = LoggerFactory.getLogger("db"); - private static JedisPool state; - private static JedisPool details; - - static { - - try { - - if (System.getenv("REDIS_STATE") != null) { - URI redisUri = new URI(System.getenv("REDIS_STATE")); - logger.info("Init redis pool: " + redisUri.toString()); - state = new JedisPool(new JedisPoolConfig(), - redisUri.getHost(), - redisUri.getPort(), - Protocol.DEFAULT_TIMEOUT, - redisUri.getUserInfo().split(":",2)[1]); - } - - if (System.getenv("REDIS_DETAILS") != null) { - URI redisUri = new URI(System.getenv("REDIS_DETAILS")); - logger.info("Init redis pool: " + redisUri.toString()); - details = new JedisPool(new JedisPoolConfig(), - redisUri.getHost(), - redisUri.getPort(), - Protocol.DEFAULT_TIMEOUT, - redisUri.getUserInfo().split(":",2)[1]); - } - - } catch (URISyntaxException e) { - logger.info("Pool is not available"); - } - } - - public static Jedis getResource(String name){ - if (state == null) return null; - - if (name.equals("state")) return state.getResource(); - if (name.equals("details")) return details.getResource(); - return null; - } -} diff --git a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisConnection.java b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisConnection.java index e7085c92..0932abdc 100644 --- a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisConnection.java +++ b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisConnection.java @@ -6,12 +6,12 @@ import org.ethereum.datasource.KeyValueDataSource; import java.util.Set; public interface RedisConnection { - + boolean isAvailable(); Set createSetFor(Class clazz, String name); Set createTransactionSet(String name); - KeyValueDataSource createKeyValueDataSource(String name); + KeyValueDataSource createDataSource(String name); } diff --git a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisConnectionImpl.java b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisConnectionImpl.java index 346d8fdd..ced78c0f 100644 --- a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisConnectionImpl.java +++ b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisConnectionImpl.java @@ -27,7 +27,7 @@ public class RedisConnectionImpl implements RedisConnection { private JedisPool jedisPool; @PostConstruct - public void init() { + public void tryConnect() { if (!SystemProperties.CONFIG.isRedisEnabled()) return; String redisCloudUrl = System.getenv("REDISCLOUD_URL"); @@ -97,8 +97,7 @@ public class RedisConnectionImpl implements RedisConnection { } @Override - public KeyValueDataSource createKeyValueDataSource(String name) { - return new RedisKeyValueDataSource(name, jedisPool, null); - + public KeyValueDataSource createDataSource(String name) { + return new RedisDataSource(name, jedisPool); } } diff --git a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisDataSource.java b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisDataSource.java new file mode 100644 index 00000000..a385dab3 --- /dev/null +++ b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisDataSource.java @@ -0,0 +1,80 @@ +package org.ethereum.datasource.redis; + +import org.ethereum.datasource.KeyValueDataSource; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; + +import java.util.Map; +import java.util.Set; + +import static org.ethereum.config.SystemProperties.CONFIG; +import static org.ethereum.util.Functional.Consumer; + +public class RedisDataSource extends RedisMap implements KeyValueDataSource { + + RedisDataSource(String namespace, JedisPool pool) { + super(namespace, pool, null, null); + } + + @Override + protected byte[] serializeKey(byte[] key) { + return key; + } + + @Override + protected byte[] deserializeKey(byte[] bytes) { + return bytes; + } + + @Override + protected byte[] serialize(byte[] value) { + return value; + } + + @Override + protected byte[] deserialize(byte[] bytes) { + return bytes; + } + + @Override + public void init() { + if (CONFIG.databaseReset()) { + pooled(new Consumer() { + @Override + public void accept(Jedis jedis) { + jedis.flushAll(); + } + }); + } + } + + @Override + public void setName(String name) { + super.setName(name); + } + + @Override + public byte[] get(byte[] key) { + return super.get(key); + } + + @Override + public void delete(final byte[] key) { + remove(key); + } + + @Override + public Set keys() { + return super.keySet(); + } + + @Override + public void updateBatch(final Map rows) { + putAll(rows); + } + + @Override + public void close() { + + } +} diff --git a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisKeyValueDataSource.java b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisKeyValueDataSource.java deleted file mode 100644 index 42d556c5..00000000 --- a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisKeyValueDataSource.java +++ /dev/null @@ -1,91 +0,0 @@ -package org.ethereum.datasource.redis; - -import org.ethereum.datasource.KeyValueDataSource; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.JedisPool; -import redis.clients.jedis.Pipeline; - -import java.util.Map; -import java.util.Set; - -import static org.ethereum.config.SystemProperties.CONFIG; -import static org.ethereum.util.Functional.*; - -public class RedisKeyValueDataSource extends RedisStorage implements KeyValueDataSource { - - RedisKeyValueDataSource(String namespace, JedisPool pool, RedisSerializer serializer) { - super(namespace, pool, serializer); - } - - @Override - public void init() { - if (CONFIG.databaseReset()) { - pooled(new Consumer() { - @Override - public void accept(Jedis jedis) { - jedis.flushAll(); - } - }); - } - } - - @Override - public void setName(String name) { - setNamespace(name); - } - - @Override - public byte[] get(final byte[] key) { - return pooledWithResult(new Function() { - @Override - public byte[] apply(Jedis jedis) { - return jedis.get(getNamespace()); - } - }); - } - - @Override - public void put(final byte[] key, final byte[] value) { - pooled(new Consumer() { - @Override - public void accept(Jedis jedis) { - jedis.set(formatKey(key), value); - } - }); - } - - @Override - public void delete(final byte[] key) { - pooled(new Consumer() { - @Override - public void accept(Jedis jedis) { - jedis.del(formatKey(key)); - } - }); - } - - @Override - public Set keys() { - return pooledWithResult(new Function>() { - @Override - public Set apply(Jedis jedis) { - return jedis.keys(formatKey("*".getBytes())); - } - }); - } - - @Override - public void updateBatch(final Map rows) { - doInPipeline(rows.entrySet(), new BiConsumer, Pipeline>() { - @Override - public void accept(Map.Entry entry, Pipeline pipeline) { - pipeline.set(formatKey(entry.getKey()), entry.getValue()); - } - }); - } - - @Override - public void close() { - - } -} diff --git a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisMap.java b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisMap.java index 679022c2..40aff499 100644 --- a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisMap.java +++ b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisMap.java @@ -5,10 +5,7 @@ import org.apache.commons.collections4.keyvalue.AbstractMapEntry; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; -import java.util.Collection; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; +import java.util.*; import static org.apache.commons.collections4.CollectionUtils.collect; import static org.ethereum.util.Functional.Consumer; @@ -18,17 +15,25 @@ public class RedisMap extends RedisStorage implements Map { private final RedisSerializer keySerializer; - RedisMap(String namespace, JedisPool pool, RedisSerializer keySerializer, RedisSerializer valueSerializer) { + public RedisMap(String namespace, JedisPool pool, RedisSerializer keySerializer, RedisSerializer valueSerializer) { super(namespace, pool, valueSerializer); this.keySerializer = keySerializer; } + protected byte[] serializeKey(K key) { + return keySerializer.serialize(key); + } + + protected K deserializeKey(byte[] input) { + return keySerializer.deserialize(input); + } + @Override public int size() { return pooledWithResult(new Function() { @Override public Integer apply(Jedis jedis) { - return jedis.hlen(getNamespace()).intValue(); + return jedis.hlen(getName()).intValue(); } }); } @@ -43,14 +48,14 @@ public class RedisMap extends RedisStorage implements Map { return pooledWithResult(new Function() { @Override public Boolean apply(Jedis jedis) { - return jedis.hexists(getNamespace(), keySerializer.serialize((K) key)); + return jedis.hexists(getName(), serializeKey((K) key)); } }); } @Override public boolean containsValue(Object value) { - return false; + return values().contains(value); } @Override @@ -58,33 +63,50 @@ public class RedisMap extends RedisStorage implements Map { return pooledWithResult(new Function() { @Override public V apply(Jedis jedis) { - byte[] value = jedis.hget(getNamespace(), keySerializer.serialize((K) key)); + byte[] value = jedis.hget(getName(), serializeKey((K) key)); return deserialize(value); } }); } @Override - public V put(K key, V value) { -/* + public V put(final K key, final V value) { return pooledWithResult(new Function() { @Override public V apply(Jedis jedis) { - return jedis.hset(getNamespace(), keySerializer.serialize(key), serialize(value)); + byte[] serializedKey = serializeKey(key); + byte[] oldValue = jedis.hget(getName(), serializedKey); + jedis.hset(getName(), serializedKey, serialize(value)); + return deserialize(oldValue); } }); -*/ - return null; } @Override - public V remove(Object key) { - return null; + public V remove(final Object key) { + return pooledWithResult(new Function() { + @Override + public V apply(Jedis jedis) { + byte[] serializedKey = serializeKey((K) key); + byte[] oldValue = jedis.hget(getName(), serializedKey); + jedis.hdel(getName(), serializedKey); + return deserialize(oldValue); + } + }); } @Override - public void putAll(Map m) { - + public void putAll(final Map m) { + pooled(new Consumer() { + @Override + public void accept(Jedis jedis) { + Map map = new HashMap(); + for (Entry entry : m.entrySet()) { + map.put(serializeKey(entry.getKey()), serialize(entry.getValue())); + } + jedis.hmset(getName(), map); + } + }); } @Override @@ -92,7 +114,7 @@ public class RedisMap extends RedisStorage implements Map { pooled(new Consumer() { @Override public void accept(Jedis jedis) { - jedis.del(getNamespace()); + jedis.del(getName()); } }); } @@ -103,10 +125,10 @@ public class RedisMap extends RedisStorage implements Map { @Override public Set apply(Jedis jedis) { Set result = new HashSet(); - collect(jedis.hkeys(getNamespace()), new Transformer() { + collect(jedis.hkeys(getName()), new Transformer() { @Override public K transform(byte[] input) { - return keySerializer.deserialize(input); + return deserializeKey(input); } }, result); return result; @@ -119,7 +141,7 @@ public class RedisMap extends RedisStorage implements Map { return pooledWithResult(new Function>() { @Override public Collection apply(Jedis jedis) { - return deserialize(jedis.hvals(getNamespace())); + return deserialize(jedis.hvals(getName())); } }); } @@ -130,10 +152,10 @@ public class RedisMap extends RedisStorage implements Map { @Override public Set> apply(Jedis jedis) { Set> result = new HashSet>(); - collect(jedis.hgetAll(getNamespace()).entrySet(), new Transformer, Entry>() { + collect(jedis.hgetAll(getName()).entrySet(), new Transformer, Entry>() { @Override public Entry transform(Entry input) { - K key = keySerializer.deserialize(input.getKey()); + K key = deserializeKey(input.getKey()); V value = deserialize(input.getValue()); return new RedisMapEntry(key, value); } diff --git a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisSet.java b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisSet.java index ba54a367..807dcb2c 100644 --- a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisSet.java +++ b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisSet.java @@ -22,7 +22,7 @@ public class RedisSet extends RedisStorage implements Set { return pooledWithResult(new Function() { @Override public Integer apply(Jedis jedis) { - return jedis.scard(getNamespace()).intValue(); + return jedis.scard(getName()).intValue(); } }); } @@ -37,30 +37,34 @@ public class RedisSet extends RedisStorage implements Set { return pooledWithResult(new Function() { @Override public Boolean apply(Jedis jedis) { - return jedis.sismember(getNamespace(), serialize(o)); + return jedis.sismember(getName(), serialize((T) o)); } }); } @Override public Iterator iterator() { + return asCollection().iterator(); + } + + private Collection asCollection() { Set members = pooledWithResult(new Function>() { @Override public Set apply(Jedis jedis) { - return jedis.smembers(getNamespace()); + return jedis.smembers(getName()); } }); - return deserialize(members).iterator(); + return deserialize(members); } @Override public Object[] toArray() { - throw new UnsupportedOperationException(); + return asCollection().toArray(); } @Override public T1[] toArray(T1[] a) { - throw new UnsupportedOperationException(); + return asCollection().toArray(a); } @Override @@ -83,7 +87,7 @@ public class RedisSet extends RedisStorage implements Set { return pooledWithResult(new Function() { @Override public Boolean apply(Jedis jedis) { - return jedis.sadd(getNamespace(), serialize(c)) == 1; + return jedis.sadd(getName(), serialize(c)) == 1; } }); } @@ -93,7 +97,7 @@ public class RedisSet extends RedisStorage implements Set { return pooledWithResult(new Function() { @Override public Boolean apply(Jedis jedis) { - return jedis.sinterstore(getNamespace(), serialize(c)) == c.size(); + return jedis.sinterstore(getName(), serialize(c)) == c.size(); } }); } @@ -103,7 +107,7 @@ public class RedisSet extends RedisStorage implements Set { return pooledWithResult(new Function() { @Override public Boolean apply(Jedis jedis) { - return jedis.srem(getNamespace(), serialize(c)) == 1; + return jedis.srem(getName(), serialize(c)) == 1; } }); } @@ -113,7 +117,7 @@ public class RedisSet extends RedisStorage implements Set { pooled(new Consumer() { @Override public void accept(Jedis jedis) { - jedis.del(getNamespace()); + jedis.del(getName()); } }); } diff --git a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisStorage.java b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisStorage.java index a20de52f..74307052 100644 --- a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisStorage.java +++ b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisStorage.java @@ -16,49 +16,33 @@ public abstract class RedisStorage { private static final Logger logger = LoggerFactory.getLogger("db"); - private String namespace; + private byte[] name; private final JedisPool pool; private final RedisSerializer serializer; - RedisStorage(String namespace, JedisPool pool, RedisSerializer serializer) { + RedisStorage(String name, JedisPool pool, RedisSerializer serializer) { + this.name = name.getBytes(); this.pool = pool; this.serializer = serializer; - - setNamespace(namespace); } - protected byte[] getNamespace() { - return namespace.getBytes(); + protected byte[] getName() { + return name; } - protected void setNamespace(String namespace) { - this.namespace = namespace + ":"; + protected void setName(String name) { + this.name = name.getBytes(); } - protected byte[] formatKey(byte[] key) { - byte[] prefix = getNamespace(); - - int length = prefix.length + key.length; - byte[] result = new byte[length]; - System.arraycopy(prefix, 0, result, 0, prefix.length); - System.arraycopy(key, 0, result, prefix.length, key.length); - return result; - } - - protected byte[] serialize(Object o) { - if (serializer.canSerialize(o)) { - return serializer.serialize((T) o); - } - - logger.warn("Cannot serialize '%s'.", o.getClass()); - return new byte[0]; + protected byte[] serialize(T o) { + return serializer.serialize(o); } protected byte[][] serialize(Collection collection) { return collect(collection, new Transformer() { @Override public byte[] transform(Object input) { - return serialize(input); + return serialize((T) input); } }).toArray(new byte[][]{}); } diff --git a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/Serializers.java b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/Serializers.java index 27b67cdb..7a0e045e 100644 --- a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/Serializers.java +++ b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/Serializers.java @@ -3,8 +3,11 @@ package org.ethereum.datasource.redis; import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.type.TypeFactory; +import org.ethereum.core.AccountState; +import org.ethereum.core.Block; import org.ethereum.core.Transaction; import org.ethereum.core.TransactionReceipt; +import org.ethereum.db.ContractDetails; import java.util.HashSet; import java.util.Set; @@ -14,13 +17,13 @@ public final class Serializers { private static abstract class BaseRedisSerializer implements RedisSerializer { public abstract boolean supports(Class aClass); - + @Override public boolean canSerialize(Object o) { return (o != null) && supports(o.getClass()); } - } - + } + private static class JacksonJsonRedisSerializer implements RedisSerializer { private final ObjectMapper objectMapper = new ObjectMapper(); @@ -68,12 +71,12 @@ public final class Serializers { @Override public byte[] serialize(TransactionReceipt transactionReceipt) { - return transactionReceipt.getEncoded(); + return (transactionReceipt == null) ? EMPTY_ARRAY : transactionReceipt.getEncoded(); } @Override public TransactionReceipt deserialize(byte[] bytes) { - return new TransactionReceipt(bytes); + return isEmpty(bytes) ? null : new TransactionReceipt(bytes); } } @@ -86,20 +89,77 @@ public final class Serializers { @Override public byte[] serialize(Transaction transaction) { - return transaction.getEncoded(); + return (transaction == null) ? EMPTY_ARRAY : transaction.getEncoded(); } @Override public Transaction deserialize(byte[] bytes) { - return new Transaction(bytes); + return isEmpty(bytes) ? null : new Transaction(bytes); } } - + private static class AccountStateSerializer extends BaseRedisSerializer { + + @Override + public boolean supports(Class aClass) { + return AccountState.class.isAssignableFrom(aClass); + } + + @Override + public byte[] serialize(AccountState accountState) { + return (accountState == null) ? EMPTY_ARRAY : accountState.getEncoded(); + } + + @Override + public AccountState deserialize(byte[] bytes) { + return isEmpty(bytes) ? null : new AccountState(bytes); + } + } + + private static class BlockSerializer extends BaseRedisSerializer { + + @Override + public boolean supports(Class aClass) { + return Block.class.isAssignableFrom(aClass); + } + + @Override + public byte[] serialize(Block block) { + return (block == null) ? EMPTY_ARRAY : block.getEncoded(); + } + + @Override + public Block deserialize(byte[] bytes) { + return isEmpty(bytes) ? null : new Block(bytes); + } + } + + private static class ContractDetailsSerializer extends BaseRedisSerializer { + + @Override + public boolean supports(Class aClass) { + return ContractDetails.class.isAssignableFrom(aClass); + } + + @Override + public byte[] serialize(ContractDetails contractDetails) { + return (contractDetails == null) ? EMPTY_ARRAY : contractDetails.getEncoded(); + } + + @Override + public ContractDetails deserialize(byte[] bytes) { + return isEmpty(bytes) ? null : new ContractDetails(bytes); + } + } + + private static final byte[] EMPTY_ARRAY = new byte[0]; private static final Set SERIALIZERS = new HashSet() {{ add(new TransactionSerializer()); add(new TransactionReceiptSerializer()); + add(new AccountStateSerializer()); + add(new BlockSerializer()); + add(new ContractDetailsSerializer()); }}; public static RedisSerializer forClass(Class clazz) { diff --git a/ethereumj-core/src/main/java/org/ethereum/db/DatabaseImpl.java b/ethereumj-core/src/main/java/org/ethereum/db/DatabaseImpl.java index 7b927187..a8454d7f 100644 --- a/ethereumj-core/src/main/java/org/ethereum/db/DatabaseImpl.java +++ b/ethereumj-core/src/main/java/org/ethereum/db/DatabaseImpl.java @@ -1,16 +1,11 @@ package org.ethereum.db; import org.ethereum.datasource.KeyValueDataSource; -import org.ethereum.facade.EthereumFactory; import org.ethereum.util.ByteUtil; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import org.spongycastle.util.encoders.Hex; -import org.springframework.beans.factory.annotation.Autowired; -import javax.xml.crypto.dsig.keyinfo.KeyValue; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -71,6 +66,7 @@ public class DatabaseImpl implements Database { @Override public void close() { + keyValueDataSource.close(); } diff --git a/ethereumj-core/src/main/java/org/ethereum/facade/CommonConfig.java b/ethereumj-core/src/main/java/org/ethereum/facade/CommonConfig.java new file mode 100644 index 00000000..69a284d8 --- /dev/null +++ b/ethereumj-core/src/main/java/org/ethereum/facade/CommonConfig.java @@ -0,0 +1,123 @@ +package org.ethereum.facade; + +import org.ethereum.config.SystemProperties; +import org.ethereum.core.Transaction; +import org.ethereum.datasource.KeyValueDataSource; +import org.ethereum.datasource.LevelDbDataSource; +import org.ethereum.datasource.redis.RedisConnection; +import org.ethereum.db.RepositoryImpl; +import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Scope; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.jdbc.datasource.DriverManagerDataSource; +import org.springframework.orm.hibernate4.LocalSessionFactoryBuilder; +import org.springframework.transaction.annotation.EnableTransactionManagement; + +import java.sql.SQLException; +import java.util.Collections; +import java.util.HashSet; +import java.util.Properties; +import java.util.Set; + +import static org.ethereum.config.SystemProperties.CONFIG; + +@Configuration +@EnableTransactionManagement +@ComponentScan(basePackages = "org.ethereum") +public class CommonConfig { + + private static final Logger logger = LoggerFactory.getLogger("general"); + + @Autowired + private RedisConnection redisConnection; + + @Bean + Repository repository(){ + return new RepositoryImpl(keyValueDataSource(), keyValueDataSource()); + } + + @Bean + @Scope("prototype") + public KeyValueDataSource keyValueDataSource(){ + if (CONFIG.getKeyValueDataSource().equals("redis")) { + if (redisConnection.isAvailable()) { + // Name will be defined before initialization + return redisConnection.createDataSource(""); + } + } + + return new LevelDbDataSource(); + } + + @Bean + public Set pendingTransactions() { + return redisConnection.isAvailable() + ? redisConnection.createTransactionSet("pendingTransactions") + : Collections.synchronizedSet(new HashSet()); + } + + @Bean + public SessionFactory sessionFactory() throws SQLException { + LocalSessionFactoryBuilder builder = + new LocalSessionFactoryBuilder(dataSource()); + builder.scanPackages("org.ethereum.db") + .addProperties(getHibernateProperties()); + + return builder.buildSessionFactory(); + } + + private Properties getHibernateProperties() { + + Properties prop = new Properties(); + + if (SystemProperties.CONFIG.databaseReset()) + prop.put("hibernate.hbm2ddl.auto", "create"); + + prop.put("hibernate.format_sql", "true"); + +// todo: useful but annoying consider define by system.properties +// prop.put("hibernate.show_sql", "true"); + prop.put("hibernate.dialect", + "org.hibernate.dialect.HSQLDialect"); + return prop; + } + + + @Bean + public DataSourceTransactionManager transactionManager() { + DataSourceTransactionManager dataSourceTransactionManager = new DataSourceTransactionManager(); + dataSourceTransactionManager.setDataSource(dataSource()); + + return dataSourceTransactionManager; + } + + @Bean(name = "dataSource") + public DriverManagerDataSource dataSource() { + + logger.info("Connecting to the block store"); + + System.setProperty("hsqldb.reconfig_logging", "false"); + + String url = + String.format("jdbc:hsqldb:file:./%s/blockchain/blockchain.db;" + + "create=%s;hsqldb.default_table_type=cached", + + SystemProperties.CONFIG.databaseDir(), + SystemProperties.CONFIG.databaseReset()); + + DriverManagerDataSource ds = new DriverManagerDataSource(); + ds.setDriverClassName("org.hsqldb.jdbcDriver"); + ds.setUrl(url); + ds.setUsername("sa"); + + + return ds; + } + +} diff --git a/ethereumj-core/src/main/java/org/ethereum/facade/DefaultConfig.java b/ethereumj-core/src/main/java/org/ethereum/facade/DefaultConfig.java index 6f5cf80a..7b7cf833 100644 --- a/ethereumj-core/src/main/java/org/ethereum/facade/DefaultConfig.java +++ b/ethereumj-core/src/main/java/org/ethereum/facade/DefaultConfig.java @@ -1,147 +1,29 @@ package org.ethereum.facade; -import org.ethereum.config.SystemProperties; -import org.ethereum.core.Transaction; -import org.ethereum.datasource.KeyValueDataSource; -import org.ethereum.datasource.LevelDbDataSource; -import org.ethereum.datasource.RedisDataSource; -import org.ethereum.datasource.redis.RedisConnection; -import org.ethereum.datasource.redis.RedisConnectionImpl; import org.ethereum.db.BlockStore; import org.ethereum.db.BlockStoreImpl; -import org.ethereum.db.RepositoryImpl; import org.hibernate.SessionFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Scope; -import org.springframework.jdbc.datasource.DataSourceTransactionManager; -import org.springframework.jdbc.datasource.DriverManagerDataSource; -import org.springframework.orm.hibernate4.LocalSessionFactoryBuilder; -import org.springframework.transaction.annotation.EnableTransactionManagement; +import org.springframework.context.annotation.Import; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; -import java.sql.SQLException; -import java.util.Collections; -import java.util.HashSet; -import java.util.Properties; -import java.util.Set; - -import static org.ethereum.config.SystemProperties.CONFIG; - /** * * @author: Roman Mandeleil * Created on: 27/01/2015 01:05 */ @Configuration -@EnableTransactionManagement -@ComponentScan(basePackages = "org.ethereum") +@Import(CommonConfig.class) public class DefaultConfig { - - private static final Logger logger = LoggerFactory.getLogger("general"); - - @Autowired - Ethereum eth; - @Bean - Repository repository(){ - return new RepositoryImpl(keyValueDataSource(), keyValueDataSource()); - } - - @Bean - @Scope("prototype") - public KeyValueDataSource keyValueDataSource(){ - - if (CONFIG.getKeyValueDataSource().equals("redis")) { - return new RedisDataSource(); - } - - return new LevelDbDataSource(); - } - - @Bean - public RedisConnection redisConnection() { - RedisConnectionImpl connection = new RedisConnectionImpl(); - connection.init(); - return connection; - } - - @Bean - public Set pendingTransactions() { - RedisConnection connection = redisConnection(); - return connection.isAvailable() - ? connection.createTransactionSet("pendingTransactions") - : Collections.synchronizedSet(new HashSet()); - } + @Autowired CommonConfig commonConfig; @Bean @Transactional(propagation = Propagation.SUPPORTS) public BlockStore blockStore(SessionFactory sessionFactory){ return new BlockStoreImpl(sessionFactory); } - - - @Bean - public SessionFactory sessionFactory() throws SQLException { - LocalSessionFactoryBuilder builder = - new LocalSessionFactoryBuilder(dataSource()); - builder.scanPackages("org.ethereum.db") - .addProperties(getHibernateProperties()); - - return builder.buildSessionFactory(); - } - - private Properties getHibernateProperties() { - - Properties prop = new Properties(); - - if (SystemProperties.CONFIG.databaseReset()) - prop.put("hibernate.hbm2ddl.auto", "create"); - - prop.put("hibernate.format_sql", "true"); - -// todo: useful but annoying consider define by system.properties -// prop.put("hibernate.show_sql", "true"); - prop.put("hibernate.dialect", - "org.hibernate.dialect.HSQLDialect"); - return prop; - } - - - @Bean - public DataSourceTransactionManager transactionManager() { - DataSourceTransactionManager dataSourceTransactionManager = new DataSourceTransactionManager(); - dataSourceTransactionManager.setDataSource(dataSource()); - - return dataSourceTransactionManager; - } - - @Bean(name = "dataSource") - public DriverManagerDataSource dataSource() { - - logger.info("Connecting to the block store"); - - System.setProperty("hsqldb.reconfig_logging", "false"); - - String url = - String.format("jdbc:hsqldb:file:./%s/blockchain/blockchain.db;" + - "create=%s;hsqldb.default_table_type=cached", - - SystemProperties.CONFIG.databaseDir(), - SystemProperties.CONFIG.databaseReset()); - - DriverManagerDataSource ds = new DriverManagerDataSource(); - ds.setDriverClassName("org.hsqldb.jdbcDriver"); - ds.setUrl(url); - ds.setUsername("sa"); - - - return ds; - } - } diff --git a/ethereumj-core/src/main/java/org/ethereum/facade/EthereumImpl.java b/ethereumj-core/src/main/java/org/ethereum/facade/EthereumImpl.java index 6d87e40f..f36b8248 100644 --- a/ethereumj-core/src/main/java/org/ethereum/facade/EthereumImpl.java +++ b/ethereumj-core/src/main/java/org/ethereum/facade/EthereumImpl.java @@ -12,25 +12,20 @@ import org.ethereum.net.server.PeerServer; import org.ethereum.net.submit.TransactionExecutor; import org.ethereum.net.submit.TransactionTask; import org.ethereum.util.ByteUtil; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Component; +import javax.annotation.PostConstruct; import java.math.BigInteger; - import java.net.InetAddress; - import java.util.HashSet; import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import javax.annotation.PostConstruct; - import static org.ethereum.config.SystemProperties.CONFIG; /** diff --git a/ethereumj-core/src/main/java/org/ethereum/facade/RemoteConfig.java b/ethereumj-core/src/main/java/org/ethereum/facade/RemoteConfig.java index 8fec9be1..31ff3611 100644 --- a/ethereumj-core/src/main/java/org/ethereum/facade/RemoteConfig.java +++ b/ethereumj-core/src/main/java/org/ethereum/facade/RemoteConfig.java @@ -1,150 +1,32 @@ package org.ethereum.facade; -import org.ethereum.config.SystemProperties; -import org.ethereum.core.Transaction; -import org.ethereum.datasource.KeyValueDataSource; -import org.ethereum.datasource.LevelDbDataSource; -import org.ethereum.datasource.RedisDataSource; -import org.ethereum.datasource.redis.RedisConnection; -import org.ethereum.datasource.redis.RedisConnectionImpl; import org.ethereum.db.BlockStore; import org.ethereum.db.InMemoryBlockStore; -import org.ethereum.db.RepositoryImpl; import org.hibernate.SessionFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Scope; -import org.springframework.jdbc.datasource.DataSourceTransactionManager; -import org.springframework.jdbc.datasource.DriverManagerDataSource; -import org.springframework.orm.hibernate4.LocalSessionFactoryBuilder; -import org.springframework.transaction.annotation.EnableTransactionManagement; +import org.springframework.context.annotation.Import; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; -import java.sql.SQLException; -import java.util.Collections; -import java.util.HashSet; -import java.util.Properties; -import java.util.Set; - -import static org.ethereum.config.SystemProperties.CONFIG; - /** * * @author: Roman Mandeleil * Created on: 27/01/2015 01:05 */ @Configuration -@EnableTransactionManagement -@ComponentScan(basePackages = "org.ethereum") +@Import(CommonConfig.class) public class RemoteConfig { - private static final Logger logger = LoggerFactory.getLogger("general"); + @Autowired CommonConfig commonConfig; - @Autowired - Ethereum eth; - - // todo: init total difficulty + // todo: init total difficulty // todo: init last 1000 blocks - - @Bean - Repository repository(){ - return new RepositoryImpl(keyValueDataSource(), keyValueDataSource()); - } - - @Bean - @Scope("prototype") - public KeyValueDataSource keyValueDataSource(){ - - if (CONFIG.getKeyValueDataSource().equals("redis")) { - return new RedisDataSource(); - } - - return new LevelDbDataSource(); - } - - @Bean - public RedisConnection redisConnection() { - RedisConnectionImpl connection = new RedisConnectionImpl(); - connection.init(); - return connection; - } - - @Bean - public Set pendingTransactions() { - RedisConnection connection = redisConnection(); - return connection.isAvailable() - ? connection.createTransactionSet("pendingTransactions") - : Collections.synchronizedSet(new HashSet()); - } @Bean @Transactional(propagation = Propagation.SUPPORTS) public BlockStore blockStore(SessionFactory sessionFactory){ return new InMemoryBlockStore(); } - - - @Bean - public SessionFactory sessionFactory() throws SQLException { - LocalSessionFactoryBuilder builder = - new LocalSessionFactoryBuilder(dataSource()); - builder.scanPackages("org.ethereum.db") - .addProperties(getHibernateProperties()); - - return builder.buildSessionFactory(); - } - - private Properties getHibernateProperties() { - - Properties prop = new Properties(); - - if (SystemProperties.CONFIG.databaseReset()) - prop.put("hibernate.hbm2ddl.auto", "create"); - - prop.put("hibernate.format_sql", "true"); - -// todo: useful but annoying consider define by system.properties -// prop.put("hibernate.show_sql", "true"); - prop.put("hibernate.dialect", - "org.hibernate.dialect.HSQLDialect"); - return prop; - } - - - @Bean - public DataSourceTransactionManager transactionManager() { - DataSourceTransactionManager dataSourceTransactionManager = new DataSourceTransactionManager(); - dataSourceTransactionManager.setDataSource(dataSource()); - - return dataSourceTransactionManager; - } - - @Bean(name = "dataSource") - public DriverManagerDataSource dataSource() { - - logger.info("Connecting to the block store"); - - System.setProperty("hsqldb.reconfig_logging", "false"); - - String url = - String.format("jdbc:hsqldb:file:./%s/blockchain/blockchain.db;" + - "create=%s;hsqldb.default_table_type=cached", - - SystemProperties.CONFIG.databaseDir(), - SystemProperties.CONFIG.databaseReset()); - - DriverManagerDataSource ds = new DriverManagerDataSource(); - ds.setDriverClassName("org.hsqldb.jdbcDriver"); - ds.setUrl(url); - ds.setUsername("sa"); - - - return ds; - } - } diff --git a/ethereumj-core/src/test/java/test/ethereum/datasource/AbstractRedisTest.java b/ethereumj-core/src/test/java/test/ethereum/datasource/AbstractRedisTest.java new file mode 100644 index 00000000..feeedeee --- /dev/null +++ b/ethereumj-core/src/test/java/test/ethereum/datasource/AbstractRedisTest.java @@ -0,0 +1,24 @@ +package test.ethereum.datasource; + +import org.ethereum.datasource.redis.RedisConnection; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.test.context.support.AnnotationConfigContextLoader; +import test.ethereum.TestContext; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(loader = AnnotationConfigContextLoader.class) +public class AbstractRedisTest { + + @Configuration + @ComponentScan(basePackages = "org.ethereum") + static class ContextConfiguration extends TestContext { } + + @Autowired + protected RedisConnection redisConnection; + +} diff --git a/ethereumj-core/src/test/java/test/ethereum/datasource/RedisConnectionTest.java b/ethereumj-core/src/test/java/test/ethereum/datasource/RedisConnectionTest.java index a1ab18b1..81f7cd9e 100644 --- a/ethereumj-core/src/test/java/test/ethereum/datasource/RedisConnectionTest.java +++ b/ethereumj-core/src/test/java/test/ethereum/datasource/RedisConnectionTest.java @@ -3,34 +3,16 @@ package test.ethereum.datasource; import org.ethereum.core.Transaction; import org.ethereum.crypto.ECKey; import org.ethereum.crypto.HashUtil; -import org.ethereum.datasource.redis.RedisConnection; import org.junit.Test; -import org.junit.runner.RunWith; import org.spongycastle.util.encoders.Hex; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import test.ethereum.TestContext; import java.math.BigInteger; import java.util.Set; import static org.junit.Assert.*; -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(loader = AnnotationConfigContextLoader.class) -public class RedisConnectionTest { +public class RedisConnectionTest extends AbstractRedisTest { - @Configuration - @ComponentScan(basePackages = "org.ethereum") - static class ContextConfiguration extends TestContext { } - - @Autowired - private RedisConnection redisConnection; - @Test public void test() { if (!redisConnection.isAvailable()) return; diff --git a/ethereumj-core/src/test/java/test/ethereum/datasource/RedisDataSourceTest.java b/ethereumj-core/src/test/java/test/ethereum/datasource/RedisDataSourceTest.java index e270e174..3ad071ee 100644 --- a/ethereumj-core/src/test/java/test/ethereum/datasource/RedisDataSourceTest.java +++ b/ethereumj-core/src/test/java/test/ethereum/datasource/RedisDataSourceTest.java @@ -1,68 +1,69 @@ package test.ethereum.datasource; -import org.ethereum.datasource.RedisDataSource; - +import org.ethereum.datasource.KeyValueDataSource; +import org.ethereum.datasource.redis.RedisDataSource; import org.junit.Assert; import org.junit.Test; - import org.spongycastle.util.encoders.Hex; -import redis.clients.jedis.exceptions.JedisConnectionException; - /** * @author Roman Mandeleil */ -public class RedisDataSourceTest { - +public class RedisDataSourceTest extends AbstractRedisTest { @Test public void testSet1() { + if (!redisConnection.isAvailable()) return; + KeyValueDataSource dataSource = createDataSource("test-state"); try { - RedisDataSource redis = new RedisDataSource(); - redis.setName("state"); - redis.init(); - byte[] key = Hex.decode("a1a2a3"); byte[] val = Hex.decode("b1b2b3"); - redis.put(key, val); - byte[] val2 = redis.get(key); + dataSource.put(key, val); + byte[] val2 = dataSource.get(key); Assert.assertEquals(Hex.toHexString(val), Hex.toHexString(val2)); - } catch (JedisConnectionException e) { - // no redis server consider test as pass + } finally { + clear(dataSource); } } @Test public void testSet2() { + if (!redisConnection.isAvailable()) return; + + KeyValueDataSource states = createDataSource("test-state"); + KeyValueDataSource details = createDataSource("test-details"); try { - RedisDataSource redis1 = new RedisDataSource(); - redis1.setName("state"); - redis1.init(); - - RedisDataSource redis2 = new RedisDataSource(); - redis2.setName("details"); - redis2.init(); - - byte[] key = Hex.decode("a1a2a3"); byte[] val1 = Hex.decode("b1b2b3"); byte[] val2 = Hex.decode("c1c2c3"); - redis1.put(key, val1); - redis2.put(key, val2); + states.put(key, val1); + details.put(key, val2); - byte[] res1 = redis1.get(key); - byte[] res2 = redis2.get(key); + byte[] res1 = states.get(key); + byte[] res2 = details.get(key); Assert.assertEquals(Hex.toHexString(val1), Hex.toHexString(res1)); Assert.assertEquals(Hex.toHexString(val2), Hex.toHexString(res2)); - } catch (JedisConnectionException e) { - // no redis server consider test as pass + } finally { + clear(states); + clear(details); } } + private KeyValueDataSource createDataSource(String name) { + KeyValueDataSource result = redisConnection.createDataSource(name); + result.setName(name); + result.init(); + return result; + } + + private void clear(KeyValueDataSource dataSource) { + ((RedisDataSource) dataSource).clear(); + } + } diff --git a/ethereumj-core/src/test/java/test/ethereum/db/MockDB.java b/ethereumj-core/src/test/java/test/ethereum/db/MockDB.java index 4608cc84..83a1ac94 100644 --- a/ethereumj-core/src/test/java/test/ethereum/db/MockDB.java +++ b/ethereumj-core/src/test/java/test/ethereum/db/MockDB.java @@ -22,15 +22,13 @@ public class MockDB implements KeyValueDataSource { @Override public byte[] get(byte[] arg0) throws DBException { - return storage.get(new ByteArrayWrapper(arg0)); } @Override - public void put(byte[] key, byte[] value) throws DBException { - - storage.put(new ByteArrayWrapper(key), value); + public byte[] put(byte[] key, byte[] value) throws DBException { + return storage.put(new ByteArrayWrapper(key), value); } /** From 8fd961ecafce04e0ee0e1346c676e9a9c64a274d Mon Sep 17 00:00:00 2001 From: eugene-shevchenko Date: Fri, 27 Mar 2015 10:52:29 +0200 Subject: [PATCH 3/8] version up --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 0de753d3..344a79bc 100644 --- a/build.gradle +++ b/build.gradle @@ -8,7 +8,7 @@ subprojects { apply plugin: 'java' group = 'org.ethereum' - version = '0.8.2-SNAPSHOT' + version = '0.8.3-SNAPSHOT' compileJava.options.encoding = 'UTF-8' compileJava.options.compilerArgs << '-XDignore.symbol.file' From 4ef20af5e0e54b02f711ffe0feb92911522de4bd Mon Sep 17 00:00:00 2001 From: eugene-shevchenko Date: Fri, 27 Mar 2015 12:49:32 +0200 Subject: [PATCH 4/8] Pending Transaction - save in external datasource --- .../datasource/redis/RedisConnection.java | 7 +- .../datasource/redis/RedisConnectionImpl.java | 18 ++- .../ethereum/datasource/redis/RedisSet.java | 12 +- .../datasource/redis/RedisStorage.java | 19 ++- .../datasource/AbstractRedisTest.java | 38 ++++- .../datasource/RedisConnectionTest.java | 91 ------------ .../datasource/RedisDataSourceTest.java | 26 +++- .../ethereum/datasource/RedisStorageTest.java | 130 ++++++++++++++++++ 8 files changed, 235 insertions(+), 106 deletions(-) delete mode 100644 ethereumj-core/src/test/java/test/ethereum/datasource/RedisConnectionTest.java create mode 100644 ethereumj-core/src/test/java/test/ethereum/datasource/RedisStorageTest.java diff --git a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisConnection.java b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisConnection.java index 0932abdc..e67f49cd 100644 --- a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisConnection.java +++ b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisConnection.java @@ -3,14 +3,19 @@ package org.ethereum.datasource.redis; import org.ethereum.core.Transaction; import org.ethereum.datasource.KeyValueDataSource; +import java.util.Map; import java.util.Set; public interface RedisConnection { - + + public static final String REDISCLOUD_URL = "REDISCLOUD_URL"; + boolean isAvailable(); Set createSetFor(Class clazz, String name); + Map createMapFor(Class keyClass, Class valueClass, String name); + Set createTransactionSet(String name); KeyValueDataSource createDataSource(String name); diff --git a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisConnectionImpl.java b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisConnectionImpl.java index ced78c0f..9bca3854 100644 --- a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisConnectionImpl.java +++ b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisConnectionImpl.java @@ -15,6 +15,7 @@ import redis.clients.jedis.Protocol; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.net.URI; +import java.util.Map; import java.util.Set; import static org.springframework.util.StringUtils.isEmpty; @@ -30,7 +31,7 @@ public class RedisConnectionImpl implements RedisConnection { public void tryConnect() { if (!SystemProperties.CONFIG.isRedisEnabled()) return; - String redisCloudUrl = System.getenv("REDISCLOUD_URL"); + String redisCloudUrl = System.getenv(REDISCLOUD_URL); if (isEmpty(redisCloudUrl)) { logger.info("Cannot connect to Redis. 'REDISCLOUD_URL' environment variable is not defined."); return; @@ -73,14 +74,16 @@ public class RedisConnectionImpl implements RedisConnection { public boolean isAvailable() { boolean available = jedisPool != null; if (available) { - Jedis jedis = jedisPool.getResource(); try { - available = jedis.isConnected(); + Jedis jedis = jedisPool.getResource(); + try { + available = jedis.isConnected(); + } finally { + jedisPool.returnResource(jedis); + } } catch (Throwable t) { logger.warn("Connection testing failed: ", t); available = false; - } finally { - jedisPool.returnResource(jedis); } } return available; @@ -90,6 +93,11 @@ public class RedisConnectionImpl implements RedisConnection { public Set createSetFor(Class clazz, String name) { return new RedisSet(name, jedisPool, Serializers.forClass(clazz)); } + + @Override + public Map createMapFor(Class keyClass, Class valueClass, String name) { + return new RedisMap(name, jedisPool, Serializers.forClass(keyClass), Serializers.forClass(valueClass)); + } @Override public Set createTransactionSet(String name) { diff --git a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisSet.java b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisSet.java index 807dcb2c..63882579 100644 --- a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisSet.java +++ b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisSet.java @@ -87,7 +87,7 @@ public class RedisSet extends RedisStorage implements Set { return pooledWithResult(new Function() { @Override public Boolean apply(Jedis jedis) { - return jedis.sadd(getName(), serialize(c)) == 1; + return jedis.sadd(getName(), serialize(c)) == c.size(); } }); } @@ -97,7 +97,13 @@ public class RedisSet extends RedisStorage implements Set { return pooledWithResult(new Function() { @Override public Boolean apply(Jedis jedis) { - return jedis.sinterstore(getName(), serialize(c)) == c.size(); + byte[] tempName = temporaryName(); + try { + jedis.sadd(tempName, serialize(c)); + return jedis.scard(getName()) != jedis.sinterstore(getName(), getName(), tempName); + } finally { + jedis.del(tempName); + } } }); } @@ -107,7 +113,7 @@ public class RedisSet extends RedisStorage implements Set { return pooledWithResult(new Function() { @Override public Boolean apply(Jedis jedis) { - return jedis.srem(getName(), serialize(c)) == 1; + return jedis.srem(getName(), serialize(c)) == c.size(); } }); } diff --git a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisStorage.java b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisStorage.java index 74307052..5303c36f 100644 --- a/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisStorage.java +++ b/ethereumj-core/src/main/java/org/ethereum/datasource/redis/RedisStorage.java @@ -1,5 +1,6 @@ package org.ethereum.datasource.redis; +import com.google.common.primitives.Bytes; import org.apache.commons.collections4.Transformer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,6 +35,14 @@ public abstract class RedisStorage { this.name = name.getBytes(); } + protected byte[] formatName(String suffix) { + return Bytes.concat(getName(), suffix.getBytes()); + } + + protected byte[] temporaryName() { + return formatName(":" + Thread.currentThread().getName() + ":" + System.currentTimeMillis()); + } + protected byte[] serialize(T o) { return serializer.serialize(o); } @@ -89,10 +98,18 @@ public abstract class RedisStorage { protected R pooledWithResult(Function function) { Jedis jedis = pool.getResource(); + Exception operationException = null; try { return function.apply(jedis); + } catch (Exception e) { + operationException = e; + throw e; } finally { - pool.returnResource(jedis); + if (operationException == null) { + pool.returnResource(jedis); + } else { + pool.returnBrokenResource(jedis); + } } } } diff --git a/ethereumj-core/src/test/java/test/ethereum/datasource/AbstractRedisTest.java b/ethereumj-core/src/test/java/test/ethereum/datasource/AbstractRedisTest.java index feeedeee..0193a5c8 100644 --- a/ethereumj-core/src/test/java/test/ethereum/datasource/AbstractRedisTest.java +++ b/ethereumj-core/src/test/java/test/ethereum/datasource/AbstractRedisTest.java @@ -1,6 +1,7 @@ package test.ethereum.datasource; import org.ethereum.datasource.redis.RedisConnection; +import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.ComponentScan; @@ -8,17 +9,50 @@ import org.springframework.context.annotation.Configuration; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import org.springframework.test.context.support.AnnotationConfigContextLoader; +import org.springframework.util.StringUtils; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.JedisPoolConfig; +import redis.clients.jedis.Protocol; import test.ethereum.TestContext; +import java.net.URI; +import java.net.URISyntaxException; + +import static org.junit.Assert.assertFalse; + @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(loader = AnnotationConfigContextLoader.class) -public class AbstractRedisTest { +public abstract class AbstractRedisTest { @Configuration @ComponentScan(basePackages = "org.ethereum") static class ContextConfiguration extends TestContext { } @Autowired - protected RedisConnection redisConnection; + private RedisConnection redisConnection; + + private Boolean connected; + + protected RedisConnection getRedisConnection() { + return redisConnection; + } + + protected Boolean isConnected() { + if (connected == null) { + String url = System.getenv(RedisConnection.REDISCLOUD_URL); + try { + Jedis jedis = new Jedis(new URI(url)); + connected = jedis.ping().equals("PONG"); + jedis.close(); + } catch (Exception e) { + System.out.printf("Cannot connect to '%s' Redis cloud.\n", url); + } + + assertFalse(connected ^ redisConnection.isAvailable()); + } + + return connected; + } } diff --git a/ethereumj-core/src/test/java/test/ethereum/datasource/RedisConnectionTest.java b/ethereumj-core/src/test/java/test/ethereum/datasource/RedisConnectionTest.java deleted file mode 100644 index 81f7cd9e..00000000 --- a/ethereumj-core/src/test/java/test/ethereum/datasource/RedisConnectionTest.java +++ /dev/null @@ -1,91 +0,0 @@ -package test.ethereum.datasource; - -import org.ethereum.core.Transaction; -import org.ethereum.crypto.ECKey; -import org.ethereum.crypto.HashUtil; -import org.junit.Test; -import org.spongycastle.util.encoders.Hex; - -import java.math.BigInteger; -import java.util.Set; - -import static org.junit.Assert.*; - -public class RedisConnectionTest extends AbstractRedisTest { - - @Test - public void test() { - if (!redisConnection.isAvailable()) return; - - Set pojos = redisConnection.createSetFor(Pojo.class, "pojos"); - - Pojo pojo = Pojo.create(1L, "test"); - pojos.add(pojo); - assertTrue(pojos.contains(pojo)); - assertEquals(1, pojos.size()); - Pojo next = pojos.iterator().next(); - assertNotNull(next); - assertEquals(pojo.getId(), next.getId()); - assertEquals(pojo.getName(), next.getName()); - assertTrue(pojos.remove(pojo)); - assertTrue(pojos.isEmpty()); - } - - @Test - public void transactionStorageTest() { - if (!redisConnection.isAvailable()) return; - - String namespace = "txnNamespace"; - Set transactions = redisConnection.createTransactionSet(namespace); - transactions.add(createTransaction("09184e72a000", "4255", "1000000000000000000000", "cat")); - transactions.add(createTransaction("09184e72a000", "4255", "1000000000000000000000", "dog")); - transactions.add(createTransaction("09184e72a000", "4255", "1000000000000000000000", "rabbit")); - - Set transactions1 = redisConnection.createTransactionSet(namespace); - transactions1.add(createTransaction("09184e72a000", "4255", "1000000000000000000000", "duck")); - transactions1.add(createTransaction("09184e72a000", "4255", "1000000000000000000000", "chicken")); - transactions1.add(createTransaction("09184e72a000", "4255", "1000000000000000000000", "cow")); - - assertEquals(6, transactions1.size()); - transactions.clear(); - assertTrue(transactions1.isEmpty()); - } - - private static class Pojo { - private long id; - private String name; - - public long getId() { - return id; - } - - public void setId(long id) { - this.id = id; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public static Pojo create(long id, String name) { - Pojo result = new Pojo(); - result.setId(id); - result.setName(name); - return result; - } - } - - public static Transaction createTransaction(String gasPrice, String gas, String val, String secret) { - - ECKey ecKey = ECKey.fromPrivate(HashUtil.sha3(secret.getBytes())); - - // Tn (nonce); Tp(pgas); Tg(gaslimi); Tt(value); Tv(value); Ti(sender); Tw; Tr; Ts - return new Transaction(null, Hex.decode(gasPrice), Hex.decode(gas), ecKey.getAddress(), - new BigInteger(val).toByteArray(), - null); - } -} \ No newline at end of file diff --git a/ethereumj-core/src/test/java/test/ethereum/datasource/RedisDataSourceTest.java b/ethereumj-core/src/test/java/test/ethereum/datasource/RedisDataSourceTest.java index 3ad071ee..9925fb56 100644 --- a/ethereumj-core/src/test/java/test/ethereum/datasource/RedisDataSourceTest.java +++ b/ethereumj-core/src/test/java/test/ethereum/datasource/RedisDataSourceTest.java @@ -1,10 +1,16 @@ package test.ethereum.datasource; import org.ethereum.datasource.KeyValueDataSource; +import org.ethereum.datasource.redis.RedisConnection; import org.ethereum.datasource.redis.RedisDataSource; import org.junit.Assert; import org.junit.Test; import org.spongycastle.util.encoders.Hex; +import redis.clients.jedis.Jedis; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Set; /** * @author Roman Mandeleil @@ -13,7 +19,7 @@ public class RedisDataSourceTest extends AbstractRedisTest { @Test public void testSet1() { - if (!redisConnection.isAvailable()) return; + if (!isConnected()) return; KeyValueDataSource dataSource = createDataSource("test-state"); try { @@ -29,9 +35,23 @@ public class RedisDataSourceTest extends AbstractRedisTest { } } + @Test + public void test() { + try { + Jedis jedis = new Jedis(new URI(System.getenv(RedisConnection.REDISCLOUD_URL))); + Long count = jedis.sinterstore("f", "f", "s"); + System.out.println(count); + Set r = jedis.smembers("f"); + System.out.println(r); + } catch (URISyntaxException e) { + e.printStackTrace(); + } + + } + @Test public void testSet2() { - if (!redisConnection.isAvailable()) return; + if (!isConnected()) return; KeyValueDataSource states = createDataSource("test-state"); KeyValueDataSource details = createDataSource("test-details"); @@ -56,7 +76,7 @@ public class RedisDataSourceTest extends AbstractRedisTest { } private KeyValueDataSource createDataSource(String name) { - KeyValueDataSource result = redisConnection.createDataSource(name); + KeyValueDataSource result = getRedisConnection().createDataSource(name); result.setName(name); result.init(); return result; diff --git a/ethereumj-core/src/test/java/test/ethereum/datasource/RedisStorageTest.java b/ethereumj-core/src/test/java/test/ethereum/datasource/RedisStorageTest.java new file mode 100644 index 00000000..97c04abc --- /dev/null +++ b/ethereumj-core/src/test/java/test/ethereum/datasource/RedisStorageTest.java @@ -0,0 +1,130 @@ +package test.ethereum.datasource; + +import org.ethereum.core.Transaction; +import org.ethereum.crypto.ECKey; +import org.ethereum.crypto.HashUtil; +import org.junit.Test; +import org.spongycastle.util.encoders.Hex; + +import java.math.BigInteger; +import java.util.Set; + +import static java.util.Arrays.asList; +import static org.junit.Assert.*; + +public class RedisStorageTest extends AbstractRedisTest { + + @Test + public void testRedisSet() { + if (!isConnected()) return; + + Pojo elephant = Pojo.create(5L, "elephant"); + Pojo lion = Pojo.create(5L, "lion"); + + Set ranch = getRedisConnection().createSetFor(Pojo.class, "ranch"); + Pojo chicken = Pojo.create(1L, "chicken"); + Pojo cow = Pojo.create(2L, "cow"); + Pojo puppy = Pojo.create(3L, "puppy"); + Pojo kitten = Pojo.create(4L, "kitten"); + + assertTrue(ranch.add(chicken)); + assertFalse(ranch.add(chicken)); + assertTrue(ranch.contains(chicken)); + assertEquals(1, ranch.size()); + + Pojo next = ranch.iterator().next(); + assertNotNull(next); + assertEquals(chicken, next); + + assertTrue(ranch.addAll(asList(cow, puppy, kitten))); + assertEquals(4, ranch.size()); + assertFalse(ranch.isEmpty()); + assertFalse(ranch.remove(elephant)); + assertFalse(ranch.removeAll(asList(cow, lion, elephant))); + assertEquals(3, ranch.size()); + + assertTrue(ranch.retainAll(asList(kitten, puppy))); + assertEquals(2, ranch.size()); + + ranch.clear(); + assertEquals(0, ranch.size()); + assertTrue(ranch.isEmpty()); + } + + @Test + public void testSeveralSetsWithOneName() { + if (!isConnected()) return; + + final String name = "testTransactions"; + Set transactions = getRedisConnection().createTransactionSet(name); + transactions.add(createTransaction("09184e72a000", "4255", "1000000000000000000000", "cat")); + transactions.add(createTransaction("09184e72a000", "4255", "1000000000000000000000", "dog")); + transactions.add(createTransaction("09184e72a000", "4255", "1000000000000000000000", "rabbit")); + + Set transactions1 = getRedisConnection().createTransactionSet(name); + transactions1.add(createTransaction("09184e72a000", "4255", "1000000000000000000000", "duck")); + transactions1.add(createTransaction("09184e72a000", "4255", "1000000000000000000000", "chicken")); + transactions1.add(createTransaction("09184e72a000", "4255", "1000000000000000000000", "cow")); + + assertEquals(6, transactions1.size()); + transactions.clear(); + assertTrue(transactions1.isEmpty()); + } + + private static class Pojo { + private Long id; + private String name; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public static Pojo create(long id, String name) { + Pojo result = new Pojo(); + result.setId(id); + result.setName(name); + return result; + } + + @Override + public boolean equals(Object obj) { + if (obj == null || !getClass().isInstance(obj)) return false; + if (this == obj) return true; + + Pojo another = (Pojo) obj; + return (another.getId() == getId()) && another.getName().equals(getName()); + } + + @Override + public int hashCode() { + int hashCode = 17; + + hashCode += ((getId() == null) ? 0 : getId().hashCode()) * 31; + hashCode += ((getName() == null) ? 0 : getName().hashCode()) * 31; + + return hashCode; + } + } + + public static Transaction createTransaction(String gasPrice, String gas, String val, String secret) { + + ECKey ecKey = ECKey.fromPrivate(HashUtil.sha3(secret.getBytes())); + + // Tn (nonce); Tp(pgas); Tg(gaslimi); Tt(value); Tv(value); Ti(sender); Tw; Tr; Ts + return new Transaction(null, Hex.decode(gasPrice), Hex.decode(gas), ecKey.getAddress(), + new BigInteger(val).toByteArray(), + null); + } +} \ No newline at end of file From 339b0e57e4ec9aa258ac9bd07f29965fe9c33214 Mon Sep 17 00:00:00 2001 From: eugene-shevchenko Date: Fri, 27 Mar 2015 12:59:54 +0200 Subject: [PATCH 5/8] Pending Transaction - save in external datasource --- .../ethereum/datasource/RedisDataSourceTest.java | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/ethereumj-core/src/test/java/test/ethereum/datasource/RedisDataSourceTest.java b/ethereumj-core/src/test/java/test/ethereum/datasource/RedisDataSourceTest.java index 9925fb56..c68ec9dd 100644 --- a/ethereumj-core/src/test/java/test/ethereum/datasource/RedisDataSourceTest.java +++ b/ethereumj-core/src/test/java/test/ethereum/datasource/RedisDataSourceTest.java @@ -35,20 +35,6 @@ public class RedisDataSourceTest extends AbstractRedisTest { } } - @Test - public void test() { - try { - Jedis jedis = new Jedis(new URI(System.getenv(RedisConnection.REDISCLOUD_URL))); - Long count = jedis.sinterstore("f", "f", "s"); - System.out.println(count); - Set r = jedis.smembers("f"); - System.out.println(r); - } catch (URISyntaxException e) { - e.printStackTrace(); - } - - } - @Test public void testSet2() { if (!isConnected()) return; From 52d9d19a1873b5dccd6ece38f2d79063b18ab220 Mon Sep 17 00:00:00 2001 From: eugene-shevchenko Date: Fri, 27 Mar 2015 17:36:43 +0200 Subject: [PATCH 6/8] redis.enabled property added --- ethereumj-core/src/main/resources/system.properties | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ethereumj-core/src/main/resources/system.properties b/ethereumj-core/src/main/resources/system.properties index 8aa459a2..fe09e247 100644 --- a/ethereumj-core/src/main/resources/system.properties +++ b/ethereumj-core/src/main/resources/system.properties @@ -153,4 +153,8 @@ root.hash.start = -1 peer.capabilities = eth, shh # Key value data source values: [leveldb/redis] -keyvalue.datasource = leveldb \ No newline at end of file +keyvalue.datasource = leveldb + +# Redis cloud enabled flag. +# Allows using RedisConnection for creating cloud based data structures. +redis.enabled=false \ No newline at end of file From f2bf688f8595e3bcd5691857ddcc4946fb6a43b8 Mon Sep 17 00:00:00 2001 From: eugene-shevchenko Date: Fri, 27 Mar 2015 22:30:24 +0200 Subject: [PATCH 7/8] Pending Transaction - save in external datasource context initialization fix --- ethereumj-core/src/main/java/org/ethereum/core/Wallet.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ethereumj-core/src/main/java/org/ethereum/core/Wallet.java b/ethereumj-core/src/main/java/org/ethereum/core/Wallet.java index 6adb7181..c16c50fe 100644 --- a/ethereumj-core/src/main/java/org/ethereum/core/Wallet.java +++ b/ethereumj-core/src/main/java/org/ethereum/core/Wallet.java @@ -12,6 +12,7 @@ import org.spongycastle.util.encoders.Hex; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.DependsOn; import org.springframework.stereotype.Component; import org.w3c.dom.Attr; @@ -48,6 +49,7 @@ import javax.xml.transform.stream.StreamResult; * New accounts can be generated and added to the wallet and existing accounts can be queried. */ @Component +@DependsOn("worldManager") public class Wallet { private Logger logger = LoggerFactory.getLogger("wallet"); From a7b0fa1c7b05b15ed104d3d0e77eee8e88d2ad0e Mon Sep 17 00:00:00 2001 From: eugene-shevchenko Date: Mon, 30 Mar 2015 12:15:35 +0300 Subject: [PATCH 8/8] Pending Transaction - save in external datasource context init logging added. --- .../org/ethereum/facade/CommonConfig.java | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/ethereumj-core/src/main/java/org/ethereum/facade/CommonConfig.java b/ethereumj-core/src/main/java/org/ethereum/facade/CommonConfig.java index 69a284d8..373226b4 100644 --- a/ethereumj-core/src/main/java/org/ethereum/facade/CommonConfig.java +++ b/ethereumj-core/src/main/java/org/ethereum/facade/CommonConfig.java @@ -38,28 +38,40 @@ public class CommonConfig { private RedisConnection redisConnection; @Bean - Repository repository(){ + Repository repository() { return new RepositoryImpl(keyValueDataSource(), keyValueDataSource()); } @Bean @Scope("prototype") - public KeyValueDataSource keyValueDataSource(){ - if (CONFIG.getKeyValueDataSource().equals("redis")) { - if (redisConnection.isAvailable()) { + public KeyValueDataSource keyValueDataSource() { + String dataSource = CONFIG.getKeyValueDataSource(); + try { + if ("redis".equals(dataSource) && redisConnection.isAvailable()) { // Name will be defined before initialization return redisConnection.createDataSource(""); } - } - return new LevelDbDataSource(); + dataSource = "leveldb"; + return new LevelDbDataSource(); + } finally { + logger.info(dataSource + " key-value data source created."); + } } @Bean public Set pendingTransactions() { - return redisConnection.isAvailable() - ? redisConnection.createTransactionSet("pendingTransactions") - : Collections.synchronizedSet(new HashSet()); + String storage = "Redis"; + try { + if (redisConnection.isAvailable()) { + return redisConnection.createTransactionSet("pendingTransactions"); + } + + storage = "In memory"; + return Collections.synchronizedSet(new HashSet()); + } finally { + logger.info(storage + " 'pendingTransactions' storage created."); + } } @Bean