status-im
/
nimbus-eth1
mirror of https://github.com/status-im/nimbus-eth1.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Populate the persistent databases with the empty RLP key. 

Also implements transactional block persistence. Two issues
in the transaction processing code have been discovered that
might affect other usages such as the CALL instruction.

The main fix gets us past block 49000.

You may need to clean up your database.
This commit is contained in:
Zahary Karadjov 2018-10-05 03:20:12 +03:00 committed by zah
parent 7459650663
commit 343cc4fa43
14 changed files with 73 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
nimbus/db/backends/caching_backend.nim
View File

@ -1,5 +1,7 @@
import ranges, eth_trie, tables, sets import
import ../storage_types ranges, tables, sets,
eth_trie/db,
../storage_types
type type
CachingDB* = ref object of RootObj CachingDB* = ref object of RootObj

29
nimbus/db/backends/rocksdb_backend.nim
View File

@ -1,4 +1,4 @@
import os, rocksdb, ranges, eth_trie/db_tracing import os, rocksdb, ranges, eth_trie/[db_tracing, constants]
import ../storage_types import ../storage_types
type type
@ -7,18 +7,6 @@ type
ChainDB* = RocksChainDB ChainDB* = RocksChainDB
proc newChainDB*(basePath: string): ChainDB =
result.new()
let
dataDir = basePath / "data"
backupsDir = basePath / "backups"
createDir(dataDir)
createDir(backupsDir)
let s = result.store.init(dataDir, backupsDir)
if not s.ok: raiseStorageInitError()
proc get*(db: ChainDB, key: openarray[byte]): seq[byte] = proc get*(db: ChainDB, key: openarray[byte]): seq[byte] =
let s = db.store.getBytes(key) let s = db.store.getBytes(key)
if s.ok: if s.ok:
@ -46,3 +34,18 @@ proc del*(db: ChainDB, key: openarray[byte]) =
proc close*(db: ChainDB) = proc close*(db: ChainDB) =
db.store.close db.store.close
proc newChainDB*(basePath: string): ChainDB =
result.new()
let
dataDir = basePath / "data"
backupsDir = basePath / "backups"
createDir(dataDir)
createDir(backupsDir)
let s = result.store.init(dataDir, backupsDir)
if not s.ok: raiseStorageInitError()
put(result, emptyRlpHash.data, emptyRlp)

6
nimbus/db/backends/sqlite_backend.nim
View File

@ -1,5 +1,5 @@
import import
os, sqlite3, ranges, ranges/ptr_arith, eth_trie/db_tracing, os, sqlite3, ranges, ranges/ptr_arith, eth_trie/[db_tracing, constants],
../storage_types ../storage_types
type type
@ -9,6 +9,8 @@ type
ChainDB* = SqliteChainDB ChainDB* = SqliteChainDB
proc put*(db: ChainDB, key, value: openarray[byte])
proc newChainDB*(basePath: string, inMemory = false): ChainDB = proc newChainDB*(basePath: string, inMemory = false): ChainDB =
result.new() result.new()
let dbPath = if inMemory: ":memory:" else: basePath / "nimbus.db" let dbPath = if inMemory: ":memory:" else: basePath / "nimbus.db"
@ -54,6 +56,8 @@ proc newChainDB*(basePath: string, inMemory = false): ChainDB =
result.deleteStmt = prepare "DELETE FROM trie_nodes WHERE key = ?;" result.deleteStmt = prepare "DELETE FROM trie_nodes WHERE key = ?;"
put(result, emptyRlpHash.data, emptyRlp)
proc bindBlob(s: Pstmt, n: int, blob: openarray[byte]): int32 = proc bindBlob(s: Pstmt, n: int, blob: openarray[byte]): int32 =
sqlite3.bind_blob(s, n.int32, blob.baseAddr, blob.len.int32, nil) sqlite3.bind_blob(s, n.int32, blob.baseAddr, blob.len.int32, nil)

2
nimbus/db/db_chain.nim
View File

@ -7,7 +7,7 @@
import import
tables, sequtils, algorithm, tables, sequtils, algorithm,
rlp, ranges, state_db, nimcrypto, eth_trie/[types, hexary], eth_common, byteutils, rlp, ranges, state_db, nimcrypto, eth_trie/[hexary, db], eth_common, byteutils,
../errors, ../block_types, ../utils/header, ../constants, ./storage_types.nim ../errors, ../block_types, ../utils/header, ../constants, ./storage_types.nim
type type

2
nimbus/db/state_db.nim
View File

@ -7,7 +7,7 @@
import import
sequtils, strformat, tables, sequtils, strformat, tables,
chronicles, eth_common, nimcrypto, rlp, eth_trie/[hexary, memdb], chronicles, eth_common, nimcrypto, rlp, eth_trie/[hexary, db],
../constants, ../errors, ../validation, ../account ../constants, ../errors, ../validation, ../account
logScope: logScope:

8
nimbus/genesis.nim
View File

@ -1,6 +1,8 @@
import db/[db_chain, state_db], genesis_alloc, eth_common, tables, stint, import
byteutils, times, config, rlp, ranges, block_types, eth_trie, times, tables,
eth_trie/memdb, account, constants, nimcrypto, chronicles eth_common, stint, byteutils, rlp, ranges, block_types, nimcrypto,
chronicles, eth_trie, eth_trie/db,
db/[db_chain, state_db], genesis_alloc, config, account, constants
type type
Genesis* = object Genesis* = object

2
nimbus/nimbus.nim
View File

@ -12,7 +12,7 @@ import
asyncdispatch2, json_rpc/rpcserver, eth_keys, asyncdispatch2, json_rpc/rpcserver, eth_keys,
eth_p2p, eth_p2p/rlpx_protocols/[eth], eth_p2p, eth_p2p/rlpx_protocols/[eth],
config, genesis, rpc/[common, p2p], p2p/chain, config, genesis, rpc/[common, p2p], p2p/chain,
eth_trie eth_trie/db
const UseSqlite = false const UseSqlite = false

12
nimbus/p2p/chain.nim
View File

@ -1,10 +1,9 @@
import ../db/[db_chain, state_db], eth_common, chronicles, ../vm_state, ../vm_types, ../transaction, ranges, import ../db/[db_chain, state_db], eth_common, chronicles, ../vm_state, ../vm_types, ../transaction, ranges,
../vm/[computation, interpreter_dispatch, message], ../constants, stint, nimcrypto, ../vm/[computation, interpreter_dispatch, message], ../constants, stint, nimcrypto,
../vm_state_transactions, ../vm_state_transactions,
eth_trie/memdb, eth_trie, rlp, eth_trie/db, eth_trie, rlp,
sugar sugar
type type
Chain* = ref object of AbstractChainDB Chain* = ref object of AbstractChainDB
db: BaseChainDB db: BaseChainDB
@ -101,7 +100,7 @@ proc processTransaction(db: var AccountStateDB, t: Transaction, sender: EthAddre
return gasUsed.u256 * t.gasPrice.u256 return gasUsed.u256 * t.gasPrice.u256
proc calcTxRoot(transactions: openarray[Transaction]): Hash256 = proc calcTxRoot(transactions: openarray[Transaction]): Hash256 =
var tr = initHexaryTrie(trieDB(newMemDB())) var tr = initHexaryTrie(newMemoryDB())
for i, t in transactions: for i, t in transactions:
tr.put(rlp.encode(i).toRange, rlp.encode(t).toRange) tr.put(rlp.encode(i).toRange, rlp.encode(t).toRange)
return tr.rootHash return tr.rootHash
@ -112,6 +111,9 @@ method persistBlocks*(c: Chain, headers: openarray[BlockHeader], bodies: openarr
let blockReward = 5.u256 * pow(10.u256, 18) # 5 ETH let blockReward = 5.u256 * pow(10.u256, 18) # 5 ETH
let transaction = c.db.db.beginTransaction()
defer: transaction.dispose()
echo "Persisting blocks: ", headers[0].blockNumber, " - ", headers[^1].blockNumber echo "Persisting blocks: ", headers[0].blockNumber, " - ", headers[^1].blockNumber
for i in 0 ..< headers.len: for i in 0 ..< headers.len:
let head = c.db.getCanonicalHead() let head = c.db.getCanonicalHead()
@ -158,6 +160,8 @@ method persistBlocks*(c: Chain, headers: openarray[BlockHeader], bodies: openarr
echo "Wrong state root in block ", headers[i].blockNumber, ". Expected: ", headers[i].stateRoot, ", Actual: ", stateDb.rootHash, " arrived from ", c.db.getCanonicalHead().stateRoot echo "Wrong state root in block ", headers[i].blockNumber, ". Expected: ", headers[i].stateRoot, ", Actual: ", stateDb.rootHash, " arrived from ", c.db.getCanonicalHead().stateRoot
assert(headers[i].stateRoot == stateDb.rootHash) assert(headers[i].stateRoot == stateDb.rootHash)
discard c.db.persistHeaderToDb(headers[i]) discard c.db.persistHeaderToDb(headers[i])
assert(c.db.getCanonicalHead().blockHash == headers[i].blockHash) assert(c.db.getCanonicalHead().blockHash == headers[i].blockHash)
transaction.commit()

7
nimbus/rpc/common.nim
View File

@ -6,10 +6,11 @@
# at your option. # at your option.
# This file may not be copied, modified, or distributed except according to # This file may not be copied, modified, or distributed except according to
# those terms. # those terms.
import strutils, nimcrypto, eth_common, stint, eth_trie/[memdb, types]
import import
json_rpc/server, ../vm_state, ../db/[db_chain, state_db], strutils,
../constants, ../config, hexstrings nimcrypto, eth_common, stint, json_rpc/server,
../vm_state, ../db/[db_chain, state_db], ../constants, ../config, hexstrings
proc setupCommonRPC*(server: RpcServer) = proc setupCommonRPC*(server: RpcServer) =
server.rpc("web3_clientVersion") do() -> string: server.rpc("web3_clientVersion") do() -> string:

35
nimbus/rpc/p2p.nim
View File

@ -6,11 +6,14 @@
# at your option. # at your option.
# This file may not be copied, modified, or distributed except according to # This file may not be copied, modified, or distributed except according to
# those terms. # those terms.
import import
nimcrypto, json_rpc/rpcserver, eth_p2p, hexstrings, strutils, stint, strutils, times,
../config, ../vm_state, ../constants, eth_trie/[memdb, types], eth_keys, nimcrypto, json_rpc/rpcserver, hexstrings, stint, byteutils, ranges/typedranges,
../db/[db_chain, state_db, storage_types], eth_common, rpc_types, byteutils, eth_common, eth_p2p, eth_keys, eth_trie/db, rlp,
ranges/typedranges, times, ../utils/header, rlp, ../transaction ../utils/header, ../transaction, ../config, ../vm_state, ../constants,
../db/[db_chain, state_db, storage_types],
rpc_types
#[ #[
Note: Note:
@ -52,14 +55,14 @@ func headerFromTag(chain:BaseChainDB, blockTag: string): BlockHeader =
result = chain.getBlockHeader(blockNum) result = chain.getBlockHeader(blockNum)
proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) = proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
func getAccountDb(header: BlockHeader, readOnly = true): AccountStateDb = func getAccountDb(header: BlockHeader, readOnly = true): AccountStateDb =
## Retrieves the account db from canonical head ## Retrieves the account db from canonical head
let vmState = newBaseVMState(header, chain) let vmState = newBaseVMState(header, chain)
result = vmState.chaindb.getStateDb(vmState.blockHeader.hash, readOnly) result = vmState.chaindb.getStateDb(vmState.blockHeader.hash, readOnly)
func accountDbFromTag(tag: string, readOnly = true): AccountStateDb = func accountDbFromTag(tag: string, readOnly = true): AccountStateDb =
result = getAccountDb(chain.headerFromTag(tag)) result = getAccountDb(chain.headerFromTag(tag))
proc getBlockBody(hash: KeccakHash): BlockBody = proc getBlockBody(hash: KeccakHash): BlockBody =
if not chain.getBlockBody(hash, result): if not chain.getBlockBody(hash, result):
@ -68,7 +71,7 @@ proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
rpcsrv.rpc("net_version") do() -> uint: rpcsrv.rpc("net_version") do() -> uint:
let conf = getConfiguration() let conf = getConfiguration()
result = conf.net.networkId result = conf.net.networkId
rpcsrv.rpc("eth_syncing") do() -> JsonNode: rpcsrv.rpc("eth_syncing") do() -> JsonNode:
## Returns SyncObject or false when not syncing. ## Returns SyncObject or false when not syncing.
# TODO: Requires PeerPool to check sync state. # TODO: Requires PeerPool to check sync state.
@ -84,7 +87,7 @@ proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
result = %sync result = %sync
else: else:
result = newJBool(false) result = newJBool(false)
rpcsrv.rpc("eth_coinbase") do() -> EthAddress: rpcsrv.rpc("eth_coinbase") do() -> EthAddress:
## Returns the current coinbase address. ## Returns the current coinbase address.
result = chain.getCanonicalHead().coinbase result = chain.getCanonicalHead().coinbase
@ -238,7 +241,7 @@ proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
## Generates and returns an estimate of how much gas is necessary to allow the transaction to complete. ## Generates and returns an estimate of how much gas is necessary to allow the transaction to complete.
## The transaction will not be added to the blockchain. Note that the estimate may be significantly more than ## The transaction will not be added to the blockchain. Note that the estimate may be significantly more than
## the amount of gas actually used by the transaction, for a variety of reasons including EVM mechanics and node performance. ## the amount of gas actually used by the transaction, for a variety of reasons including EVM mechanics and node performance.
## ##
## call: the transaction call object. ## call: the transaction call object.
## quantityTag: integer block number, or the string "latest", "earliest" or "pending", see the default block parameter. ## quantityTag: integer block number, or the string "latest", "earliest" or "pending", see the default block parameter.
## Returns the amount of gas used. ## Returns the amount of gas used.
@ -256,7 +259,7 @@ proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
startIdx = 0 startIdx = 0
for i in 0 ..< blockBody.uncles.len: for i in 0 ..< blockBody.uncles.len:
rawData[startIdx .. startIdx + 32] = blockBody.uncles[i].hash.data rawData[startIdx .. startIdx + 32] = blockBody.uncles[i].hash.data
startIdx += 32 startIdx += 32
result.sha3Uncles = keccak256.digest(rawData) result.sha3Uncles = keccak256.digest(rawData)
result.logsBloom = some(header.bloom) result.logsBloom = some(header.bloom)
@ -317,7 +320,7 @@ proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
result.gasPrice = transaction.gasPrice result.gasPrice = transaction.gasPrice
result.gas = accountGas result.gas = accountGas
result.input = transaction.payload result.input = transaction.payload
rpcsrv.rpc("eth_getTransactionByHash") do(data: HexDataStr) -> TransactionObject: rpcsrv.rpc("eth_getTransactionByHash") do(data: HexDataStr) -> TransactionObject:
## Returns the information about a transaction requested by transaction hash. ## Returns the information about a transaction requested by transaction hash.
## ##
@ -393,7 +396,7 @@ proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
idx.inc idx.inc
rpcsrv.rpc("eth_getUncleByBlockHashAndIndex") do(data: HexDataStr, quantity: int) -> Option[BlockObject]: rpcsrv.rpc("eth_getUncleByBlockHashAndIndex") do(data: HexDataStr, quantity: int) -> Option[BlockObject]:
## Returns information about a uncle of a block by hash and uncle index position. ## Returns information about a uncle of a block by hash and uncle index position.
## ##
## data: hash of block. ## data: hash of block.
## quantity: the uncle's index position. ## quantity: the uncle's index position.
@ -428,12 +431,12 @@ proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
## [A] "A in first position (and anything after)" ## [A] "A in first position (and anything after)"
## [null, B] "anything in first position AND B in second position (and anything after)" ## [null, B] "anything in first position AND B in second position (and anything after)"
## [A, B] "A in first position AND B in second position (and anything after)" ## [A, B] "A in first position AND B in second position (and anything after)"
## [[A, B], [A, B]] "(A OR B) in first position AND (A OR B) in second position (and anything after)" ## [[A, B], [A, B]] "(A OR B) in first position AND (A OR B) in second position (and anything after)"
## ##
## filterOptions: settings for this filter. ## filterOptions: settings for this filter.
## Returns integer filter id. ## Returns integer filter id.
discard discard
rpcsrv.rpc("eth_newBlockFilter") do() -> int: rpcsrv.rpc("eth_newBlockFilter") do() -> int:
## Creates a filter in the node, to notify when a new block arrives. ## Creates a filter in the node, to notify when a new block arrives.
## To check if the state has changed, call eth_getFilterChanges. ## To check if the state has changed, call eth_getFilterChanges.
@ -449,7 +452,7 @@ proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
discard discard
rpcsrv.rpc("eth_uninstallFilter") do(filterId: int) -> bool: rpcsrv.rpc("eth_uninstallFilter") do(filterId: int) -> bool:
## Uninstalls a filter with given id. Should always be called when watch is no longer needed. ## Uninstalls a filter with given id. Should always be called when watch is no longer needed.
## Additonally Filters timeout when they aren't requested with eth_getFilterChanges for a period of time. ## Additonally Filters timeout when they aren't requested with eth_getFilterChanges for a period of time.
## ##
## filterId: The filter id. ## filterId: The filter id.

9
tests/test_caching_db_backend.nim
View File

@ -1,5 +1,8 @@
#[
import ../nimbus/db/backends/caching_backend, eth_trie, eth_trie/memdb, unittest import
unittest,
eth_trie/db,
../nimbus/db/backends/caching_backend
let let
key1 = [0.byte, 0, 1] key1 = [0.byte, 0, 1]
@ -42,4 +45,4 @@ suite "Caching DB backend":
mdb.get(key2) == @value2 mdb.get(key2) == @value2
mdb.get(key3) == @value3 mdb.get(key3) == @value3
key4 notin mdb key4 notin mdb
]#

2
tests/test_generalstate_json.nim
View File

@ -8,7 +8,7 @@
import import
unittest, strformat, strutils, tables, json, ospaths, times, unittest, strformat, strutils, tables, json, ospaths, times,
byteutils, ranges/typedranges, nimcrypto/[keccak, hash], byteutils, ranges/typedranges, nimcrypto/[keccak, hash],
rlp, eth_trie/[types, memdb], eth_common, rlp, eth_trie/db, eth_common,
eth_keys, eth_keys,
./test_helpers, ./test_helpers,
../nimbus/[constants, errors], ../nimbus/[constants, errors],

3
tests/test_opcode.nim
View File

@ -7,7 +7,7 @@
import import
unittest, tables, parseutils, byteutils, unittest, tables, parseutils, byteutils,
eth_trie/[types, memdb], eth_common/eth_types, eth_trie/db, eth_common/eth_types,
../nimbus/[constants, vm_types, vm_state], ../nimbus/[constants, vm_types, vm_state],
../nimbus/vm/interpreter, ../nimbus/vm/interpreter,
../nimbus/utils/header, ../nimbus/utils/header,
@ -18,7 +18,6 @@ from eth_common import GasInt
proc testCode(code: string, initialGas: GasInt, blockNum: UInt256): BaseComputation = proc testCode(code: string, initialGas: GasInt, blockNum: UInt256): BaseComputation =
let header = BlockHeader(blockNumber: blockNum) let header = BlockHeader(blockNumber: blockNum)
var memDb = newMemDB()
var vmState = newBaseVMState(header, newBaseChainDB(newMemoryDb())) var vmState = newBaseVMState(header, newBaseChainDB(newMemoryDb()))
# coinbase: "", # coinbase: "",

3
tests/test_vm_json.nim
View File

@ -8,7 +8,7 @@
import import
unittest, strformat, strutils, sequtils, tables, json, ospaths, times, unittest, strformat, strutils, sequtils, tables, json, ospaths, times,
byteutils, ranges/typedranges, nimcrypto/[keccak, hash], byteutils, ranges/typedranges, nimcrypto/[keccak, hash],
rlp, eth_trie/[types, memdb], eth_common, rlp, eth_trie/db, eth_common,
./test_helpers, ./test_helpers,
../nimbus/[constants, errors], ../nimbus/[constants, errors],
../nimbus/[vm_state, vm_types], ../nimbus/[vm_state, vm_types],
@ -39,7 +39,6 @@ proc testFixture(fixtures: JsonNode, testStatusIMPL: var TestStatus) =
stateRoot: emptyRlpHash stateRoot: emptyRlpHash
) )
var memDb = newMemDB()
var vmState = newBaseVMState(header, newBaseChainDB(newMemoryDB())) var vmState = newBaseVMState(header, newBaseChainDB(newMemoryDB()))
let fexec = fixture["exec"] let fexec = fixture["exec"]
var code: seq[byte] var code: seq[byte]