status-im
/
nimbus-eth1
mirror of https://github.com/status-im/nimbus-eth1.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Use simpler schema when writing transactions, receipts, and withdrawals (#2420) 

* Use simpler schema when writing transactions, receipts, and withdrawals

Using MPT not only slow but also take up more spaces than needed.
Aristo will remove older tries and only keep the last block tries.
Using simpler schema will avoid those problems.

* Rename getTransaction to getTransactionByIndex
This commit is contained in:
andri lim 2024-06-29 12:43:17 +07:00 committed by GitHub
parent 55ebd70d1e
commit c24affadee
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 187 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hive_integration/nodocker/engine/node.nim
View File

@ -108,11 +108,11 @@ proc setBlock*(c: ChainRef; blk: EthBlock): Result[void, string] =
return err("Could not persist header")
try:
c.db.persistTransactions(header.number, blk.transactions)
c.db.persistReceipts(vmState.receipts)
c.db.persistTransactions(header.number, header.txRoot, blk.transactions)
c.db.persistReceipts(header.receiptsRoot, vmState.receipts)
if blk.withdrawals.isSome:
c.db.persistWithdrawals(blk.withdrawals.get)
c.db.persistWithdrawals(header.withdrawalsRoot, blk.withdrawals.get)
except CatchableError as exc:
return err(exc.msg)

13
nimbus/core/chain/forked_chain.nim
View File

@ -169,16 +169,21 @@ proc replaySegment(c: ForkedChainRef, target: Hash256) =
proc writeBaggage(c: ForkedChainRef, target: Hash256) =
# Write baggage from base+1 to target block
template header(): BlockHeader =
blk.blk.header
shouldNotKeyError:
var prevHash = target
while prevHash != c.baseHash:
let blk = c.blocks[prevHash]
c.db.persistTransactions(blk.blk.header.number, blk.blk.transactions)
c.db.persistReceipts(blk.receipts)
c.db.persistTransactions(header.number, header.txRoot, blk.blk.transactions)
c.db.persistReceipts(header.receiptsRoot, blk.receipts)
discard c.db.persistUncles(blk.blk.uncles)
if blk.blk.withdrawals.isSome:
c.db.persistWithdrawals(blk.blk.withdrawals.get)
prevHash = blk.blk.header.parentHash
c.db.persistWithdrawals(
header.withdrawalsRoot.expect("WithdrawalsRoot should be verified before"),
blk.blk.withdrawals.get)
prevHash = header.parentHash
func updateBase(c: ForkedChainRef,
newBaseHash: Hash256,

8
nimbus/core/chain/persist_blocks.nim
View File

@ -170,13 +170,15 @@ proc persistBlocksImpl(
return err("Could not persist header")
if NoPersistTransactions notin flags:
c.db.persistTransactions(header.number, blk.transactions)
c.db.persistTransactions(header.number, header.txRoot, blk.transactions)
if NoPersistReceipts notin flags:
c.db.persistReceipts(vmState.receipts)
c.db.persistReceipts(header.receiptsRoot, vmState.receipts)
if NoPersistWithdrawals notin flags and blk.withdrawals.isSome:
c.db.persistWithdrawals(blk.withdrawals.get)
c.db.persistWithdrawals(
header.withdrawalsRoot.expect("WithdrawalsRoot should be verified before"),
blk.withdrawals.get)
# update currentBlock *after* we persist it
# so the rpc return consistent result

3
nimbus/db/core_db/base/base_desc.nim
View File

@ -61,9 +61,6 @@ type
CoreDbColType* = enum
CtGeneric = 2 # columns smaller than 2 are not provided
CtReceipts
CtTxs
CtWithdrawals
CoreDbCaptFlags* {.pure.} = enum
PersistPut

170
nimbus/db/core_db/core_apps.nim
View File

@ -114,30 +114,23 @@ iterator findNewAncestors(
iterator getBlockTransactionData*(
db: CoreDbRef;
transactionRoot: Hash256;
txRoot: Hash256;
): Blob =
const info = "getBlockTransactionData()"
block body:
if transactionRoot == EMPTY_ROOT_HASH:
if txRoot == EMPTY_ROOT_HASH:
break body
let
transactionDb = db.ctx.getColumn CtTxs
state = transactionDb.state(updateOk=true).valueOr:
raiseAssert info & ": " & $$error
if state != transactionRoot:
warn logTxt info, transactionRoot, state, error="state mismatch"
break body
var transactionIdx = 0'u64
while true:
let transactionKey = rlp.encode(transactionIdx)
let data = transactionDb.fetch(transactionKey).valueOr:
if error.error != MptNotFound:
warn logTxt info, transactionRoot,
transactionKey, action="fetch()", error=($$error)
break body
yield data
inc transactionIdx
let kvt = db.newKvt()
for idx in 0'u16..<uint16.high:
let key = hashIndexKey(txRoot, idx)
let txData = kvt.getOrEmpty(key).valueOr:
warn logTxt "getBlockTransactionData()",
txRoot, key, action="getOrEmpty()", error=($$error)
break body
if txData.len == 0:
break body
yield txData
iterator getBlockTransactions*(
db: CoreDbRef;
@ -158,32 +151,24 @@ iterator getBlockTransactionHashes*(
for encodedTx in db.getBlockTransactionData(blockHeader.txRoot):
yield keccakHash(encodedTx)
iterator getWithdrawalsData*(
iterator getWithdrawals*(
db: CoreDbRef;
withdrawalsRoot: Hash256;
): Blob =
const info = "getWithdrawalsData()"
): Withdrawal {.raises: [RlpError].} =
block body:
if withdrawalsRoot == EMPTY_ROOT_HASH:
break body
let
wddb = db.ctx.getColumn CtWithdrawals
state = wddb.state(updateOk=true).valueOr:
raiseAssert info & ": " & $$error
if state != withdrawalsRoot:
warn logTxt info, withdrawalsRoot, state, error="state mismatch"
break body
var idx = 0
while true:
let wdKey = rlp.encode(idx.uint)
let data = wddb.fetch(wdKey).valueOr:
if error.error != MptNotFound:
warn logTxt "getWithdrawalsData()",
withdrawalsRoot, wdKey, action="fetch()", error=($$error)
break body
yield data
inc idx
let kvt = db.newKvt()
for idx in 0'u16..<uint16.high:
let key = hashIndexKey(withdrawalsRoot, idx)
let data = kvt.getOrEmpty(key).valueOr:
warn logTxt "getWithdrawals()",
withdrawalsRoot, key, action="getOrEmpty()", error=($$error)
break body
if data.len == 0:
break body
yield rlp.decode(data, Withdrawal)
iterator getReceipts*(
db: CoreDbRef;
@ -194,23 +179,17 @@ iterator getReceipts*(
block body:
if receiptsRoot == EMPTY_ROOT_HASH:
break body
let
receiptDb = db.ctx.getColumn CtReceipts
state = receiptDb.state(updateOk=true).valueOr:
raiseAssert info & ": " & $$error
if state != receiptsRoot:
warn logTxt info, receiptsRoot, state, error="state mismatch"
let kvt = db.newKvt()
for idx in 0'u16..<uint16.high:
let key = hashIndexKey(receiptsRoot, idx)
let data = kvt.getOrEmpty(key).valueOr:
warn logTxt "getReceipts()",
receiptsRoot, key, action="getOrEmpty()", error=($$error)
break body
var receiptIdx = 0
while true:
let receiptKey = rlp.encode(receiptIdx.uint)
let receiptData = receiptDb.fetch(receiptKey).valueOr:
if error.error != MptNotFound:
warn logTxt "getWithdrawalsData()",
receiptsRoot, receiptKey, action="hasKey()", error=($$error)
if data.len == 0:
break body
yield rlp.decode(receiptData, Receipt)
inc receiptIdx
yield rlp.decode(data, Receipt)
# ------------------------------------------------------------------------------
# Private helpers
@ -535,6 +514,7 @@ proc addBlockNumberToHashLookup*(
proc persistTransactions*(
db: CoreDbRef;
blockNumber: BlockNumber;
txRoot: Hash256;
transactions: openArray[Transaction];
) =
const
@ -543,19 +523,16 @@ proc persistTransactions*(
if transactions.len == 0:
return
let
mpt = db.ctx.getColumn(CtTxs, clearData=true)
kvt = db.newKvt()
let kvt = db.newKvt()
for idx, tx in transactions:
let
encodedKey = rlp.encode(idx.uint)
encodedTx = rlp.encode(tx)
txHash = keccakHash(encodedTx)
blockKey = transactionHashToBlockKey(txHash)
txKey: TransactionKey = (blockNumber, idx.uint)
mpt.merge(encodedKey, encodedTx).isOkOr:
warn logTxt info, idx, action="merge()", error=($$error)
key = hashIndexKey(txRoot, idx.uint16)
kvt.put(key, encodedTx).isOkOr:
warn logTxt info, idx, action="put()", error=($$error)
return
kvt.put(blockKey.toOpenArray, rlp.encode(txKey)).isOkOr:
trace logTxt info, blockKey, action="put()", error=($$error)
@ -579,32 +556,29 @@ proc forgetHistory*(
# delete blockHash->header, stateRoot->blockNum
discard kvt.del(genericHashKey(blockHash).toOpenArray)
proc getTransaction*(
proc getTransactionByIndex*(
db: CoreDbRef;
txRoot: Hash256;
txIndex: uint64;
txIndex: uint16;
res: var Transaction;
): bool =
const
info = "getTransaction()"
let
clearOk = txRoot == EMPTY_ROOT_HASH
mpt = db.ctx.getColumn(CtTxs, clearData=clearOk)
if not clearOk:
let state = mpt.state(updateOk=true).valueOr:
raiseAssert info & ": " & $$error
if state != txRoot:
warn logTxt info, txRoot, state, error="state mismatch"
let kvt = db.newKvt()
let key = hashIndexKey(txRoot, txIndex)
let txData = kvt.getOrEmpty(key).valueOr:
warn logTxt "getTransaction()",
txRoot, key, action="getOrEmpty()", error=($$error)
return false
let
txData = mpt.fetch(rlp.encode(txIndex)).valueOr:
if error.error != MptNotFound:
warn logTxt info, txIndex, error=($$error)
if txData.len == 0:
return false
try:
res = rlp.decode(txData, Transaction)
except RlpError as e:
warn logTxt info, txRoot, action="rlp.decode()", name=($e.name), msg=e.msg
except RlpError as exc:
warn logTxt info,
txRoot, action="rlp.decode()", error=exc.msg
return false
true
@ -614,24 +588,19 @@ proc getTransactionCount*(
): int =
const
info = "getTransactionCount()"
let
clearOk = txRoot == EMPTY_ROOT_HASH
mpt = db.ctx.getColumn(CtTxs, clearData=clearOk)
if not clearOk:
let state = mpt.state(updateOk=true).valueOr:
raiseAssert info & ": " & $$error
if state != txRoot:
warn logTxt info, txRoot, state, error="state mismatch"
return 0
var txCount = 0
let kvt = db.newKvt()
var txCount = 0'u16
while true:
let hasPath = mpt.hasPath(rlp.encode(txCount.uint)).valueOr:
warn logTxt info, txCount, action="hasPath()", error=($$error)
let key = hashIndexKey(txRoot, txCount)
let yes = kvt.hasKey(key).valueOr:
warn logTxt info,
txRoot, key, action="hasKey()", error=($$error)
return 0
if hasPath:
if yes:
inc txCount
else:
return txCount
return txCount.int
doAssert(false, "unreachable")
@ -667,15 +636,17 @@ proc getUncles*(
proc persistWithdrawals*(
db: CoreDbRef;
withdrawalsRoot: Hash256;
withdrawals: openArray[Withdrawal];
) =
const info = "persistWithdrawals()"
if withdrawals.len == 0:
return
let mpt = db.ctx.getColumn(CtWithdrawals, clearData=true)
let kvt = db.newKvt()
for idx, wd in withdrawals:
mpt.merge(rlp.encode(idx.uint), rlp.encode(wd)).isOkOr:
warn logTxt info, idx, error=($$error)
let key = hashIndexKey(withdrawalsRoot, idx.uint16)
kvt.put(key, rlp.encode(wd)).isOkOr:
warn logTxt info, idx, action="put()", error=($$error)
return
proc getWithdrawals*(
@ -683,8 +654,8 @@ proc getWithdrawals*(
withdrawalsRoot: Hash256;
): seq[Withdrawal]
{.gcsafe, raises: [RlpError].} =
for encodedWd in db.getWithdrawalsData(withdrawalsRoot):
result.add(rlp.decode(encodedWd, Withdrawal))
for wd in db.getWithdrawals(withdrawalsRoot):
result.add(wd)
proc getTransactions*(
db: CoreDbRef;
@ -837,14 +808,17 @@ proc setHead*(
proc persistReceipts*(
db: CoreDbRef;
receiptsRoot: Hash256;
receipts: openArray[Receipt];
) =
const info = "persistReceipts()"
if receipts.len == 0:
return
let mpt = db.ctx.getColumn(CtReceipts, clearData=true)
let kvt = db.newKvt()
for idx, rec in receipts:
mpt.merge(rlp.encode(idx.uint), rlp.encode(rec)).isOkOr:
let key = hashIndexKey(receiptsRoot, idx.uint16)
kvt.put(key, rlp.encode(rec)).isOkOr:
warn logTxt info, idx, action="merge()", error=($$error)
proc getReceipts*(

7
nimbus/db/storage_types.nim
View File

@ -33,6 +33,8 @@ type
data*: array[33, byte]
dataEndPos*: uint8 # the last populated position in the data
HashIndexKey* = array[34, byte]
func genericHashKey*(h: Hash256): DbKey {.inline.} =
result.data[0] = byte ord(genericHash)
result.data[1 .. 32] = h.data
@ -102,6 +104,11 @@ func skeletonBodyKey*(h: Hash256): DbKey {.inline.} =
result.data[1 .. 32] = h.data
result.dataEndPos = uint8 32
func hashIndexKey*(hash: Hash256, index: uint16): HashIndexKey =
result[0..31] = hash.data
result[32] = byte(index and 0xFF)
result[33] = byte((index shl 8) and 0xFF)
template toOpenArray*(k: DbKey): openArray[byte] =
k.data.toOpenArray(0, int(k.dataEndPos))

2
nimbus/graphql/ethapi.nim
View File

@ -315,7 +315,7 @@ proc getWithdrawals(ctx: GraphqlContextRef, header: common.BlockHeader): RespRes
proc getTxAt(ctx: GraphqlContextRef, header: common.BlockHeader, index: uint64): RespResult =
try:
var tx: Transaction
if getTransaction(ctx.chainDB, header.txRoot, index, tx):
if getTransactionByIndex(ctx.chainDB, header.txRoot, index.uint16, tx):
let txn = txNode(ctx, tx, index, header.number, header.baseFeePerGas)
var i = 0'u64

16
nimbus/rpc/p2p.nim
View File

@ -11,8 +11,12 @@
import
std/[sequtils, times, tables, typetraits],
json_rpc/rpcserver, stint, stew/byteutils,
json_serialization, web3/conversions, json_serialization/stew/results,
json_rpc/rpcserver,
stint,
stew/byteutils,
json_serialization,
web3/conversions,
json_serialization/stew/results,
eth/common/eth_types_json_serialization,
eth/[keys, rlp, p2p],
".."/[transaction, evm/state, constants],
@ -414,7 +418,7 @@ proc setupEthRpc*(
let header = chainDB.getBlockHeader(txDetails.blockNumber)
var tx: Transaction
if chainDB.getTransaction(header.txRoot, txDetails.index, tx):
if chainDB.getTransactionByIndex(header.txRoot, uint16(txDetails.index), tx):
result = populateTransactionObject(tx, Opt.some(header), Opt.some(txDetails.index))
server.rpc("eth_getTransactionByBlockHashAndIndex") do(data: Web3Hash, quantity: Web3Quantity) -> TransactionObject:
@ -429,7 +433,7 @@ proc setupEthRpc*(
return nil
var tx: Transaction
if chainDB.getTransaction(header.txRoot, index, tx):
if chainDB.getTransactionByIndex(header.txRoot, uint16(index), tx):
result = populateTransactionObject(tx, Opt.some(header), Opt.some(index))
else:
result = nil
@ -444,7 +448,7 @@ proc setupEthRpc*(
index = uint64(quantity)
var tx: Transaction
if chainDB.getTransaction(header.txRoot, index, tx):
if chainDB.getTransactionByIndex(header.txRoot, uint16(index), tx):
result = populateTransactionObject(tx, Opt.some(header), Opt.some(index))
else:
result = nil
@ -461,7 +465,7 @@ proc setupEthRpc*(
let header = chainDB.getBlockHeader(txDetails.blockNumber)
var tx: Transaction
if not chainDB.getTransaction(header.txRoot, txDetails.index, tx):
if not chainDB.getTransactionByIndex(header.txRoot, uint16(txDetails.index), tx):
return nil
var

2
premix/prestate.nim
View File

@ -24,7 +24,7 @@ proc generatePrestate*(nimbus, geth: JsonNode, blockNumber: BlockNumber, parent:
kvt = chainDB.newKvt()
discard chainDB.setHead(parent, true)
chainDB.persistTransactions(blockNumber, blk.transactions)
chainDB.persistTransactions(blockNumber, header.txRoot, blk.transactions)
discard chainDB.persistUncles(blk.uncles)
kvt.put(genericHashKey(headerHash).toOpenArray, rlp.encode(header)).isOkOr:

8
tests/test_rpc.nim
View File

@ -87,8 +87,8 @@ proc persistFixtureBlock(chainDB: CoreDbRef) =
# Manually inserting header to avoid any parent checks
discard chainDB.newKvt.put(genericHashKey(header.blockHash).toOpenArray, rlp.encode(header))
chainDB.addBlockNumberToHashLookup(header)
chainDB.persistTransactions(header.number, getBlockBody4514995().transactions)
chainDB.persistReceipts(getReceipts4514995())
chainDB.persistTransactions(header.number, header.txRoot, getBlockBody4514995().transactions)
chainDB.persistReceipts(header.receiptsRoot, getReceipts4514995())
proc setupEnv(com: CommonRef, signer, ks2: EthAddress, ctx: EthContext): TestEnv =
var
@ -153,9 +153,9 @@ proc setupEnv(com: CommonRef, signer, ks2: EthAddress, ctx: EthContext): TestEnv
signedTx1 = signTransaction(unsignedTx1, acc.privateKey, com.chainId, eip155)
signedTx2 = signTransaction(unsignedTx2, acc.privateKey, com.chainId, eip155)
txs = [signedTx1, signedTx2]
com.db.persistTransactions(blockNumber, txs)
let txRoot = com.db.ctx.getColumn(CtTxs).state(updateOk=true).valueOr(EMPTY_ROOT_HASH)
let txRoot = calcTxRoot(txs)
com.db.persistTransactions(blockNumber, txRoot, txs)
vmState.receipts = newSeq[Receipt](txs.len)
vmState.cumulativeGasUsed = 0

72
tests/test_txpool2.nim
View File

@ -337,6 +337,77 @@ proc runTxHeadDelta(noisy = true) =
balance = sdb.getBalance(recipient)
check balance == expected
proc runGetBlockBodyTest() =
var
env = initEnv(Cancun)
blockTime = EthTime.now()
parentHeader: BlockHeader
currentHeader: BlockHeader
suite "Test get parent transactions after persistBlock":
test "TxPool create first block":
let
tx1 = env.makeTx(recipient, 1.u256)
tx2 = env.makeTx(recipient, 2.u256)
check env.xp.addLocal(PooledTransaction(tx: tx1), true).isOk
check env.xp.addLocal(PooledTransaction(tx: tx2), true).isOk
env.com.pos.prevRandao = prevRandao
env.com.pos.feeRecipient = feeRecipient
env.com.pos.timestamp = blockTime
let r = env.xp.assembleBlock()
if r.isErr:
check false
return
let blk = r.get.blk
check env.chain.persistBlocks([blk]).isOk
parentHeader = blk.header
check env.xp.smartHead(parentHeader)
check blk.transactions.len == 2
test "TxPool create second block":
let
tx1 = env.makeTx(recipient, 3.u256)
tx2 = env.makeTx(recipient, 4.u256)
tx3 = env.makeTx(recipient, 5.u256)
check env.xp.addLocal(PooledTransaction(tx: tx1), true).isOk
check env.xp.addLocal(PooledTransaction(tx: tx2), true).isOk
check env.xp.addLocal(PooledTransaction(tx: tx3), true).isOk
env.com.pos.prevRandao = prevRandao
env.com.pos.feeRecipient = feeRecipient
env.com.pos.timestamp = blockTime + 1
let r = env.xp.assembleBlock()
if r.isErr:
check false
return
let blk = r.get.blk
check env.chain.persistBlocks([blk]).isOk
currentHeader = blk.header
check env.xp.smartHead(currentHeader)
check blk.transactions.len == 3
test "Get current block body":
var body: BlockBody
check env.com.db.getBlockBody(currentHeader, body)
check body.transactions.len == 3
check env.com.db.getReceipts(currentHeader.receiptsRoot).len == 3
check env.com.db.getTransactionCount(currentHeader.txRoot) == 3
test "Get parent block body":
# Make sure parent baggage doesn't swept away by aristo
var body: BlockBody
check env.com.db.getBlockBody(parentHeader, body)
check body.transactions.len == 2
check env.com.db.getReceipts(parentHeader.receiptsRoot).len == 2
check env.com.db.getTransactionCount(parentHeader.txRoot) == 2
proc txPool2Main*() =
const
noisy = defined(debug)
@ -346,6 +417,7 @@ proc txPool2Main*() =
runTxPoolPosTest()
runTxPoolBlobhashTest()
noisy.runTxHeadDelta
runGetBlockBodyTest()
when isMainModule:
txPool2Main()