Add block bodies to the propagation and lookups (#975)

* Add block bodies to the propagation and lookups

- Read and propagate block bodies next to the headers
- Add block bodies content (via lookups) to the eth_getBlockByHash
call
- Test the above in test_portal_testnet

* Fix storage/propagation of block bodies

- Data format is an actual block: [header, txs, uncles], which
requires some adjustment to store the block body
- Added also `eth_getBlockTransactionCountByHash` json rpc call
This commit is contained in:
Kim De Mey 2022-02-22 11:52:44 +01:00 committed by GitHub
parent d1127d77b1
commit 6f21c232ae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 211 additions and 70 deletions

View File

@ -10,7 +10,7 @@
import
json_serialization, json_serialization/std/tables,
stew/[byteutils, io2, results], nimcrypto/keccak, chronos, chronicles,
eth/rlp,
eth/[rlp, common/eth_types],
./content_db,
./network/wire/portal_protocol,
./network/history/history_content
@ -45,50 +45,98 @@ proc readBlockData*(dataFile: string): Result[BlockDataTable, string] =
iterator blockHashes*(blockData: BlockDataTable): BlockHash =
for k,v in blockData:
var blockHash: BlockHash
try:
var blockHash: BlockHash
blockHash.data = hexToByteArray[sizeof(BlockHash)](k)
yield blockHash
except ValueError as e:
error "Invalid hex for block hash", error = e.msg, number = v.number
continue
iterator blockHeaders*(
blockData: BlockDataTable, verify = false): (ContentKey, seq[byte]) =
yield blockHash
iterator blocks*(
blockData: BlockDataTable, verify = false): seq[(ContentKey, seq[byte])] =
for k,v in blockData:
try:
var rlp = rlpFromHex(v.rlp)
var res: seq[(ContentKey, seq[byte])]
if rlp.enterList():
# List that contains 3 items: Block header, body and receipts.
# Only make block header available for now.
# When we want others, can use `rlp.skipElem()` and `rlp.rawData()`.
var rlp =
try:
rlpFromHex(v.rlp)
except ValueError as e:
error "Invalid hex for rlp data", error = e.msg, number = v.number
continue
# Prepare content key
var blockHash: BlockHash
# The data is currently formatted as an rlp encoded `EthBlock`, thus
# containing header, txs and uncles: [header, txs, uncles]. No receipts are
# available.
# TODO: Change to format to rlp data as it gets stored and send over the
# network over the network. I.e. [header, [txs, uncles], receipts]
if rlp.enterList():
var blockHash: BlockHash
try:
blockHash.data = hexToByteArray[sizeof(BlockHash)](k)
except ValueError as e:
error "Invalid hex for block hash", error = e.msg, number = v.number
continue
let contentKey = ContentKey(
contentType: blockHeader,
blockHeaderKey: ContentKeyType(chainId: 1'u16, blockHash: blockHash))
let contentKeyType =
ContentKeyType(chainId: 1'u16, blockHash: blockHash)
# If wanted we can verify the hash for the corresponding header
try:
# If wanted the hash for the corresponding header can be verified
if verify:
if keccak256.digest(rlp.rawData()) != blockHash:
error "Data is not matching hash, skipping"
error "Data is not matching hash, skipping", number = v.number
continue
yield (contentKey, @(rlp.rawData()))
except CatchableError as e:
error "Failed decoding block hash or data", error = e.msg,
number = v.number
block:
let contentKey = ContentKey(
contentType: blockHeader,
blockHeaderKey: contentKeyType)
res.add((contentKey, @(rlp.rawData())))
rlp.skipElem()
block:
let contentKey = ContentKey(
contentType: blockBody,
blockBodyKey: contentKeyType)
# Note: Temporary until the data format gets changed.
let blockBody = BlockBody(
transactions: rlp.read(seq[Transaction]),
uncles: rlp.read(seq[BlockHeader]))
let rlpdata = encode(blockBody)
echo rlpdata.toHex()
res.add((contentKey, rlpdata))
# res.add((contentKey, @(rlp.rawData())))
# rlp.skipElem()
# Note: No receipts yet in the data set
# block:
# let contentKey = ContentKey(
# contentType: receipts,
# receiptsKey: contentKeyType)
# res.add((contentKey, @(rlp.rawData())))
# rlp.skipElem()
except RlpError as e:
error "Invalid rlp data", number = v.number, error = e.msg
continue
yield res
else:
error "Item is not a valid rlp list", number = v.number
proc populateHistoryDb*(
db: ContentDB, dataFile: string, verify = false): Result[void, string] =
let blockData = ? readBlockData(dataFile)
for k,v in blockHeaders(blockData, verify):
# Note: This is the slowest part due to the hashing that takes place.
db.put(history_content.toContentId(k), v)
for b in blocks(blockData, verify):
for value in b:
# Note: This is the slowest part due to the hashing that takes place.
db.put(history_content.toContentId(value[0]), value[1])
ok()
@ -98,13 +146,14 @@ proc propagateHistoryDb*(
let blockData = readBlockData(dataFile)
if blockData.isOk():
for k,v in blockHeaders(blockData.get(), verify):
# Note: This is the slowest part due to the hashing that takes place.
p.contentDB.put(history_content.toContentId(k), v)
for b in blocks(blockData.get(), verify):
for value in b:
# Note: This is the slowest part due to the hashing that takes place.
p.contentDB.put(history_content.toContentId(value[0]), value[1])
# TODO: This call will get the content we just stored in the db, so it
# might be an improvement to directly pass it.
await p.neighborhoodGossip(ContentKeysList(@[encode(k)]))
# TODO: This call will get the content we just stored in the db, so it
# might be an improvement to directly pass it.
await p.neighborhoodGossip(ContentKeysList(@[encode(value[0])]))
return ok()
else:
return err(blockData.error)

View File

@ -8,13 +8,14 @@
{.push raises: [Defect].}
import
std/times,
json_rpc/[rpcproxy, rpcserver],
std/[times, sequtils],
json_rpc/[rpcproxy, rpcserver], nimcrypto/[hash, keccak],
web3/conversions, # sigh, for FixedBytes marshalling
eth/common/eth_types,
eth/[common/eth_types, rlp],
# TODO: Using the Nimbus json-rpc helpers, but they could use some rework as
# they bring a whole lot of other stuff with them.
../../nimbus/rpc/[rpc_types, hexstrings, rpc_utils],
../../nimbus/errors, # for ValidationError, should be exported instead
../network/history/[history_network, history_content]
# Subset of Eth JSON-RPC API: https://eth.wiki/json-rpc/API
@ -29,7 +30,10 @@ import
# Note: Similar as `populateBlockObject` from rpc_utils, but more limited as
# there is currently only access to the block header.
proc buildBlockObject*(header: BlockHeader): BlockObject =
proc buildBlockObject*(
header: BlockHeader, body: BlockBody,
fullTx = true, isUncle = false):
BlockObject {.raises: [Defect, ValidationError].} =
let blockHash = header.blockHash
result.number = some(encodeQuantity(header.blockNumber))
@ -57,6 +61,19 @@ proc buildBlockObject*(header: BlockHeader): BlockObject =
result.gasUsed = encodeQuantity(header.gasUsed.uint64)
result.timestamp = encodeQuantity(header.timeStamp.toUnix.uint64)
if not isUncle:
result.uncles = body.uncles.map(proc(h: BlockHeader): Hash256 = h.blockHash)
if fullTx:
var i = 0
for tx in body.transactions:
# ValidationError from tx.getSender in populateTransactionObject
result.transactions.add %(populateTransactionObject(tx, header, i))
inc i
else:
for tx in body.transactions:
result.transactions.add %(keccak256.digest(rlp.encode(tx)))
proc installEthApiHandlers*(
# Currently only HistoryNetwork needed, later we might want a master object
# holding all the networks.
@ -80,7 +97,7 @@ proc installEthApiHandlers*(
rpcServerWithProxy.registerProxyMethod("eth_getBlockByNumber")
rpcServerWithProxy.registerProxyMethod("eth_getBlockTransactionCountByHash")
# rpcServerWithProxy.registerProxyMethod("eth_getBlockTransactionCountByHash")
rpcServerWithProxy.registerProxyMethod("eth_getBlockTransactionCountByNumber")
@ -150,17 +167,62 @@ proc installEthApiHandlers*(
## Note: transactions and uncles are currently not implemented.
##
## Returns BlockObject or nil when no block was found.
let blockHash = data.toHash()
let
blockHash = data.toHash()
contentKeyType = ContentKeyType(chainId: 1'u16, blockHash: blockHash)
let contentKey = ContentKey(
contentType: blockHeader,
blockHeaderKey: ContentKeyType(chainId: 1'u16, blockHash: blockHash))
contentKeyHeader =
ContentKey(contentType: blockHeader, blockHeaderKey: contentKeyType)
contentKeyBody =
ContentKey(contentType: blockBody, blockBodyKey: contentKeyType)
let content = await historyNetwork.getContent(contentKey)
if content.isSome():
var rlp = rlpFromBytes(content.get())
let blockHeader = rlp.read(BlockHeader)
let headerContent = await historyNetwork.getContent(contentKeyHeader)
if headerContent.isNone():
return none(BlockObject)
return some(buildBlockObject(blockHeader))
var rlp = rlpFromBytes(headerContent.get())
let blockHeader = rlp.read(BlockHeader)
let bodyContent = await historyNetwork.getContent(contentKeyBody)
if bodyContent.isSome():
var rlp = rlpFromBytes(bodyContent.get())
let blockBody = rlp.read(BlockBody)
return some(buildBlockObject(blockHeader, blockBody))
else:
return none(BlockObject)
rpcServerWithProxy.rpc("eth_getBlockTransactionCountByHash") do(
data: EthHashStr) -> HexQuantityStr:
## Returns the number of transactions in a block from a block matching the
## given block hash.
##
## data: hash of a block
## Returns integer of the number of transactions in this block.
let
blockHash = data.toHash()
contentKeyType = ContentKeyType(chainId: 1'u16, blockHash: blockHash)
contentKeyBody =
ContentKey(contentType: blockBody, blockBodyKey: contentKeyType)
let bodyContent = await historyNetwork.getContent(contentKeyBody)
if bodyContent.isSome():
var rlp = rlpFromBytes(bodyContent.get())
let blockBody = rlp.read(BlockBody)
var txCount:uint = 0
for tx in blockBody.transactions:
txCount.inc()
return encodeQuantity(txCount)
else:
raise newException(ValueError, "Could not find block with requested hash")
# Note: can't implement this yet as the fluffy node doesn't know the relation
# of tx hash -> block number -> block hash, in order to get the receipt
# from from the block with that block hash. The Canonical Indices Network
# would need to be implemented to get this information.
# rpcServerWithProxy.rpc("eth_getTransactionReceipt") do(
# data: EthHashStr) -> Option[ReceiptObject]:

View File

@ -1,22 +0,0 @@
{
"0x88e96d4537bea4d9c05d12549907b32561d3bf31f45aae734cdc119f13406cb6": {
"rlp": "0xf90216f90211a0d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d493479405a56e2d52c817161883f50c441c3228cfe54d9fa0d67e4d450343046425ae4271474353857ab860dbc0a1dde64b41b5cd3a532bf3a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008503ff80000001821388808455ba422499476574682f76312e302e302f6c696e75782f676f312e342e32a0969b900de27b6ac6a67742365dd65f55a0526c41fd18e1b16f1a1215c2e66f5988539bd4979fef1ec4c0c0",
"number": 1
},
"0xb495a1d7e6663152ae92708da4843337b958146015a2802f4193a410044698c9": {
"rlp": "0xf9021df90218a088e96d4537bea4d9c05d12549907b32561d3bf31f45aae734cdc119f13406cb6a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d4934794dd2f1e6e498202e86d8f5442af596580a4f03c2ca04943d941637411107494da9ec8bc04359d731bfd08b72b4d0edcbd4cd2ecb341a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008503ff00100002821388808455ba4241a0476574682f76312e302e302d30636463373634372f6c696e75782f676f312e34a02f0790c5aa31ab94195e1f6443d645af5b75c46c04fbf9911711198a0ce8fdda88b853fa261a86aa9ec0c0",
"number": 2
},
"0x3d6122660cc824376f11ee842f83addc3525e2dd6756b9bcf0affa6aa88cf741": {
"rlp": "0xf90434f90218a0b495a1d7e6663152ae92708da4843337b958146015a2802f4193a410044698c9a06b17b938c6e4ef18b26ad81b9ca3515f27fd9c4e82aac56a1fd8eab288785e41945088d623ba0fcf0131e0897a91734a4d83596aa0a076ab0b899e8387436ff2658e2988f83cbf1af1590b9fe9feca3714f8d1824940a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008503fe802ffe03821388808455ba4260a0476574682f76312e302e302d66633739643332642f6c696e75782f676f312e34a065e12eec23fe6555e6bcdb47aa25269ae106e5f16b54e1e92dcee25e1c8ad037882e9344e0cbde83cec0f90215f90212a0d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d4934794c8ebccc5f5689fa8659d83713341e5ad19349448a01e6e030581fd1873b4784280859cd3b3c04aa85520f08c304cf5ee63d3935adda056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008503ff80000001821388808455ba42429a59617465732052616e64616c6c202d2045746865724e696e6a61a0f8c94dfe61cf26dcdf8cffeda337cf6a903d65c449d7691a022837f6e2d994598868b769c5451a7aea",
"number": 3
},
"0x23adf5a3be0f5235b36941bcb29b62504278ec5b9cdfa277b992ba4a7a3cd3a2": {
"rlp": "0xf90434f90212a03d6122660cc824376f11ee842f83addc3525e2dd6756b9bcf0affa6aa88cf741a083a8da8965660cb6bdf0c37f1b111778e49753c4213bf7c3e280fccfde89f2b594c8ebccc5f5689fa8659d83713341e5ad19349448a0e6d9f6e95a05ee69719c718c6157d0759049ef3dffdba2d48f015d7c8b9933d8a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008503fe005ff904821388808455ba427d9a59617465732052616e64616c6c202d2045746865724e696e6a61a006ba40902198357cbeac24a86b2ef11e9fdff48d28a421a0055e26476e3ac59f88c2535b5efca9bee0c0f9021bf90218a0d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347945088d623ba0fcf0131e0897a91734a4d83596aa0a09a6597b26adc0e5915cfcca537ba493a647cad1c3c923d406cdec6ca49a0a06da056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008503ff80000001821388808455ba4237a0476574682f76312e302e302d66633739643332642f6c696e75782f676f312e34a0d045b852770160da169ec793ec0c6e6ff562e473b2bf3f8192dc59842e36f75488db821a775bf9dace",
"number": 4
},
"0xf37c632d361e0a93f08ba29b1a2c708d9caa3ee19d1ee8d2a02612bffe49f0a9": {
"rlp": "0xf90216f90211a023adf5a3be0f5235b36941bcb29b62504278ec5b9cdfa277b992ba4a7a3cd3a2a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d493479405a56e2d52c817161883f50c441c3228cfe54d9fa04470f3dc1cc8097394a4ae85302eac3368462b3c1cfa523ffca942c1dd478220a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008503fe80200405821388808455ba428399476574682f76312e302e302f6c696e75782f676f312e342e32a017b85b5ec310c4868249fa2f378c83b4f330e2d897e5373a8195946c71d1d19e88fba9d0cff9dc5cf3c0c0",
"number": 5
}
}

File diff suppressed because one or more lines are too long

View File

@ -9,7 +9,7 @@ import
std/sequtils,
unittest2, testutils, confutils, chronos,
eth/p2p/discoveryv5/random2, eth/keys,
../../nimbus/rpc/hexstrings,
../../nimbus/rpc/[hexstrings, rpc_types],
../rpc/portal_rpc_client,
../rpc/eth_rpc_client,
../populate_db
@ -182,7 +182,7 @@ procSuite "Portal testnet tests":
await client.close()
nodeInfos.add(nodeInfo)
const dataFile = "./fluffy/scripts/test_data/mainnet_blocks_1-5.json"
const dataFile = "./fluffy/scripts/test_data/mainnet_blocks_selected.json"
# This will fill the first node its db with blocks from the data file. Next,
# this node wil offer all these blocks their headers one by one.
check (await clients[0].portal_history_propagate(dataFile))
@ -198,6 +198,16 @@ procSuite "Portal testnet tests":
let content = await client.eth_getBlockByHash(
hash.ethHashStr(), false)
check content.isSome()
check content.get().hash.get() == hash
let blockObj = content.get()
check blockObj.hash.get() == hash
for tx in blockObj.transactions:
var txObj: TransactionObject
tx.fromJson("tx", txObj)
check txObj.blockHash.get() == hash
# TODO: Check ommersHash, need the headers and not just the hashes
# for uncle in blockObj.uncles:
# discard
await client.close()