Add block bodies to the propagation and lookups (#975)

* Add block bodies to the propagation and lookups

- Read and propagate block bodies next to the headers
- Add block bodies content (via lookups) to the eth_getBlockByHash
call
- Test the above in test_portal_testnet

* Fix storage/propagation of block bodies

- Data format is an actual block: [header, txs, uncles], which
requires some adjustment to store the block body
- Added also `eth_getBlockTransactionCountByHash` json rpc call

fluffy/populate_db.nim

@@ -10,7 +10,7 @@
 import
   json_serialization, json_serialization/std/tables,
   stew/[byteutils, io2, results], nimcrypto/keccak, chronos, chronicles,
-  eth/rlp,
+  eth/[rlp, common/eth_types],
   ./content_db,
   ./network/wire/portal_protocol,
   ./network/history/history_content
@@ -45,50 +45,98 @@ proc readBlockData*(dataFile: string): Result[BlockDataTable, string] =
 
 iterator blockHashes*(blockData: BlockDataTable): BlockHash =
   for k,v in blockData:
-    try:
     var blockHash: BlockHash
+    try:
       blockHash.data = hexToByteArray[sizeof(BlockHash)](k)
-      yield blockHash
     except ValueError as e:
       error "Invalid hex for block hash", error = e.msg, number = v.number
-
-iterator blockHeaders*(
-    blockData: BlockDataTable, verify = false): (ContentKey, seq[byte]) =
-  for k,v in blockData:
-    try:
-      var rlp = rlpFromHex(v.rlp)
-
-      if rlp.enterList():
-        # List that contains 3 items: Block header, body and receipts.
-        # Only make block header available for now.
-        # When we want others, can use `rlp.skipElem()` and `rlp.rawData()`.
-
-        # Prepare content key
-        var blockHash: BlockHash
-        blockHash.data = hexToByteArray[sizeof(BlockHash)](k)
-
-        let contentKey = ContentKey(
-          contentType: blockHeader,
-          blockHeaderKey: ContentKeyType(chainId: 1'u16, blockHash: blockHash))
-
-        # If wanted we can verify the hash for the corresponding header
-        if verify:
-          if keccak256.digest(rlp.rawData()) != blockHash:
-            error "Data is not matching hash, skipping"
       continue
 
-        yield (contentKey, @(rlp.rawData()))
+    yield blockHash
-    except CatchableError as e:
+
-      error "Failed decoding block hash or data", error = e.msg,
+iterator blocks*(
-        number = v.number
+    blockData: BlockDataTable, verify = false): seq[(ContentKey, seq[byte])] =
+  for k,v in blockData:
+    var res: seq[(ContentKey, seq[byte])]
+
+    var rlp =
+      try:
+        rlpFromHex(v.rlp)
+      except ValueError as e:
+        error "Invalid hex for rlp data", error = e.msg, number = v.number
+        continue
+
+    # The data is currently formatted as an rlp encoded `EthBlock`, thus
+    # containing header, txs and uncles: [header, txs, uncles]. No receipts are
+    # available.
+    # TODO: Change to format to rlp data as it gets stored and send over the
+    # network over the network. I.e. [header, [txs, uncles], receipts]
+    if rlp.enterList():
+      var blockHash: BlockHash
+      try:
+        blockHash.data = hexToByteArray[sizeof(BlockHash)](k)
+      except ValueError as e:
+        error "Invalid hex for block hash", error = e.msg, number = v.number
+        continue
+
+      let contentKeyType =
+        ContentKeyType(chainId: 1'u16, blockHash: blockHash)
+
+      try:
+        # If wanted the hash for the corresponding header can be verified
+        if verify:
+          if keccak256.digest(rlp.rawData()) != blockHash:
+            error "Data is not matching hash, skipping", number = v.number
+            continue
+
+        block:
+          let contentKey = ContentKey(
+            contentType: blockHeader,
+            blockHeaderKey: contentKeyType)
+
+          res.add((contentKey, @(rlp.rawData())))
+          rlp.skipElem()
+
+        block:
+          let contentKey = ContentKey(
+            contentType: blockBody,
+            blockBodyKey: contentKeyType)
+
+          # Note: Temporary until the data format gets changed.
+          let blockBody = BlockBody(
+            transactions: rlp.read(seq[Transaction]),
+            uncles: rlp.read(seq[BlockHeader]))
+          let rlpdata = encode(blockBody)
+          echo rlpdata.toHex()
+          res.add((contentKey, rlpdata))
+          # res.add((contentKey, @(rlp.rawData())))
+          # rlp.skipElem()
+
+        # Note: No receipts yet in the data set
+        # block:
+          # let contentKey = ContentKey(
+          #   contentType: receipts,
+          #   receiptsKey: contentKeyType)
+
+          # res.add((contentKey, @(rlp.rawData())))
+          # rlp.skipElem()
+
+      except RlpError as e:
+        error "Invalid rlp data", number = v.number, error = e.msg
+        continue
+
+      yield res
+    else:
+      error "Item is not a valid rlp list", number = v.number
 
 proc populateHistoryDb*(
     db: ContentDB, dataFile: string, verify = false): Result[void, string] =
   let blockData = ? readBlockData(dataFile)
 
-  for k,v in blockHeaders(blockData, verify):
+  for b in blocks(blockData, verify):
+    for value in b:
       # Note: This is the slowest part due to the hashing that takes place.
-    db.put(history_content.toContentId(k), v)
+      db.put(history_content.toContentId(value[0]), value[1])
 
   ok()
 
@@ -98,13 +146,14 @@ proc propagateHistoryDb*(
   let blockData = readBlockData(dataFile)
 
   if blockData.isOk():
-    for k,v in blockHeaders(blockData.get(), verify):
+    for b in blocks(blockData.get(), verify):
+      for value in b:
         # Note: This is the slowest part due to the hashing that takes place.
-      p.contentDB.put(history_content.toContentId(k), v)
+        p.contentDB.put(history_content.toContentId(value[0]), value[1])
 
         # TODO: This call will get the content we just stored in the db, so it
         # might be an improvement to directly pass it.
-      await p.neighborhoodGossip(ContentKeysList(@[encode(k)]))
+        await p.neighborhoodGossip(ContentKeysList(@[encode(value[0])]))
     return ok()
   else:
     return err(blockData.error)

fluffy/rpc/rpc_eth_api.nim

@@ -8,13 +8,14 @@
 {.push raises: [Defect].}
 
 import
-  std/times,
+  std/[times, sequtils],
-  json_rpc/[rpcproxy, rpcserver],
+  json_rpc/[rpcproxy, rpcserver], nimcrypto/[hash, keccak],
   web3/conversions, # sigh, for FixedBytes marshalling
-  eth/common/eth_types,
+  eth/[common/eth_types, rlp],
   # TODO: Using the Nimbus json-rpc helpers, but they could use some rework as
   # they bring a whole lot of other stuff with them.
   ../../nimbus/rpc/[rpc_types, hexstrings, rpc_utils],
+  ../../nimbus/errors, # for ValidationError, should be exported instead
   ../network/history/[history_network, history_content]
 
 # Subset of Eth JSON-RPC API: https://eth.wiki/json-rpc/API
@@ -29,7 +30,10 @@ import
 
 # Note: Similar as `populateBlockObject` from rpc_utils, but more limited as
 # there is currently only access to the block header.
-proc buildBlockObject*(header: BlockHeader): BlockObject =
+proc buildBlockObject*(
+    header: BlockHeader, body: BlockBody,
+    fullTx = true, isUncle = false):
+    BlockObject {.raises: [Defect, ValidationError].} =
   let blockHash = header.blockHash
 
   result.number = some(encodeQuantity(header.blockNumber))
@@ -57,6 +61,19 @@ proc buildBlockObject*(header: BlockHeader): BlockObject =
   result.gasUsed   = encodeQuantity(header.gasUsed.uint64)
   result.timestamp = encodeQuantity(header.timeStamp.toUnix.uint64)
 
+  if not isUncle:
+    result.uncles = body.uncles.map(proc(h: BlockHeader): Hash256 = h.blockHash)
+
+    if fullTx:
+      var i = 0
+      for tx in body.transactions:
+        # ValidationError from tx.getSender in populateTransactionObject
+        result.transactions.add %(populateTransactionObject(tx, header, i))
+        inc i
+    else:
+      for tx in body.transactions:
+        result.transactions.add %(keccak256.digest(rlp.encode(tx)))
+
 proc installEthApiHandlers*(
     # Currently only HistoryNetwork needed, later we might want a master object
     # holding all the networks.
@@ -80,7 +97,7 @@ proc installEthApiHandlers*(
 
   rpcServerWithProxy.registerProxyMethod("eth_getBlockByNumber")
 
-  rpcServerWithProxy.registerProxyMethod("eth_getBlockTransactionCountByHash")
+  # rpcServerWithProxy.registerProxyMethod("eth_getBlockTransactionCountByHash")
 
   rpcServerWithProxy.registerProxyMethod("eth_getBlockTransactionCountByNumber")
 
@@ -150,17 +167,62 @@ proc installEthApiHandlers*(
     ## Note: transactions and uncles are currently not implemented.
     ##
     ## Returns BlockObject or nil when no block was found.
-    let blockHash = data.toHash()
+    let
+      blockHash = data.toHash()
+      contentKeyType = ContentKeyType(chainId: 1'u16, blockHash: blockHash)
 
-    let contentKey = ContentKey(
+      contentKeyHeader =
-      contentType: blockHeader,
+        ContentKey(contentType: blockHeader, blockHeaderKey: contentKeyType)
-      blockHeaderKey: ContentKeyType(chainId: 1'u16, blockHash: blockHash))
+      contentKeyBody =
+        ContentKey(contentType: blockBody, blockBodyKey: contentKeyType)
 
-    let content = await historyNetwork.getContent(contentKey)
+    let headerContent = await historyNetwork.getContent(contentKeyHeader)
-    if content.isSome():
+    if headerContent.isNone():
-      var rlp = rlpFromBytes(content.get())
+      return none(BlockObject)
+
+    var rlp = rlpFromBytes(headerContent.get())
     let blockHeader = rlp.read(BlockHeader)
 
-      return some(buildBlockObject(blockHeader))
+    let bodyContent = await historyNetwork.getContent(contentKeyBody)
+
+    if bodyContent.isSome():
+      var rlp = rlpFromBytes(bodyContent.get())
+      let blockBody = rlp.read(BlockBody)
+
+      return some(buildBlockObject(blockHeader, blockBody))
     else:
       return none(BlockObject)
+
+  rpcServerWithProxy.rpc("eth_getBlockTransactionCountByHash") do(
+      data: EthHashStr) -> HexQuantityStr:
+    ## Returns the number of transactions in a block from a block matching the
+    ## given block hash.
+    ##
+    ## data: hash of a block
+    ## Returns integer of the number of transactions in this block.
+    let
+      blockHash = data.toHash()
+      contentKeyType = ContentKeyType(chainId: 1'u16, blockHash: blockHash)
+      contentKeyBody =
+        ContentKey(contentType: blockBody, blockBodyKey: contentKeyType)
+
+    let bodyContent = await historyNetwork.getContent(contentKeyBody)
+
+    if bodyContent.isSome():
+      var rlp = rlpFromBytes(bodyContent.get())
+      let blockBody = rlp.read(BlockBody)
+
+      var txCount:uint = 0
+      for tx in blockBody.transactions:
+        txCount.inc()
+
+      return encodeQuantity(txCount)
+    else:
+      raise newException(ValueError, "Could not find block with requested hash")
+
+  # Note: can't implement this yet as the fluffy node doesn't know the relation
+  # of tx hash -> block number -> block hash, in order to get the receipt
+  # from from the block with that block hash. The Canonical Indices Network
+  # would need to be implemented to get this information.
+  # rpcServerWithProxy.rpc("eth_getTransactionReceipt") do(
+  #     data: EthHashStr) -> Option[ReceiptObject]:

fluffy/scripts/test_data/mainnet_blocks_1-5.json

@@ -1,22 +0,0 @@
-{
-  "0x88e96d4537bea4d9c05d12549907b32561d3bf31f45aae734cdc119f13406cb6": {
-    "rlp": "0xf90216f90211a0d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d493479405a56e2d52c817161883f50c441c3228cfe54d9fa0d67e4d450343046425ae4271474353857ab860dbc0a1dde64b41b5cd3a532bf3a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008503ff80000001821388808455ba422499476574682f76312e302e302f6c696e75782f676f312e342e32a0969b900de27b6ac6a67742365dd65f55a0526c41fd18e1b16f1a1215c2e66f5988539bd4979fef1ec4c0c0",
-    "number": 1
-  },
-  "0xb495a1d7e6663152ae92708da4843337b958146015a2802f4193a410044698c9": {
-    "rlp": "0xf9021df90218a088e96d4537bea4d9c05d12549907b32561d3bf31f45aae734cdc119f13406cb6a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d4934794dd2f1e6e498202e86d8f5442af596580a4f03c2ca04943d941637411107494da9ec8bc04359d731bfd08b72b4d0edcbd4cd2ecb341a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008503ff00100002821388808455ba4241a0476574682f76312e302e302d30636463373634372f6c696e75782f676f312e34a02f0790c5aa31ab94195e1f6443d645af5b75c46c04fbf9911711198a0ce8fdda88b853fa261a86aa9ec0c0",
-    "number": 2
-  },
-  "0x3d6122660cc824376f11ee842f83addc3525e2dd6756b9bcf0affa6aa88cf741": {
-    "rlp": "0xf90434f90218a0b495a1d7e6663152ae92708da4843337b958146015a2802f4193a410044698c9a06b17b938c6e4ef18b26ad81b9ca3515f27fd9c4e82aac56a1fd8eab288785e41945088d623ba0fcf0131e0897a91734a4d83596aa0a076ab0b899e8387436ff2658e2988f83cbf1af1590b9fe9feca3714f8d1824940a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008503fe802ffe03821388808455ba4260a0476574682f76312e302e302d66633739643332642f6c696e75782f676f312e34a065e12eec23fe6555e6bcdb47aa25269ae106e5f16b54e1e92dcee25e1c8ad037882e9344e0cbde83cec0f90215f90212a0d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d4934794c8ebccc5f5689fa8659d83713341e5ad19349448a01e6e030581fd1873b4784280859cd3b3c04aa85520f08c304cf5ee63d3935adda056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008503ff80000001821388808455ba42429a59617465732052616e64616c6c202d2045746865724e696e6a61a0f8c94dfe61cf26dcdf8cffeda337cf6a903d65c449d7691a022837f6e2d994598868b769c5451a7aea",
-    "number": 3
-  },
-  "0x23adf5a3be0f5235b36941bcb29b62504278ec5b9cdfa277b992ba4a7a3cd3a2": {
-    "rlp": "0xf90434f90212a03d6122660cc824376f11ee842f83addc3525e2dd6756b9bcf0affa6aa88cf741a083a8da8965660cb6bdf0c37f1b111778e49753c4213bf7c3e280fccfde89f2b594c8ebccc5f5689fa8659d83713341e5ad19349448a0e6d9f6e95a05ee69719c718c6157d0759049ef3dffdba2d48f015d7c8b9933d8a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008503fe005ff904821388808455ba427d9a59617465732052616e64616c6c202d2045746865724e696e6a61a006ba40902198357cbeac24a86b2ef11e9fdff48d28a421a0055e26476e3ac59f88c2535b5efca9bee0c0f9021bf90218a0d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347945088d623ba0fcf0131e0897a91734a4d83596aa0a09a6597b26adc0e5915cfcca537ba493a647cad1c3c923d406cdec6ca49a0a06da056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008503ff80000001821388808455ba4237a0476574682f76312e302e302d66633739643332642f6c696e75782f676f312e34a0d045b852770160da169ec793ec0c6e6ff562e473b2bf3f8192dc59842e36f75488db821a775bf9dace",
-    "number": 4
-  },
-  "0xf37c632d361e0a93f08ba29b1a2c708d9caa3ee19d1ee8d2a02612bffe49f0a9": {
-    "rlp": "0xf90216f90211a023adf5a3be0f5235b36941bcb29b62504278ec5b9cdfa277b992ba4a7a3cd3a2a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d493479405a56e2d52c817161883f50c441c3228cfe54d9fa04470f3dc1cc8097394a4ae85302eac3368462b3c1cfa523ffca942c1dd478220a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008503fe80200405821388808455ba428399476574682f76312e302e302f6c696e75782f676f312e342e32a017b85b5ec310c4868249fa2f378c83b4f330e2d897e5373a8195946c71d1d19e88fba9d0cff9dc5cf3c0c0",
-    "number": 5
-  }
-}

fluffy/scripts/test_portal_testnet.nim

@@ -9,7 +9,7 @@ import
   std/sequtils,
   unittest2, testutils, confutils, chronos,
   eth/p2p/discoveryv5/random2, eth/keys,
-  ../../nimbus/rpc/hexstrings,
+  ../../nimbus/rpc/[hexstrings, rpc_types],
   ../rpc/portal_rpc_client,
   ../rpc/eth_rpc_client,
   ../populate_db
@@ -182,7 +182,7 @@ procSuite "Portal testnet tests":
       await client.close()
       nodeInfos.add(nodeInfo)
 
-    const dataFile = "./fluffy/scripts/test_data/mainnet_blocks_1-5.json"
+    const dataFile = "./fluffy/scripts/test_data/mainnet_blocks_selected.json"
     # This will fill the first node its db with blocks from the data file. Next,
     # this node wil offer all these blocks their headers one by one.
     check (await clients[0].portal_history_propagate(dataFile))
@@ -198,6 +198,16 @@ procSuite "Portal testnet tests":
         let content = await client.eth_getBlockByHash(
           hash.ethHashStr(), false)
         check content.isSome()
-        check content.get().hash.get() == hash
+        let blockObj = content.get()
+        check blockObj.hash.get() == hash
+
+        for tx in blockObj.transactions:
+          var txObj: TransactionObject
+          tx.fromJson("tx", txObj)
+          check txObj.blockHash.get() == hash
+
+        # TODO: Check ommersHash, need the headers and not just the hashes
+        # for uncle in blockObj.uncles:
+        #   discard
 
       await client.close()