Add getting block by number through accumulator (#1182)

fluffy/network/history/accumulator.nim

@@ -37,6 +37,26 @@ type
     historicalEpochs*: List[Bytes32, maxHistoricalEpochs]
     currentEpoch*: EpochAccumulator
 
+  BlockHashResultType* = enum
+    BHash, HEpoch, UnknownBlockNumber
+
+  BlockHashResult* = object
+    case kind*: BlockHashResultType
+    of BHash:
+      blockHash*: BlockHash
+    of HEpoch:
+      epochHash*: Bytes32
+      epochIndex*: uint64
+      blockRelativeIndex*: uint64
+    of UnknownBlockNumber:
+      discard
+
+proc newEmptyAccumulator*(): Accumulator =
+  return Accumulator(
+    historicalEpochs:  List[Bytes32, maxHistoricalEpochs].init(@[]),
+    currentEpoch: List[HeaderRecord, epochSize].init(@[])
+  )
+
 func updateAccumulator*(a: var Accumulator, header: BlockHeader) =
   let lastTotalDifficulty =
     if a.currentEpoch.len() == 0:
@@ -91,18 +111,28 @@ func buildAccumulatorData*(headers: seq[BlockHeader]):
 ## Calls and helper calls for building header proofs and verifying headers
 ## against the Accumulator and the header proofs.
 
+func inCurrentEpoch*(bn: uint64, a: Accumulator): bool =
+  bn > uint64(a.historicalEpochs.len() * epochSize) - 1
+
 func inCurrentEpoch*(header: BlockHeader, a: Accumulator): bool =
   let blockNumber = header.blockNumber.truncate(uint64)
+  return inCurrentEpoch(blockNumber, a)
 
-  blockNumber > uint64(a.historicalEpochs.len() * epochSize) - 1
+func getEpochIndex*(bn: uint64): uint64 =
+  bn div epochSize
 
 func getEpochIndex*(header: BlockHeader): uint64 =
+  let blockNumber = header.blockNumber.truncate(uint64)
   ## Get the index for the historical epochs
-  header.blockNumber.truncate(uint64) div epochSize
+  return getEpochIndex(blockNumber)
+
+func getHeaderRecordIndex(bn: uint64, epochIndex: uint64): uint64 =
+  ## Get the relative header index for the epoch accumulator
+  uint64(bn - epochIndex * epochSize)
 
 func getHeaderRecordIndex*(header: BlockHeader, epochIndex: uint64): uint64 =
   ## Get the relative header index for the epoch accumulator
-  uint64(header.blockNumber.truncate(uint64) - epochIndex * epochSize)
+  return getHeaderRecordIndex(header.blockNumber.truncate(uint64), epochIndex)
 
 func verifyProof*(
     a: Accumulator, proof: openArray[Digest], header: BlockHeader): bool =
@@ -140,3 +170,21 @@ proc verifyHeader*(
         err("Proof verification failed")
     else:
       err("Need proof to verify header")
+
+func getHeaderHashForBlockNumber*(a: Accumulator, bn: UInt256): BlockHashResult=
+  let blockNumber = bn.truncate(uint64)
+  if inCurrentEpoch(blockNumber, a):
+    let relIndex = blockNumber - uint64(a.historicalEpochs.len()) * epochSize
+
+    if relIndex > uint64(a.currentEpoch.len() - 1):
+      return BlockHashResult(kind: UnknownBlockNumber)
+
+    return BlockHashResult(kind: BHash, blockHash: a.currentEpoch[relIndex].blockHash)
+  else:
+    let epochIndex = getEpochIndex(blockNumber)
+    return BlockHashResult(
+      kind: HEpoch,
+      epochHash: a.historicalEpochs[epochIndex],
+      epochIndex: epochIndex,
+      blockRelativeIndex: getHeaderRecordIndex(blockNumber, epochIndex)
+    )

fluffy/network/history/history_network.nim

@@ -505,6 +505,44 @@ proc getEpochAccumulator(
 
   return none(EpochAccumulator)
 
+proc getBlock*(
+    n: HistoryNetwork, chainId: uint16, bn: Uint256):
+    Future[Result[Option[Block], string]] {.async.} =
+
+  # TODO for now checking accumulator only in db, we could also ask our
+  # peers for it.
+  let accumulatorOpt = n.contentDB.getAccumulator()
+
+  if accumulatorOpt.isNone():
+    return err("Master accumulator not found in database")
+
+  let accumulator = accumulatorOpt.unsafeGet()
+
+  let hashResponse = accumulator.getHeaderHashForBlockNumber(bn)
+
+  case hashResponse.kind
+  of BHash:
+    # we got header hash in current epoch accumulator, try to retrieve it from network
+    let blockResponse = await n.getBlock(chainId, hashResponse.blockHash)
+    return ok(blockResponse)
+  of HEpoch:
+    let digest = Digest(data: hashResponse.epochHash)
+
+    let epochOpt = await n.getEpochAccumulator(digest)
+
+    if epochOpt.isNone():
+      return err("Cannot retrieve epoch accumulator for given block number")
+
+    let
+      epoch = epochOpt.unsafeGet()
+      blockHash = epoch[hashResponse.blockRelativeIndex].blockHash
+
+    let maybeBlock = await n.getBlock(chainId, blockHash)
+
+    return ok(maybeBlock)
+  of UnknownBlockNumber:
+    return err("Block number not included in master accumulator")
+
 proc getInitialMasterAccumulator*(
     n: HistoryNetwork):
     Future[bool] {.async.} =

fluffy/rpc/rpc_calls/rpc_eth_calls.nim

@@ -1,2 +1,3 @@
 proc eth_getBlockByHash(data: EthHashStr, fullTransactions: bool): Option[BlockObject]
+proc eth_getBlockByNumber(quantityTag: string, fullTransactions: bool): Option[BlockObject]
 proc eth_getLogs(filterOptions: FilterOptions): seq[FilterLog]

fluffy/rpc/rpc_eth_api.nim

@@ -126,7 +126,7 @@ proc installEthApiHandlers*(
 
   # rpcServerWithProxy.registerProxyMethod("eth_getBlockByHash")
 
-  rpcServerWithProxy.registerProxyMethod("eth_getBlockByNumber")
+  # rpcServerWithProxy.registerProxyMethod("eth_getBlockByNumber")
 
   # rpcServerWithProxy.registerProxyMethod("eth_getBlockTransactionCountByHash")
 
@@ -207,6 +207,29 @@ proc installEthApiHandlers*(
       let (header, body) = blockRes.unsafeGet()
       return some(BlockObject.init(header, body))
 
+  # TODO add test to local testnet, it requires addin proper handling accumulators
+  # in testnet
+  rpcServerWithProxy.rpc("eth_getBlockByNumber") do(quantityTag: string, fullTransactions: bool) -> Option[BlockObject]:
+    # TODO for now support only numeric queries, as it is not obvious how to retrieve
+    # pending or even latest block.
+    if not isValidHexQuantity(quantityTag):
+      raise newException(ValueError, "Provided tag should be valid hex number")
+
+    let blockNum = fromHex(UInt256, quantityTag)
+
+    let blockResult = await historyNetwork.getBlock(1'u16, blockNum)
+
+    if blockResult.isOk():
+      let maybeBlock = blockResult.get()
+
+      if maybeBlock.isNone():
+        return none(BlockObject)
+      else:
+        let (header, body) = maybeBlock.unsafeGet()
+        return some(BlockObject.init(header, body))
+    else:
+       raise newException(ValueError, blockResult.error)
+
   rpcServerWithProxy.rpc("eth_getBlockTransactionCountByHash") do(
       data: EthHashStr) -> HexQuantityStr:
     ## Returns the number of transactions in a block from a block matching the

fluffy/tests/all_fluffy_tests.nim

@@ -15,6 +15,7 @@ import
   ./test_state_network,
   ./test_history_content,
   ./test_history_validation,
+  ./test_history_network,
   ./test_header_content,
   ./test_accumulator,
   ./test_content_db,

fluffy/tests/test_accumulator.nim

@@ -106,3 +106,57 @@ suite "Header Accumulator":
           blockNumber: i.stuint(256), difficulty: 2.stuint(256))
 
         check db.verifyHeader(header, none(seq[Digest])).isErr()
+
+  test "Header Accumulator header hash for blocknumber":
+    var acc = newEmptyAccumulator()
+
+    let numEpochs = 2
+    let numHeadersInCurrentEpoch = 5
+    let numHeaders = numEpochs * epochSize + numHeadersInCurrentEpoch
+
+    var headerHashes: seq[Hash256] = @[]
+
+    for i in 0..numHeaders:
+      var bh = BlockHeader()
+      bh.blockNumber = u256(i)
+      bh.difficulty = u256(1)
+      headerHashes.add(bh.blockHash())
+      acc.updateAccumulator(bh)
+
+    # get valid response for epoch 0
+    block:
+      for i in 0..epochSize-1:
+        let res = acc.getHeaderHashForBlockNumber(u256(i))
+        check:
+          res.kind == HEpoch
+          res.epochIndex == 0
+
+    # get valid response for epoch 1
+    block:
+      for i in epochSize..(2 * epochSize)-1:
+        let res = acc.getHeaderHashForBlockNumber(u256(i))
+        check:
+          res.kind == HEpoch
+          res.epochIndex == 1
+
+    # get valid response from current epoch (epoch 3)
+    block:
+      for i in (2 * epochSize)..(2 * epochSize) + numHeadersInCurrentEpoch:
+        let res = acc.getHeaderHashForBlockNumber(u256(i))
+        check:
+          res.kind == BHash
+          res.blockHash == headerHashes[i]
+
+    # get valid response when getting unknown hash
+    block:
+      let firstUknownBlockNumber = (2 * epochSize) + numHeadersInCurrentEpoch + 1
+      let res = acc.getHeaderHashForBlockNumber(u256(firstUknownBlockNumber))
+
+      check:
+        res.kind == UnknownBlockNumber
+
+      let res1 = acc.getHeaderHashForBlockNumber(u256(3 * epochSize))
+      check:
+        res1.kind == UnknownBlockNumber
+
+

fluffy/tests/test_history_network.nim

@@ -0,0 +1,127 @@
+# Nimbus - Portal Network
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+import
+  std/os,
+  testutils/unittests, chronos,
+  eth/p2p/discoveryv5/protocol as discv5_protocol, eth/p2p/discoveryv5/routing_table,
+  eth/common/eth_types,
+  eth/rlp,
+  ../network/wire/[portal_protocol, portal_stream, portal_protocol_config],
+  ../network/history/[history_network, accumulator, history_content],
+  ../../nimbus/constants,
+  ../content_db,
+  ./test_helpers
+
+
+type HistoryNode = ref object
+  discoveryProtocol*: discv5_protocol.Protocol
+  historyNetwork*: HistoryNetwork
+
+proc newHistoryNode(rng: ref HmacDrbgContext, port: int): HistoryNode =
+  let
+    node = initDiscoveryNode(rng, PrivateKey.random(rng[]), localAddress(port))
+    db = ContentDB.new("", uint32.high, inMemory = true)
+    socketConfig = SocketConfig.init(
+      incomingSocketReceiveTimeout = none(Duration),
+      payloadSize = uint32(maxUtpPayloadSize)
+    )
+    hn = HistoryNetwork.new(node, db)
+    streamTransport = UtpDiscv5Protocol.new(
+      node,
+      utpProtocolId,
+      registerIncomingSocketCallback(@[hn.portalProtocol.stream]),
+      allowRegisteredIdCallback(@[hn.portalProtocol.stream]),
+      socketConfig
+    )
+
+  hn.setStreamTransport(streamTransport)
+
+  return HistoryNode(discoveryProtocol: node, historyNetwork: hn)
+
+proc portalWireProtocol(hn: HistoryNode): PortalProtocol =
+  hn.historyNetwork.portalProtocol
+
+proc localNodeInfo(hn: HistoryNode): Node =
+  hn.discoveryProtocol.localNode
+
+proc start(hn: HistoryNode) =
+  hn.historyNetwork.start()
+
+proc stop(hn: HistoryNode) {.async.} =
+  hn.historyNetwork.stop()
+  await hn.discoveryProtocol.closeWait()
+
+procSuite "History Content Network":
+  let rng = newRng()
+  asyncTest "Get block by block number":
+    let
+      historyNode1 = newHistoryNode(rng, 20302)
+      historyNode2 = newHistoryNode(rng, 20303)
+
+    historyNode1.start()
+    historyNode2.start()
+
+    # enough headers so there will be at least two epochs
+    let numHeaders = 9000
+    var headers: seq[BlockHeader]
+    for i in 0..numHeaders:
+      var bh = BlockHeader()
+      bh.blockNumber = u256(i)
+      bh.difficulty = u256(i)
+      # empty so that we won't care about creating fake block bodies
+      bh.ommersHash = EMPTY_UNCLE_HASH
+      bh.txRoot = BLANK_ROOT_HASH
+      headers.add(bh)
+
+    let masterAccumulator = buildAccumulator(headers)
+    let epochAccumulators = buildAccumulatorData(headers)
+
+    # both nodes start with the same master accumulator, but only node2 have all
+    # headers and all epoch accumulators
+    await historyNode1.historyNetwork.initMasterAccumulator(some(masterAccumulator))
+    await historyNode2.historyNetwork.initMasterAccumulator(some(masterAccumulator))
+
+    for h in headers:
+      let headerHash = h.blockHash()
+      let bk = BlockKey(chainId: 1'u16, blockHash: headerHash)
+      let ck = ContentKey(contentType: blockHeader, blockHeaderKey: bk)
+      let ci = toContentId(ck)
+      let headerEncoded = rlp.encode(h)
+      historyNode2.portalWireProtocol().storeContent(ci, headerEncoded)
+
+    for ad in epochAccumulators:
+      let (ck, epochAccumulator) = ad
+      let id = toContentId(ck)
+      let bytes = SSZ.encode(epochAccumulator)
+      historyNode2.portalWireProtocol().storeContent(id, bytes)
+
+    check historyNode1.portalWireProtocol().addNode(historyNode2.localNodeInfo()) == Added
+    check historyNode2.portalWireProtocol().addNode(historyNode1.localNodeInfo()) == Added
+
+    check (await historyNode1.portalWireProtocol().ping(historyNode2.localNodeInfo())).isOk()
+    check (await historyNode2.portalWireProtocol().ping(historyNode1.localNodeInfo())).isOk()
+
+
+    for i in 0..numHeaders:
+      let blockResponse = await historyNode1.historyNetwork.getBlock(1'u16, u256(i))
+
+      check:
+        blockResponse.isOk()
+
+      let blockOpt = blockResponse.get()
+
+      check:
+        blockOpt.isSome()
+
+      let (blockHeader, blockBody) = blockOpt.unsafeGet()
+
+      check:
+        blockHeader == headers[i]
+
+    await historyNode1.stop()
+    await historyNode2.stop()