Additional seed endpoints (#1164)

* Additional seed endpoints

fluffy/network/history/history_network.nim

@@ -314,6 +314,11 @@ proc getBlockHeader*(
 proc getBlockBody*(
     n: HistoryNetwork, chainId: uint16, hash: BlockHash, header: BlockHeader):
     Future[Option[BlockBody]] {.async.} =
+
+  # Got header with empty body, no need to make any db calls or network requests
+  if header.txRoot == BLANK_ROOT_HASH and header.ommersHash == EMPTY_UNCLE_HASH:
+    return some(BlockBody(transactions: @[], uncles: @[]))
+
   let
     (keyEncoded, contentId) = getEncodedKeyForContent(blockBody, chainId, hash)
     bodyFromDb = n.getContentFromDb(BlockBody, contentId)
@@ -325,8 +330,10 @@ proc getBlockBody*(
   for i in 0..<requestRetries:
     let bodyContentLookup =
       await n.portalProtocol.contentLookup(keyEncoded, contentId)
+
     if bodyContentLookup.isNone():
       warn "Failed fetching block body from the network", hash
+      return none(BlockBody)
 
     let bodyContent = bodyContentLookup.unsafeGet()
 
@@ -493,7 +500,6 @@ proc processContentLoop(n: HistoryNetwork) {.async.} =
 
       # content passed here can have less items then contentKeys, but not more.
       for i, contentItem in contentItems:
-        echo contentItem.len()
         let contentKey = contentKeys[i]
         if await n.validateContent(contentItem, contentKey):
           let contentIdOpt = n.portalProtocol.toContentId(contentKey)

fluffy/network/wire/portal_protocol.nim

@@ -16,7 +16,7 @@ import
   bearssl, ssz_serialization, metrics, faststreams,
   eth/rlp, eth/p2p/discoveryv5/[protocol, node, enr, routing_table, random2,
     nodes_verification, lru],
-  ../../content_db,
+  ".."/../[content_db, seed_db],
   "."/[portal_stream, portal_protocol_config],
   ./messages
 
@@ -1329,3 +1329,118 @@ proc resolve*(p: PortalProtocol, id: NodeId): Future[Option[Node]] {.async.} =
         return some(n)
 
   return node
+
+proc resolveWithRadius*(p: PortalProtocol, id: NodeId): Future[Option[(Node, UInt256)]] {.async.} =
+  ## Resolve a `Node` based on provided `NodeId`, also try to establish what
+  ## is known radius of found node.
+  ##
+  ## This will first look in the own routing table. If the node is known, it
+  ## will try to contact if for newer information. If node is not known or it
+  ## does not reply, a lookup is done to see if it can find a (newer) record of
+  ## the node on the network.
+  ##
+  ## If node is found, radius will be first checked in radius cache, it radius
+  ## is not known node will be pinged to establish what is its current radius
+  ##
+
+  let n = await p.resolve(id)
+
+  if n.isNone():
+    return none((Node, UInt256))
+
+  let node = n.unsafeGet()
+
+  let r = p.radiusCache.get(id)
+
+  if r.isSome():
+    return some((node, r.unsafeGet()))
+
+  let pongResult = await p.ping(node)
+
+  if pongResult.isOk():
+    let maybeRadius = p.radiusCache.get(id)
+
+    # After successful ping radius should already be in cache, but for the unlikely
+    # case that it is not, check it just to be sure.
+    # TODO: rafactor ping to return node radius.
+    if maybeRadius.isNone():
+      return none((Node, UInt256))
+
+    # If pong is successful, radius of the node should definitly be in local
+    # radius cache
+    return some((node, maybeRadius.unsafeGet()))
+  else:
+    return none((Node, UInt256))
+
+proc offerContentInNodeRange*(
+    p: PortalProtocol,
+    seedDbPath: string,
+    nodeId: NodeId,
+    max: uint32,
+    starting: uint32):  Future[PortalResult[void]] {.async.} =
+  ## Offers `max` closest elements starting from `starting` index to peer
+  ## with given `nodeId`.
+  ## Maximum value of `max` is 64 , as this is limit for single offer.
+  ## `starting` argument is needed as seed_db is read only, so if there is
+  ## more content in peer range than max, then to offer 64 closest elements
+  # it needs to be set to 0. To offer next 64 elements it need to be set to
+  # 64 etc.
+
+  let maybePathAndDbName = getDbBasePathAndName(seedDbPath)
+
+  if maybePathAndDbName.isNone():
+    return err("Provided path is not valid sqlite database path")
+
+  let (dbPath, dbName) = maybePathAndDbName.unsafeGet()
+
+  let maybeNodeAndRadius = await p.resolveWithRadius(nodeId)
+
+  if maybeNodeAndRadius.isNone():
+    return err("Could not find node with provided nodeId")
+
+  let
+    db = SeedDb.new(path = dbPath, name = dbName)
+    (node, radius) = maybeNodeAndRadius.unsafeGet()
+    content = db.getContentInRange(node.id, radius, int64(max), int64(starting))
+
+  # We got all we wanted from seed_db, it can be closed now.
+  db.close()
+
+  var ci: seq[ContentInfo]
+
+  for cont in content:
+    let k = ByteList.init(cont.contentKey)
+    let info = ContentInfo(contentKey: k, content: cont.content)
+    ci.add(info)
+
+  let offerResult = await p.offer(node, ci)
+
+  # waiting for offer result, by the end of this call remote node should
+  # have received offered content
+  return offerResult
+
+proc storeContentInNodeRange*(
+    p: PortalProtocol,
+    seedDbPath: string,
+    max: uint32,
+    starting: uint32): PortalResult[void] =
+  let maybePathAndDbName = getDbBasePathAndName(seedDbPath)
+
+  if maybePathAndDbName.isNone():
+    return err("Provided path is not valid sqlite database path")
+
+  let (dbPath, dbName) = maybePathAndDbName.unsafeGet()
+
+  let
+    localRadius = p.dataRadius
+    db = SeedDb.new(path = dbPath, name = dbName)
+    localId = p.localNode.id
+    contentInRange = db.getContentInRange(localId, localRadius, int64(max), int64(starting))
+
+  db.close()
+
+  for contentData in contentInRange:
+    let cid = UInt256.fromBytesBE(contentData.contentId)
+    p.storeContent(cid, contentData.content)
+
+  return ok()

fluffy/populate_db.nim

@@ -14,7 +14,7 @@ import
   # TODO: `NetworkId` should not be in these private types
   eth/p2p/private/p2p_types,
   ../nimbus/[chain_config, genesis],
-  ./content_db,
+  "."/[content_db, seed_db],
   ./network/wire/portal_protocol,
   ./network/history/history_content
 
@@ -118,6 +118,15 @@ iterator blocks*(
     else:
       error "Failed reading block from block data", error = res.error
 
+iterator blocksContent*(
+    blockData: BlockDataTable, verify = false): (ContentId, seq[byte], seq[byte]) =
+  for b in blocks(blockData, verify):
+    for value in b:
+      if len(value[1]) > 0:
+        let ckBytes = history_content.encode(value[0])
+        let contentId = history_content.toContentId(ckBytes)
+        yield (contentId, ckBytes.asSeq(), value[1])
+
 func readBlockHeader*(blockData: BlockData): Result[BlockHeader, string] =
   var rlp =
     try:

fluffy/rpc/rpc_calls/rpc_portal_debug_calls.nim

@@ -3,3 +3,13 @@ proc portal_history_store(contentKey: string, content: string): bool
 proc portal_history_storeContent(dataFile: string): bool
 proc portal_history_propagate(dataFile: string): bool
 proc portal_history_propagateBlock(dataFile: string, blockHash: string): bool
+proc portal_history_storeContentInNodeRange(
+  dbPath: string,
+  max: uint32,
+  starting: uint32): bool
+
+proc portal_history_offerContentInNodeRange(
+  dbPath: string,
+  nodeId: NodeId,
+  max: uint32,
+  starting: uint32): bool

fluffy/rpc/rpc_portal_debug_api.nim

@@ -10,7 +10,7 @@
 import
   json_rpc/[rpcproxy, rpcserver], stew/byteutils,
   ../network/wire/portal_protocol,
-  ../content_db
+  ".."/[content_db, seed_db]
 
 export rpcserver
 
@@ -54,3 +54,29 @@ proc installPortalDebugApiHandlers*(
       return true
     else:
       raise newException(ValueError, $res.error)
+
+  rpcServer.rpc("portal_" & network & "_storeContentInNodeRange") do(
+      dbPath: string,
+      max: uint32,
+      starting: uint32) -> bool:
+
+    let storeResult = p.storeContentInNodeRange(dbPath, max, starting)
+
+    if storeResult.isOk():
+      return true
+    else:
+      raise newException(ValueError, $storeResult.error)
+
+  rpcServer.rpc("portal_" & network & "_offerContentInNodeRange") do(
+      dbPath: string,
+      nodeId: NodeId,
+      max: uint32,
+      starting: uint32) -> bool:
+      # waiting for offer result, by the end of this call remote node should
+      # have received offered content
+      let offerResult = await p.offerContentInNodeRange(dbPath, nodeId, max, starting)
+
+      if offerResult.isOk():
+        return true
+      else:
+        raise newException(ValueError, $offerResult.error)

fluffy/scripts/test_portal_testnet.nim

@@ -6,15 +6,20 @@
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
 import
+  os,
   std/sequtils,
   unittest2, testutils, confutils, chronos,
   eth/p2p/discoveryv5/random2, eth/keys,
   ../../nimbus/rpc/[hexstrings, rpc_types],
   ../rpc/portal_rpc_client,
   ../rpc/eth_rpc_client,
-  ../populate_db
+  ".."/[populate_db, seed_db]
 
 type
+  FutureCallback[A] = proc (): Future[A] {.gcsafe, raises: [Defect].}
+
+  CheckCallback[A] = proc (a: A): bool {.gcsafe, raises: [Defect].}
+
   PortalTestnetConf* = object
     nodeCount* {.
       defaultValue: 17
@@ -42,6 +47,42 @@ proc connectToRpcServers(config: PortalTestnetConf):
 
   return clients
 
+proc withRetries[A](
+  f: FutureCallback[A],
+  check: CheckCallback[A],
+  numRetries: int,
+  initialWait: Duration): Future[A] {.async.} =
+  ## Retries given future callback until either:
+  ## it returns successfuly and given check is true
+  ## or
+  ## function reaches max specified retries
+
+  var tries = 0
+  var currentDuration = initialWait
+
+  while true:
+    try:
+      let res = await f()
+
+      if check(res):
+        return res
+    except CatchableError as exc:
+      inc tries
+      if tries > numRetries:
+        # if we reached max number of retries fail
+        raise exc
+
+    # wait before new retry
+    await sleepAsync(currentDuration)
+    currentDuration = currentDuration * 2
+
+# Sometimes we need to wait till data will be propagated over the network.
+# To avoid long sleeps, this combinator can be used to retry some calls until
+# success or until some condition hold (or both)
+proc retryUntilDataPropagated[A](f: FutureCallback[A], c: CheckCallback[A]): Future[A] =
+  # some reasonable limits, which will cause waits as: 1, 2, 4, 8, 16 seconds
+  return withRetries(f, c, 5, seconds(1))
+
 # Note:
 # When doing json-rpc requests following `RpcPostError` can occur:
 # "Failed to send POST Request with JSON-RPC." when a `HttpClientRequestRef`
@@ -188,12 +229,6 @@ procSuite "Portal testnet tests":
     check (await clients[0].portal_history_propagate(dataFile))
     await clients[0].close()
 
-    # Note: Sleeping to make a test work is never great. Here it is needed
-    # because the data needs to propagate over the nodes. What one could do is
-    # add a json-rpc debug proc that returns whether the offer queue is empty or
-    # not. And then poll every node until all nodes have an empty queue.
-    await sleepAsync(60.seconds)
-
     let blockData = readBlockDataTable(dataFile)
     check blockData.isOk()
 
@@ -201,8 +236,23 @@ procSuite "Portal testnet tests":
       # Note: Once there is the Canonical Indices Network, we don't need to
       # access this file anymore here for the block hashes.
       for hash in blockData.get().blockHashes():
-        let content = await client.eth_getBlockByHash(
+
-          hash.ethHashStr(), false)
+        # Note: More flexible approach instead of generic retries could be to
+        # add a json-rpc debug proc that returns whether the offer queue is empty or
+        # not. And then poll every node until all nodes have an empty queue.
+
+        let content = await retryUntilDataPropagated(
+          proc (): Future[Option[BlockObject]] {.async.} =
+            try:
+              let res = await client.eth_getBlockByHash(hash.ethHashStr(), false)
+              await client.close()
+              return res
+            except CatchableError as exc:
+              await client.close()
+              raise exc
+          ,
+          proc (mc: Option[BlockObject]): bool = return mc.isSome()
+        )
         check content.isSome()
         let blockObj = content.get()
         check blockObj.hash.get() == hash
@@ -216,7 +266,18 @@ procSuite "Portal testnet tests":
           blockHash: some(hash)
         )
 
-        let logs = await client.eth_getLogs(filterOptions)
+        let logs = await retryUntilDataPropagated(
+          proc (): Future[seq[FilterLog]] {.async.} =
+            try:
+              let res = await client.eth_getLogs(filterOptions)
+              await client.close()
+              return res
+            except CatchableError as exc:
+              await client.close()
+              raise exc
+          ,
+          proc (mc: seq[FilterLog]): bool = return true
+        )
 
         for l in logs:
           check:
@@ -227,3 +288,75 @@ procSuite "Portal testnet tests":
         #   discard
 
       await client.close()
+
+  asyncTest "Portal History - Propagate content from seed db":
+    let clients = await connectToRpcServers(config)
+
+    var nodeInfos: seq[NodeInfo]
+    for client in clients:
+      let nodeInfo = await client.portal_history_nodeInfo()
+      await client.close()
+      nodeInfos.add(nodeInfo)
+
+    const dataPath = "./fluffy/tests/blocks/mainnet_blocks_1000000_1000020.json"
+
+    # path for temporary db, separate dir is used as sqlite usually also creates
+    # wal files, and we do not want for those to linger in filesystem
+    const tempDbPath = "./fluffy/tests/blocks/tempDir/mainnet_blocks_1000000_1000020.sqlite3"
+
+    let (dbFile, dbName) = getDbBasePathAndName(tempDbPath).unsafeGet()
+
+    let blockData = readBlockDataTable(dataPath)
+    check blockData.isOk()
+    let bd = blockData.get()
+
+    createDir(dbFile)
+    let db = SeedDb.new(path = dbFile, name = dbName)
+
+    try:
+      let lastNodeIdx = len(nodeInfos) - 1
+
+      # populate temp database from json file
+      for t in blocksContent(bd, false):
+        db.put(t[0], t[1], t[2])
+
+      # store content in node0 database
+      check (await clients[0].portal_history_storeContentInNodeRange(tempDbPath, 100, 0))
+      await clients[0].close()
+
+      # offer content to node 1..63
+      for i in 1..lastNodeIdx:
+        let receipientId = nodeInfos[i].nodeId
+        check (await clients[0].portal_history_offerContentInNodeRange(tempDbPath, receipientId, 64, 0))
+        await clients[0].close()
+
+      for client in clients:
+        # Note: Once there is the Canonical Indices Network, we don't need to
+        # access this file anymore here for the block hashes.
+        for hash in bd.blockHashes():
+          let content = await retryUntilDataPropagated(
+            proc (): Future[Option[BlockObject]] {.async.} =
+              try:
+                let res = await client.eth_getBlockByHash(hash.ethHashStr(), false)
+                await client.close()
+                return res
+              except CatchableError as exc:
+                await client.close()
+                raise exc
+            ,
+            proc (mc: Option[BlockObject]): bool = return mc.isSome()
+          )
+          check content.isSome()
+
+          let blockObj = content.get()
+          check blockObj.hash.get() == hash
+
+          for tx in blockObj.transactions:
+            var txObj: TransactionObject
+            tx.fromJson("tx", txObj)
+            check txObj.blockHash.get() == hash
+
+        await client.close()
+    finally:
+      db.close()
+      removeDir(dbFile)

fluffy/seed_db.nim

@@ -8,7 +8,8 @@
 {.push raises: [Defect].}
 
 import
-  std/options,
+  std/[options, os],
+  strutils,
   eth/db/kvstore,
   eth/db/kvstore_sqlite3,
   stint
@@ -31,7 +32,7 @@ type
     store: SqStoreRef
     putStmt: SqliteStmt[(array[32, byte], seq[byte], seq[byte]), void]
     getStmt: SqliteStmt[array[32, byte], ContentData]
-    getInRangeStmt: SqliteStmt[(array[32, byte], array[32, byte], int64), ContentDataDist]
+    getInRangeStmt: SqliteStmt[(array[32, byte], array[32, byte], int64, int64), ContentDataDist]
 
 func xorDistance(
   a: openArray[byte],
@@ -54,6 +55,18 @@ template expectDb(x: auto): untyped =
   # full disk - this requires manual intervention, so we'll panic for now
   x.expect("working database (disk broken/full?)")
 
+proc getDbBasePathAndName*(path: string): Option[(string, string)] =
+  let (basePath, name) = splitPath(path)
+  if len(basePath) > 0 and len(name) > 0 and name.endsWith(".sqlite3"):
+    let nameAndExt = rsplit(name, ".", 1)
+
+    if len(nameAndExt) < 2 and len(nameAndExt[0]) == 0:
+      return none((string, string))
+
+    return some((basePath, nameAndExt[0]))
+  else:
+    return none((string, string))
+
 proc new*(T: type SeedDb, path: string, name: string, inMemory = false): SeedDb =
   let db =
     if inMemory:
@@ -94,9 +107,10 @@ proc new*(T: type SeedDb, path: string, name: string, inMemory = false): SeedDb
         SELECT contentid, contentkey, content, xorDistance(?, contentid) as distance
         FROM seed_data
         WHERE distance <= ?
-        LIMIT ?;
+        LIMIT ?
+        OFFSET ?;
       """,
-      (array[32, byte], array[32, byte], int64),
+      (array[32, byte], array[32, byte], int64, int64),
       ContentDataDist
     ).get()
 
@@ -125,13 +139,25 @@ proc getContentInRange*(
     db: SeedDb,
     nodeId: UInt256,
     nodeRadius: UInt256,
-    max: int64): seq[ContentDataDist] =
+    max: int64,
+    offset: int64): seq[ContentDataDist] =
+  ## Return `max` amount of content in `nodeId` range, starting from `offset` position
+  ## i.e using `offset` 0 will return `max` closest items, using `offset` `10` will
+  ## will retrun `max` closest items except first 10
 
   var res: seq[ContentDataDist] = @[]
   var cd: ContentDataDist
-  for e in db.getInRangeStmt.exec((nodeId.toByteArrayBE(), nodeRadius.toByteArrayBE(), max), cd):
+  for e in db.getInRangeStmt.exec((nodeId.toByteArrayBE(), nodeRadius.toByteArrayBE(), max, offset), cd):
     res.add(cd)
   return res
 
+proc getContentInRange*(
+    db: SeedDb,
+    nodeId: UInt256,
+    nodeRadius: UInt256,
+    max: int64): seq[ContentDataDist] =
+  ## Return `max` amount of content in `nodeId` range, starting from closest content
+  return db.getContentInRange(nodeId, nodeRadius, max, 0)
+
 proc close*(db: SeedDb) =
   db.store.close()