Additional seed endpoints (#1164)

* Additional seed endpoints
This commit is contained in:
KonradStaniec 2022-07-20 12:46:42 +02:00 committed by GitHub
parent 4721fc7a54
commit aa945f1ed9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 473 additions and 20 deletions

View File

@ -314,6 +314,11 @@ proc getBlockHeader*(
proc getBlockBody*(
n: HistoryNetwork, chainId: uint16, hash: BlockHash, header: BlockHeader):
Future[Option[BlockBody]] {.async.} =
# Got header with empty body, no need to make any db calls or network requests
if header.txRoot == BLANK_ROOT_HASH and header.ommersHash == EMPTY_UNCLE_HASH:
return some(BlockBody(transactions: @[], uncles: @[]))
let
(keyEncoded, contentId) = getEncodedKeyForContent(blockBody, chainId, hash)
bodyFromDb = n.getContentFromDb(BlockBody, contentId)
@ -325,8 +330,10 @@ proc getBlockBody*(
for i in 0..<requestRetries:
let bodyContentLookup =
await n.portalProtocol.contentLookup(keyEncoded, contentId)
if bodyContentLookup.isNone():
warn "Failed fetching block body from the network", hash
return none(BlockBody)
let bodyContent = bodyContentLookup.unsafeGet()
@ -493,7 +500,6 @@ proc processContentLoop(n: HistoryNetwork) {.async.} =
# content passed here can have less items then contentKeys, but not more.
for i, contentItem in contentItems:
echo contentItem.len()
let contentKey = contentKeys[i]
if await n.validateContent(contentItem, contentKey):
let contentIdOpt = n.portalProtocol.toContentId(contentKey)

View File

@ -16,7 +16,7 @@ import
bearssl, ssz_serialization, metrics, faststreams,
eth/rlp, eth/p2p/discoveryv5/[protocol, node, enr, routing_table, random2,
nodes_verification, lru],
../../content_db,
".."/../[content_db, seed_db],
"."/[portal_stream, portal_protocol_config],
./messages
@ -1329,3 +1329,118 @@ proc resolve*(p: PortalProtocol, id: NodeId): Future[Option[Node]] {.async.} =
return some(n)
return node
proc resolveWithRadius*(p: PortalProtocol, id: NodeId): Future[Option[(Node, UInt256)]] {.async.} =
## Resolve a `Node` based on provided `NodeId`, also try to establish what
## is known radius of found node.
##
## This will first look in the own routing table. If the node is known, it
## will try to contact if for newer information. If node is not known or it
## does not reply, a lookup is done to see if it can find a (newer) record of
## the node on the network.
##
## If node is found, radius will be first checked in radius cache, it radius
## is not known node will be pinged to establish what is its current radius
##
let n = await p.resolve(id)
if n.isNone():
return none((Node, UInt256))
let node = n.unsafeGet()
let r = p.radiusCache.get(id)
if r.isSome():
return some((node, r.unsafeGet()))
let pongResult = await p.ping(node)
if pongResult.isOk():
let maybeRadius = p.radiusCache.get(id)
# After successful ping radius should already be in cache, but for the unlikely
# case that it is not, check it just to be sure.
# TODO: rafactor ping to return node radius.
if maybeRadius.isNone():
return none((Node, UInt256))
# If pong is successful, radius of the node should definitly be in local
# radius cache
return some((node, maybeRadius.unsafeGet()))
else:
return none((Node, UInt256))
proc offerContentInNodeRange*(
p: PortalProtocol,
seedDbPath: string,
nodeId: NodeId,
max: uint32,
starting: uint32): Future[PortalResult[void]] {.async.} =
## Offers `max` closest elements starting from `starting` index to peer
## with given `nodeId`.
## Maximum value of `max` is 64 , as this is limit for single offer.
## `starting` argument is needed as seed_db is read only, so if there is
## more content in peer range than max, then to offer 64 closest elements
# it needs to be set to 0. To offer next 64 elements it need to be set to
# 64 etc.
let maybePathAndDbName = getDbBasePathAndName(seedDbPath)
if maybePathAndDbName.isNone():
return err("Provided path is not valid sqlite database path")
let (dbPath, dbName) = maybePathAndDbName.unsafeGet()
let maybeNodeAndRadius = await p.resolveWithRadius(nodeId)
if maybeNodeAndRadius.isNone():
return err("Could not find node with provided nodeId")
let
db = SeedDb.new(path = dbPath, name = dbName)
(node, radius) = maybeNodeAndRadius.unsafeGet()
content = db.getContentInRange(node.id, radius, int64(max), int64(starting))
# We got all we wanted from seed_db, it can be closed now.
db.close()
var ci: seq[ContentInfo]
for cont in content:
let k = ByteList.init(cont.contentKey)
let info = ContentInfo(contentKey: k, content: cont.content)
ci.add(info)
let offerResult = await p.offer(node, ci)
# waiting for offer result, by the end of this call remote node should
# have received offered content
return offerResult
proc storeContentInNodeRange*(
p: PortalProtocol,
seedDbPath: string,
max: uint32,
starting: uint32): PortalResult[void] =
let maybePathAndDbName = getDbBasePathAndName(seedDbPath)
if maybePathAndDbName.isNone():
return err("Provided path is not valid sqlite database path")
let (dbPath, dbName) = maybePathAndDbName.unsafeGet()
let
localRadius = p.dataRadius
db = SeedDb.new(path = dbPath, name = dbName)
localId = p.localNode.id
contentInRange = db.getContentInRange(localId, localRadius, int64(max), int64(starting))
db.close()
for contentData in contentInRange:
let cid = UInt256.fromBytesBE(contentData.contentId)
p.storeContent(cid, contentData.content)
return ok()

View File

@ -14,7 +14,7 @@ import
# TODO: `NetworkId` should not be in these private types
eth/p2p/private/p2p_types,
../nimbus/[chain_config, genesis],
./content_db,
"."/[content_db, seed_db],
./network/wire/portal_protocol,
./network/history/history_content
@ -118,6 +118,15 @@ iterator blocks*(
else:
error "Failed reading block from block data", error = res.error
iterator blocksContent*(
blockData: BlockDataTable, verify = false): (ContentId, seq[byte], seq[byte]) =
for b in blocks(blockData, verify):
for value in b:
if len(value[1]) > 0:
let ckBytes = history_content.encode(value[0])
let contentId = history_content.toContentId(ckBytes)
yield (contentId, ckBytes.asSeq(), value[1])
func readBlockHeader*(blockData: BlockData): Result[BlockHeader, string] =
var rlp =
try:

View File

@ -3,3 +3,13 @@ proc portal_history_store(contentKey: string, content: string): bool
proc portal_history_storeContent(dataFile: string): bool
proc portal_history_propagate(dataFile: string): bool
proc portal_history_propagateBlock(dataFile: string, blockHash: string): bool
proc portal_history_storeContentInNodeRange(
dbPath: string,
max: uint32,
starting: uint32): bool
proc portal_history_offerContentInNodeRange(
dbPath: string,
nodeId: NodeId,
max: uint32,
starting: uint32): bool

View File

@ -10,7 +10,7 @@
import
json_rpc/[rpcproxy, rpcserver], stew/byteutils,
../network/wire/portal_protocol,
../content_db
".."/[content_db, seed_db]
export rpcserver
@ -54,3 +54,29 @@ proc installPortalDebugApiHandlers*(
return true
else:
raise newException(ValueError, $res.error)
rpcServer.rpc("portal_" & network & "_storeContentInNodeRange") do(
dbPath: string,
max: uint32,
starting: uint32) -> bool:
let storeResult = p.storeContentInNodeRange(dbPath, max, starting)
if storeResult.isOk():
return true
else:
raise newException(ValueError, $storeResult.error)
rpcServer.rpc("portal_" & network & "_offerContentInNodeRange") do(
dbPath: string,
nodeId: NodeId,
max: uint32,
starting: uint32) -> bool:
# waiting for offer result, by the end of this call remote node should
# have received offered content
let offerResult = await p.offerContentInNodeRange(dbPath, nodeId, max, starting)
if offerResult.isOk():
return true
else:
raise newException(ValueError, $offerResult.error)

View File

@ -6,15 +6,20 @@
# at your option. This file may not be copied, modified, or distributed except according to those terms.
import
os,
std/sequtils,
unittest2, testutils, confutils, chronos,
eth/p2p/discoveryv5/random2, eth/keys,
../../nimbus/rpc/[hexstrings, rpc_types],
../rpc/portal_rpc_client,
../rpc/eth_rpc_client,
../populate_db
".."/[populate_db, seed_db]
type
FutureCallback[A] = proc (): Future[A] {.gcsafe, raises: [Defect].}
CheckCallback[A] = proc (a: A): bool {.gcsafe, raises: [Defect].}
PortalTestnetConf* = object
nodeCount* {.
defaultValue: 17
@ -42,6 +47,42 @@ proc connectToRpcServers(config: PortalTestnetConf):
return clients
proc withRetries[A](
f: FutureCallback[A],
check: CheckCallback[A],
numRetries: int,
initialWait: Duration): Future[A] {.async.} =
## Retries given future callback until either:
## it returns successfuly and given check is true
## or
## function reaches max specified retries
var tries = 0
var currentDuration = initialWait
while true:
try:
let res = await f()
if check(res):
return res
except CatchableError as exc:
inc tries
if tries > numRetries:
# if we reached max number of retries fail
raise exc
# wait before new retry
await sleepAsync(currentDuration)
currentDuration = currentDuration * 2
# Sometimes we need to wait till data will be propagated over the network.
# To avoid long sleeps, this combinator can be used to retry some calls until
# success or until some condition hold (or both)
proc retryUntilDataPropagated[A](f: FutureCallback[A], c: CheckCallback[A]): Future[A] =
# some reasonable limits, which will cause waits as: 1, 2, 4, 8, 16 seconds
return withRetries(f, c, 5, seconds(1))
# Note:
# When doing json-rpc requests following `RpcPostError` can occur:
# "Failed to send POST Request with JSON-RPC." when a `HttpClientRequestRef`
@ -188,12 +229,6 @@ procSuite "Portal testnet tests":
check (await clients[0].portal_history_propagate(dataFile))
await clients[0].close()
# Note: Sleeping to make a test work is never great. Here it is needed
# because the data needs to propagate over the nodes. What one could do is
# add a json-rpc debug proc that returns whether the offer queue is empty or
# not. And then poll every node until all nodes have an empty queue.
await sleepAsync(60.seconds)
let blockData = readBlockDataTable(dataFile)
check blockData.isOk()
@ -201,8 +236,23 @@ procSuite "Portal testnet tests":
# Note: Once there is the Canonical Indices Network, we don't need to
# access this file anymore here for the block hashes.
for hash in blockData.get().blockHashes():
let content = await client.eth_getBlockByHash(
hash.ethHashStr(), false)
# Note: More flexible approach instead of generic retries could be to
# add a json-rpc debug proc that returns whether the offer queue is empty or
# not. And then poll every node until all nodes have an empty queue.
let content = await retryUntilDataPropagated(
proc (): Future[Option[BlockObject]] {.async.} =
try:
let res = await client.eth_getBlockByHash(hash.ethHashStr(), false)
await client.close()
return res
except CatchableError as exc:
await client.close()
raise exc
,
proc (mc: Option[BlockObject]): bool = return mc.isSome()
)
check content.isSome()
let blockObj = content.get()
check blockObj.hash.get() == hash
@ -216,7 +266,18 @@ procSuite "Portal testnet tests":
blockHash: some(hash)
)
let logs = await client.eth_getLogs(filterOptions)
let logs = await retryUntilDataPropagated(
proc (): Future[seq[FilterLog]] {.async.} =
try:
let res = await client.eth_getLogs(filterOptions)
await client.close()
return res
except CatchableError as exc:
await client.close()
raise exc
,
proc (mc: seq[FilterLog]): bool = return true
)
for l in logs:
check:
@ -227,3 +288,75 @@ procSuite "Portal testnet tests":
# discard
await client.close()
asyncTest "Portal History - Propagate content from seed db":
let clients = await connectToRpcServers(config)
var nodeInfos: seq[NodeInfo]
for client in clients:
let nodeInfo = await client.portal_history_nodeInfo()
await client.close()
nodeInfos.add(nodeInfo)
const dataPath = "./fluffy/tests/blocks/mainnet_blocks_1000000_1000020.json"
# path for temporary db, separate dir is used as sqlite usually also creates
# wal files, and we do not want for those to linger in filesystem
const tempDbPath = "./fluffy/tests/blocks/tempDir/mainnet_blocks_1000000_1000020.sqlite3"
let (dbFile, dbName) = getDbBasePathAndName(tempDbPath).unsafeGet()
let blockData = readBlockDataTable(dataPath)
check blockData.isOk()
let bd = blockData.get()
createDir(dbFile)
let db = SeedDb.new(path = dbFile, name = dbName)
try:
let lastNodeIdx = len(nodeInfos) - 1
# populate temp database from json file
for t in blocksContent(bd, false):
db.put(t[0], t[1], t[2])
# store content in node0 database
check (await clients[0].portal_history_storeContentInNodeRange(tempDbPath, 100, 0))
await clients[0].close()
# offer content to node 1..63
for i in 1..lastNodeIdx:
let receipientId = nodeInfos[i].nodeId
check (await clients[0].portal_history_offerContentInNodeRange(tempDbPath, receipientId, 64, 0))
await clients[0].close()
for client in clients:
# Note: Once there is the Canonical Indices Network, we don't need to
# access this file anymore here for the block hashes.
for hash in bd.blockHashes():
let content = await retryUntilDataPropagated(
proc (): Future[Option[BlockObject]] {.async.} =
try:
let res = await client.eth_getBlockByHash(hash.ethHashStr(), false)
await client.close()
return res
except CatchableError as exc:
await client.close()
raise exc
,
proc (mc: Option[BlockObject]): bool = return mc.isSome()
)
check content.isSome()
let blockObj = content.get()
check blockObj.hash.get() == hash
for tx in blockObj.transactions:
var txObj: TransactionObject
tx.fromJson("tx", txObj)
check txObj.blockHash.get() == hash
await client.close()
finally:
db.close()
removeDir(dbFile)

View File

@ -8,7 +8,8 @@
{.push raises: [Defect].}
import
std/options,
std/[options, os],
strutils,
eth/db/kvstore,
eth/db/kvstore_sqlite3,
stint
@ -31,7 +32,7 @@ type
store: SqStoreRef
putStmt: SqliteStmt[(array[32, byte], seq[byte], seq[byte]), void]
getStmt: SqliteStmt[array[32, byte], ContentData]
getInRangeStmt: SqliteStmt[(array[32, byte], array[32, byte], int64), ContentDataDist]
getInRangeStmt: SqliteStmt[(array[32, byte], array[32, byte], int64, int64), ContentDataDist]
func xorDistance(
a: openArray[byte],
@ -54,6 +55,18 @@ template expectDb(x: auto): untyped =
# full disk - this requires manual intervention, so we'll panic for now
x.expect("working database (disk broken/full?)")
proc getDbBasePathAndName*(path: string): Option[(string, string)] =
let (basePath, name) = splitPath(path)
if len(basePath) > 0 and len(name) > 0 and name.endsWith(".sqlite3"):
let nameAndExt = rsplit(name, ".", 1)
if len(nameAndExt) < 2 and len(nameAndExt[0]) == 0:
return none((string, string))
return some((basePath, nameAndExt[0]))
else:
return none((string, string))
proc new*(T: type SeedDb, path: string, name: string, inMemory = false): SeedDb =
let db =
if inMemory:
@ -94,9 +107,10 @@ proc new*(T: type SeedDb, path: string, name: string, inMemory = false): SeedDb
SELECT contentid, contentkey, content, xorDistance(?, contentid) as distance
FROM seed_data
WHERE distance <= ?
LIMIT ?;
LIMIT ?
OFFSET ?;
""",
(array[32, byte], array[32, byte], int64),
(array[32, byte], array[32, byte], int64, int64),
ContentDataDist
).get()
@ -125,13 +139,25 @@ proc getContentInRange*(
db: SeedDb,
nodeId: UInt256,
nodeRadius: UInt256,
max: int64): seq[ContentDataDist] =
max: int64,
offset: int64): seq[ContentDataDist] =
## Return `max` amount of content in `nodeId` range, starting from `offset` position
## i.e using `offset` 0 will return `max` closest items, using `offset` `10` will
## will retrun `max` closest items except first 10
var res: seq[ContentDataDist] = @[]
var cd: ContentDataDist
for e in db.getInRangeStmt.exec((nodeId.toByteArrayBE(), nodeRadius.toByteArrayBE(), max), cd):
for e in db.getInRangeStmt.exec((nodeId.toByteArrayBE(), nodeRadius.toByteArrayBE(), max, offset), cd):
res.add(cd)
return res
proc getContentInRange*(
db: SeedDb,
nodeId: UInt256,
nodeRadius: UInt256,
max: int64): seq[ContentDataDist] =
## Return `max` amount of content in `nodeId` range, starting from closest content
return db.getContentInRange(nodeId, nodeRadius, max, 0)
proc close*(db: SeedDb) =
db.store.close()

File diff suppressed because one or more lines are too long