310 lines
11 KiB
Nim
310 lines
11 KiB
Nim
# Fluffy
|
|
# Copyright (c) 2022-2024 Status Research & Development GmbH
|
|
# Licensed and distributed under either of
|
|
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
|
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
|
# at your option. This file may not be copied, modified, or distributed except according to those terms.
|
|
|
|
import
|
|
testutils/unittests,
|
|
chronos,
|
|
eth/p2p/discoveryv5/protocol as discv5_protocol,
|
|
eth/p2p/discoveryv5/routing_table,
|
|
eth/common/eth_types_rlp,
|
|
eth/rlp,
|
|
../network/wire/[portal_protocol, portal_stream, portal_protocol_config],
|
|
../network/history/[history_network, accumulator, history_content],
|
|
../database/content_db,
|
|
./test_helpers
|
|
|
|
type HistoryNode = ref object
|
|
discoveryProtocol*: discv5_protocol.Protocol
|
|
historyNetwork*: HistoryNetwork
|
|
|
|
proc newHistoryNode(
|
|
rng: ref HmacDrbgContext, port: int, accumulator: FinishedAccumulator
|
|
): HistoryNode =
|
|
let
|
|
node = initDiscoveryNode(rng, PrivateKey.random(rng[]), localAddress(port))
|
|
db = ContentDB.new(
|
|
"", uint32.high, RadiusConfig(kind: Dynamic), node.localNode.id, inMemory = true
|
|
)
|
|
streamManager = StreamManager.new(node)
|
|
historyNetwork =
|
|
HistoryNetwork.new(PortalNetwork.none, node, db, streamManager, accumulator)
|
|
|
|
return HistoryNode(discoveryProtocol: node, historyNetwork: historyNetwork)
|
|
|
|
proc portalProtocol(hn: HistoryNode): PortalProtocol =
|
|
hn.historyNetwork.portalProtocol
|
|
|
|
proc localNode(hn: HistoryNode): Node =
|
|
hn.discoveryProtocol.localNode
|
|
|
|
proc start(hn: HistoryNode) =
|
|
hn.historyNetwork.start()
|
|
|
|
proc stop(hn: HistoryNode) {.async.} =
|
|
hn.historyNetwork.stop()
|
|
await hn.discoveryProtocol.closeWait()
|
|
|
|
proc containsId(hn: HistoryNode, contentId: ContentId): bool =
|
|
return hn.historyNetwork.contentDB.get(contentId).isSome()
|
|
|
|
proc createEmptyHeaders(fromNum: int, toNum: int): seq[BlockHeader] =
|
|
var headers: seq[BlockHeader]
|
|
for i in fromNum .. toNum:
|
|
var bh = BlockHeader()
|
|
bh.number = BlockNumber(i)
|
|
bh.difficulty = u256(i)
|
|
# empty so that we won't care about creating fake block bodies
|
|
bh.ommersHash = EMPTY_UNCLE_HASH
|
|
bh.txRoot = EMPTY_ROOT_HASH
|
|
headers.add(bh)
|
|
return headers
|
|
|
|
proc headersToContentKV(headersWithProof: seq[BlockHeaderWithProof]): seq[ContentKV] =
|
|
var contentKVs: seq[ContentKV]
|
|
for headerWithProof in headersWithProof:
|
|
let
|
|
# TODO: Decoding step could be avoided
|
|
header = rlp.decode(headerWithProof.header.asSeq(), BlockHeader)
|
|
headerHash = header.blockHash()
|
|
blockKey = BlockKey(blockHash: headerHash)
|
|
contentKey =
|
|
encode(ContentKey(contentType: blockHeader, blockHeaderKey: blockKey))
|
|
contentKV =
|
|
ContentKV(contentKey: contentKey, content: SSZ.encode(headerWithProof))
|
|
contentKVs.add(contentKV)
|
|
return contentKVs
|
|
|
|
procSuite "History Content Network":
|
|
let rng = newRng()
|
|
|
|
asyncTest "Get Block by Number":
|
|
const
|
|
lastBlockNumber = mergeBlockNumber - 1
|
|
|
|
headersToTest = [
|
|
0,
|
|
EPOCH_SIZE - 1,
|
|
EPOCH_SIZE,
|
|
EPOCH_SIZE * 2 - 1,
|
|
EPOCH_SIZE * 2,
|
|
EPOCH_SIZE * 3 - 1,
|
|
EPOCH_SIZE * 3,
|
|
EPOCH_SIZE * 3 + 1,
|
|
int(lastBlockNumber),
|
|
]
|
|
|
|
let headers = createEmptyHeaders(0, int(lastBlockNumber))
|
|
let accumulatorRes = buildAccumulatorData(headers)
|
|
check accumulatorRes.isOk()
|
|
|
|
let
|
|
(masterAccumulator, epochRecords) = accumulatorRes.get()
|
|
historyNode1 = newHistoryNode(rng, 20302, masterAccumulator)
|
|
historyNode2 = newHistoryNode(rng, 20303, masterAccumulator)
|
|
|
|
var selectedHeaders: seq[BlockHeader]
|
|
for i in headersToTest:
|
|
selectedHeaders.add(headers[i])
|
|
|
|
let headersWithProof = buildHeadersWithProof(selectedHeaders, epochRecords)
|
|
|
|
check headersWithProof.isOk()
|
|
|
|
# Only node 2 stores the headers and all epoch accumulators.
|
|
for headerWithProof in headersWithProof.get():
|
|
let
|
|
header = rlp.decode(headerWithProof.header.asSeq(), BlockHeader)
|
|
headerHash = header.blockHash()
|
|
blockKey = BlockKey(blockHash: headerHash)
|
|
contentKey = ContentKey(contentType: blockHeader, blockHeaderKey: blockKey)
|
|
encKey = encode(contentKey)
|
|
contentId = toContentId(contentKey)
|
|
historyNode2.portalProtocol().storeContent(
|
|
encKey, contentId, SSZ.encode(headerWithProof)
|
|
)
|
|
|
|
# Need to store the epoch accumulators to be able to do the block to hash
|
|
# mapping
|
|
for epochRecord in epochRecords:
|
|
let
|
|
rootHash = epochRecord.hash_tree_root()
|
|
contentKey = ContentKey(
|
|
contentType: ContentType.epochRecord,
|
|
epochRecordKey: EpochRecordKey(epochHash: rootHash),
|
|
)
|
|
encKey = encode(contentKey)
|
|
contentId = toContentId(contentKey)
|
|
historyNode2.portalProtocol().storeContent(
|
|
encKey, contentId, SSZ.encode(epochRecord)
|
|
)
|
|
|
|
check:
|
|
historyNode1.portalProtocol().addNode(historyNode2.localNode()) == Added
|
|
historyNode2.portalProtocol().addNode(historyNode1.localNode()) == Added
|
|
|
|
(await historyNode1.portalProtocol().ping(historyNode2.localNode())).isOk()
|
|
(await historyNode2.portalProtocol().ping(historyNode1.localNode())).isOk()
|
|
|
|
for i in headersToTest:
|
|
let blockResponse = await historyNode1.historyNetwork.getBlock(u256(i))
|
|
|
|
check blockResponse.isOk()
|
|
|
|
let blockOpt = blockResponse.get()
|
|
|
|
check blockOpt.isSome()
|
|
|
|
let (blockHeader, blockBody) = blockOpt.unsafeGet()
|
|
|
|
check blockHeader == headers[i]
|
|
|
|
await historyNode1.stop()
|
|
await historyNode2.stop()
|
|
|
|
asyncTest "Offer - Maximum Content Keys in 1 Message":
|
|
# Need to provide enough headers to have the accumulator "finished".
|
|
const lastBlockNumber = int(mergeBlockNumber - 1)
|
|
|
|
let headers = createEmptyHeaders(0, lastBlockNumber)
|
|
let accumulatorRes = buildAccumulatorData(headers)
|
|
check accumulatorRes.isOk()
|
|
|
|
let
|
|
(masterAccumulator, epochRecords) = accumulatorRes.get()
|
|
historyNode1 = newHistoryNode(rng, 20302, masterAccumulator)
|
|
historyNode2 = newHistoryNode(rng, 20303, masterAccumulator)
|
|
|
|
check:
|
|
historyNode1.portalProtocol().addNode(historyNode2.localNode()) == Added
|
|
historyNode2.portalProtocol().addNode(historyNode1.localNode()) == Added
|
|
|
|
(await historyNode1.portalProtocol().ping(historyNode2.localNode())).isOk()
|
|
(await historyNode2.portalProtocol().ping(historyNode1.localNode())).isOk()
|
|
|
|
# Need to run start to get the processContentLoop running
|
|
historyNode1.start()
|
|
historyNode2.start()
|
|
|
|
let maxOfferedHistoryContent =
|
|
getMaxOfferedContentKeys(uint32(len(PortalProtocolId)), maxContentKeySize)
|
|
|
|
let headersWithProof =
|
|
buildHeadersWithProof(headers[0 .. maxOfferedHistoryContent], epochRecords)
|
|
check headersWithProof.isOk()
|
|
|
|
# This is one header more than maxOfferedHistoryContent
|
|
let contentKVs = headersToContentKV(headersWithProof.get())
|
|
|
|
# node 1 will offer the content so it needs to have it in its database
|
|
for contentKV in contentKVs:
|
|
let id = toContentId(contentKV.contentKey)
|
|
historyNode1.portalProtocol.storeContent(
|
|
contentKV.contentKey, id, contentKV.content
|
|
)
|
|
|
|
# Offering 1 content item too much which should result in a discv5 packet
|
|
# that is too large and thus not get any response.
|
|
block:
|
|
let offerResult =
|
|
await historyNode1.portalProtocol.offer(historyNode2.localNode(), contentKVs)
|
|
|
|
# Fail due timeout, as remote side must drop the too large discv5 packet
|
|
check offerResult.isErr()
|
|
|
|
for contentKV in contentKVs:
|
|
let id = toContentId(contentKV.contentKey)
|
|
check historyNode2.containsId(id) == false
|
|
|
|
# One content key less should make offer be succesful and should result
|
|
# in the content being transferred and stored on the other node.
|
|
block:
|
|
let offerResult = await historyNode1.portalProtocol.offer(
|
|
historyNode2.localNode(), contentKVs[0 ..< maxOfferedHistoryContent]
|
|
)
|
|
|
|
check offerResult.isOk()
|
|
|
|
# Make sure the content got processed out of content queue
|
|
while not historyNode2.historyNetwork.contentQueue.empty():
|
|
await sleepAsync(1.milliseconds)
|
|
|
|
# Note: It seems something changed in chronos, causing different behavior.
|
|
# Seems that validateContent called through processContentLoop used to
|
|
# run immediatly in case of a "non async shortpath". This is no longer the
|
|
# case and causes the content not yet to be validated and thus stored at
|
|
# this step. Add an await here so that the store can happen.
|
|
await sleepAsync(100.milliseconds)
|
|
|
|
for i, contentKV in contentKVs:
|
|
let id = toContentId(contentKV.contentKey)
|
|
if i < len(contentKVs) - 1:
|
|
check historyNode2.containsId(id) == true
|
|
else:
|
|
check historyNode2.containsId(id) == false
|
|
|
|
await historyNode1.stop()
|
|
await historyNode2.stop()
|
|
|
|
asyncTest "Offer - Headers with No Historical Epochs - Stopped at Merge Block":
|
|
const
|
|
lastBlockNumber = int(mergeBlockNumber - 1)
|
|
headersToTest =
|
|
[0, 1, EPOCH_SIZE div 2, EPOCH_SIZE - 1, lastBlockNumber - 1, lastBlockNumber]
|
|
|
|
let headers = createEmptyHeaders(0, lastBlockNumber)
|
|
let accumulatorRes = buildAccumulatorData(headers)
|
|
check accumulatorRes.isOk()
|
|
|
|
let
|
|
(masterAccumulator, epochRecords) = accumulatorRes.get()
|
|
historyNode1 = newHistoryNode(rng, 20302, masterAccumulator)
|
|
historyNode2 = newHistoryNode(rng, 20303, masterAccumulator)
|
|
|
|
check:
|
|
historyNode1.portalProtocol().addNode(historyNode2.localNode()) == Added
|
|
historyNode2.portalProtocol().addNode(historyNode1.localNode()) == Added
|
|
|
|
(await historyNode1.portalProtocol().ping(historyNode2.localNode())).isOk()
|
|
(await historyNode2.portalProtocol().ping(historyNode1.localNode())).isOk()
|
|
|
|
# Need to run start to get the processContentLoop running
|
|
historyNode1.start()
|
|
historyNode2.start()
|
|
|
|
var selectedHeaders: seq[BlockHeader]
|
|
for i in headersToTest:
|
|
selectedHeaders.add(headers[i])
|
|
|
|
let headersWithProof = buildHeadersWithProof(selectedHeaders, epochRecords)
|
|
check headersWithProof.isOk()
|
|
|
|
let contentKVs = headersToContentKV(headersWithProof.get())
|
|
|
|
for contentKV in contentKVs:
|
|
let id = toContentId(contentKV.contentKey)
|
|
historyNode1.portalProtocol.storeContent(
|
|
contentKV.contentKey, id, contentKV.content
|
|
)
|
|
|
|
let offerResult =
|
|
await historyNode1.portalProtocol.offer(historyNode2.localNode(), @[contentKV])
|
|
|
|
check offerResult.isOk()
|
|
|
|
# Make sure the content got processed out of content queue
|
|
while not historyNode2.historyNetwork.contentQueue.empty():
|
|
await sleepAsync(1.milliseconds)
|
|
|
|
await sleepAsync(100.milliseconds)
|
|
|
|
for contentKV in contentKVs:
|
|
let id = toContentId(contentKV.contentKey)
|
|
check historyNode2.containsId(id) == true
|
|
|
|
await historyNode1.stop()
|
|
await historyNode2.stop()
|