State network content retrieval (#2033)

* initial rework of contentLoop for state network

* prepare content for storage and retrieval

* modify test to account for retrieval data
This commit is contained in:
Daniel Sobol 2024-02-19 17:35:53 +03:00 committed by GitHub
parent 402a3eda73
commit 47b254d8b5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 171 additions and 87 deletions

View File

@ -98,6 +98,33 @@ type
ContractCodeRetrieval* = object
code*: Bytecode
OfferContentValueType* = enum
accountTrieNodeOffer,
contractTrieNodeOffer,
contractCodeOffer,
OfferContentValue* = object
case contentType*: ContentType
of unused:
discard
of accountTrieNode:
accountTrieNode*: AccountTrieNodeOffer
of contractTrieNode:
contractTrieNode*: ContractTrieNodeOffer
of contractCode:
contractCode*: ContractCodeOffer
RetrievalContentValue* = object
case contentType*: ContentType
of unused:
discard
of accountTrieNode:
accountTrieNode*: AccountTrieNodeRetrieval
of contractTrieNode:
contractTrieNode*: ContractTrieNodeRetrieval
of contractCode:
contractCode*: ContractCodeRetrieval
func encode*(contentKey: ContentKey): ByteList =
doAssert(contentKey.contentType != unused)
ByteList.init(SSZ.encode(contentKey))
@ -125,6 +152,60 @@ func toContentId*(contentKey: ByteList): ContentId =
func toContentId*(contentKey: ContentKey): ContentId =
toContentId(encode(contentKey))
func offerContentToRetrievalContent*(offerContent: OfferContentValue): RetrievalContentValue =
case offerContent.contentType:
of unused:
raiseAssert "Converting content with unused content type"
of accountTrieNode:
RetrievalContentValue(
contentType: accountTrieNode,
accountTrieNode: AccountTrieNodeRetrieval(node: offerContent.accountTrieNode.proof[^1])
) # TODO implement properly
of contractTrieNode:
RetrievalContentValue(
contentType: contractTrieNode,
contractTrieNode: ContractTrieNodeRetrieval(node: offerContent.contractTrieNode.storageProof[^1])
) # TODO implement properly
of contractCode:
RetrievalContentValue(
contentType: contractCode,
contractCode: ContractCodeRetrieval(code: offerContent.contractCode.code)
)
func encode*(content: RetrievalContentValue): seq[byte] =
case content.contentType:
of unused:
raiseAssert "Encoding content with unused content type"
of accountTrieNode:
SSZ.encode(content.accountTrieNode)
of contractTrieNode:
SSZ.encode(content.contractTrieNode)
of contractCode:
SSZ.encode(content.contractCode)
func decodeKV*(contentKey: ByteList, contentValue: seq[byte]): Opt[(ContentKey, OfferContentValue)] =
const empty = Opt.none((ContentKey, OfferContentValue))
let
key = contentKey.decode().valueOr:
return empty
value = case key.contentType:
of unused:
return empty
of accountTrieNode:
let val = decodeSsz(contentValue, AccountTrieNodeOffer).valueOr:
return empty
OfferContentValue(contentType: accountTrieNode, accountTrieNode: val)
of contractTrieNode:
let val = decodeSsz(contentValue, ContractTrieNodeOffer).valueOr:
return empty
OfferContentValue(contentType: contractTrieNode, contractTrieNode: val)
of contractCode:
let val = decodeSsz(contentValue, ContractCodeOffer).valueOr:
return empty
OfferContentValue(contentType: contractCode, contractCode: val)
Opt.some((key, value))
func packNibbles*(nibbles: seq[byte]): Nibbles =
doAssert(nibbles.len() <= MAX_UNPACKED_NIBBLES_LEN, "Can't pack more than 64 nibbles")

View File

@ -60,70 +60,39 @@ proc getContent*(n: StateNetwork, key: ContentKey):
# domain types.
return Opt.some(contentResult.content)
proc validateAccountTrieNode(key: ContentKey, contentValue: seq[byte]): bool =
let value = decodeSsz(contentValue, AccountTrieNodeOffer).valueOr:
warn "Received invalid account trie proof", error
return false
proc validateAccountTrieNode(key: ContentKey, contentValue: OfferContentValue): bool =
true
proc validateContractTrieNode(key: ContentKey, contentValue: seq[byte]): bool =
let value = decodeSsz(contentValue, ContractTrieNodeOffer).valueOr:
warn "Received invalid contract trie proof", error
return false
proc validateContractTrieNode(key: ContentKey, contentValue: OfferContentValue): bool =
true
proc validateContractCode(key: ContentKey, contentValue: seq[byte]): bool =
let value = decodeSsz(contentValue, ContractCodeOffer).valueOr:
warn "Received invalid contract code", error
return false
proc validateContractCode(key: ContentKey, contentValue: OfferContentValue): bool =
true
proc validateContent*(
n: StateNetwork,
contentKey: ByteList,
contentValue: seq[byte]): bool =
let key = contentKey.decode().valueOr:
return false
case key.contentType:
contentKey: ContentKey,
contentValue: OfferContentValue): bool =
case contentKey.contentType:
of unused:
warn "Received content with unused content type"
false
of accountTrieNode:
validateAccountTrieNode(key, contentValue)
validateAccountTrieNode(contentKey, contentValue)
of contractTrieNode:
validateContractTrieNode(key, contentValue)
validateContractTrieNode(contentKey, contentValue)
of contractCode:
validateContractCode(key, contentValue)
proc validateContent(
n: StateNetwork,
contentKeys: ContentKeysList,
contentValues: seq[seq[byte]]): bool =
for i, contentValue in contentValues:
let contentKey = contentKeys[i]
if n.validateContent(contentKey, contentValue):
let contentId = n.portalProtocol.toContentId(contentKey).valueOr:
error "Received offered content with invalid content key", contentKey
return false
n.portalProtocol.storeContent(contentKey, contentId, contentValue)
info "Received offered content validated successfully", contentKey
else:
error "Received offered content failed validation", contentKey
return false
validateContractCode(contentKey, contentValue)
proc recursiveGossipAccountTrieNode(
p: PortalProtocol,
maybeSrcNodeId: Opt[NodeId],
decodedKey: ContentKey,
contentValue: seq[byte]
decodedValue: AccountTrieNodeOffer
): Future[void] {.async.} =
var nibbles = decodedKey.accountTrieNodeKey.path.unpackNibbles()
let decodedValue = decodeSsz(contentValue, AccountTrieNodeOffer).valueOr:
raiseAssert "Received offered content failed validation"
var proof = decodedValue.proof
var
nibbles = decodedKey.accountTrieNodeKey.path.unpackNibbles()
proof = decodedValue.proof
discard nibbles.pop()
discard (distinctBase proof).pop()
let
@ -144,32 +113,28 @@ proc recursiveGossipContractTrieNode(
p: PortalProtocol,
maybeSrcNodeId: Opt[NodeId],
decodedKey: ContentKey,
contentValue: seq[byte]
decodedValue: ContractTrieNodeOffer
): Future[void] {.async.} =
return
proc gossipContent*(
p: PortalProtocol,
maybeSrcNodeId: Opt[NodeId],
contentKeys: ContentKeysList,
contentValues: seq[seq[byte]]
contentKey: ByteList,
decodedKey: ContentKey,
contentValue: seq[byte],
decodedValue: OfferContentValue
): Future[void] {.async.} =
for i, contentValue in contentValues:
let
contentKey = contentKeys[i]
decodedKey = contentKey.decode().valueOr:
raiseAssert "Received offered content with invalid content key"
case decodedKey.contentType:
of unused:
warn("Gossiping content with unused content type")
continue
raiseAssert "Gossiping content with unused content type"
of accountTrieNode:
await recursiveGossipAccountTrieNode(p, maybeSrcNodeId, decodedKey, contentValue)
await recursiveGossipAccountTrieNode(p, maybeSrcNodeId, decodedKey, decodedValue.accountTrieNode)
of contractTrieNode:
await recursiveGossipContractTrieNode(p, maybeSrcNodeId, decodedKey, contentValue)
await recursiveGossipContractTrieNode(p, maybeSrcNodeId, decodedKey, decodedValue.contractTrieNode)
of contractCode:
await neighborhoodGossipDiscardPeers(
p, maybeSrcNodeId, ContentKeysList.init(@[contentKey]), @[contentValue]
await p.neighborhoodGossipDiscardPeers(
maybeSrcNodeId, ContentKeysList.init(@[contentKey]), @[contentValue]
)
proc new*(
@ -201,13 +166,32 @@ proc processContentLoop(n: StateNetwork) {.async.} =
try:
while true:
let (maybeSrcNodeId, contentKeys, contentValues) = await n.contentQueue.popFirst()
if n.validateContent(contentKeys, contentValues):
for i, contentValue in contentValues:
let
contentKey = contentKeys[i]
(decodedKey, decodedValue) = decodeKV(contentKey, contentValue).valueOr:
error "Unable to decode offered Key/Value"
continue
if validateContent(decodedKey, decodedValue):
let
valueForRetrieval = decodedValue.offerContentToRetrievalContent().encode()
contentId = n.portalProtocol.toContentId(contentKey).valueOr:
error "Received offered content with invalid content key", contentKey
continue
n.portalProtocol.storeContent(contentKey, contentId, valueForRetrieval)
info "Received offered content validated successfully", contentKey
await gossipContent(
n.portalProtocol,
maybeSrcNodeId,
contentKeys,
contentValues
contentKey,
decodedKey,
contentValue,
decodedValue
)
else:
error "Received offered content failed validation", contentKey
except CancelledError:
trace "processContentLoop canceled"

View File

@ -39,30 +39,49 @@ procSuite "State Network Gossip":
proto.start()
clients.add(proto)
for i, pair in recursiveGossipSteps[0..^2]:
for i in 0..numOfClients-1:
let
currentNode = clients[i]
nextNode = clients[i+1]
key = ByteList.init(pair.content_key.hexToSeqByte())
decodedKey = key.decode().valueOr:
raiseAssert "Cannot decode key"
nextKey = ByteList.init(recursiveGossipSteps[1].content_key.hexToSeqByte())
decodedNextKey = nextKey.decode().valueOr:
raiseAssert "Cannot decode key"
value = pair.content_value.hexToSeqByte()
nextValue = recursiveGossipSteps[1].content_value.hexToSeqByte()
check:
currentNode.portalProtocol.addNode(nextNode.portalProtocol.localNode) == Added
(await currentNode.portalProtocol.ping(nextNode.portalProtocol.localNode)).isOk()
await currentNode.portalProtocol.gossipContent(Opt.none(NodeId), ContentKeysList.init(@[key]), @[value])
await sleepAsync(100.milliseconds)
let gossipedValue = await nextNode.getContent(decodedNextKey)
for i in 0..numOfClients-1:
let
pair = recursiveGossipSteps[i]
currentNode = clients[i]
nextNode = clients[i+1]
check:
gossipedValue.isSome()
gossipedValue.get() == nextValue
key = ByteList.init(pair.content_key.hexToSeqByte())
decodedKey = key.decode().valueOr:
raiseAssert "Cannot decode key"
nextKey = ByteList.init(recursiveGossipSteps[1].content_key.hexToSeqByte())
decodedNextKey = nextKey.decode().valueOr:
raiseAssert "Cannot decode key"
value = pair.content_value.hexToSeqByte()
decodedValue = SSZ.decode(value, AccountTrieNodeOffer)
offerValue = OfferContentValue(contentType: accountTrieNode, accountTrieNode: decodedValue)
nextValue = recursiveGossipSteps[1].content_value.hexToSeqByte()
nextDecodedValue = SSZ.decode(nextValue, AccountTrieNodeOffer)
nextOfferValue = OfferContentValue(contentType: accountTrieNode, accountTrieNode: nextDecodedValue)
nextRetrievalValue = nextOfferValue.offerContentToRetrievalContent().encode()
if i == 0:
await currentNode.portalProtocol.gossipContent(
Opt.none(NodeId),
key,
decodedKey,
value,
offerValue
)
await sleepAsync(100.milliseconds) #TODO figure out how to get rid of this sleep
check (await nextNode.getContent(decodedNextKey)) == Opt.some(nextRetrievalValue)
for i in 0..numOfClients:
await clients[i].portalProtocol.baseProtocol.closeWait()