State network content retrieval (#2033)
* initial rework of contentLoop for state network * prepare content for storage and retrieval * modify test to account for retrieval data
This commit is contained in:
Daniel Sobol
GitHub
parent
402a3eda73
commit
47b254d8b5
|
|
@ -98,6 +98,33 @@ type
|
|||
ContractCodeRetrieval* = object
|
||||
code*: Bytecode
|
||||
|
||||
OfferContentValueType* = enum
|
||||
accountTrieNodeOffer,
|
||||
contractTrieNodeOffer,
|
||||
contractCodeOffer,
|
||||
|
||||
OfferContentValue* = object
|
||||
case contentType*: ContentType
|
||||
of unused:
|
||||
discard
|
||||
of accountTrieNode:
|
||||
accountTrieNode*: AccountTrieNodeOffer
|
||||
of contractTrieNode:
|
||||
contractTrieNode*: ContractTrieNodeOffer
|
||||
of contractCode:
|
||||
contractCode*: ContractCodeOffer
|
||||
|
||||
RetrievalContentValue* = object
|
||||
case contentType*: ContentType
|
||||
of unused:
|
||||
discard
|
||||
of accountTrieNode:
|
||||
accountTrieNode*: AccountTrieNodeRetrieval
|
||||
of contractTrieNode:
|
||||
contractTrieNode*: ContractTrieNodeRetrieval
|
||||
of contractCode:
|
||||
contractCode*: ContractCodeRetrieval
|
||||
|
||||
func encode*(contentKey: ContentKey): ByteList =
|
||||
doAssert(contentKey.contentType != unused)
|
||||
ByteList.init(SSZ.encode(contentKey))
|
||||
|
|
@ -125,6 +152,60 @@ func toContentId*(contentKey: ByteList): ContentId =
|
|||
func toContentId*(contentKey: ContentKey): ContentId =
|
||||
toContentId(encode(contentKey))
|
||||
|
||||
func offerContentToRetrievalContent*(offerContent: OfferContentValue): RetrievalContentValue =
|
||||
case offerContent.contentType:
|
||||
of unused:
|
||||
raiseAssert "Converting content with unused content type"
|
||||
of accountTrieNode:
|
||||
RetrievalContentValue(
|
||||
contentType: accountTrieNode,
|
||||
accountTrieNode: AccountTrieNodeRetrieval(node: offerContent.accountTrieNode.proof[^1])
|
||||
) # TODO implement properly
|
||||
of contractTrieNode:
|
||||
RetrievalContentValue(
|
||||
contentType: contractTrieNode,
|
||||
contractTrieNode: ContractTrieNodeRetrieval(node: offerContent.contractTrieNode.storageProof[^1])
|
||||
) # TODO implement properly
|
||||
of contractCode:
|
||||
RetrievalContentValue(
|
||||
contentType: contractCode,
|
||||
contractCode: ContractCodeRetrieval(code: offerContent.contractCode.code)
|
||||
)
|
||||
|
||||
func encode*(content: RetrievalContentValue): seq[byte] =
|
||||
case content.contentType:
|
||||
of unused:
|
||||
raiseAssert "Encoding content with unused content type"
|
||||
of accountTrieNode:
|
||||
SSZ.encode(content.accountTrieNode)
|
||||
of contractTrieNode:
|
||||
SSZ.encode(content.contractTrieNode)
|
||||
of contractCode:
|
||||
SSZ.encode(content.contractCode)
|
||||
|
||||
func decodeKV*(contentKey: ByteList, contentValue: seq[byte]): Opt[(ContentKey, OfferContentValue)] =
|
||||
const empty = Opt.none((ContentKey, OfferContentValue))
|
||||
let
|
||||
key = contentKey.decode().valueOr:
|
||||
return empty
|
||||
value = case key.contentType:
|
||||
of unused:
|
||||
return empty
|
||||
of accountTrieNode:
|
||||
let val = decodeSsz(contentValue, AccountTrieNodeOffer).valueOr:
|
||||
return empty
|
||||
OfferContentValue(contentType: accountTrieNode, accountTrieNode: val)
|
||||
of contractTrieNode:
|
||||
let val = decodeSsz(contentValue, ContractTrieNodeOffer).valueOr:
|
||||
return empty
|
||||
OfferContentValue(contentType: contractTrieNode, contractTrieNode: val)
|
||||
of contractCode:
|
||||
let val = decodeSsz(contentValue, ContractCodeOffer).valueOr:
|
||||
return empty
|
||||
OfferContentValue(contentType: contractCode, contractCode: val)
|
||||
|
||||
Opt.some((key, value))
|
||||
|
||||
func packNibbles*(nibbles: seq[byte]): Nibbles =
|
||||
doAssert(nibbles.len() <= MAX_UNPACKED_NIBBLES_LEN, "Can't pack more than 64 nibbles")
|
||||
|
||||
|
|
|
|||
|
|
@ -60,70 +60,39 @@ proc getContent*(n: StateNetwork, key: ContentKey):
|
|||
# domain types.
|
||||
return Opt.some(contentResult.content)
|
||||
|
||||
proc validateAccountTrieNode(key: ContentKey, contentValue: seq[byte]): bool =
|
||||
let value = decodeSsz(contentValue, AccountTrieNodeOffer).valueOr:
|
||||
warn "Received invalid account trie proof", error
|
||||
return false
|
||||
proc validateAccountTrieNode(key: ContentKey, contentValue: OfferContentValue): bool =
|
||||
true
|
||||
|
||||
proc validateContractTrieNode(key: ContentKey, contentValue: seq[byte]): bool =
|
||||
let value = decodeSsz(contentValue, ContractTrieNodeOffer).valueOr:
|
||||
warn "Received invalid contract trie proof", error
|
||||
return false
|
||||
proc validateContractTrieNode(key: ContentKey, contentValue: OfferContentValue): bool =
|
||||
true
|
||||
|
||||
proc validateContractCode(key: ContentKey, contentValue: seq[byte]): bool =
|
||||
let value = decodeSsz(contentValue, ContractCodeOffer).valueOr:
|
||||
warn "Received invalid contract code", error
|
||||
return false
|
||||
proc validateContractCode(key: ContentKey, contentValue: OfferContentValue): bool =
|
||||
true
|
||||
|
||||
proc validateContent*(
|
||||
n: StateNetwork,
|
||||
contentKey: ByteList,
|
||||
contentValue: seq[byte]): bool =
|
||||
let key = contentKey.decode().valueOr:
|
||||
return false
|
||||
|
||||
case key.contentType:
|
||||
contentKey: ContentKey,
|
||||
contentValue: OfferContentValue): bool =
|
||||
case contentKey.contentType:
|
||||
of unused:
|
||||
warn "Received content with unused content type"
|
||||
false
|
||||
of accountTrieNode:
|
||||
validateAccountTrieNode(key, contentValue)
|
||||
validateAccountTrieNode(contentKey, contentValue)
|
||||
of contractTrieNode:
|
||||
validateContractTrieNode(key, contentValue)
|
||||
validateContractTrieNode(contentKey, contentValue)
|
||||
of contractCode:
|
||||
validateContractCode(key, contentValue)
|
||||
|
||||
proc validateContent(
|
||||
n: StateNetwork,
|
||||
contentKeys: ContentKeysList,
|
||||
contentValues: seq[seq[byte]]): bool =
|
||||
for i, contentValue in contentValues:
|
||||
let contentKey = contentKeys[i]
|
||||
if n.validateContent(contentKey, contentValue):
|
||||
let contentId = n.portalProtocol.toContentId(contentKey).valueOr:
|
||||
error "Received offered content with invalid content key", contentKey
|
||||
return false
|
||||
|
||||
n.portalProtocol.storeContent(contentKey, contentId, contentValue)
|
||||
|
||||
info "Received offered content validated successfully", contentKey
|
||||
else:
|
||||
error "Received offered content failed validation", contentKey
|
||||
return false
|
||||
validateContractCode(contentKey, contentValue)
|
||||
|
||||
proc recursiveGossipAccountTrieNode(
|
||||
p: PortalProtocol,
|
||||
maybeSrcNodeId: Opt[NodeId],
|
||||
decodedKey: ContentKey,
|
||||
contentValue: seq[byte]
|
||||
decodedValue: AccountTrieNodeOffer
|
||||
): Future[void] {.async.} =
|
||||
var nibbles = decodedKey.accountTrieNodeKey.path.unpackNibbles()
|
||||
let decodedValue = decodeSsz(contentValue, AccountTrieNodeOffer).valueOr:
|
||||
raiseAssert "Received offered content failed validation"
|
||||
var proof = decodedValue.proof
|
||||
var
|
||||
nibbles = decodedKey.accountTrieNodeKey.path.unpackNibbles()
|
||||
proof = decodedValue.proof
|
||||
|
||||
discard nibbles.pop()
|
||||
discard (distinctBase proof).pop()
|
||||
let
|
||||
|
|
@ -144,33 +113,29 @@ proc recursiveGossipContractTrieNode(
|
|||
p: PortalProtocol,
|
||||
maybeSrcNodeId: Opt[NodeId],
|
||||
decodedKey: ContentKey,
|
||||
contentValue: seq[byte]
|
||||
decodedValue: ContractTrieNodeOffer
|
||||
): Future[void] {.async.} =
|
||||
return
|
||||
|
||||
proc gossipContent*(
|
||||
p: PortalProtocol,
|
||||
maybeSrcNodeId: Opt[NodeId],
|
||||
contentKeys: ContentKeysList,
|
||||
contentValues: seq[seq[byte]]
|
||||
contentKey: ByteList,
|
||||
decodedKey: ContentKey,
|
||||
contentValue: seq[byte],
|
||||
decodedValue: OfferContentValue
|
||||
): Future[void] {.async.} =
|
||||
for i, contentValue in contentValues:
|
||||
let
|
||||
contentKey = contentKeys[i]
|
||||
decodedKey = contentKey.decode().valueOr:
|
||||
raiseAssert "Received offered content with invalid content key"
|
||||
case decodedKey.contentType:
|
||||
of unused:
|
||||
warn("Gossiping content with unused content type")
|
||||
continue
|
||||
of accountTrieNode:
|
||||
await recursiveGossipAccountTrieNode(p, maybeSrcNodeId, decodedKey, contentValue)
|
||||
of contractTrieNode:
|
||||
await recursiveGossipContractTrieNode(p, maybeSrcNodeId, decodedKey, contentValue)
|
||||
of contractCode:
|
||||
await neighborhoodGossipDiscardPeers(
|
||||
p, maybeSrcNodeId, ContentKeysList.init(@[contentKey]), @[contentValue]
|
||||
)
|
||||
case decodedKey.contentType:
|
||||
of unused:
|
||||
raiseAssert "Gossiping content with unused content type"
|
||||
of accountTrieNode:
|
||||
await recursiveGossipAccountTrieNode(p, maybeSrcNodeId, decodedKey, decodedValue.accountTrieNode)
|
||||
of contractTrieNode:
|
||||
await recursiveGossipContractTrieNode(p, maybeSrcNodeId, decodedKey, decodedValue.contractTrieNode)
|
||||
of contractCode:
|
||||
await p.neighborhoodGossipDiscardPeers(
|
||||
maybeSrcNodeId, ContentKeysList.init(@[contentKey]), @[contentValue]
|
||||
)
|
||||
|
||||
proc new*(
|
||||
T: type StateNetwork,
|
||||
|
|
@ -201,13 +166,32 @@ proc processContentLoop(n: StateNetwork) {.async.} =
|
|||
try:
|
||||
while true:
|
||||
let (maybeSrcNodeId, contentKeys, contentValues) = await n.contentQueue.popFirst()
|
||||
if n.validateContent(contentKeys, contentValues):
|
||||
await gossipContent(
|
||||
n.portalProtocol,
|
||||
maybeSrcNodeId,
|
||||
contentKeys,
|
||||
contentValues
|
||||
for i, contentValue in contentValues:
|
||||
let
|
||||
contentKey = contentKeys[i]
|
||||
(decodedKey, decodedValue) = decodeKV(contentKey, contentValue).valueOr:
|
||||
error "Unable to decode offered Key/Value"
|
||||
continue
|
||||
if validateContent(decodedKey, decodedValue):
|
||||
let
|
||||
valueForRetrieval = decodedValue.offerContentToRetrievalContent().encode()
|
||||
contentId = n.portalProtocol.toContentId(contentKey).valueOr:
|
||||
error "Received offered content with invalid content key", contentKey
|
||||
continue
|
||||
|
||||
n.portalProtocol.storeContent(contentKey, contentId, valueForRetrieval)
|
||||
info "Received offered content validated successfully", contentKey
|
||||
|
||||
await gossipContent(
|
||||
n.portalProtocol,
|
||||
maybeSrcNodeId,
|
||||
contentKey,
|
||||
decodedKey,
|
||||
contentValue,
|
||||
decodedValue
|
||||
)
|
||||
else:
|
||||
error "Received offered content failed validation", contentKey
|
||||
except CancelledError:
|
||||
trace "processContentLoop canceled"
|
||||
|
||||
|
|
|
|||
|
|
@ -39,30 +39,49 @@ procSuite "State Network Gossip":
|
|||
proto.start()
|
||||
clients.add(proto)
|
||||
|
||||
for i, pair in recursiveGossipSteps[0..^2]:
|
||||
for i in 0..numOfClients-1:
|
||||
let
|
||||
currentNode = clients[i]
|
||||
nextNode = clients[i+1]
|
||||
key = ByteList.init(pair.content_key.hexToSeqByte())
|
||||
decodedKey = key.decode().valueOr:
|
||||
raiseAssert "Cannot decode key"
|
||||
nextKey = ByteList.init(recursiveGossipSteps[1].content_key.hexToSeqByte())
|
||||
decodedNextKey = nextKey.decode().valueOr:
|
||||
raiseAssert "Cannot decode key"
|
||||
value = pair.content_value.hexToSeqByte()
|
||||
nextValue = recursiveGossipSteps[1].content_value.hexToSeqByte()
|
||||
|
||||
check:
|
||||
currentNode.portalProtocol.addNode(nextNode.portalProtocol.localNode) == Added
|
||||
(await currentNode.portalProtocol.ping(nextNode.portalProtocol.localNode)).isOk()
|
||||
|
||||
await currentNode.portalProtocol.gossipContent(Opt.none(NodeId), ContentKeysList.init(@[key]), @[value])
|
||||
await sleepAsync(100.milliseconds)
|
||||
let gossipedValue = await nextNode.getContent(decodedNextKey)
|
||||
for i in 0..numOfClients-1:
|
||||
let
|
||||
pair = recursiveGossipSteps[i]
|
||||
currentNode = clients[i]
|
||||
nextNode = clients[i+1]
|
||||
|
||||
check:
|
||||
gossipedValue.isSome()
|
||||
gossipedValue.get() == nextValue
|
||||
key = ByteList.init(pair.content_key.hexToSeqByte())
|
||||
decodedKey = key.decode().valueOr:
|
||||
raiseAssert "Cannot decode key"
|
||||
|
||||
nextKey = ByteList.init(recursiveGossipSteps[1].content_key.hexToSeqByte())
|
||||
decodedNextKey = nextKey.decode().valueOr:
|
||||
raiseAssert "Cannot decode key"
|
||||
|
||||
value = pair.content_value.hexToSeqByte()
|
||||
decodedValue = SSZ.decode(value, AccountTrieNodeOffer)
|
||||
offerValue = OfferContentValue(contentType: accountTrieNode, accountTrieNode: decodedValue)
|
||||
|
||||
nextValue = recursiveGossipSteps[1].content_value.hexToSeqByte()
|
||||
nextDecodedValue = SSZ.decode(nextValue, AccountTrieNodeOffer)
|
||||
nextOfferValue = OfferContentValue(contentType: accountTrieNode, accountTrieNode: nextDecodedValue)
|
||||
nextRetrievalValue = nextOfferValue.offerContentToRetrievalContent().encode()
|
||||
|
||||
if i == 0:
|
||||
await currentNode.portalProtocol.gossipContent(
|
||||
Opt.none(NodeId),
|
||||
key,
|
||||
decodedKey,
|
||||
value,
|
||||
offerValue
|
||||
)
|
||||
|
||||
await sleepAsync(100.milliseconds) #TODO figure out how to get rid of this sleep
|
||||
|
||||
check (await nextNode.getContent(decodedNextKey)) == Opt.some(nextRetrievalValue)
|
||||
|
||||
for i in 0..numOfClients:
|
||||
await clients[i].portalProtocol.baseProtocol.closeWait()
|
||||
|
|
|
|||
Loading…
Reference in New Issue