From 47b254d8b565e2868d2d4ec8dabadc95c8b16798 Mon Sep 17 00:00:00 2001 From: Daniel Sobol Date: Mon, 19 Feb 2024 17:35:53 +0300 Subject: [PATCH] State network content retrieval (#2033) * initial rework of contentLoop for state network * prepare content for storage and retrieval * modify test to account for retrieval data --- fluffy/network/state/state_content.nim | 81 +++++++++++ fluffy/network/state/state_network.nim | 126 ++++++++---------- .../test_state_network_gossip.nim | 51 ++++--- 3 files changed, 171 insertions(+), 87 deletions(-) diff --git a/fluffy/network/state/state_content.nim b/fluffy/network/state/state_content.nim index 038e436bc..8ee5155a3 100644 --- a/fluffy/network/state/state_content.nim +++ b/fluffy/network/state/state_content.nim @@ -98,6 +98,33 @@ type ContractCodeRetrieval* = object code*: Bytecode + OfferContentValueType* = enum + accountTrieNodeOffer, + contractTrieNodeOffer, + contractCodeOffer, + + OfferContentValue* = object + case contentType*: ContentType + of unused: + discard + of accountTrieNode: + accountTrieNode*: AccountTrieNodeOffer + of contractTrieNode: + contractTrieNode*: ContractTrieNodeOffer + of contractCode: + contractCode*: ContractCodeOffer + + RetrievalContentValue* = object + case contentType*: ContentType + of unused: + discard + of accountTrieNode: + accountTrieNode*: AccountTrieNodeRetrieval + of contractTrieNode: + contractTrieNode*: ContractTrieNodeRetrieval + of contractCode: + contractCode*: ContractCodeRetrieval + func encode*(contentKey: ContentKey): ByteList = doAssert(contentKey.contentType != unused) ByteList.init(SSZ.encode(contentKey)) @@ -125,6 +152,60 @@ func toContentId*(contentKey: ByteList): ContentId = func toContentId*(contentKey: ContentKey): ContentId = toContentId(encode(contentKey)) +func offerContentToRetrievalContent*(offerContent: OfferContentValue): RetrievalContentValue = + case offerContent.contentType: + of unused: + raiseAssert "Converting content with unused content type" + of accountTrieNode: + RetrievalContentValue( + contentType: accountTrieNode, + accountTrieNode: AccountTrieNodeRetrieval(node: offerContent.accountTrieNode.proof[^1]) + ) # TODO implement properly + of contractTrieNode: + RetrievalContentValue( + contentType: contractTrieNode, + contractTrieNode: ContractTrieNodeRetrieval(node: offerContent.contractTrieNode.storageProof[^1]) + ) # TODO implement properly + of contractCode: + RetrievalContentValue( + contentType: contractCode, + contractCode: ContractCodeRetrieval(code: offerContent.contractCode.code) + ) + +func encode*(content: RetrievalContentValue): seq[byte] = + case content.contentType: + of unused: + raiseAssert "Encoding content with unused content type" + of accountTrieNode: + SSZ.encode(content.accountTrieNode) + of contractTrieNode: + SSZ.encode(content.contractTrieNode) + of contractCode: + SSZ.encode(content.contractCode) + +func decodeKV*(contentKey: ByteList, contentValue: seq[byte]): Opt[(ContentKey, OfferContentValue)] = + const empty = Opt.none((ContentKey, OfferContentValue)) + let + key = contentKey.decode().valueOr: + return empty + value = case key.contentType: + of unused: + return empty + of accountTrieNode: + let val = decodeSsz(contentValue, AccountTrieNodeOffer).valueOr: + return empty + OfferContentValue(contentType: accountTrieNode, accountTrieNode: val) + of contractTrieNode: + let val = decodeSsz(contentValue, ContractTrieNodeOffer).valueOr: + return empty + OfferContentValue(contentType: contractTrieNode, contractTrieNode: val) + of contractCode: + let val = decodeSsz(contentValue, ContractCodeOffer).valueOr: + return empty + OfferContentValue(contentType: contractCode, contractCode: val) + + Opt.some((key, value)) + func packNibbles*(nibbles: seq[byte]): Nibbles = doAssert(nibbles.len() <= MAX_UNPACKED_NIBBLES_LEN, "Can't pack more than 64 nibbles") diff --git a/fluffy/network/state/state_network.nim b/fluffy/network/state/state_network.nim index ce015c4f5..2a23ab653 100644 --- a/fluffy/network/state/state_network.nim +++ b/fluffy/network/state/state_network.nim @@ -60,70 +60,39 @@ proc getContent*(n: StateNetwork, key: ContentKey): # domain types. return Opt.some(contentResult.content) -proc validateAccountTrieNode(key: ContentKey, contentValue: seq[byte]): bool = - let value = decodeSsz(contentValue, AccountTrieNodeOffer).valueOr: - warn "Received invalid account trie proof", error - return false +proc validateAccountTrieNode(key: ContentKey, contentValue: OfferContentValue): bool = true -proc validateContractTrieNode(key: ContentKey, contentValue: seq[byte]): bool = - let value = decodeSsz(contentValue, ContractTrieNodeOffer).valueOr: - warn "Received invalid contract trie proof", error - return false +proc validateContractTrieNode(key: ContentKey, contentValue: OfferContentValue): bool = true -proc validateContractCode(key: ContentKey, contentValue: seq[byte]): bool = - let value = decodeSsz(contentValue, ContractCodeOffer).valueOr: - warn "Received invalid contract code", error - return false +proc validateContractCode(key: ContentKey, contentValue: OfferContentValue): bool = true proc validateContent*( - n: StateNetwork, - contentKey: ByteList, - contentValue: seq[byte]): bool = - let key = contentKey.decode().valueOr: - return false - - case key.contentType: + contentKey: ContentKey, + contentValue: OfferContentValue): bool = + case contentKey.contentType: of unused: warn "Received content with unused content type" false of accountTrieNode: - validateAccountTrieNode(key, contentValue) + validateAccountTrieNode(contentKey, contentValue) of contractTrieNode: - validateContractTrieNode(key, contentValue) + validateContractTrieNode(contentKey, contentValue) of contractCode: - validateContractCode(key, contentValue) - -proc validateContent( - n: StateNetwork, - contentKeys: ContentKeysList, - contentValues: seq[seq[byte]]): bool = - for i, contentValue in contentValues: - let contentKey = contentKeys[i] - if n.validateContent(contentKey, contentValue): - let contentId = n.portalProtocol.toContentId(contentKey).valueOr: - error "Received offered content with invalid content key", contentKey - return false - - n.portalProtocol.storeContent(contentKey, contentId, contentValue) - - info "Received offered content validated successfully", contentKey - else: - error "Received offered content failed validation", contentKey - return false + validateContractCode(contentKey, contentValue) proc recursiveGossipAccountTrieNode( p: PortalProtocol, maybeSrcNodeId: Opt[NodeId], decodedKey: ContentKey, - contentValue: seq[byte] + decodedValue: AccountTrieNodeOffer ): Future[void] {.async.} = - var nibbles = decodedKey.accountTrieNodeKey.path.unpackNibbles() - let decodedValue = decodeSsz(contentValue, AccountTrieNodeOffer).valueOr: - raiseAssert "Received offered content failed validation" - var proof = decodedValue.proof + var + nibbles = decodedKey.accountTrieNodeKey.path.unpackNibbles() + proof = decodedValue.proof + discard nibbles.pop() discard (distinctBase proof).pop() let @@ -144,33 +113,29 @@ proc recursiveGossipContractTrieNode( p: PortalProtocol, maybeSrcNodeId: Opt[NodeId], decodedKey: ContentKey, - contentValue: seq[byte] + decodedValue: ContractTrieNodeOffer ): Future[void] {.async.} = return proc gossipContent*( p: PortalProtocol, maybeSrcNodeId: Opt[NodeId], - contentKeys: ContentKeysList, - contentValues: seq[seq[byte]] + contentKey: ByteList, + decodedKey: ContentKey, + contentValue: seq[byte], + decodedValue: OfferContentValue ): Future[void] {.async.} = - for i, contentValue in contentValues: - let - contentKey = contentKeys[i] - decodedKey = contentKey.decode().valueOr: - raiseAssert "Received offered content with invalid content key" - case decodedKey.contentType: - of unused: - warn("Gossiping content with unused content type") - continue - of accountTrieNode: - await recursiveGossipAccountTrieNode(p, maybeSrcNodeId, decodedKey, contentValue) - of contractTrieNode: - await recursiveGossipContractTrieNode(p, maybeSrcNodeId, decodedKey, contentValue) - of contractCode: - await neighborhoodGossipDiscardPeers( - p, maybeSrcNodeId, ContentKeysList.init(@[contentKey]), @[contentValue] - ) + case decodedKey.contentType: + of unused: + raiseAssert "Gossiping content with unused content type" + of accountTrieNode: + await recursiveGossipAccountTrieNode(p, maybeSrcNodeId, decodedKey, decodedValue.accountTrieNode) + of contractTrieNode: + await recursiveGossipContractTrieNode(p, maybeSrcNodeId, decodedKey, decodedValue.contractTrieNode) + of contractCode: + await p.neighborhoodGossipDiscardPeers( + maybeSrcNodeId, ContentKeysList.init(@[contentKey]), @[contentValue] + ) proc new*( T: type StateNetwork, @@ -201,13 +166,32 @@ proc processContentLoop(n: StateNetwork) {.async.} = try: while true: let (maybeSrcNodeId, contentKeys, contentValues) = await n.contentQueue.popFirst() - if n.validateContent(contentKeys, contentValues): - await gossipContent( - n.portalProtocol, - maybeSrcNodeId, - contentKeys, - contentValues + for i, contentValue in contentValues: + let + contentKey = contentKeys[i] + (decodedKey, decodedValue) = decodeKV(contentKey, contentValue).valueOr: + error "Unable to decode offered Key/Value" + continue + if validateContent(decodedKey, decodedValue): + let + valueForRetrieval = decodedValue.offerContentToRetrievalContent().encode() + contentId = n.portalProtocol.toContentId(contentKey).valueOr: + error "Received offered content with invalid content key", contentKey + continue + + n.portalProtocol.storeContent(contentKey, contentId, valueForRetrieval) + info "Received offered content validated successfully", contentKey + + await gossipContent( + n.portalProtocol, + maybeSrcNodeId, + contentKey, + decodedKey, + contentValue, + decodedValue ) + else: + error "Received offered content failed validation", contentKey except CancelledError: trace "processContentLoop canceled" diff --git a/fluffy/tests/state_network_tests/test_state_network_gossip.nim b/fluffy/tests/state_network_tests/test_state_network_gossip.nim index 1f1552c66..b4075d216 100644 --- a/fluffy/tests/state_network_tests/test_state_network_gossip.nim +++ b/fluffy/tests/state_network_tests/test_state_network_gossip.nim @@ -39,30 +39,49 @@ procSuite "State Network Gossip": proto.start() clients.add(proto) - for i, pair in recursiveGossipSteps[0..^2]: + for i in 0..numOfClients-1: let currentNode = clients[i] nextNode = clients[i+1] - key = ByteList.init(pair.content_key.hexToSeqByte()) - decodedKey = key.decode().valueOr: - raiseAssert "Cannot decode key" - nextKey = ByteList.init(recursiveGossipSteps[1].content_key.hexToSeqByte()) - decodedNextKey = nextKey.decode().valueOr: - raiseAssert "Cannot decode key" - value = pair.content_value.hexToSeqByte() - nextValue = recursiveGossipSteps[1].content_value.hexToSeqByte() - check: currentNode.portalProtocol.addNode(nextNode.portalProtocol.localNode) == Added (await currentNode.portalProtocol.ping(nextNode.portalProtocol.localNode)).isOk() - await currentNode.portalProtocol.gossipContent(Opt.none(NodeId), ContentKeysList.init(@[key]), @[value]) - await sleepAsync(100.milliseconds) - let gossipedValue = await nextNode.getContent(decodedNextKey) + for i in 0..numOfClients-1: + let + pair = recursiveGossipSteps[i] + currentNode = clients[i] + nextNode = clients[i+1] - check: - gossipedValue.isSome() - gossipedValue.get() == nextValue + key = ByteList.init(pair.content_key.hexToSeqByte()) + decodedKey = key.decode().valueOr: + raiseAssert "Cannot decode key" + + nextKey = ByteList.init(recursiveGossipSteps[1].content_key.hexToSeqByte()) + decodedNextKey = nextKey.decode().valueOr: + raiseAssert "Cannot decode key" + + value = pair.content_value.hexToSeqByte() + decodedValue = SSZ.decode(value, AccountTrieNodeOffer) + offerValue = OfferContentValue(contentType: accountTrieNode, accountTrieNode: decodedValue) + + nextValue = recursiveGossipSteps[1].content_value.hexToSeqByte() + nextDecodedValue = SSZ.decode(nextValue, AccountTrieNodeOffer) + nextOfferValue = OfferContentValue(contentType: accountTrieNode, accountTrieNode: nextDecodedValue) + nextRetrievalValue = nextOfferValue.offerContentToRetrievalContent().encode() + + if i == 0: + await currentNode.portalProtocol.gossipContent( + Opt.none(NodeId), + key, + decodedKey, + value, + offerValue + ) + + await sleepAsync(100.milliseconds) #TODO figure out how to get rid of this sleep + + check (await nextNode.getContent(decodedNextKey)) == Opt.some(nextRetrievalValue) for i in 0..numOfClients: await clients[i].portalProtocol.baseProtocol.closeWait()