From 23a43d1d15c50e6c42464d7a7402ee32ff2c5baa Mon Sep 17 00:00:00 2001 From: bhartnett <51288821+bhartnett@users.noreply.github.com> Date: Thu, 28 Nov 2024 20:33:57 +0800 Subject: [PATCH] Fluffy: Implement poke in state network (#2750) * Implement poke in state network. --- fluffy/network/state/content/content_keys.nim | 14 +- .../network/state/content/content_values.nim | 55 ++++-- fluffy/network/state/state_endpoints.nim | 181 ++++++++++++++---- fluffy/network/state/state_network.nim | 45 +++-- fluffy/network/state/state_validation.nim | 75 +++++--- fluffy/rpc/rpc_portal_state_api.nim | 6 +- .../test_state_endpoints_genesis.nim | 12 +- .../test_state_gossip_getparent_genesis.nim | 16 +- .../test_state_gossip_getparent_vectors.nim | 4 +- .../test_state_gossip_gossipoffer_vectors.nim | 12 +- ...est_state_network_offercontent_vectors.nim | 24 +-- 11 files changed, 298 insertions(+), 146 deletions(-) diff --git a/fluffy/network/state/content/content_keys.nim b/fluffy/network/state/content/content_keys.nim index 467702470..c1b10fde7 100644 --- a/fluffy/network/state/content/content_keys.nim +++ b/fluffy/network/state/content/content_keys.nim @@ -60,12 +60,12 @@ type ContentKeyType* = AccountTrieNodeKey | ContractTrieNodeKey | ContractCodeKey -func init*(T: type AccountTrieNodeKey, path: Nibbles, nodeHash: Hash32): T {.inline.} = +func init*(T: type AccountTrieNodeKey, path: Nibbles, nodeHash: Hash32): T = T(path: path, nodeHash: nodeHash) func init*( T: type ContractTrieNodeKey, addressHash: Hash32, path: Nibbles, nodeHash: Hash32 -): T {.inline.} = +): T = T(addressHash: addressHash, path: path, nodeHash: nodeHash) func init*( @@ -73,13 +73,13 @@ func init*( ): T {.inline.} = T(addressHash: addressHash, codeHash: codeHash) -func toContentKey*(key: AccountTrieNodeKey): ContentKey {.inline.} = +template toContentKey*(key: AccountTrieNodeKey): ContentKey = ContentKey(contentType: accountTrieNode, accountTrieNodeKey: key) -func toContentKey*(key: ContractTrieNodeKey): ContentKey {.inline.} = +template toContentKey*(key: ContractTrieNodeKey): ContentKey = ContentKey(contentType: contractTrieNode, contractTrieNodeKey: key) -func toContentKey*(key: ContractCodeKey): ContentKey {.inline.} = +template toContentKey*(key: ContractCodeKey): ContentKey = ContentKey(contentType: contractCode, contractCodeKey: key) proc readSszBytes*(data: openArray[byte], val: var ContentKey) {.raises: [SszError].} = @@ -89,7 +89,7 @@ proc readSszBytes*(data: openArray[byte], val: var ContentKey) {.raises: [SszErr readSszValue(data, val) -func encode*(contentKey: ContentKey): ContentKeyByteList {.inline.} = +func encode*(contentKey: ContentKey): ContentKeyByteList = doAssert(contentKey.contentType != unused) ContentKeyByteList.init(SSZ.encode(contentKey)) @@ -101,6 +101,6 @@ func decode*( return err("ContentKey contentType: unused") ok(key) -func toContentId*(contentKey: ContentKeyByteList): ContentId {.inline.} = +func toContentId*(contentKey: ContentKeyByteList): ContentId = let idHash = sha256.digest(contentKey.asSeq()) readUintBE[256](idHash.data) diff --git a/fluffy/network/state/content/content_values.nim b/fluffy/network/state/content/content_values.nim index e1736c74a..4e33162a8 100644 --- a/fluffy/network/state/content/content_values.nim +++ b/fluffy/network/state/content/content_values.nim @@ -52,12 +52,10 @@ type AccountTrieNodeRetrieval | ContractTrieNodeRetrieval | ContractCodeRetrieval ContentValueType* = ContentOfferType | ContentRetrievalType -func init*( - T: type AccountTrieNodeOffer, proof: TrieProof, blockHash: Hash32 -): T {.inline.} = +func init*(T: type AccountTrieNodeOffer, proof: TrieProof, blockHash: Hash32): T = T(proof: proof, blockHash: blockHash) -func init*(T: type AccountTrieNodeRetrieval, node: TrieNode): T {.inline.} = +func init*(T: type AccountTrieNodeRetrieval, node: TrieNode): T = T(node: node) func init*( @@ -65,10 +63,10 @@ func init*( storageProof: TrieProof, accountProof: TrieProof, blockHash: Hash32, -): T {.inline.} = +): T = T(storageProof: storageProof, accountProof: accountProof, blockHash: blockHash) -func init*(T: type ContractTrieNodeRetrieval, node: TrieNode): T {.inline.} = +func init*(T: type ContractTrieNodeRetrieval, node: TrieNode): T = T(node: node) func init*( @@ -76,35 +74,50 @@ func init*( code: Bytecode, accountProof: TrieProof, blockHash: Hash32, -): T {.inline.} = +): T = T(code: code, accountProof: accountProof, blockHash: blockHash) -func init*(T: type ContractCodeRetrieval, code: Bytecode): T {.inline.} = +func init*(T: type ContractCodeRetrieval, code: Bytecode): T = T(code: code) -func toRetrievalValue*( - offer: AccountTrieNodeOffer -): AccountTrieNodeRetrieval {.inline.} = +template toRetrieval*(offer: AccountTrieNodeOffer): AccountTrieNodeRetrieval = AccountTrieNodeRetrieval.init(offer.proof[^1]) -func toRetrievalValue*( - offer: ContractTrieNodeOffer -): ContractTrieNodeRetrieval {.inline.} = +template toRetrieval*(offer: ContractTrieNodeOffer): ContractTrieNodeRetrieval = ContractTrieNodeRetrieval.init(offer.storageProof[^1]) -func toRetrievalValue*(offer: ContractCodeOffer): ContractCodeRetrieval {.inline.} = +template toRetrieval*(offer: ContractCodeOffer): ContractCodeRetrieval = ContractCodeRetrieval.init(offer.code) -func empty*(T: type TrieProof): T {.inline.} = +func toOffer*( + retrieval: AccountTrieNodeRetrieval, parent: AccountTrieNodeOffer +): AccountTrieNodeOffer = + var proof = parent.proof + let added = proof.add(retrieval.node) + doAssert(added) + AccountTrieNodeOffer.init(proof, parent.blockHash) + +func toOffer*( + retrieval: ContractTrieNodeRetrieval, parent: ContractTrieNodeOffer +): ContractTrieNodeOffer = + var proof = parent.storageProof + let added = proof.add(retrieval.node) + doAssert(added) + ContractTrieNodeOffer.init(proof, parent.accountProof, parent.blockHash) + +func toOffer*( + retrieval: ContractCodeRetrieval, parent: ContractCodeOffer +): ContractCodeOffer = + ContractCodeOffer.init(retrieval.code, parent.accountProof, parent.blockHash) + +template empty*(T: type TrieProof): T = T.init(@[]) -func empty*(T: type Bytecode): T {.inline.} = +template empty*(T: type Bytecode): T = T(@[]) -func encode*(value: ContentValueType): seq[byte] {.inline.} = +func encode*(value: ContentValueType): seq[byte] = SSZ.encode(value) -func decode*( - T: type ContentValueType, bytes: openArray[byte] -): Result[T, string] {.inline.} = +func decode*(T: type ContentValueType, bytes: openArray[byte]): Result[T, string] = decodeSsz(bytes, T) diff --git a/fluffy/network/state/state_endpoints.nim b/fluffy/network/state/state_endpoints.nim index 5295805ca..211c4bd88 100644 --- a/fluffy/network/state/state_endpoints.nim +++ b/fluffy/network/state/state_endpoints.nim @@ -16,7 +16,9 @@ import ./state_network, ./state_utils -export results, state_network +from eth/common/eth_types_rlp import rlpHash + +export results, state_network, hashes, addresses logScope: topics = "portal_state" @@ -72,7 +74,10 @@ proc getNextNodeHash( raiseAssert(e.msg) proc getAccountProof( - n: StateNetwork, stateRoot: Hash32, address: Address + n: StateNetwork, + stateRoot: Hash32, + address: Address, + maybeBlockHash: Opt[Hash32], # required for poke ): Future[Result[(TrieProof, bool), string]] {.async: (raises: [CancelledError]).} = let nibbles = address.toPath().unpackNibbles() @@ -81,15 +86,26 @@ proc getAccountProof( key = AccountTrieNodeKey.init(Nibbles.empty(), stateRoot) proof = TrieProof.empty() - while nibblesIdx < nibbles.len(): - let accountTrieNode = (await n.getAccountTrieNode(key)).valueOr: - return err("Failed to get account trie node when building account proof") + # Construct the parent offer which is used to provide the data needed to + # implement poke after the account trie node has been retrieved + maybeParentOffer = + if maybeBlockHash.isSome(): + Opt.some(AccountTrieNodeOffer.init(proof, maybeBlockHash.get())) + else: + Opt.none(AccountTrieNodeOffer) + while nibblesIdx < nibbles.len(): let + accountTrieNode = (await n.getAccountTrieNode(key, maybeParentOffer)).valueOr: + return err("Failed to get account trie node when building account proof") trieNode = accountTrieNode.node added = proof.add(trieNode) doAssert(added) + if maybeParentOffer.isSome(): + let added = maybeParentOffer.get().proof.add(trieNode) + doAssert(added) + let (nextPath, nextNodeHash) = trieNode.getNextNodeHash(nibbles, nibblesIdx).valueOr: break @@ -99,7 +115,12 @@ proc getAccountProof( ok((proof, nibblesIdx == nibbles.len())) proc getStorageProof( - n: StateNetwork, storageRoot: Hash32, address: Address, slotKey: UInt256 + n: StateNetwork, + storageRoot: Hash32, + address: Address, + slotKey: UInt256, + maybeBlockHash: Opt[Hash32], + maybeAccProof: Opt[TrieProof], ): Future[Result[(TrieProof, bool), string]] {.async: (raises: [CancelledError]).} = let nibbles = slotKey.toPath().unpackNibbles() @@ -109,15 +130,29 @@ proc getStorageProof( key = ContractTrieNodeKey.init(addressHash, Nibbles.empty(), storageRoot) proof = TrieProof.empty() - while nibblesIdx < nibbles.len(): - let contractTrieNode = (await n.getContractTrieNode(key)).valueOr: - return err("Failed to get contract trie node when building account proof") + # Construct the parent offer which is used to provide the data needed to + # implement poke after the contract trie node has been retrieved + maybeParentOffer = + if maybeBlockHash.isSome(): + doAssert maybeAccProof.isSome() + Opt.some( + ContractTrieNodeOffer.init(proof, maybeAccProof.get(), maybeBlockHash.get()) + ) + else: + Opt.none(ContractTrieNodeOffer) + while nibblesIdx < nibbles.len(): let + contractTrieNode = (await n.getContractTrieNode(key, maybeParentOffer)).valueOr: + return err("Failed to get contract trie node when building storage proof") trieNode = contractTrieNode.node added = proof.add(trieNode) doAssert(added) + if maybeParentOffer.isSome(): + let added = maybeParentOffer.get().storageProof.add(trieNode) + doAssert(added) + let (nextPath, nextNodeHash) = trieNode.getNextNodeHash(nibbles, nibblesIdx).valueOr: break @@ -127,9 +162,11 @@ proc getStorageProof( ok((proof, nibblesIdx == nibbles.len())) proc getAccount( - n: StateNetwork, stateRoot: Hash32, address: Address + n: StateNetwork, stateRoot: Hash32, address: Address, maybeBlockHash: Opt[Hash32] ): Future[Opt[Account]] {.async: (raises: [CancelledError]).} = - let (accountProof, exists) = (await n.getAccountProof(stateRoot, address)).valueOr: + let (accountProof, exists) = ( + await n.getAccountProof(stateRoot, address, maybeBlockHash) + ).valueOr: warn "Failed to get account proof", error = error return Opt.none(Account) @@ -145,39 +182,64 @@ proc getAccount( Opt.some(account) proc getBalanceByStateRoot*( - n: StateNetwork, stateRoot: Hash32, address: Address + n: StateNetwork, + stateRoot: Hash32, + address: Address, + maybeBlockHash = Opt.none(Hash32), ): Future[Opt[UInt256]] {.async: (raises: [CancelledError]).} = - let account = (await n.getAccount(stateRoot, address)).valueOr: + let account = (await n.getAccount(stateRoot, address, maybeBlockHash)).valueOr: return Opt.none(UInt256) Opt.some(account.balance) proc getTransactionCountByStateRoot*( - n: StateNetwork, stateRoot: Hash32, address: Address + n: StateNetwork, + stateRoot: Hash32, + address: Address, + maybeBlockHash = Opt.none(Hash32), ): Future[Opt[AccountNonce]] {.async: (raises: [CancelledError]).} = - let account = (await n.getAccount(stateRoot, address)).valueOr: + let account = (await n.getAccount(stateRoot, address, maybeBlockHash)).valueOr: return Opt.none(AccountNonce) Opt.some(account.nonce) proc getStorageAtByStateRoot*( - n: StateNetwork, stateRoot: Hash32, address: Address, slotKey: UInt256 + n: StateNetwork, + stateRoot: Hash32, + address: Address, + slotKey: UInt256, + maybeBlockHash = Opt.none(Hash32), ): Future[Opt[UInt256]] {.async: (raises: [CancelledError]).} = - let account = (await n.getAccount(stateRoot, address)).valueOr: + let (accountProof, exists) = ( + await n.getAccountProof(stateRoot, address, maybeBlockHash) + ).valueOr: + warn "Failed to get account proof", error = error return Opt.none(UInt256) + let account = + if exists: + accountProof.toAccount().valueOr: + error "Failed to get account from accountProof" + return Opt.none(UInt256) + else: + info "Account doesn't exist, returning default account" + # return an empty account if the account doesn't exist + EMPTY_ACCOUNT + if account.storageRoot == EMPTY_ROOT_HASH: info "Storage doesn't exist, returning default storage value" # return zero if the storage doesn't exist return Opt.some(0.u256) - let (storageProof, exists) = ( - await n.getStorageProof(account.storageRoot, address, slotKey) + let (storageProof, slotExists) = ( + await n.getStorageProof( + account.storageRoot, address, slotKey, maybeBlockHash, Opt.some(accountProof) + ) ).valueOr: warn "Failed to get storage proof", error = error return Opt.none(UInt256) - if not exists: + if not slotExists: info "Slot doesn't exist, returning default storage value" # return zero if the slot doesn't exist return Opt.some(0.u256) @@ -189,11 +251,27 @@ proc getStorageAtByStateRoot*( Opt.some(slotValue) proc getCodeByStateRoot*( - n: StateNetwork, stateRoot: Hash32, address: Address + n: StateNetwork, + stateRoot: Hash32, + address: Address, + maybeBlockHash = Opt.none(Hash32), ): Future[Opt[Bytecode]] {.async: (raises: [CancelledError]).} = - let account = (await n.getAccount(stateRoot, address)).valueOr: + let (accountProof, exists) = ( + await n.getAccountProof(stateRoot, address, maybeBlockHash) + ).valueOr: + warn "Failed to get account proof", error = error return Opt.none(Bytecode) + let account = + if exists: + accountProof.toAccount().valueOr: + error "Failed to get account from accountProof" + return Opt.none(Bytecode) + else: + info "Account doesn't exist, returning default account" + # return an empty account if the account doesn't exist + EMPTY_ACCOUNT + if account.codeHash == EMPTY_CODE_HASH: info "Code doesn't exist, returning default code value" # return empty bytecode if the code doesn't exist @@ -201,7 +279,14 @@ proc getCodeByStateRoot*( let contractCodeKey = ContractCodeKey.init(keccak256(address.data), account.codeHash) - contractCodeRetrieval = (await n.getContractCode(contractCodeKey)).valueOr: + maybeParentOffer = + if maybeBlockHash.isSome(): + Opt.some( + ContractCodeOffer.init(Bytecode.empty(), accountProof, maybeBlockHash.get()) + ) + else: + Opt.none(ContractCodeOffer) + contractCodeRetrieval = (await n.getContractCode(contractCodeKey, maybeParentOffer)).valueOr: warn "Failed to get contract code" return Opt.none(Bytecode) @@ -214,10 +299,16 @@ type Proofs* = ref object slotProofs*: seq[TrieProof] proc getProofsByStateRoot*( - n: StateNetwork, stateRoot: Hash32, address: Address, slotKeys: seq[UInt256] + n: StateNetwork, + stateRoot: Hash32, + address: Address, + slotKeys: seq[UInt256], + maybeBlockHash = Opt.none(Hash32), ): Future[Opt[Proofs]] {.async: (raises: [CancelledError]).} = let - (accountProof, accountExists) = (await n.getAccountProof(stateRoot, address)).valueOr: + (accountProof, accountExists) = ( + await n.getAccountProof(stateRoot, address, maybeBlockHash) + ).valueOr: warn "Failed to get account proof", error = error return Opt.none(Proofs) account = @@ -241,7 +332,9 @@ proc getProofsByStateRoot*( let (storageProof, slotExists) = ( - await n.getStorageProof(account.storageRoot, address, slotKey) + await n.getStorageProof( + account.storageRoot, address, slotKey, maybeBlockHash, Opt.some(accountProof) + ) ).valueOr: warn "Failed to get storage proof", error = error return Opt.none(Proofs) @@ -265,41 +358,45 @@ proc getProofsByStateRoot*( proc getBalance*( n: StateNetwork, blockNumOrHash: uint64 | Hash32, address: Address ): Future[Opt[UInt256]] {.async: (raises: [CancelledError]).} = - let stateRoot = (await n.getStateRootByBlockNumOrHash(blockNumOrHash)).valueOr: - warn "Failed to get state root by block number or hash", blockNumOrHash + let header = (await n.getBlockHeaderByBlockNumOrHash(blockNumOrHash)).valueOr: + warn "Failed to get block header by block number or hash", blockNumOrHash return Opt.none(UInt256) - await n.getBalanceByStateRoot(stateRoot, address) + await n.getBalanceByStateRoot(header.stateRoot, address, Opt.some(header.rlpHash())) # Used by: eth_getTransactionCount proc getTransactionCount*( n: StateNetwork, blockNumOrHash: uint64 | Hash32, address: Address ): Future[Opt[AccountNonce]] {.async: (raises: [CancelledError]).} = - let stateRoot = (await n.getStateRootByBlockNumOrHash(blockNumOrHash)).valueOr: - warn "Failed to get state root by block number or hash", blockNumOrHash + let header = (await n.getBlockHeaderByBlockNumOrHash(blockNumOrHash)).valueOr: + warn "Failed to get block header by block number or hash", blockNumOrHash return Opt.none(AccountNonce) - await n.getTransactionCountByStateRoot(stateRoot, address) + await n.getTransactionCountByStateRoot( + header.stateRoot, address, Opt.some(header.rlpHash()) + ) # Used by: eth_getStorageAt proc getStorageAt*( n: StateNetwork, blockNumOrHash: uint64 | Hash32, address: Address, slotKey: UInt256 ): Future[Opt[UInt256]] {.async: (raises: [CancelledError]).} = - let stateRoot = (await n.getStateRootByBlockNumOrHash(blockNumOrHash)).valueOr: - warn "Failed to get state root by block number or hash", blockNumOrHash + let header = (await n.getBlockHeaderByBlockNumOrHash(blockNumOrHash)).valueOr: + warn "Failed to get block header by block number or hash", blockNumOrHash return Opt.none(UInt256) - await n.getStorageAtByStateRoot(stateRoot, address, slotKey) + await n.getStorageAtByStateRoot( + header.stateRoot, address, slotKey, Opt.some(header.rlpHash()) + ) # Used by: eth_getCode proc getCode*( n: StateNetwork, blockNumOrHash: uint64 | Hash32, address: Address ): Future[Opt[Bytecode]] {.async: (raises: [CancelledError]).} = - let stateRoot = (await n.getStateRootByBlockNumOrHash(blockNumOrHash)).valueOr: - warn "Failed to get state root by block number or hash", blockNumOrHash + let header = (await n.getBlockHeaderByBlockNumOrHash(blockNumOrHash)).valueOr: + warn "Failed to get block header by block number or hash", blockNumOrHash return Opt.none(Bytecode) - await n.getCodeByStateRoot(stateRoot, address) + await n.getCodeByStateRoot(header.stateRoot, address, Opt.some(header.rlpHash())) # Used by: eth_getProof proc getProofs*( @@ -308,8 +405,10 @@ proc getProofs*( address: Address, slotKeys: seq[UInt256], ): Future[Opt[Proofs]] {.async: (raises: [CancelledError]).} = - let stateRoot = (await n.getStateRootByBlockNumOrHash(blockNumOrHash)).valueOr: - warn "Failed to get state root by block number or hash", blockNumOrHash + let header = (await n.getBlockHeaderByBlockNumOrHash(blockNumOrHash)).valueOr: + warn "Failed to get block header by block number or hash", blockNumOrHash return Opt.none(Proofs) - await n.getProofsByStateRoot(stateRoot, address, slotKeys) + await n.getProofsByStateRoot( + header.stateRoot, address, slotKeys, Opt.some(header.rlpHash()) + ) diff --git a/fluffy/network/state/state_network.nim b/fluffy/network/state/state_network.nim index da01a7368..1531d5372 100644 --- a/fluffy/network/state/state_network.nim +++ b/fluffy/network/state/state_network.nim @@ -81,8 +81,9 @@ proc new*( proc getContent( n: StateNetwork, - key: AccountTrieNodeKey | ContractTrieNodeKey | ContractCodeKey, + key: ContentKeyType, V: type ContentRetrievalType, + maybeParentOffer: Opt[ContentOfferType], ): Future[Opt[V]] {.async: (raises: [CancelledError]).} = let contentKeyBytes = key.toContentKey().encode() @@ -118,42 +119,54 @@ proc getContent( contentKeyBytes, contentId, contentValueBytes, cacheContent = true ) + if maybeParentOffer.isSome(): + let offer = contentValue.toOffer(maybeParentOffer.get()) + n.portalProtocol.triggerPoke( + contentLookupResult.nodesInterestedInContent, contentKeyBytes, offer.encode() + ) + return Opt.some(contentValue) # Content was requested `1 + requestRetries` times and all failed on validation Opt.none(V) proc getAccountTrieNode*( - n: StateNetwork, key: AccountTrieNodeKey + n: StateNetwork, + key: AccountTrieNodeKey, + maybeParentOffer = Opt.none(AccountTrieNodeOffer), ): Future[Opt[AccountTrieNodeRetrieval]] {. async: (raw: true, raises: [CancelledError]) .} = - n.getContent(key, AccountTrieNodeRetrieval) + n.getContent(key, AccountTrieNodeRetrieval, maybeParentOffer) proc getContractTrieNode*( - n: StateNetwork, key: ContractTrieNodeKey + n: StateNetwork, + key: ContractTrieNodeKey, + maybeParentOffer = Opt.none(ContractTrieNodeOffer), ): Future[Opt[ContractTrieNodeRetrieval]] {. async: (raw: true, raises: [CancelledError]) .} = - n.getContent(key, ContractTrieNodeRetrieval) + n.getContent(key, ContractTrieNodeRetrieval, maybeParentOffer) proc getContractCode*( - n: StateNetwork, key: ContractCodeKey + n: StateNetwork, + key: ContractCodeKey, + maybeParentOffer = Opt.none(ContractCodeOffer), ): Future[Opt[ContractCodeRetrieval]] {.async: (raw: true, raises: [CancelledError]).} = - n.getContent(key, ContractCodeRetrieval) + n.getContent(key, ContractCodeRetrieval, maybeParentOffer) -proc getStateRootByBlockNumOrHash*( +proc getBlockHeaderByBlockNumOrHash*( n: StateNetwork, blockNumOrHash: uint64 | Hash32 -): Future[Opt[Hash32]] {.async: (raises: [CancelledError]).} = +): Future[Opt[Header]] {.async: (raises: [CancelledError]).} = let hn = n.historyNetwork.valueOr: warn "History network is not available" - return Opt.none(Hash32) + return Opt.none(Header) let header = (await hn.getVerifiedBlockHeader(blockNumOrHash)).valueOr: warn "Failed to get block header from history", blockNumOrHash - return Opt.none(Hash32) + return Opt.none(Header) - Opt.some(header.stateRoot) + Opt.some(header) proc processOffer*( n: StateNetwork, @@ -168,9 +181,9 @@ proc processOffer*( return err("Unable to decode offered content value") validationRes = if n.validateStateIsCanonical: - let stateRoot = (await n.getStateRootByBlockNumOrHash(contentValue.blockHash)).valueOr: - return err("Failed to get state root by block hash") - validateOffer(Opt.some(stateRoot), contentKey, contentValue) + let header = (await n.getBlockHeaderByBlockNumOrHash(contentValue.blockHash)).valueOr: + return err("Failed to get block header by hash") + validateOffer(Opt.some(header.stateRoot), contentKey, contentValue) else: # Skip state root validation validateOffer(Opt.none(Hash32), contentKey, contentValue) @@ -182,7 +195,7 @@ proc processOffer*( return err("Received offered content with invalid content key") n.portalProtocol.storeContent( - contentKeyBytes, contentId, contentValue.toRetrievalValue().encode() + contentKeyBytes, contentId, contentValue.toRetrieval().encode() ) await gossipOffer( diff --git a/fluffy/network/state/state_validation.nim b/fluffy/network/state/state_validation.nim index 3bbe69fa3..18e9e2400 100644 --- a/fluffy/network/state/state_validation.nim +++ b/fluffy/network/state/state_validation.nim @@ -146,7 +146,7 @@ func validateOffer*( ): Result[void, string] = ?validateTrieProof(trustedStateRoot, key.path, offer.proof) - validateRetrieval(key, offer.toRetrievalValue()) + validateRetrieval(key, offer.toRetrieval()) func validateOffer*( trustedStateRoot: Opt[Hash32], @@ -164,7 +164,7 @@ func validateOffer*( ?validateTrieProof(Opt.some(account.storageRoot), key.path, offer.storageProof) - validateRetrieval(key, offer.toRetrievalValue()) + validateRetrieval(key, offer.toRetrieval()) func validateOffer*( trustedStateRoot: Opt[Hash32], key: ContractCodeKey, offer: ContractCodeOffer @@ -180,7 +180,10 @@ func validateOffer*( if not offer.code.hashEquals(account.codeHash): return err("hash of bytecode doesn't match the code hash in the account proof") - validateRetrieval(key, offer.toRetrievalValue()) + validateRetrieval(key, offer.toRetrieval()) + +# Local validations that check the structure of the content keys and values. +# None of the validations below check if the data is canonical or not func validateGetContentKey*( keyBytes: ContentKeyByteList @@ -204,24 +207,52 @@ func validateRetrieval*( let retrieval = ?ContractCodeRetrieval.decode(contentBytes) validateRetrieval(key.contractCodeKey, retrieval) -func validateOfferGetValue*( - trustedStateRoot: Opt[Hash32], key: ContentKey, contentBytes: seq[byte] +func validateRetrievalGetOffer*( + key: ContentKey, contentBytes: seq[byte], parentContentBytes: seq[byte] ): Result[seq[byte], string] = - let value = - case key.contentType - of unused: - raiseAssert("ContentKey contentType: unused") - of accountTrieNode: - let offer = ?AccountTrieNodeOffer.decode(contentBytes) - ?validateOffer(trustedStateRoot, key.accountTrieNodeKey, offer) - offer.toRetrievalValue.encode() - of contractTrieNode: - let offer = ?ContractTrieNodeOffer.decode(contentBytes) - ?validateOffer(trustedStateRoot, key.contractTrieNodeKey, offer) - offer.toRetrievalValue.encode() - of contractCode: - let offer = ?ContractCodeOffer.decode(contentBytes) - ?validateOffer(trustedStateRoot, key.contractCodeKey, offer) - offer.toRetrievalValue.encode() + case key.contentType + of unused: + raiseAssert("ContentKey contentType: unused") + of accountTrieNode: + let + retrieval = ?AccountTrieNodeRetrieval.decode(contentBytes) + parentOffer = ?AccountTrieNodeOffer.decode(parentContentBytes) + offer = retrieval.toOffer(parentOffer) + ?validateRetrieval(key.accountTrieNodeKey, retrieval) + ?validateOffer(Opt.none(Hash32), key.accountTrieNodeKey, offer) + ok(offer.encode()) + of contractTrieNode: + let + retrieval = ?ContractTrieNodeRetrieval.decode(contentBytes) + parentOffer = ?ContractTrieNodeOffer.decode(parentContentBytes) + offer = retrieval.toOffer(parentOffer) + ?validateRetrieval(key.contractTrieNodeKey, retrieval) + ?validateOffer(Opt.none(Hash32), key.contractTrieNodeKey, offer) + ok(offer.encode()) + of contractCode: + let + retrieval = ?ContractCodeRetrieval.decode(contentBytes) + parentOffer = ?ContractCodeOffer.decode(parentContentBytes) + offer = retrieval.toOffer(parentOffer) + ?validateRetrieval(key.contractCodeKey, retrieval) + ?validateOffer(Opt.none(Hash32), key.contractCodeKey, offer) + ok(offer.encode()) - ok(value) +func validateOfferGetRetrieval*( + key: ContentKey, contentBytes: seq[byte] +): Result[seq[byte], string] = + case key.contentType + of unused: + raiseAssert("ContentKey contentType: unused") + of accountTrieNode: + let offer = ?AccountTrieNodeOffer.decode(contentBytes) + ?validateOffer(Opt.none(Hash32), key.accountTrieNodeKey, offer) + ok(offer.toRetrieval.encode()) + of contractTrieNode: + let offer = ?ContractTrieNodeOffer.decode(contentBytes) + ?validateOffer(Opt.none(Hash32), key.contractTrieNodeKey, offer) + ok(offer.toRetrieval.encode()) + of contractCode: + let offer = ?ContractCodeOffer.decode(contentBytes) + ?validateOffer(Opt.none(Hash32), key.contractCodeKey, offer) + ok(offer.toRetrieval.encode()) diff --git a/fluffy/rpc/rpc_portal_state_api.nim b/fluffy/rpc/rpc_portal_state_api.nim index c35ae0d27..e6c8ebdd1 100644 --- a/fluffy/rpc/rpc_portal_state_api.nim +++ b/fluffy/rpc/rpc_portal_state_api.nim @@ -74,7 +74,7 @@ proc installPortalStateApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) = contentBytes = hexToSeqByte(contentItem[1]) contentKV = ContentKV(contentKey: keyBytes, content: contentBytes) - discard validateOfferGetValue(Opt.none(Hash32), key, contentBytes).valueOr: + discard validateOfferGetRetrieval(key, contentBytes).valueOr: raise invalidValueErr() contentItemsToOffer.add(contentKV) @@ -134,7 +134,7 @@ proc installPortalStateApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) = (key, contentId) = validateGetContentKey(keyBytes).valueOr: raise invalidKeyErr() contentBytes = hexToSeqByte(content) - contentValue = validateOfferGetValue(Opt.none(Hash32), key, contentBytes).valueOr: + contentValue = validateOfferGetRetrieval(key, contentBytes).valueOr: raise invalidValueErr() p.storeContent(keyBytes, contentId, contentValue) @@ -156,7 +156,7 @@ proc installPortalStateApiHandlers*(rpcServer: RpcServer, p: PortalProtocol) = (key, contentId) = validateGetContentKey(keyBytes).valueOr: raise invalidKeyErr() contentBytes = hexToSeqByte(content) - contentValue = validateOfferGetValue(Opt.none(Hash32), key, contentBytes).valueOr: + contentValue = validateOfferGetRetrieval(key, contentBytes).valueOr: raise invalidValueErr() p.storeContent(keyBytes, contentId, contentValue) diff --git a/fluffy/tests/state_network_tests/test_state_endpoints_genesis.nim b/fluffy/tests/state_network_tests/test_state_endpoints_genesis.nim index 915a7809c..534227b53 100644 --- a/fluffy/tests/state_network_tests/test_state_endpoints_genesis.nim +++ b/fluffy/tests/state_network_tests/test_state_endpoints_genesis.nim @@ -41,7 +41,7 @@ suite "State Endpoints - Genesis JSON Files": # store the account leaf node let contentKey = key.toContentKey().encode() stateNode.portalProtocol.storeContent( - contentKey, contentKey.toContentId(), offer.toRetrievalValue().encode() + contentKey, contentKey.toContentId(), offer.toRetrieval().encode() ) # store the account parent nodes / all remaining nodes @@ -52,7 +52,7 @@ suite "State Endpoints - Genesis JSON Files": stateNode.portalProtocol.storeContent( parentContentKey, parentContentKey.toContentId(), - parent.offer.toRetrievalValue().encode(), + parent.offer.toRetrieval().encode(), ) for i in proof.low ..< proof.high - 1: @@ -62,7 +62,7 @@ suite "State Endpoints - Genesis JSON Files": stateNode.portalProtocol.storeContent( parentContentKey, parentContentKey.toContentId(), - parent.offer.toRetrievalValue().encode(), + parent.offer.toRetrieval().encode(), ) proc setupCodeInDb( @@ -101,7 +101,7 @@ suite "State Endpoints - Genesis JSON Files": # store the contract storage leaf node let contentKey = key.toContentKey().encode() stateNode.portalProtocol.storeContent( - contentKey, contentKey.toContentId(), offer.toRetrievalValue().encode() + contentKey, contentKey.toContentId(), offer.toRetrieval().encode() ) # store the remaining contract storage nodes @@ -112,7 +112,7 @@ suite "State Endpoints - Genesis JSON Files": stateNode.portalProtocol.storeContent( parentContentKey, parentContentKey.toContentId(), - parent.offer.toRetrievalValue().encode(), + parent.offer.toRetrieval().encode(), ) for i in storageProof.low ..< storageProof.high - 1: @@ -122,7 +122,7 @@ suite "State Endpoints - Genesis JSON Files": stateNode.portalProtocol.storeContent( parentContentKey, parentContentKey.toContentId(), - parent.offer.toRetrievalValue().encode(), + parent.offer.toRetrieval().encode(), ) asyncTest "Test getBalance, getTransactionCount, getStorageAt and getCode using JSON files": diff --git a/fluffy/tests/state_network_tests/test_state_gossip_getparent_genesis.nim b/fluffy/tests/state_network_tests/test_state_gossip_getparent_genesis.nim index c4610aec0..7cfa82ea0 100644 --- a/fluffy/tests/state_network_tests/test_state_gossip_getparent_genesis.nim +++ b/fluffy/tests/state_network_tests/test_state_gossip_getparent_genesis.nim @@ -39,13 +39,13 @@ suite "State Gossip getParent - Genesis JSON Files": offer = AccountTrieNodeOffer(proof: proof) var db = newMemoryDB() - db.put(key.nodeHash.data, offer.toRetrievalValue().node.asSeq()) + db.put(key.nodeHash.data, offer.toRetrieval().node.asSeq()) # validate each parent offer until getting to the root node var parent = offer.withKey(key).getParent() check validateOffer(Opt.some(accountState.rootHash()), parent.key, parent.offer) .isOk() - db.put(parent.key.nodeHash.data, parent.offer.toRetrievalValue().node.asSeq()) + db.put(parent.key.nodeHash.data, parent.offer.toRetrieval().node.asSeq()) for i in proof.low ..< proof.high - 1: parent = parent.getParent() @@ -53,7 +53,7 @@ suite "State Gossip getParent - Genesis JSON Files": Opt.some(accountState.rootHash()), parent.key, parent.offer ) .isOk() - db.put(parent.key.nodeHash.data, parent.offer.toRetrievalValue().node.asSeq()) + db.put(parent.key.nodeHash.data, parent.offer.toRetrieval().node.asSeq()) # after putting all parent nodes into the trie, verify can lookup the leaf let @@ -94,7 +94,7 @@ suite "State Gossip getParent - Genesis JSON Files": ) var db = newMemoryDB() - db.put(key.nodeHash.data, offer.toRetrievalValue().node.asSeq()) + db.put(key.nodeHash.data, offer.toRetrieval().node.asSeq()) # validate each parent offer until getting to the root node var parent = offer.withKey(key).getParent() @@ -102,9 +102,7 @@ suite "State Gossip getParent - Genesis JSON Files": Opt.some(accountState.rootHash()), parent.key, parent.offer ) .isOk() - db.put( - parent.key.nodeHash.data, parent.offer.toRetrievalValue().node.asSeq() - ) + db.put(parent.key.nodeHash.data, parent.offer.toRetrieval().node.asSeq()) for i in storageProof.low ..< storageProof.high - 1: parent = parent.getParent() @@ -112,9 +110,7 @@ suite "State Gossip getParent - Genesis JSON Files": Opt.some(accountState.rootHash()), parent.key, parent.offer ) .isOk() - db.put( - parent.key.nodeHash.data, parent.offer.toRetrievalValue().node.asSeq() - ) + db.put(parent.key.nodeHash.data, parent.offer.toRetrieval().node.asSeq()) # after putting all parent nodes into the trie, verify can lookup the leaf let diff --git a/fluffy/tests/state_network_tests/test_state_gossip_getparent_vectors.nim b/fluffy/tests/state_network_tests/test_state_gossip_getparent_vectors.nim index 07c57bca4..68390292b 100644 --- a/fluffy/tests/state_network_tests/test_state_gossip_getparent_vectors.nim +++ b/fluffy/tests/state_network_tests/test_state_gossip_getparent_vectors.nim @@ -39,7 +39,7 @@ suite "State Gossip getParent - Test Vectors": parentKey.toContentKey().encode() == parentTestData.content_key.hexToSeqByte().ContentKeyByteList parentOffer.encode() == parentTestData.content_value_offer.hexToSeqByte() - parentOffer.toRetrievalValue().encode() == + parentOffer.toRetrieval().encode() == parentTestData.content_value_retrieval.hexToSeqByte() test "Check contract storage trie node parent matches expected recursive gossip": @@ -68,5 +68,5 @@ suite "State Gossip getParent - Test Vectors": parentKey.toContentKey().encode() == parentTestData.content_key.hexToSeqByte().ContentKeyByteList parentOffer.encode() == parentTestData.content_value_offer.hexToSeqByte() - parentOffer.toRetrievalValue().encode() == + parentOffer.toRetrieval().encode() == parentTestData.content_value_retrieval.hexToSeqByte() diff --git a/fluffy/tests/state_network_tests/test_state_gossip_gossipoffer_vectors.nim b/fluffy/tests/state_network_tests/test_state_gossip_gossipoffer_vectors.nim index eefe5437f..f33832004 100644 --- a/fluffy/tests/state_network_tests/test_state_gossip_gossipoffer_vectors.nim +++ b/fluffy/tests/state_network_tests/test_state_gossip_gossipoffer_vectors.nim @@ -78,8 +78,8 @@ procSuite "State Gossip - Gossip Offer": check: stateNode2.containsId(contentId) res1.isOk() - res1.get() == contentValue.toRetrievalValue() - res1.get().node == contentValue.toRetrievalValue().node + res1.get() == contentValue.toRetrieval() + res1.get().node == contentValue.toRetrieval().node # check that the parent offer was not received by the second state instance let res2 = await stateNode2.stateNetwork.getAccountTrieNode( @@ -147,8 +147,8 @@ procSuite "State Gossip - Gossip Offer": check: stateNode2.containsId(contentId) res1.isOk() - res1.get() == contentValue.toRetrievalValue() - res1.get().node == contentValue.toRetrievalValue().node + res1.get() == contentValue.toRetrieval() + res1.get().node == contentValue.toRetrieval().node # check that the offer parent was not received by the second state instance let res2 = await stateNode2.stateNetwork.getContractTrieNode( @@ -205,8 +205,8 @@ procSuite "State Gossip - Gossip Offer": check: stateNode2.containsId(contentId) res1.isOk() - res1.get() == contentValue.toRetrievalValue() - res1.get().code == contentValue.toRetrievalValue().code + res1.get() == contentValue.toRetrieval() + res1.get().code == contentValue.toRetrieval().code await stateNode1.stop() await stateNode2.stop() diff --git a/fluffy/tests/state_network_tests/test_state_network_offercontent_vectors.nim b/fluffy/tests/state_network_tests/test_state_network_offercontent_vectors.nim index 8b140ba8b..360c7af07 100644 --- a/fluffy/tests/state_network_tests/test_state_network_offercontent_vectors.nim +++ b/fluffy/tests/state_network_tests/test_state_network_offercontent_vectors.nim @@ -90,8 +90,8 @@ procSuite "State Network - Offer Content": check: stateNode1.containsId(contentId) getRes.isOk() - getRes.get() == contentValue.toRetrievalValue() - getRes.get().node == contentValue.toRetrievalValue().node + getRes.get() == contentValue.toRetrieval() + getRes.get().node == contentValue.toRetrieval().node await stateNode1.stop() @@ -159,8 +159,8 @@ procSuite "State Network - Offer Content": check: stateNode1.containsId(contentId) getRes.isOk() - getRes.get() == contentValue.toRetrievalValue() - getRes.get().node == contentValue.toRetrievalValue().node + getRes.get() == contentValue.toRetrieval() + getRes.get().node == contentValue.toRetrieval().node await stateNode1.stop() @@ -227,8 +227,8 @@ procSuite "State Network - Offer Content": check: stateNode1.containsId(contentId) getRes.isOk() - getRes.get() == contentValue.toRetrievalValue() - getRes.get().code == contentValue.toRetrievalValue().code + getRes.get() == contentValue.toRetrieval() + getRes.get().code == contentValue.toRetrieval().code await stateNode1.stop() @@ -277,8 +277,8 @@ procSuite "State Network - Offer Content": check: stateNode2.containsId(contentId) getRes.isOk() - getRes.get() == contentValue.toRetrievalValue() - getRes.get().node == contentValue.toRetrievalValue().node + getRes.get() == contentValue.toRetrieval() + getRes.get().node == contentValue.toRetrieval().node await stateNode1.stop() await stateNode2.stop() @@ -327,8 +327,8 @@ procSuite "State Network - Offer Content": check: stateNode2.containsId(contentId) getRes.isOk() - getRes.get() == contentValue.toRetrievalValue() - getRes.get().node == contentValue.toRetrievalValue().node + getRes.get() == contentValue.toRetrieval() + getRes.get().node == contentValue.toRetrieval().node await stateNode1.stop() await stateNode2.stop() @@ -376,8 +376,8 @@ procSuite "State Network - Offer Content": check: stateNode2.containsId(contentId) getRes.isOk() - getRes.get() == contentValue.toRetrievalValue() - getRes.get().code == contentValue.toRetrievalValue().code + getRes.get() == contentValue.toRetrieval() + getRes.get().code == contentValue.toRetrieval().code await stateNode1.stop() await stateNode2.stop()