Fluffy recursive gossip improvements, fixes and tests. (#2231)

* Refactor get parent gossip code and add Nibbles helper function. * Add logging to state gossip. * Unit test recursive gossip using state gossip getParent functions. * Add recursive gossip genesis json test and fix bug in state gossip getParent. * Add Nibbles len function.
2025-02-02 23:35:31 +00:00 · 2024-05-28 08:45:30 +08:00 · 2024-05-28 08:45:30 +08:00 · 9354cb8411
commit 9354cb8411
parent f932c8df22
11 changed files with 383 additions and 94 deletions
--- a/fluffy/network/state/content/nibbles.nim
+++ b/fluffy/network/state/content/nibbles.nim
@ -25,6 +25,7 @@ const
  MAX_UNPACKED_NIBBLES_LEN = 64

 type Nibbles* = List[byte, MAX_PACKED_NIBBLES_LEN]
+type UnpackedNibbles* = seq[byte]

 func init*(T: type Nibbles, packed: openArray[byte], isEven: bool): T =
  doAssert(packed.len() <= MAX_PACKED_NIBBLES_LEN)
@ -79,7 +80,7 @@ func packNibbles*(unpacked: openArray[byte]): Nibbles =

  Nibbles(output)

-func unpackNibbles*(packed: Nibbles): seq[byte] =
+func unpackNibbles*(packed: Nibbles): UnpackedNibbles =
  doAssert(packed.len() <= MAX_PACKED_NIBBLES_LEN, "Packed nibbles length is too long")

  var output = newSeqOfCap[byte](packed.len() * 2)
@ -98,4 +99,17 @@ func unpackNibbles*(packed: Nibbles): seq[byte] =
      output.add(first)
      output.add(second)

-  output
+  move(output)
+
+func len(packed: Nibbles): int =
+  let lenExclPrefix = (packed.len() - 1) * 2
+
+  if packed[0] == 0x00: # is even length
+    lenExclPrefix
+  else:
+    lenExclPrefix + 1
+
+func dropN*(unpacked: UnpackedNibbles, num: int): UnpackedNibbles =
+  var nibbles = unpacked
+  nibbles.setLen(nibbles.len() - num)
+  move(nibbles)
--- a/fluffy/network/state/state_gossip.nim
+++ b/fluffy/network/state/state_gossip.nim
@ -19,57 +19,71 @@ export results, state_content
 logScope:
  topics = "portal_state"

-func getParent(nibbles: Nibbles, proof: TrieProof): (Nibbles, TrieProof) =
-  doAssert(nibbles.len() > 0, "nibbles too short")
-  doAssert(proof.len() > 1, "proof too short")
+type ProofWithPath = tuple[path: Nibbles, proof: TrieProof]
+
+type AccountTrieOfferWithKey* =
+  tuple[key: AccountTrieNodeKey, offer: AccountTrieNodeOffer]
+
+type ContractTrieOfferWithKey* =
+  tuple[key: ContractTrieNodeKey, offer: ContractTrieNodeOffer]
+
+func withPath(proof: TrieProof, path: Nibbles): ProofWithPath =
+  (path: path, proof: proof)
+
+func withKey*(
+    offer: AccountTrieNodeOffer, key: AccountTrieNodeKey
+): AccountTrieOfferWithKey =
+  (key: key, offer: offer)
+
+func withKey*(
+    offer: ContractTrieNodeOffer, key: ContractTrieNodeKey
+): ContractTrieOfferWithKey =
+  (key: key, offer: offer)
+
+func getParent(p: ProofWithPath): ProofWithPath =
+  doAssert(p.path.len() > 0, "nibbles too short")
+  doAssert(p.proof.len() > 1, "proof too short")

  let
-    parentProof = TrieProof.init(proof[0 ..^ 2])
+    parentProof = TrieProof.init(p.proof[0 ..^ 2])
    parentEndNode = rlpFromBytes(parentProof[^1].asSeq())

  # the trie proof should have already been validated when receiving the offer content
  doAssert(parentEndNode.listLen() == 2 or parentEndNode.listLen() == 17)

-  var unpackedNibbles = nibbles.unpackNibbles()
+  var unpackedNibbles = p.path.unpackNibbles()

  if parentEndNode.listLen() == 17:
    # branch node so only need to remove a single nibble
-    unpackedNibbles.setLen(unpackedNibbles.len() - 1)
-    return (unpackedNibbles.packNibbles(), parentProof)
+    return parentProof.withPath(unpackedNibbles.dropN(1).packNibbles())

  # leaf or extension node so we need to remove one or more nibbles
-  let (_, isEven, prefixNibbles) = decodePrefix(parentEndNode.listElem(0))
+  let prefixNibbles = decodePrefix(parentEndNode.listElem(0))[2]

-  var removeCount = (prefixNibbles.len() - 1) * 2
-  if not isEven:
-    inc removeCount
+  parentProof.withPath(unpackedNibbles.dropN(prefixNibbles.len()).packNibbles())

-  unpackedNibbles.setLen(unpackedNibbles.len() - removeCount)
-  (unpackedNibbles.packNibbles(), parentProof)
-
-func getParent*(
-    key: AccountTrieNodeKey, offer: AccountTrieNodeOffer
-): (AccountTrieNodeKey, AccountTrieNodeOffer) =
+func getParent*(offerWithKey: AccountTrieOfferWithKey): AccountTrieOfferWithKey =
  let
-    (parentNibbles, parentProof) = getParent(key.path, offer.proof)
-    parentKey =
-      AccountTrieNodeKey.init(parentNibbles, keccakHash(parentProof[^1].asSeq()))
+    (key, offer) = offerWithKey
+    (parentPath, parentProof) = offer.proof.withPath(key.path).getParent()
+
+    parentKey = AccountTrieNodeKey.init(parentPath, keccakHash(parentProof[^1].asSeq()))
    parentOffer = AccountTrieNodeOffer.init(parentProof, offer.blockHash)

-  (parentKey, parentOffer)
+  parentOffer.withKey(parentKey)

-func getParent*(
-    key: ContractTrieNodeKey, offer: ContractTrieNodeOffer
-): (ContractTrieNodeKey, ContractTrieNodeOffer) =
+func getParent*(offerWithKey: ContractTrieOfferWithKey): ContractTrieOfferWithKey =
  let
-    (parentNibbles, parentProof) = getParent(key.path, offer.storageProof)
+    (key, offer) = offerWithKey
+    (parentPath, parentProof) = offer.storageProof.withPath(key.path).getParent()
+
    parentKey = ContractTrieNodeKey.init(
-      key.address, parentNibbles, keccakHash(parentProof[^1].asSeq())
+      key.address, parentPath, keccakHash(parentProof[^1].asSeq())
    )
    parentOffer =
      ContractTrieNodeOffer.init(parentProof, offer.accountProof, offer.blockHash)

-  (parentKey, parentOffer)
+  parentOffer.withKey(parentKey)

 proc gossipOffer*(
    p: PortalProtocol,
@ -79,20 +93,25 @@ proc gossipOffer*(
    key: AccountTrieNodeKey,
    offer: AccountTrieNodeOffer,
 ) {.async.} =
-  asyncSpawn p.neighborhoodGossipDiscardPeers(
+  let req1Peers = await p.neighborhoodGossip(
    srcNodeId, ContentKeysList.init(@[keyBytes]), @[offerBytes]
  )
+  info "Offered content gossipped successfully with peers", keyBytes, peers = req1Peers

  # root node, recursive gossip is finished
  if key.path.unpackNibbles().len() == 0:
    return

-  let (parentKey, parentOffer) = getParent(key, offer)
-  asyncSpawn p.neighborhoodGossipDiscardPeers(
-    srcNodeId,
-    ContentKeysList.init(@[parentKey.toContentKey().encode()]),
-    @[parentOffer.encode()],
-  )
+  # continue the recursive gossip by sharing the parent offer with peers
+  let
+    (parentKey, parentOffer) = offer.withKey(key).getParent()
+    parentKeyBytes = parentKey.toContentKey().encode()
+    req2Peers = await p.neighborhoodGossip(
+      srcNodeId, ContentKeysList.init(@[parentKeyBytes]), @[parentOffer.encode()]
+    )
+
+  info "Offered content parent gossipped successfully with peers",
+    parentKeyBytes, peers = req2Peers

 proc gossipOffer*(
    p: PortalProtocol,
@ -102,20 +121,25 @@ proc gossipOffer*(
    key: ContractTrieNodeKey,
    offer: ContractTrieNodeOffer,
 ) {.async.} =
-  asyncSpawn p.neighborhoodGossipDiscardPeers(
+  let req1Peers = await p.neighborhoodGossip(
    srcNodeId, ContentKeysList.init(@[keyBytes]), @[offerBytes]
  )
+  info "Offered content gossipped successfully with peers", keyBytes, peers = req1Peers

  # root node, recursive gossip is finished
  if key.path.unpackNibbles().len() == 0:
    return

-  let (parentKey, parentOffer) = getParent(key, offer)
-  asyncSpawn p.neighborhoodGossipDiscardPeers(
-    srcNodeId,
-    ContentKeysList.init(@[parentKey.toContentKey().encode()]),
-    @[parentOffer.encode()],
-  )
+  # continue the recursive gossip by sharing the parent offer with peers
+  let
+    (parentKey, parentOffer) = offer.withKey(key).getParent()
+    parentKeyBytes = parentKey.toContentKey().encode()
+    req2Peers = await p.neighborhoodGossip(
+      srcNodeId, ContentKeysList.init(@[parentKeyBytes]), @[parentOffer.encode()]
+    )
+
+  info "Offered content parent gossipped successfully with peers",
+    parentKeyBytes, peers = req2Peers

 proc gossipOffer*(
    p: PortalProtocol,
@ -125,6 +149,7 @@ proc gossipOffer*(
    key: ContractCodeKey,
    offer: ContractCodeOffer,
 ) {.async.} =
-  asyncSpawn p.neighborhoodGossipDiscardPeers(
+  let peers = await p.neighborhoodGossip(
    srcNodeId, ContentKeysList.init(@[keyBytes]), @[offerBytes]
  )
+  info "Offered content gossipped successfully with peers", keyBytes, peers
--- a/fluffy/network/state/state_network.nim
+++ b/fluffy/network/state/state_network.nim
@ -162,7 +162,7 @@ proc processOffer(

  let res = validateOffer(stateRoot, contentKey, contentValue)
  if res.isErr():
-    return err("Received offered content failed validation: " & res.error())
+    return err("Offered content failed validation: " & res.error())

  let contentId = n.portalProtocol.toContentId(contentKeyBytes).valueOr:
    return err("Received offered content with invalid content key")
@ -170,7 +170,7 @@ proc processOffer(
  n.portalProtocol.storeContent(
    contentKeyBytes, contentId, contentValue.toRetrievalValue().encode()
  )
-  info "Received offered content validated successfully", contentKeyBytes
+  info "Offered content validated successfully", contentKeyBytes

  asyncSpawn gossipOffer(
    n.portalProtocol, maybeSrcNodeId, contentKeyBytes, contentValueBytes, contentKey,
--- a/fluffy/network/state/state_validation.nim
+++ b/fluffy/network/state/state_validation.nim
@ -59,7 +59,7 @@ proc validateTrieProof*(
      if isLastNode:
        break
      else:
-        return err("empty nibbles but proof has more nodes")
+        return err("proof has more nodes then expected for given path")

    case thisNodeRlp.listLen()
    of 2:
--- a/fluffy/tests/state_network_tests/all_state_network_tests.nim
+++ b/fluffy/tests/state_network_tests/all_state_network_tests.nim
@ -9,10 +9,12 @@

 import
  ./test_state_content_keys,
-  ./test_state_content_values,
  ./test_state_content_nibbles,
-  ./test_state_network,
+  ./test_state_content_values,
  #./test_state_network_gossip,
-  ./test_state_validation,
+  ./test_state_network,
+  ./test_state_recursivegossip_genesis,
+  ./test_state_recursivegossip_vectors,
  ./test_state_validation_genesis,
-  ./test_state_validation_trieproof
+  ./test_state_validation_trieproof,
+  ./test_state_validation_vectors
--- a/fluffy/tests/state_network_tests/state_test_helpers.nim
+++ b/fluffy/tests/state_network_tests/state_test_helpers.nim
@ -11,7 +11,45 @@ import
  std/[sugar, sequtils],
  eth/[common, trie, trie/db],
  ../../nimbus/common/chain_config,
-  ../../network/state/[state_content, state_utils]
+  ../../network/state/[state_content, state_utils],
+  ../../eth_data/yaml_utils
+
+export yaml_utils
+
+const testVectorDir* = "./vendor/portal-spec-tests/tests/mainnet/state/validation/"
+
+type
+  YamlTrieNodeRecursiveGossipKV* = ref object
+    content_key*: string
+    content_value_offer*: string
+    content_value_retrieval*: string
+
+  YamlTrieNodeKV* = object
+    state_root*: string
+    content_key*: string
+    content_value_offer*: string
+    content_value_retrieval*: string
+    recursive_gossip*: YamlTrieNodeRecursiveGossipKV
+
+  YamlTrieNodeKVs* = seq[YamlTrieNodeKV]
+
+  YamlContractBytecodeKV* = object
+    state_root*: string
+    content_key*: string
+    content_value_offer*: string
+    content_value_retrieval*: string
+
+  YamlContractBytecodeKVs* = seq[YamlContractBytecodeKV]
+
+  YamlRecursiveGossipKV* = object
+    content_key*: string
+    content_value*: string
+
+  YamlRecursiveGossipData* = object
+    state_root*: string
+    recursive_gossip*: seq[YamlRecursiveGossipKV]
+
+  YamlRecursiveGossipKVs* = seq[YamlRecursiveGossipData]

 func asNibbles*(key: openArray[byte], isEven = true): Nibbles =
  Nibbles.init(key, isEven)
@ -28,8 +66,7 @@ func removeLeafKeyEndNibbles*(
  var unpackedNibbles = nibbles.unpackNibbles()
  doAssert(unpackedNibbles[^leafPrefix.len() .. ^1] == leafPrefix)

-  unpackedNibbles.setLen(unpackedNibbles.len() - leafPrefix.len())
-  unpackedNibbles.packNibbles()
+  unpackedNibbles.dropN(leafPrefix.len()).packNibbles()

 func asTrieProof*(branch: openArray[seq[byte]]): TrieProof =
  TrieProof.init(branch.map(node => TrieNode.init(node)))
@ -38,6 +75,8 @@ proc getTrieProof*(
    state: HexaryTrie, key: openArray[byte]
 ): TrieProof {.raises: [RlpError].} =
  let branch = state.getBranch(key)
+  # for node in branch:
+  #   debugEcho rlp.decode(node)
  branch.asTrieProof()

 proc generateAccountProof*(
--- a/fluffy/tests/state_network_tests/test_state_recursivegossip_genesis.nim
+++ b/fluffy/tests/state_network_tests/test_state_recursivegossip_genesis.nim
@ -0,0 +1,109 @@
+# Nimbus
+# Copyright (c) 2023-2024 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.push raises: [].}
+
+import
+  std/os,
+  unittest2,
+  stew/results,
+  eth/[common, trie, trie/db],
+  ../../../nimbus/common/chain_config,
+  ../../network/state/[state_content, state_validation, state_gossip, state_utils],
+  ./state_test_helpers
+
+suite "State Recursive Gossip - Genesis JSON Files":
+  let genesisFiles = [
+    "berlin2000.json", "calaveras.json", "chainid1.json", "chainid7.json",
+    "devnet4.json", "devnet5.json", "holesky.json", "mainshadow1.json", "merge.json",
+  ]
+
+  test "Recursive gossip account leaf nodes":
+    for file in genesisFiles:
+      let
+        accounts = getGenesisAlloc("fluffy" / "tests" / "custom_genesis" / file)
+        (accountState, _) = accounts.toState()
+
+      for address, account in accounts:
+        let
+          proof = accountState.generateAccountProof(address)
+          leafNode = proof[^1]
+          addressHash = keccakHash(address).data
+          path = removeLeafKeyEndNibbles(Nibbles.init(addressHash, true), leafNode)
+          key = AccountTrieNodeKey.init(path, keccakHash(leafNode.asSeq()))
+          offer = AccountTrieNodeOffer(proof: proof)
+
+        var db = newMemoryDB()
+        db.put(key.nodeHash.data, offer.toRetrievalValue().node.asSeq())
+
+        # validate each parent offer until getting to the root node
+        var parent = offer.withKey(key).getParent()
+        check validateOffer(accountState.rootHash(), parent.key, parent.offer).isOk()
+        db.put(parent.key.nodeHash.data, parent.offer.toRetrievalValue().node.asSeq())
+
+        for i in proof.low ..< proof.high - 1:
+          parent = parent.getParent()
+          check validateOffer(accountState.rootHash(), parent.key, parent.offer).isOk()
+          db.put(parent.key.nodeHash.data, parent.offer.toRetrievalValue().node.asSeq())
+
+        # after putting all parent nodes into the trie, verify can lookup the leaf
+        let
+          trie = initHexaryTrie(db, accountState.rootHash())
+          expectedAcc = rlpDecodeAccountTrieNode(leafNode).get()
+          accBytes = trie.get(addressHash)
+        check rlp.decode(accBytes, Account) == expectedAcc
+
+  test "Recursive gossip contract storage leaf nodes":
+    for file in genesisFiles:
+      let
+        accounts = getGenesisAlloc("fluffy" / "tests" / "custom_genesis" / file)
+        (accountState, storageStates) = accounts.toState()
+
+      for address, account in accounts:
+        let accountProof = accountState.generateAccountProof(address)
+
+        if account.code.len() > 0:
+          let storageState = storageStates[address]
+
+          for slotKey, slotValue in account.storage:
+            let
+              storageProof = storageState.generateStorageProof(slotKey)
+              leafNode = storageProof[^1]
+              slotKeyHash = keccakHash(toBytesBE(slotKey)).data
+              path = removeLeafKeyEndNibbles(
+                Nibbles.init(keccakHash(toBytesBE(slotKey)).data, true), leafNode
+              )
+              key = ContractTrieNodeKey(
+                address: address, path: path, nodeHash: keccakHash(leafNode.asSeq())
+              )
+              offer = ContractTrieNodeOffer(
+                storageProof: storageProof, accountProof: accountProof
+              )
+
+            var db = newMemoryDB()
+            db.put(key.nodeHash.data, offer.toRetrievalValue().node.asSeq())
+
+            # validate each parent offer until getting to the root node
+            var parent = offer.withKey(key).getParent()
+            check validateOffer(accountState.rootHash(), parent.key, parent.offer).isOk()
+            db.put(
+              parent.key.nodeHash.data, parent.offer.toRetrievalValue().node.asSeq()
+            )
+
+            for i in storageProof.low ..< storageProof.high - 1:
+              parent = parent.getParent()
+              check validateOffer(accountState.rootHash(), parent.key, parent.offer)
+              .isOk()
+              db.put(
+                parent.key.nodeHash.data, parent.offer.toRetrievalValue().node.asSeq()
+              )
+
+            # after putting all parent nodes into the trie, verify can lookup the leaf
+            let
+              trie = initHexaryTrie(db, storageState.rootHash())
+              expectedSlotBytes = rlpFromBytes(leafNode.asSeq()).listElem(1).toBytes()
+            check trie.get(slotKeyHash) == expectedSlotBytes
--- a/fluffy/tests/state_network_tests/test_state_recursivegossip_vectors.nim
+++ b/fluffy/tests/state_network_tests/test_state_recursivegossip_vectors.nim
@ -0,0 +1,135 @@
+# Fluffy
+# Copyright (c) 2023-2024 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+import
+  std/[os, strutils],
+  results,
+  unittest2,
+  stew/byteutils,
+  eth/common,
+  ../../common/common_utils,
+  ../../network/state/[state_content, state_gossip],
+  ./state_test_helpers
+
+suite "State Recursive Gossip - Test Vectors":
+  test "Check account trie node parent matches expected recursive gossip":
+    const file = testVectorDir / "account_trie_node.yaml"
+
+    let testCase = YamlTrieNodeKVs.loadFromYaml(file).valueOr:
+      raiseAssert "Cannot read test vector: " & error
+
+    for i, testData in testCase:
+      var stateRoot = KeccakHash.fromBytes(testData.state_root.hexToSeqByte())
+
+      let key = ContentKey.decode(testData.content_key.hexToSeqByte().ByteList).get()
+      let offer =
+        AccountTrieNodeOffer.decode(testData.content_value_offer.hexToSeqByte()).get()
+
+      if i == 1: # second test case only has root node and no recursive gossip
+        doAssertRaises(AssertionDefect):
+          discard offer.withKey(key.accountTrieNodeKey).getParent()
+        continue
+
+      let (parentKey, parentOffer) = offer.withKey(key.accountTrieNodeKey).getParent()
+      check:
+        parentKey.path.unpackNibbles().len() <
+          key.accountTrieNodeKey.path.unpackNibbles().len()
+        parentOffer.proof.len() == offer.proof.len() - 1
+        parentKey.toContentKey().encode() ==
+          testData.recursive_gossip.content_key.hexToSeqByte().ByteList
+        parentOffer.encode() ==
+          testData.recursive_gossip.content_value_offer.hexToSeqByte()
+        parentOffer.toRetrievalValue().encode() ==
+          testData.recursive_gossip.content_value_retrieval.hexToSeqByte()
+
+  test "Check contract storage trie node parent matches expected recursive gossip":
+    const file = testVectorDir / "contract_storage_trie_node.yaml"
+
+    let testCase = YamlTrieNodeKVs.loadFromYaml(file).valueOr:
+      raiseAssert "Cannot read test vector: " & error
+
+    for i, testData in testCase:
+      var stateRoot = KeccakHash.fromBytes(testData.state_root.hexToSeqByte())
+
+      let key = ContentKey.decode(testData.content_key.hexToSeqByte().ByteList).get()
+      let offer =
+        ContractTrieNodeOffer.decode(testData.content_value_offer.hexToSeqByte()).get()
+
+      if i == 1: # second test case only has root node and no recursive gossip
+        doAssertRaises(AssertionDefect):
+          discard offer.withKey(key.contractTrieNodeKey).getParent()
+        continue
+
+      let (parentKey, parentOffer) = offer.withKey(key.contractTrieNodeKey).getParent()
+      check:
+        parentKey.path.unpackNibbles().len() <
+          key.contractTrieNodeKey.path.unpackNibbles().len()
+        parentOffer.storageProof.len() == offer.storageProof.len() - 1
+        parentKey.toContentKey().encode() ==
+          testData.recursive_gossip.content_key.hexToSeqByte().ByteList
+        parentOffer.encode() ==
+          testData.recursive_gossip.content_value_offer.hexToSeqByte()
+        parentOffer.toRetrievalValue().encode() ==
+          testData.recursive_gossip.content_value_retrieval.hexToSeqByte()
+
+  test "Check each account trie node parent matches expected recursive gossip":
+    const file = testVectorDir / "recursive_gossip.yaml"
+
+    let testCase = YamlRecursiveGossipKVs.loadFromYaml(file).valueOr:
+      raiseAssert "Cannot read test vector: " & error
+
+    for i, testData in testCase:
+      if i == 1:
+        continue
+
+      for j in 0 ..< testData.recursive_gossip.high:
+        let
+          key = ContentKey
+            .decode(testData.recursive_gossip[j].content_key.hexToSeqByte().ByteList)
+            .get()
+          offer = AccountTrieNodeOffer
+            .decode(testData.recursive_gossip[j].content_value.hexToSeqByte())
+            .get()
+          (parentKey, parentOffer) = offer.withKey(key.accountTrieNodeKey).getParent()
+
+        check:
+          parentKey.path.unpackNibbles().len() <
+            key.accountTrieNodeKey.path.unpackNibbles().len()
+          parentOffer.proof.len() == offer.proof.len() - 1
+          parentKey.toContentKey().encode() ==
+            testData.recursive_gossip[j + 1].content_key.hexToSeqByte().ByteList
+          parentOffer.encode() ==
+            testData.recursive_gossip[j + 1].content_value.hexToSeqByte()
+
+  test "Check each contract trie node parent matches expected recursive gossip":
+    const file = testVectorDir / "recursive_gossip.yaml"
+
+    let testCase = YamlRecursiveGossipKVs.loadFromYaml(file).valueOr:
+      raiseAssert "Cannot read test vector: " & error
+
+    for i, testData in testCase:
+      if i != 1:
+        continue
+
+      for j in 0 ..< testData.recursive_gossip.high:
+        let
+          key = ContentKey
+            .decode(testData.recursive_gossip[j].content_key.hexToSeqByte().ByteList)
+            .get()
+          offer = ContractTrieNodeOffer
+            .decode(testData.recursive_gossip[j].content_value.hexToSeqByte())
+            .get()
+          (parentKey, parentOffer) = offer.withKey(key.contractTrieNodeKey).getParent()
+
+        check:
+          parentKey.path.unpackNibbles().len() <
+            key.contractTrieNodeKey.path.unpackNibbles().len()
+          parentOffer.storageProof.len() == offer.storageProof.len() - 1
+          parentKey.toContentKey().encode() ==
+            testData.recursive_gossip[j + 1].content_key.hexToSeqByte().ByteList
+          parentOffer.encode() ==
+            testData.recursive_gossip[j + 1].content_value.hexToSeqByte()
--- a/fluffy/tests/state_network_tests/test_state_validation_genesis.nim
+++ b/fluffy/tests/state_network_tests/test_state_validation_genesis.nim
@ -126,7 +126,7 @@ template checkInvalidProofsWithBadValue(
          )
        check proofResult.isErr()

-suite "State Proof Verification Tests":
+suite "State Validation - Genesis JSON Files":
  let genesisFiles = [
    "berlin2000.json", "calaveras.json", "chainid1.json", "chainid7.json",
    "devnet4.json", "devnet5.json", "holesky.json", "mainshadow1.json", "merge.json",
--- a/fluffy/tests/state_network_tests/test_state_validation_trieproof.nim
+++ b/fluffy/tests/state_network_tests/test_state_validation_trieproof.nim
@ -23,7 +23,7 @@ proc getKeyBytes(i: int): seq[byte] =
  let hash = keccakHash(u256(i).toBytesBE())
  return toSeq(hash.data)

-suite "MPT trie proof verification":
+suite "State Validation - validateTrieProof":
  test "Validate proof for existing value":
    let numValues = 1000
    var trie = initHexaryTrie(newMemoryDB())
--- a/fluffy/tests/state_network_tests/test_state_validation_vectors.nim
+++ b/fluffy/tests/state_network_tests/test_state_validation_vectors.nim
@ -12,45 +12,10 @@ import
  stew/byteutils,
  eth/common,
  ../../common/common_utils,
-  ../../network/state/state_content,
-  ../../network/state/state_validation,
-  ../../eth_data/yaml_utils
+  ../../network/state/[state_content, state_validation],
+  ./state_test_helpers

-const testVectorDir = "./vendor/portal-spec-tests/tests/mainnet/state/validation/"
-
-type YamlTrieNodeRecursiveGossipKV = ref object
-  content_key: string
-  content_value_offer: string
-  content_value_retrieval: string
-
-type YamlTrieNodeKV = object
-  state_root: string
-  content_key: string
-  content_value_offer: string
-  content_value_retrieval: string
-  recursive_gossip: YamlTrieNodeRecursiveGossipKV
-
-type YamlTrieNodeKVs = seq[YamlTrieNodeKV]
-
-type YamlContractBytecodeKV = object
-  state_root: string
-  content_key: string
-  content_value_offer: string
-  content_value_retrieval: string
-
-type YamlContractBytecodeKVs = seq[YamlContractBytecodeKV]
-
-type YamlRecursiveGossipKV = object
-  content_key: string
-  content_value: string
-
-type YamlRecursiveGossipData = object
-  state_root: string
-  recursive_gossip: seq[YamlRecursiveGossipKV]
-
-type YamlRecursiveGossipKVs = seq[YamlRecursiveGossipData]
-
-suite "State Validation":
+suite "State Validation - Test Vectors":
  # Retrieval validation tests

  test "Validate valid AccountTrieNodeRetrieval nodes":