Implement building offers from proofs.

fluffy/network/state/content/nibbles.nim

@@ -102,7 +102,7 @@ func unpackNibbles*(packed: Nibbles): UnpackedNibbles =
       output.add(first)
       output.add(second)
 
-  move(output)
+  ensureMove(output)
 
 func len(packed: Nibbles): int =
   let lenExclPrefix = (packed.len() - 1) * 2
@@ -115,4 +115,4 @@ func len(packed: Nibbles): int =
 func dropN*(unpacked: UnpackedNibbles, num: int): UnpackedNibbles =
   var nibbles = unpacked
   nibbles.setLen(nibbles.len() - num)
-  move(nibbles)
+  ensureMove(nibbles)

fluffy/tools/portal_bridge/portal_bridge_state.nim

@@ -12,13 +12,15 @@ import
   chronicles,
   chronos,
   stint,
+  # stew/byteutils,
   web3/[eth_api, eth_api_types],
   results,
   eth/common/[eth_types, eth_types_rlp],
   ../../../nimbus/common/chain_config,
+  ../../common/common_utils,
   ../../rpc/rpc_calls/rpc_trace_calls,
   ../../network/state/state_content,
-  ./state_bridge/[database, state_diff, world_state_helper],
+  ./state_bridge/[database, state_diff, world_state_helper, offers_builder],
   ./[portal_bridge_conf, portal_bridge_common]
 
 type BlockDataRef = ref object
@@ -118,6 +120,23 @@ proc runBackfillBuildBlockOffersLoop(
         except CatchableError as e:
           raiseAssert(e.msg) # Should never happen
     ws.applyGenesisAccounts(genesisAccounts)
+
+    let genesisBlockHash = KeccakHash.fromHex(
+      "d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"
+    )
+
+    var builder = OffersBuilderRef.init(ws, genesisBlockHash)
+    builder.buildBlockOffers()
+
+    await blockOffersQueue.addLast(
+      BlockOffersRef(
+        blockNumber: 0.uint64,
+        accountTrieOffers: builder.getAccountTrieOffers(),
+        contractTrieOffers: builder.getContractTrieOffers(),
+        contractCodeOffers: builder.getContractCodeOffers(),
+      )
+    )
+
     ws
 
   while true:
@@ -130,6 +149,9 @@ proc runBackfillBuildBlockOffersLoop(
     # because the DatabaseRef currently only supports reading and writing to/from
     # a single active transaction.
     db.withTransaction:
+      defer:
+        worldState.clearPreimages()
+
       for stateDiff in blockData.stateDiffs:
         worldState.applyStateDiff(stateDiff)
       let
@@ -139,24 +161,23 @@ proc runBackfillBuildBlockOffersLoop(
           blockData.uncleBlocks.mapIt((EthAddress(it.miner), it.number.uint64))
       worldState.applyBlockRewards(minerData, uncleMinersData)
 
-      for address, proof in worldState.updatedAccountProofs():
-        # echo "Account Address: ", address
-        # echo "Account Proof.len(): ", proof.len()
+      doAssert(blockData.blockObject.stateRoot.bytes() == worldState.stateRoot.data)
+      trace "State diffs successfully applied to block number:",
+        blockNumber = blockData.blockNumber
 
-        for slotHash, storageProof in worldState.updatedStorageProofs(address):
-          debug "Storage SlotHash: ", slotHash
-          debug "Storage Proof.len(): ", storageProofLen = storageProof.len()
-          debug "Storage Proof: ", storageProof
+      var builder = OffersBuilderRef.init(
+        worldState, KeccakHash.fromBytes(blockData.blockObject.hash.bytes())
+      )
+      builder.buildBlockOffers()
 
-      for address, code in worldState.updatedBytecode():
-        debug "Bytecode Address: ", address
-        debug "Bytecode: ", code
-
-      worldState.clearPreimages()
-
-    doAssert(blockData.blockObject.stateRoot.bytes() == worldState.stateRoot.data)
-    trace "State diffs successfully applied to block number:",
-      blockNumber = blockData.blockNumber
+      await blockOffersQueue.addLast(
+        BlockOffersRef(
+          blockNumber: blockData.blockNumber,
+          accountTrieOffers: builder.getAccountTrieOffers(),
+          contractTrieOffers: builder.getContractTrieOffers(),
+          contractCodeOffers: builder.getContractCodeOffers(),
+        )
+      )
 
 proc runBackfillMetricsLoop(
     blockDataQueue: AsyncQueue[BlockDataRef],

fluffy/tools/portal_bridge/state_bridge/offers_builder.nim

@@ -0,0 +1,93 @@
+# Fluffy
+# Copyright (c) 2024 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.push raises: [].}
+
+import
+  std/[sequtils, sugar], eth/common, ../../../network/state/state_content, ./world_state
+
+type OffersBuilderRef* = object
+  worldState: WorldStateRef
+  blockHash: BlockHash
+  accountTrieOffers: seq[(AccountTrieNodeKey, AccountTrieNodeOffer)]
+  contractTrieOffers: seq[(ContractTrieNodeKey, ContractTrieNodeOffer)]
+  contractCodeOffers: seq[(ContractCodeKey, ContractCodeOffer)]
+
+proc init*(
+    T: type OffersBuilderRef, worldState: WorldStateRef, blockHash: BlockHash
+): T =
+  T(worldState: worldState, blockHash: blockHash)
+
+proc toTrieProof(proof: seq[seq[byte]]): TrieProof =
+  TrieProof.init(proof.map((node) => TrieNode.init(node)))
+
+proc buildAccountTrieNodeOffer(
+    builder: var OffersBuilderRef, address: EthAddress, proof: TrieProof
+) =
+  let
+    path = Nibbles.init(worldState.toAccountKey(address).data, isEven = true)
+    offerKey = AccountTrieNodeKey.init(path, keccakHash(proof[^1].asSeq()))
+    offerValue = AccountTrieNodeOffer.init(proof, builder.blockHash)
+
+  builder.accountTrieOffers.add((offerKey, offerValue))
+
+proc buildContractTrieNodeOffer(
+    builder: var OffersBuilderRef,
+    address: EthAddress,
+    slotHash: SlotKeyHash,
+    storageProof: TrieProof,
+    accountProof: TrieProof,
+) =
+  let
+    path = Nibbles.init(slotHash.data, isEven = true)
+    offerKey =
+      ContractTrieNodeKey.init(address, path, keccakHash(storageProof[^1].asSeq()))
+    offerValue =
+      ContractTrieNodeOffer.init(storageProof, accountProof, builder.blockHash)
+
+  builder.contractTrieOffers.add((offerKey, offerValue))
+
+proc buildContractCodeOffer(
+    builder: var OffersBuilderRef,
+    address: EthAddress,
+    code: seq[byte],
+    accountProof: TrieProof,
+) =
+  let
+    offerKey = ContractCodeKey.init(address, keccakHash(code))
+    offerValue =
+      ContractCodeOffer.init(Bytecode.init(code), accountProof, builder.blockHash)
+
+  builder.contractCodeOffers.add((offerKey, offerValue))
+
+proc buildBlockOffers*(builder: var OffersBuilderRef) =
+  for address, proof in builder.worldState.updatedAccountProofs():
+    let accountProof = toTrieProof(proof)
+    builder.buildAccountTrieNodeOffer(address, accountProof)
+
+    for slotHash, sProof in builder.worldState.updatedStorageProofs(address):
+      let storageProof = toTrieProof(sProof)
+      builder.buildContractTrieNodeOffer(address, slotHash, storageProof, accountProof)
+
+    let code = builder.worldState.getUpdatedBytecode(address)
+    if code.len() > 0:
+      builder.buildContractCodeOffer(address, code, accountProof)
+
+proc getAccountTrieOffers*(
+    builder: OffersBuilderRef
+): seq[(AccountTrieNodeKey, AccountTrieNodeOffer)] =
+  builder.accountTrieOffers
+
+proc getContractTrieOffers*(
+    builder: OffersBuilderRef
+): seq[(ContractTrieNodeKey, ContractTrieNodeOffer)] =
+  builder.contractTrieOffers
+
+proc getContractCodeOffers*(
+    builder: OffersBuilderRef
+): seq[(ContractCodeKey, ContractCodeOffer)] =
+  builder.contractCodeOffers

fluffy/tools/portal_bridge/state_bridge/world_state.nim

@@ -72,18 +72,19 @@ proc init*(T: type WorldStateRef, db: DatabaseRef): T =
 template stateRoot*(state: WorldStateRef): KeccakHash =
   state.accountsTrie.rootHash()
 
-template toAccountKey(address: EthAddress): AddressHash =
+template toAccountKey*(address: EthAddress): AddressHash =
   keccakHash(address)
 
-template toStorageKey(slotKey: UInt256): SlotKeyHash =
+template toStorageKey*(slotKey: UInt256): SlotKeyHash =
   keccakHash(toBytesBE(slotKey))
 
-proc getAccountPreImage(state: WorldStateRef, accountKey: AddressHash): EthAddress =
+proc getAccountPreImage*(state: WorldStateRef, accountKey: AddressHash): EthAddress =
   doAssert(state.accountPreimages.contains(accountKey))
   state.accountPreimages.getOrDefault(accountKey)
 
 proc getAccount*(state: WorldStateRef, address: EthAddress): AccountState =
   let accountKey = toAccountKey(address)
+  state.accountPreimages[accountKey] = address
 
   try:
     if state.accountsTrie.contains(accountKey.data):
@@ -96,6 +97,7 @@ proc getAccount*(state: WorldStateRef, address: EthAddress): AccountState =
 
 proc setAccount*(state: WorldStateRef, address: EthAddress, accState: AccountState) =
   let accountKey = toAccountKey(address)
+  state.accountPreimages[accountKey] = address
 
   try:
     if not state.storageTries.contains(accountKey):
@@ -119,14 +121,12 @@ proc setAccount*(state: WorldStateRef, address: EthAddress, accState: AccountSta
       accountToSave.codeHash = keccakHash(accState.code)
 
     state.accountsTrie.put(accountKey.data, rlp.encode(accountToSave))
-
-    # TODO: Cleanup the preimages between each transaction or store them in the database
-    state.accountPreimages[accountKey] = address
   except RlpError as e:
     raiseAssert(e.msg) # should never happen unless the database is corrupted
 
 proc deleteAccount*(state: WorldStateRef, address: EthAddress) =
   let accountKey = toAccountKey(address)
+  state.accountPreimages[accountKey] = address
 
   try:
     state.accountsTrie.del(accountKey.data)
@@ -136,7 +136,8 @@ proc deleteAccount*(state: WorldStateRef, address: EthAddress) =
     raiseAssert(e.msg) # should never happen unless the database is corrupted
 
 proc clearPreimages*(state: WorldStateRef) =
-  state.accountPreimages.clear()
+  discard # TODO: fix this
+  #state.accountPreimages.clear()
 
 # Returns the account proofs for all the updated accounts from the last transaction
 iterator updatedAccountProofs*(state: WorldStateRef): (EthAddress, seq[seq[byte]]) =
@@ -171,11 +172,14 @@ iterator updatedStorageProofs*(
   except RlpError as e:
     raiseAssert(e.msg) # should never happen unless the database is corrupted
 
+proc getUpdatedBytecode*(state: WorldStateRef, address: EthAddress): seq[byte] =
+  state.db.getBytecodeUpdatedCache().get(toAccountKey(address).data)
+
 # Returns the updated bytecode for all accounts from the last transaction
-iterator updatedBytecode*(state: WorldStateRef): (EthAddress, seq[byte]) =
-  let updatedCache = state.db.getBytecodeUpdatedCache()
-  for key, value in updatedCache.pairsInMemoryDB():
-    if key == emptyRlpHash.data:
-      continue # skip the empty node created on initialization
-    let address = state.getAccountPreImage(KeccakHash.fromBytes(key))
-    yield (address, value)
+# iterator updatedBytecode*(state: WorldStateRef): (EthAddress, seq[byte]) =
+#   let updatedCache = state.db.getBytecodeUpdatedCache()
+#   for key, value in updatedCache.pairsInMemoryDB():
+#     if key == emptyRlpHash.data:
+#       continue # skip the empty node created on initialization
+#     let address = state.getAccountPreImage(KeccakHash.fromBytes(key))
+#     yield (address, value)