Fluffy State Bridge - State Gossip via Portal JSON-RPC (#2535)

* Create block offers queue and collect account preimages.

* Implement iterators to return account and storage proofs and bytecode from updatedCaches.

* Implement building offers from proofs.

* Refactor BlockDataRef type to only include required fields.

* Store block data in database.

* Improve state diff types.

* Implement start state backfill from specific block.

* Record last persisted block number in database.

* Persist account preimages in db.

* Apply state updates for DAO hard fork.

* Implement state gossip of block offers via portal JSON RPC.

fluffy/network/state/content/content_keys.nim

@@ -62,6 +62,8 @@ type
     of contractCode:
       contractCodeKey*: ContractCodeKey
 
+  ContentKeyType* = AccountTrieNodeKey | ContractTrieNodeKey | ContractCodeKey
+
 func init*(T: type AccountTrieNodeKey, path: Nibbles, nodeHash: NodeHash): T =
   AccountTrieNodeKey(path: path, nodeHash: nodeHash)
 

fluffy/network/state/content/content_values.nim

@@ -15,9 +15,9 @@ import results, eth/common/eth_types, ssz_serialization, ../../../common/common_
 export ssz_serialization, common_types, hash, results
 
 const
-  MAX_TRIE_NODE_LEN = 1024
+  MAX_TRIE_NODE_LEN* = 1024
-  MAX_TRIE_PROOF_LEN = 65
+  MAX_TRIE_PROOF_LEN* = 65
-  MAX_BYTECODE_LEN = 32768
+  MAX_BYTECODE_LEN* = 32768
 
 type
   TrieNode* = List[byte, MAX_TRIE_NODE_LEN]

fluffy/network/state/content/nibbles.nim

@@ -102,7 +102,7 @@ func unpackNibbles*(packed: Nibbles): UnpackedNibbles =
       output.add(first)
       output.add(second)
 
-  move(output)
+  ensureMove(output)
 
 func len(packed: Nibbles): int =
   let lenExclPrefix = (packed.len() - 1) * 2
@@ -115,4 +115,4 @@ func len(packed: Nibbles): int =
 func dropN*(unpacked: UnpackedNibbles, num: int): UnpackedNibbles =
   var nibbles = unpacked
   nibbles.setLen(nibbles.len() - num)
-  move(nibbles)
+  ensureMove(nibbles)

fluffy/network/state/state_gossip.nim

@@ -29,17 +29,12 @@ type AccountTrieOfferWithKey* =
 type ContractTrieOfferWithKey* =
   tuple[key: ContractTrieNodeKey, offer: ContractTrieNodeOffer]
 
+type ContractCodeOfferWithKey* = tuple[key: ContractCodeKey, offer: ContractCodeOffer]
+
 func withPath(proof: TrieProof, path: Nibbles): ProofWithPath =
   (path: path, proof: proof)
 
-func withKey*(
+func withKey*(offer: ContentOfferType, key: ContentKeyType): auto =
-    offer: AccountTrieNodeOffer, key: AccountTrieNodeKey
-): AccountTrieOfferWithKey =
-  (key: key, offer: offer)
-
-func withKey*(
-    offer: ContractTrieNodeOffer, key: ContractTrieNodeKey
-): ContractTrieOfferWithKey =
   (key: key, offer: offer)
 
 func getParent(p: ProofWithPath): ProofWithPath =

fluffy/network/state/state_utils.nim

@@ -70,6 +70,20 @@ func toSlot*(storageProof: TrieProof): Result[UInt256, string] {.inline.} =
 
   rlpDecodeContractTrieNode(storageProof[^1])
 
+func removeLeafKeyEndNibbles*(
+    nibbles: Nibbles, leafNode: TrieNode
+): Nibbles {.raises: RlpError.} =
+  let nodeRlp = rlpFromBytes(leafNode.asSeq())
+  doAssert(nodeRlp.listLen() == 2)
+  let (_, isLeaf, prefix) = decodePrefix(nodeRlp.listElem(0))
+  doAssert(isLeaf)
+
+  let leafPrefix = prefix.unpackNibbles()
+  var unpackedNibbles = nibbles.unpackNibbles()
+  doAssert(unpackedNibbles[^leafPrefix.len() .. ^1] == leafPrefix)
+
+  unpackedNibbles.dropN(leafPrefix.len()).packNibbles()
+
 func toPath*(hash: KeccakHash): Nibbles {.inline.} =
   Nibbles.init(hash.data, isEven = true)
 

fluffy/tests/state_network_tests/state_test_helpers.nim

@@ -61,20 +61,6 @@ type
 func asNibbles*(key: openArray[byte], isEven = true): Nibbles =
   Nibbles.init(key, isEven)
 
-func removeLeafKeyEndNibbles*(
-    nibbles: Nibbles, leafNode: TrieNode
-): Nibbles {.raises: [RlpError].} =
-  let nodeRlp = rlpFromBytes(leafNode.asSeq())
-  doAssert(nodeRlp.listLen() == 2)
-  let (_, isLeaf, prefix) = decodePrefix(nodeRlp.listElem(0))
-  doAssert(isLeaf)
-
-  let leafPrefix = prefix.unpackNibbles()
-  var unpackedNibbles = nibbles.unpackNibbles()
-  doAssert(unpackedNibbles[^leafPrefix.len() .. ^1] == leafPrefix)
-
-  unpackedNibbles.dropN(leafPrefix.len()).packNibbles()
-
 func asTrieProof*(branch: openArray[seq[byte]]): TrieProof =
   TrieProof.init(branch.map(node => TrieNode.init(node)))
 

fluffy/tests/state_network_tests/test_state_validation_genesis.nim

@@ -13,7 +13,7 @@ import
   results,
   eth/[common, trie, trie/trie_defs],
   ../../../nimbus/common/chain_config,
-  ../../network/state/[state_content, state_validation],
+  ../../network/state/[state_content, state_validation, state_utils],
   ./state_test_helpers
 
 template checkValidProofsForExistingLeafs(

fluffy/tools/portal_bridge/portal_bridge_conf.nim

@@ -141,10 +141,10 @@ type
         name: "state-dir"
       .}: InputDir
 
-      # TODO: support starting from a specific block. Currently this is not possible using the existing HexaryTrie library.
+      startBlockNumber* {.
-      # startBlockNumber* {.
+        desc: "The block number to start from", defaultValue: 1, name: "start-block"
-      #   desc: "The block number to start from", defaultValue: 1, name: "start-block"
+      .}: uint64
-      # .}: uint64
+
       verifyState* {.
         desc: "Verify the fetched state before gossiping it into the network",
         defaultValue: true,

fluffy/tools/portal_bridge/portal_bridge_state.nim

@@ -12,92 +12,144 @@ import
   chronicles,
   chronos,
   stint,
+  stew/byteutils,
   web3/[eth_api, eth_api_types],
   results,
   eth/common/[eth_types, eth_types_rlp],
   ../../../nimbus/common/chain_config,
+  ../../common/common_utils,
   ../../rpc/rpc_calls/rpc_trace_calls,
-  ./state_bridge/[database, state_diff, world_state_helper],
+  ../../rpc/portal_rpc_client,
+  ../../network/state/[state_content, state_gossip],
+  ./state_bridge/[database, state_diff, world_state_helper, offers_builder],
   ./[portal_bridge_conf, portal_bridge_common]
 
 type BlockData = object
   blockNumber: uint64
-  blockObject: BlockObject
+  blockHash: KeccakHash
-  stateDiffs: seq[StateDiffRef]
+  miner: EthAddress
-  uncleBlocks: seq[BlockObject]
+  uncles: seq[tuple[miner: EthAddress, blockNumber: uint64]]
+  parentStateRoot: KeccakHash
+  stateRoot: KeccakHash
+  stateDiffs: seq[TransactionDiff]
+
+type BlockOffersRef = ref object
+  blockNumber: uint64
+  accountTrieOffers: seq[AccountTrieOfferWithKey]
+  contractTrieOffers: seq[ContractTrieOfferWithKey]
+  contractCodeOffers: seq[ContractCodeOfferWithKey]
+
+proc getBlockData(db: DatabaseRef, blockNumber: uint64): Opt[BlockData] =
+  let blockDataBytes = db.get(rlp.encode(blockNumber))
+  if blockDataBytes.len() == 0:
+    return Opt.none(BlockData)
+
+  try:
+    Opt.some(rlp.decode(blockDataBytes, BlockData))
+  except RlpError as e:
+    raiseAssert(e.msg) # Should never happen
+
+proc putBlockData(
+    db: DatabaseRef, blockNumber: uint64, blockData: BlockData
+) {.inline.} =
+  db.put(rlp.encode(blockNumber), rlp.encode(blockData))
+
+proc getLastPersistedBlockNumber(db: DatabaseRef): Opt[uint64] =
+  let blockNumberBytes = db.get(rlp.encode("lastPersistedBlockNumber"))
+  if blockNumberBytes.len() == 0:
+    return Opt.none(uint64)
+
+  try:
+    Opt.some(rlp.decode(blockNumberBytes, uint64))
+  except RlpError as e:
+    raiseAssert(e.msg) # Should never happen
+
+proc putLastPersistedBlockNumber(db: DatabaseRef, blockNumber: uint64) {.inline.} =
+  db.put(rlp.encode("lastPersistedBlockNumber"), rlp.encode(blockNumber))
 
 proc runBackfillCollectBlockDataLoop(
+    db: DatabaseRef,
     blockDataQueue: AsyncQueue[BlockData],
     web3Client: RpcClient,
     startBlockNumber: uint64,
 ) {.async: (raises: [CancelledError]).} =
-  debug "Starting state backfill collect block data loop"
+  info "Starting state backfill collect block data loop"
 
-  var currentBlockNumber = startBlockNumber
+  let parentBlock = (
+    await web3Client.getBlockByNumber(blockId(startBlockNumber - 1.uint64), false)
+  ).valueOr:
+    raiseAssert("Failed to get parent block")
+
+  var
+    parentStateRoot = parentBlock.stateRoot
+    currentBlockNumber = startBlockNumber
 
   while true:
     if currentBlockNumber mod 10000 == 0:
       info "Collecting block data for block number: ", blockNumber = currentBlockNumber
 
+    let blockData = db.getBlockData(currentBlockNumber).valueOr:
+      # block data doesn't exist in db so we fetch it via RPC
       let
         blockId = blockId(currentBlockNumber)
-      blockRequest = web3Client.getBlockByNumber(blockId, false)
+        blockObject = (await web3Client.getBlockByNumber(blockId, false)).valueOr:
-      stateDiffsRequest = web3Client.getStateDiffsByBlockNumber(blockId)
-
-      blockObject = (await blockRequest).valueOr:
           error "Failed to get block", error
           await sleepAsync(1.seconds)
           continue
-
+        stateDiffs = (await web3Client.getStateDiffsByBlockNumber(blockId)).valueOr:
-    var uncleBlockRequests: seq[Future[Result[BlockObject, string]]]
-    for i in 0 .. blockObject.uncles.high:
-      uncleBlockRequests.add(
-        web3Client.getUncleByBlockNumberAndIndex(blockId, i.Quantity)
-      )
-
-    let stateDiffs = (await stateDiffsRequest).valueOr:
           error "Failed to get state diffs", error
           await sleepAsync(1.seconds)
           continue
 
       var uncleBlocks: seq[BlockObject]
-    for uncleBlockRequest in uncleBlockRequests:
+      for i in 0 .. blockObject.uncles.high:
-      try:
+        let uncleBlock = (
-        let uncleBlock = (await uncleBlockRequest).valueOr:
+          await web3Client.getUncleByBlockNumberAndIndex(blockId, i.Quantity)
-          error "Failed to get uncle blocks", error
+        ).valueOr:
+          error "Failed to get uncle block", error
           await sleepAsync(1.seconds)
-          break
-        uncleBlocks.add(uncleBlock)
-      except CatchableError as e:
-        error "Failed to get uncleBlockRequest", error = e.msg
-        break
-
-    if uncleBlocks.len() < uncleBlockRequests.len():
           continue
+        uncleBlocks.add(uncleBlock)
 
       let blockData = BlockData(
         blockNumber: currentBlockNumber,
-      blockObject: blockObject,
+        blockHash: KeccakHash.fromBytes(blockObject.hash.bytes()),
+        miner: blockObject.miner.EthAddress,
+        uncles: uncleBlocks.mapIt((it.miner.EthAddress, it.number.uint64)),
+        parentStateRoot: KeccakHash.fromBytes(parentStateRoot.bytes()),
+        stateRoot: KeccakHash.fromBytes(blockObject.stateRoot.bytes()),
         stateDiffs: stateDiffs,
-      uncleBlocks: uncleBlocks,
       )
-    await blockDataQueue.addLast(blockData)
+      db.putBlockData(currentBlockNumber, blockData)
 
+      parentStateRoot = blockObject.stateRoot
+      blockData
+
+    await blockDataQueue.addLast(blockData)
     inc currentBlockNumber
 
-proc runBackfillBuildStateLoop(
+proc runBackfillBuildBlockOffersLoop(
-    blockDataQueue: AsyncQueue[BlockData], stateDir: string
+    db: DatabaseRef,
+    blockDataQueue: AsyncQueue[BlockData],
+    blockOffersQueue: AsyncQueue[BlockOffersRef],
 ) {.async: (raises: [CancelledError]).} =
-  debug "Starting state backfill build state loop"
+  info "Starting state backfill build block offers loop"
 
-  let db = DatabaseRef.init(stateDir).get()
+  # wait for the first block data to be put on the queue
-  defer:
+  # so that we can access the first block once available
-    db.close()
+  while blockDataQueue.empty():
+    await sleepAsync(100.milliseconds)
+  # peek but don't remove it so that it can be processed later
+  let firstBlock = blockDataQueue[0]
 
-  let worldState = db.withTransaction:
+  # Only apply genesis accounts if starting from block 1
-    let
+  if firstBlock.blockNumber == 1:
+    info "Building state for genesis"
+
+    db.withTransaction:
       # Requires an active transaction because it writes an emptyRlp node
       # to the accounts HexaryTrie on initialization
+      let
         ws = WorldStateRef.init(db)
         genesisAccounts =
           try:
@@ -105,7 +157,24 @@ proc runBackfillBuildStateLoop(
           except CatchableError as e:
             raiseAssert(e.msg) # Should never happen
       ws.applyGenesisAccounts(genesisAccounts)
-    ws
+
+      let genesisBlockHash = KeccakHash.fromHex(
+        "d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"
+      )
+      var builder = OffersBuilderRef.init(ws, genesisBlockHash)
+      builder.buildBlockOffers()
+
+      await blockOffersQueue.addLast(
+        BlockOffersRef(
+          blockNumber: 0.uint64,
+          accountTrieOffers: builder.getAccountTrieOffers(),
+          contractTrieOffers: builder.getContractTrieOffers(),
+          contractCodeOffers: builder.getContractCodeOffers(),
+        )
+      )
+
+  # Load the world state using the parent state root
+  let worldState = WorldStateRef.init(db, firstBlock.parentStateRoot)
 
   while true:
     let blockData = await blockDataQueue.popFirst()
@@ -113,33 +182,112 @@ proc runBackfillBuildStateLoop(
     if blockData.blockNumber mod 10000 == 0:
       info "Building state for block number: ", blockNumber = blockData.blockNumber
 
+    # For now all WorldStateRef functions need to be inside a transaction
+    # because the DatabaseRef backends currently only supports reading and
+    # writing to/from a single active transaction.
     db.withTransaction:
       for stateDiff in blockData.stateDiffs:
         worldState.applyStateDiff(stateDiff)
-      let
-        minerData =
-          (EthAddress(blockData.blockObject.miner), blockData.blockObject.number.uint64)
-        uncleMinersData =
-          blockData.uncleBlocks.mapIt((EthAddress(it.miner), it.number.uint64))
-      worldState.applyBlockRewards(minerData, uncleMinersData)
 
-    doAssert(blockData.blockObject.stateRoot.bytes() == worldState.stateRoot.data)
+      worldState.applyBlockRewards(
+        (blockData.miner, blockData.blockNumber), blockData.uncles
+      )
+
+      if blockData.blockNumber == 1_920_000:
+        info "Applying state updates for DAO hard fork"
+        worldState.applyDAOHardFork()
+
+      doAssert(
+        worldState.stateRoot == blockData.stateRoot,
+        "State root mismatch at block number: " & $blockData.blockNumber,
+      )
       trace "State diffs successfully applied to block number:",
         blockNumber = blockData.blockNumber
 
-proc runBackfillMetricsLoop(
+      # worldState.verifyProofs(blockData.parentStateRoot, blockData.stateRoot)
-    blockDataQueue: AsyncQueue[BlockData]
+
+      var builder = OffersBuilderRef.init(worldState, blockData.blockHash)
+      builder.buildBlockOffers()
+
+      await blockOffersQueue.addLast(
+        BlockOffersRef(
+          blockNumber: blockData.blockNumber,
+          accountTrieOffers: builder.getAccountTrieOffers(),
+          contractTrieOffers: builder.getContractTrieOffers(),
+          contractCodeOffers: builder.getContractCodeOffers(),
+        )
+      )
+
+    # After commit of the above db transaction which stores the updated account state
+    # then we store the last persisted block number in the database so that we can use it
+    # to enable restarting from this block if needed
+    db.putLastPersistedBlockNumber(blockData.blockNumber)
+
+proc gossipOffer(
+    portalClient: RpcClient,
+    offerWithKey:
+      AccountTrieOfferWithKey | ContractTrieOfferWithKey | ContractCodeOfferWithKey,
 ) {.async: (raises: [CancelledError]).} =
-  debug "Starting state backfill metrics loop"
+  let
+    keyBytes = offerWithKey.key.toContentKey().encode().asSeq()
+    offerBytes = offerWithKey.offer.encode()
+  try:
+    let numPeers =
+      await portalClient.portal_stateGossip(keyBytes.to0xHex(), offerBytes.to0xHex())
+    debug "Gossiping offer to peers: ", offerKey = keyBytes.to0xHex(), numPeers
+  except CatchableError as e:
+    raiseAssert(e.msg) # Should never happen
+
+proc recursiveGossipOffer(
+    portalClient: RpcClient,
+    offerWithKey: AccountTrieOfferWithKey | ContractTrieOfferWithKey,
+) {.async: (raises: [CancelledError]).} =
+  await portalClient.gossipOffer(offerWithKey)
+
+  # root node, recursive gossip is finished
+  if offerWithKey.key.path.unpackNibbles().len() == 0:
+    return
+
+  # continue the recursive gossip by sharing the parent offer with peers
+  await portalClient.recursiveGossipOffer(offerWithKey.getParent())
+
+proc runBackfillGossipBlockOffersLoop(
+    blockOffersQueue: AsyncQueue[BlockOffersRef], portalClient: RpcClient
+) {.async: (raises: [CancelledError]).} =
+  info "Starting state backfill gossip block offers loop"
+
+  while true:
+    let blockOffers = await blockOffersQueue.popFirst()
+
+    for offerWithKey in blockOffers.accountTrieOffers:
+      await portalClient.recursiveGossipOffer(offerWithKey)
+
+    for offerWithKey in blockOffers.contractTrieOffers:
+      await portalClient.recursiveGossipOffer(offerWithKey)
+
+    for offerWithKey in blockOffers.contractCodeOffers:
+      await portalClient.gossipOffer(offerWithKey)
+
+proc runBackfillMetricsLoop(
+    blockDataQueue: AsyncQueue[BlockData], blockOffersQueue: AsyncQueue[BlockOffersRef]
+) {.async: (raises: [CancelledError]).} =
+  info "Starting state backfill metrics loop"
 
   while true:
     await sleepAsync(10.seconds)
-    info "Block data queue length: ", queueLen = blockDataQueue.len()
+    info "Block data queue length: ", blockDataQueueLen = blockDataQueue.len()
+    info "Block offers queue length: ", blockOffersQueueLen = blockOffersQueue.len()
 
 proc runState*(config: PortalBridgeConf) =
   let
-    #portalClient = newRpcClientConnect(config.portalRpcUrl)
+    portalClient = newRpcClientConnect(config.portalRpcUrl)
     web3Client = newRpcClientConnect(config.web3UrlState)
+    db = DatabaseRef.init(config.stateDir.string).get()
+  defer:
+    db.close()
+
+  if web3Client of RpcHttpClient:
+    warn "Using a WebSocket connection to the JSON-RPC API is recommended to improve performance"
 
   # TODO:
   # Here we'd want to implement initially a loop that backfills the state
@@ -154,20 +302,34 @@ proc runState*(config: PortalBridgeConf) =
   # inside the bridge, and getting the blocks from era1 files.
 
   if config.backfillState:
-    const startBlockNumber = 1
+    let maybeLastPersistedBlock = db.getLastPersistedBlockNumber()
-      # This will become a parameter in the config once we can support it
+    if maybeLastPersistedBlock.isSome():
-    info "Starting state backfill from block number: ", startBlockNumber
+      info "Last persisted block found in the database: ",
+        lastPersistedBlock = maybeLastPersistedBlock.get()
+      if config.startBlockNumber < 1 or
+          config.startBlockNumber > maybeLastPersistedBlock.get():
+        warn "Start block must be set to a value between 1 and the last persisted block"
+        quit QuitFailure
+    else:
+      info "No last persisted block found in the database"
+      if config.startBlockNumber != 1:
+        warn "Start block must be set to 1"
+        quit QuitFailure
+
+    info "Starting state backfill from block number: ",
+      startBlockNumber = config.startBlockNumber
 
     const bufferSize = 1000 # Should we make this configurable?
-    let blockDataQueue = newAsyncQueue[BlockData](bufferSize)
+    let
+      blockDataQueue = newAsyncQueue[BlockData](bufferSize)
+      blockOffersQueue = newAsyncQueue[BlockOffersRef](bufferSize)
 
     asyncSpawn runBackfillCollectBlockDataLoop(
-      blockDataQueue, web3Client, startBlockNumber
+      db, blockDataQueue, web3Client, config.startBlockNumber
     )
-
+    asyncSpawn runBackfillBuildBlockOffersLoop(db, blockDataQueue, blockOffersQueue)
-    asyncSpawn runBackfillBuildStateLoop(blockDataQueue, config.stateDir.string)
+    asyncSpawn runBackfillGossipBlockOffersLoop(blockOffersQueue, portalClient)
-
+    asyncSpawn runBackfillMetricsLoop(blockDataQueue, blockOffersQueue)
-    asyncSpawn runBackfillMetricsLoop(blockDataQueue)
 
   while true:
     poll()

fluffy/tools/portal_bridge/state_bridge/database.nim

@@ -14,27 +14,36 @@ export results, db
 const COL_FAMILY_NAME_ACCOUNTS = "A"
 const COL_FAMILY_NAME_STORAGE = "S"
 const COL_FAMILY_NAME_BYTECODE = "B"
+const COL_FAMILY_NAME_PREIMAGES = "P"
 
-const COL_FAMILY_NAMES =
+const COL_FAMILY_NAMES = [
-  [COL_FAMILY_NAME_ACCOUNTS, COL_FAMILY_NAME_STORAGE, COL_FAMILY_NAME_BYTECODE]
+  COL_FAMILY_NAME_ACCOUNTS, COL_FAMILY_NAME_STORAGE, COL_FAMILY_NAME_BYTECODE,
+  COL_FAMILY_NAME_PREIMAGES,
+]
 
 type
   AccountsBackendRef = ref object of RootObj
     cfHandle: ColFamilyHandleRef
     tx: TransactionRef
-    updatedCache: TableRef[seq[byte], seq[byte]]
+    updatedCache: TrieDatabaseRef
 
   StorageBackendRef = ref object of RootObj
     cfHandle: ColFamilyHandleRef
     tx: TransactionRef
-    updatedCache: TableRef[seq[byte], seq[byte]]
+    updatedCache: TrieDatabaseRef
 
   BytecodeBackendRef = ref object of RootObj
     cfHandle: ColFamilyHandleRef
     tx: TransactionRef
-    updatedCache: TableRef[seq[byte], seq[byte]]
+    updatedCache: TrieDatabaseRef
 
-  DatabaseBackendRef = AccountsBackendRef | StorageBackendRef | BytecodeBackendRef
+  PreimagesBackendRef = ref object of RootObj
+    cfHandle: ColFamilyHandleRef
+    tx: TransactionRef
+    updatedCache: TrieDatabaseRef
+
+  DatabaseBackendRef =
+    AccountsBackendRef | StorageBackendRef | BytecodeBackendRef | PreimagesBackendRef
 
   DatabaseRef* = ref object
     rocksDb: OptimisticTxDbRef
@@ -42,6 +51,7 @@ type
     accountsBackend: AccountsBackendRef
     storageBackend: StorageBackendRef
     bytecodeBackend: BytecodeBackendRef
+    preimagesBackend: PreimagesBackendRef
 
 proc init*(T: type DatabaseRef, baseDir: string): Result[T, string] =
   let dbPath = baseDir / "db"
@@ -69,6 +79,9 @@ proc init*(T: type DatabaseRef, baseDir: string): Result[T, string] =
     bytecodeBackend = BytecodeBackendRef(
       cfHandle: db.getColFamilyHandle(COL_FAMILY_NAME_BYTECODE).get()
     )
+    preimagesBackend = PreimagesBackendRef(
+      cfHandle: db.getColFamilyHandle(COL_FAMILY_NAME_PREIMAGES).get()
+    )
 
   ok(
     T(
@@ -77,6 +90,7 @@ proc init*(T: type DatabaseRef, baseDir: string): Result[T, string] =
       accountsBackend: accountsBackend,
       storageBackend: storageBackend,
       bytecodeBackend: bytecodeBackend,
+      preimagesBackend: preimagesBackend,
     )
   )
 
@@ -92,7 +106,8 @@ proc put(
     dbBackend: DatabaseBackendRef, key, val: openArray[byte]
 ) {.gcsafe, raises: [].} =
   doAssert dbBackend.tx.put(key, val, dbBackend.cfHandle).isOk()
-  dbBackend.updatedCache[@key] = @val
+  if not dbBackend.updatedCache.isNil():
+    dbBackend.updatedCache.put(key, val)
 
 proc get(
     dbBackend: DatabaseBackendRef, key: openArray[byte]
@@ -111,15 +126,47 @@ proc del(
   else:
     false
 
-proc getAccountsBackend*(db: DatabaseRef): TrieDatabaseRef =
+proc getAccountsBackend*(db: DatabaseRef): TrieDatabaseRef {.inline.} =
   trieDB(db.accountsBackend)
 
-proc getStorageBackend*(db: DatabaseRef): TrieDatabaseRef =
+proc getStorageBackend*(db: DatabaseRef): TrieDatabaseRef {.inline.} =
   trieDB(db.storageBackend)
 
-proc getBytecodeBackend*(db: DatabaseRef): TrieDatabaseRef =
+proc getBytecodeBackend*(db: DatabaseRef): TrieDatabaseRef {.inline.} =
   trieDB(db.bytecodeBackend)
 
+proc getPreimagesBackend*(db: DatabaseRef): TrieDatabaseRef {.inline.} =
+  trieDB(db.preimagesBackend)
+
+proc getAccountsUpdatedCache*(db: DatabaseRef): TrieDatabaseRef {.inline.} =
+  db.accountsBackend.updatedCache
+
+proc getStorageUpdatedCache*(db: DatabaseRef): TrieDatabaseRef {.inline.} =
+  db.storageBackend.updatedCache
+
+proc getBytecodeUpdatedCache*(db: DatabaseRef): TrieDatabaseRef {.inline.} =
+  db.bytecodeBackend.updatedCache
+
+proc put*(db: DatabaseRef, key, val: openArray[byte]) =
+  let tx = db.rocksDb.beginTransaction()
+  defer:
+    tx.close()
+
+  # using default column family
+  doAssert tx.put(key, val).isOk()
+  doAssert tx.commit().isOk()
+
+proc get*(db: DatabaseRef, key: openArray[byte]): seq[byte] =
+  let tx = db.rocksDb.beginTransaction()
+  defer:
+    tx.close()
+
+  # using default column family
+  if tx.get(key, onData).get():
+    tx.get(key).get()
+  else:
+    @[]
+
 proc beginTransaction*(db: DatabaseRef): Result[void, string] =
   if not db.pendingTransaction.isNil():
     return err("DatabaseRef: Pending transaction already in progress")
@@ -129,10 +176,12 @@ proc beginTransaction*(db: DatabaseRef): Result[void, string] =
   db.accountsBackend.tx = tx
   db.storageBackend.tx = tx
   db.bytecodeBackend.tx = tx
+  db.preimagesBackend.tx = tx
 
-  db.accountsBackend.updatedCache = newTable[seq[byte], seq[byte]]()
+  db.accountsBackend.updatedCache = newMemoryDB()
-  db.storageBackend.updatedCache = newTable[seq[byte], seq[byte]]()
+  db.storageBackend.updatedCache = newMemoryDB()
-  db.bytecodeBackend.updatedCache = newTable[seq[byte], seq[byte]]()
+  db.bytecodeBackend.updatedCache = newMemoryDB()
+  db.preimagesBackend.updatedCache = nil # not used
 
   ok()
 
@@ -165,15 +214,6 @@ template withTransaction*(db: DatabaseRef, body: untyped): auto =
   finally:
     db.commitTransaction().expect("Transaction should be commited")
 
-template accountsBackendUpdatedCache*(db: DatabaseRef): TableRef[seq[byte], seq[byte]] =
-  db.accountsBackend.updatedCache
-
-template storageBackendUpdatedCache*(db: DatabaseRef): TableRef[seq[byte], seq[byte]] =
-  db.storageBackend.updatedCache
-
-template bytecodeBackendUpdatedCache*(db: DatabaseRef): TableRef[seq[byte], seq[byte]] =
-  db.bytecodeBackend.updatedCache
-
 proc close*(db: DatabaseRef) =
   if not db.pendingTransaction.isNil():
     discard db.rollbackTransaction()

fluffy/tools/portal_bridge/state_bridge/offers_builder.nim

@@ -0,0 +1,96 @@
+# Fluffy
+# Copyright (c) 2024 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.push raises: [].}
+
+import
+  std/[sequtils, sugar],
+  eth/common,
+  ../../../network/state/[state_content, state_utils, state_gossip],
+  ./world_state
+
+type OffersBuilderRef* = ref object
+  worldState: WorldStateRef
+  blockHash: BlockHash
+  accountTrieOffers: seq[AccountTrieOfferWithKey]
+  contractTrieOffers: seq[ContractTrieOfferWithKey]
+  contractCodeOffers: seq[ContractCodeOfferWithKey]
+
+proc init*(
+    T: type OffersBuilderRef, worldState: WorldStateRef, blockHash: BlockHash
+): T =
+  T(worldState: worldState, blockHash: blockHash)
+
+proc toTrieProof(proof: seq[seq[byte]]): TrieProof =
+  TrieProof.init(proof.map((node) => TrieNode.init(node)))
+
+proc buildAccountTrieNodeOffer(
+    builder: var OffersBuilderRef, address: EthAddress, proof: TrieProof
+) =
+  try:
+    let
+      path = removeLeafKeyEndNibbles(
+        Nibbles.init(worldState.toAccountKey(address).data, isEven = true), proof[^1]
+      )
+      offerKey = AccountTrieNodeKey.init(path, keccakHash(proof[^1].asSeq()))
+      offerValue = AccountTrieNodeOffer.init(proof, builder.blockHash)
+
+    builder.accountTrieOffers.add(offerValue.withKey(offerKey))
+  except RlpError as e:
+    raiseAssert(e.msg) # Should never happen
+
+proc buildContractTrieNodeOffer(
+    builder: var OffersBuilderRef,
+    address: EthAddress,
+    slotHash: SlotKeyHash,
+    storageProof: TrieProof,
+    accountProof: TrieProof,
+) =
+  let
+    path = Nibbles.init(slotHash.data, isEven = true)
+    offerKey =
+      ContractTrieNodeKey.init(address, path, keccakHash(storageProof[^1].asSeq()))
+    offerValue =
+      ContractTrieNodeOffer.init(storageProof, accountProof, builder.blockHash)
+
+  builder.contractTrieOffers.add(offerValue.withKey(offerKey))
+
+proc buildContractCodeOffer(
+    builder: var OffersBuilderRef,
+    address: EthAddress,
+    code: seq[byte],
+    accountProof: TrieProof,
+) =
+  let
+    #bytecode = Bytelist.init(code) # This fails to compile for some reason
+    bytecode = List[byte, MAX_BYTECODE_LEN](code)
+    offerKey = ContractCodeKey.init(address, keccakHash(code))
+    offerValue = ContractCodeOffer.init(bytecode, accountProof, builder.blockHash)
+
+  builder.contractCodeOffers.add(offerValue.withKey(offerKey))
+
+proc buildBlockOffers*(builder: var OffersBuilderRef) =
+  for address, proof in builder.worldState.updatedAccountProofs():
+    let accountProof = toTrieProof(proof)
+    builder.buildAccountTrieNodeOffer(address, accountProof)
+
+    for slotHash, sProof in builder.worldState.updatedStorageProofs(address):
+      let storageProof = toTrieProof(sProof)
+      builder.buildContractTrieNodeOffer(address, slotHash, storageProof, accountProof)
+
+    let code = builder.worldState.getUpdatedBytecode(address)
+    if code.len() > 0:
+      builder.buildContractCodeOffer(address, code, accountProof)
+
+proc getAccountTrieOffers*(builder: OffersBuilderRef): seq[AccountTrieOfferWithKey] =
+  builder.accountTrieOffers
+
+proc getContractTrieOffers*(builder: OffersBuilderRef): seq[ContractTrieOfferWithKey] =
+  builder.contractTrieOffers
+
+proc getContractCodeOffers*(builder: OffersBuilderRef): seq[ContractCodeOfferWithKey] =
+  builder.contractCodeOffers

fluffy/tools/portal_bridge/state_bridge/state_diff.nim

@@ -30,11 +30,16 @@ type
     before*: StateValue
     after*: StateValue
 
-  StateDiffRef* = ref object
+  SlotDiff* = tuple[slotKey: UInt256, slotValueDiff: StateValueDiff[UInt256]]
-    balances*: Table[EthAddress, StateValueDiff[UInt256]]
+
-    nonces*: Table[EthAddress, StateValueDiff[AccountNonce]]
+  AccountDiff* = object
-    storage*: Table[EthAddress, Table[UInt256, StateValueDiff[UInt256]]]
+    address*: EthAddress
-    code*: Table[EthAddress, StateValueDiff[Code]]
+    balanceDiff*: StateValueDiff[UInt256]
+    nonceDiff*: StateValueDiff[AccountNonce]
+    storageDiff*: seq[SlotDiff]
+    codeDiff*: StateValueDiff[Code]
+
+  TransactionDiff* = seq[AccountDiff]
 
 proc toStateValue(T: type UInt256, hex: string): T {.raises: [ValueError].} =
   UInt256.fromHex(hex)
@@ -68,45 +73,49 @@ proc toStateValueDiff(
   else:
     doAssert false # unreachable
 
-proc toStateDiff(stateDiffJson: JsonNode): StateDiffRef {.raises: [ValueError].} =
+proc toTransactionDiff(
-  let stateDiff = StateDiffRef()
+    stateDiffJson: JsonNode
+): TransactionDiff {.raises: [ValueError].} =
+  var txDiff = newSeqOfCap[AccountDiff](stateDiffJson.len())
 
-  for addrJson, accJson in stateDiffJson.pairs:
+  for addrJson, accJson in stateDiffJson:
-    let address = EthAddress.fromHex(addrJson)
+    let storageDiffJson = accJson["storage"]
+    var storageDiff = newSeqOfCap[SlotDiff](storageDiffJson.len())
 
-    stateDiff.balances[address] = toStateValueDiff(accJson["balance"], UInt256)
+    for slotKeyJson, slotValueJson in storageDiffJson:
-    stateDiff.nonces[address] = toStateValueDiff(accJson["nonce"], AccountNonce)
+      storageDiff.add(
-    stateDiff.code[address] = toStateValueDiff(accJson["code"], Code)
+        (UInt256.fromHex(slotKeyJson), toStateValueDiff(slotValueJson, UInt256))
+      )
 
-    let storageDiff = accJson["storage"]
+    let accountDiff = AccountDiff(
-    var accountStorage: Table[UInt256, StateValueDiff[UInt256]]
+      address: EthAddress.fromHex(addrJson),
+      balanceDiff: toStateValueDiff(accJson["balance"], UInt256),
+      nonceDiff: toStateValueDiff(accJson["nonce"], AccountNonce),
+      storageDiff: storageDiff,
+      codeDiff: toStateValueDiff(accJson["code"], Code),
+    )
+    txDiff.add(accountDiff)
 
-    for slotKeyJson, slotValueJson in storageDiff.pairs:
+  txDiff
-      let slotKey = UInt256.fromHex(slotKeyJson)
-      accountStorage[slotKey] = toStateValueDiff(slotValueJson, UInt256)
 
-    stateDiff.storage[address] = ensureMove(accountStorage)
+proc toTransactionDiffs(
-
-  stateDiff
-
-proc toStateDiffs(
     blockTraceJson: JsonNode
-): seq[StateDiffRef] {.raises: [ValueError].} =
+): seq[TransactionDiff] {.raises: [ValueError].} =
-  var stateDiffs = newSeqOfCap[StateDiffRef](blockTraceJson.len())
+  var txDiffs = newSeqOfCap[TransactionDiff](blockTraceJson.len())
   for blockTrace in blockTraceJson:
-    stateDiffs.add(blockTrace["stateDiff"].toStateDiff())
+    txDiffs.add(blockTrace["stateDiff"].toTransactionDiff())
 
-  stateDiffs
+  txDiffs
 
 proc getStateDiffsByBlockNumber*(
     client: RpcClient, blockId: BlockIdentifier
-): Future[Result[seq[StateDiffRef], string]] {.async: (raises: []).} =
+): Future[Result[seq[TransactionDiff], string]] {.async: (raises: []).} =
   const traceOpts = @["stateDiff"]
 
   try:
     let blockTraceJson = await client.trace_replayBlockTransactions(blockId, traceOpts)
     if blockTraceJson.isNil:
       return err("EL failed to provide requested state diff")
-    ok(blockTraceJson.toStateDiffs())
+    ok(blockTraceJson.toTransactionDiffs())
   except CatchableError as e:
     return err("EL JSON-RPC trace_replayBlockTransactions failed: " & e.msg)

fluffy/tools/portal_bridge/state_bridge/world_state.nim

@@ -9,9 +9,9 @@
 
 import
   stint,
-  eth/[common, trie, trie/db],
+  eth/[common, trie, trie/db, trie/trie_defs],
   eth/common/[eth_types, eth_types_rlp],
-  ../../../common/common_types,
+  ../../../common/[common_types, common_utils],
   ./database
 
 # Account State definition
@@ -22,22 +22,30 @@ type AccountState* = ref object
   code: seq[byte]
   codeUpdated: bool
 
-proc init(T: type AccountState, account = newAccount()): T =
+proc init(T: type AccountState, account = newAccount()): T {.inline.} =
   T(account: account, codeUpdated: false)
 
-proc setBalance*(accState: var AccountState, balance: UInt256) =
+proc getBalance*(accState: AccountState): UInt256 {.inline.} =
+  accState.account.balance
+
+proc getNonce*(accState: AccountState): AccountNonce {.inline.} =
+  accState.account.nonce
+
+proc setBalance*(accState: var AccountState, balance: UInt256) {.inline.} =
   accState.account.balance = balance
 
-proc addBalance*(accState: var AccountState, balance: UInt256) =
+proc addBalance*(accState: var AccountState, balance: UInt256) {.inline.} =
   accState.account.balance += balance
 
-proc setNonce*(accState: var AccountState, nonce: AccountNonce) =
+proc setNonce*(accState: var AccountState, nonce: AccountNonce) {.inline.} =
   accState.account.nonce = nonce
 
-proc setStorage*(accState: var AccountState, slotKey: UInt256, slotValue: UInt256) =
+proc setStorage*(
+    accState: var AccountState, slotKey: UInt256, slotValue: UInt256
+) {.inline.} =
   accState.storageUpdates[slotKey] = slotValue
 
-proc deleteStorage*(accState: var AccountState, slotKey: UInt256) =
+proc deleteStorage*(accState: var AccountState, slotKey: UInt256) {.inline.} =
   # setting to zero has the effect of deleting the slot
   accState.setStorage(slotKey, 0.u256)
 
@@ -48,32 +56,59 @@ proc setCode*(accState: var AccountState, code: seq[byte]) =
 # World State definition
 
 type
-  AccountHash = KeccakHash
+  AddressHash* = KeccakHash
-  SlotKeyHash = KeccakHash
+  SlotKeyHash* = KeccakHash
 
   WorldStateRef* = ref object
+    db: DatabaseRef
     accountsTrie: HexaryTrie
-    storageTries: TableRef[AccountHash, HexaryTrie]
+    storageTries: TableRef[AddressHash, HexaryTrie]
     storageDb: TrieDatabaseRef
-    bytecodeDb: TrieDatabaseRef # maps AccountHash -> seq[byte]
+    bytecodeDb: TrieDatabaseRef # maps AddressHash -> seq[byte]
+    preimagesDb: TrieDatabaseRef # maps AddressHash -> EthAddress
 
-proc init*(T: type WorldStateRef, db: DatabaseRef): T =
+proc init*(
+    T: type WorldStateRef, db: DatabaseRef, accountsTrieRoot: KeccakHash = emptyRlpHash
+): T =
   WorldStateRef(
-    accountsTrie: initHexaryTrie(db.getAccountsBackend(), isPruning = false),
+    db: db,
-    storageTries: newTable[AccountHash, HexaryTrie](),
+    accountsTrie:
+      if accountsTrieRoot == emptyRlpHash:
+        initHexaryTrie(db.getAccountsBackend(), isPruning = false)
+      else:
+        initHexaryTrie(db.getAccountsBackend(), accountsTrieRoot, isPruning = false),
+    storageTries: newTable[AddressHash, HexaryTrie](),
     storageDb: db.getStorageBackend(),
     bytecodeDb: db.getBytecodeBackend(),
+    preimagesDb: db.getPreimagesBackend(),
   )
 
-template stateRoot*(state: WorldStateRef): KeccakHash =
+proc stateRoot*(state: WorldStateRef): KeccakHash {.inline.} =
   state.accountsTrie.rootHash()
 
-template toAccountKey(address: EthAddress): AccountHash =
+proc toAccountKey*(address: EthAddress): AddressHash {.inline.} =
   keccakHash(address)
 
-template toStorageKey(slotKey: UInt256): SlotKeyHash =
+proc toStorageKey*(slotKey: UInt256): SlotKeyHash {.inline.} =
   keccakHash(toBytesBE(slotKey))
 
+proc getAccountPreimage(state: WorldStateRef, accountKey: AddressHash): EthAddress =
+  doAssert(
+    state.preimagesDb.contains(rlp.encode(accountKey)),
+    "No account preimage with address hash: " & $accountKey,
+  )
+  let addressBytes = state.preimagesDb.get(rlp.encode(accountKey))
+
+  try:
+    rlp.decode(addressBytes, EthAddress)
+  except RlpError as e:
+    raiseAssert(e.msg) # Should never happen
+
+proc setAccountPreimage(
+    state: WorldStateRef, accountKey: AddressHash, address: EthAddress
+) =
+  state.preimagesDb.put(rlp.encode(accountKey), rlp.encode(address))
+
 proc getAccount*(state: WorldStateRef, address: EthAddress): AccountState =
   let accountKey = toAccountKey(address)
 
@@ -88,11 +123,17 @@ proc getAccount*(state: WorldStateRef, address: EthAddress): AccountState =
 
 proc setAccount*(state: WorldStateRef, address: EthAddress, accState: AccountState) =
   let accountKey = toAccountKey(address)
+  state.setAccountPreimage(accountKey, address)
 
   try:
     if not state.storageTries.contains(accountKey):
+      if accState.account.storageRoot == EMPTY_ROOT_HASH:
         state.storageTries[accountKey] =
           initHexaryTrie(state.storageDb, isPruning = false)
+      else:
+        state.storageTries[accountKey] = initHexaryTrie(
+          state.storageDb, accState.account.storageRoot, isPruning = false
+        )
 
     var storageTrie = state.storageTries.getOrDefault(accountKey)
     for k, v in accState.storageUpdates:
@@ -123,3 +164,41 @@ proc deleteAccount*(state: WorldStateRef, address: EthAddress) =
     state.bytecodeDb.del(accountKey.data)
   except RlpError as e:
     raiseAssert(e.msg) # should never happen unless the database is corrupted
+
+# Returns the account proofs for all the updated accounts from the last transaction
+iterator updatedAccountProofs*(state: WorldStateRef): (EthAddress, seq[seq[byte]]) =
+  let trie = initHexaryTrie(
+    state.db.getAccountsUpdatedCache(), state.stateRoot(), isPruning = false
+  )
+
+  try:
+    for key in trie.keys():
+      if key.len() == 0:
+        continue # skip the empty node created on initialization
+      let address = state.getAccountPreimage(KeccakHash.fromBytes(key))
+      yield (address, trie.getBranch(key))
+  except RlpError as e:
+    raiseAssert(e.msg) # should never happen unless the database is corrupted
+
+# Returns the storage proofs for the updated slots for the given account from the last transaction
+iterator updatedStorageProofs*(
+    state: WorldStateRef, address: EthAddress
+): (SlotKeyHash, seq[seq[byte]]) =
+  let accState = state.getAccount(address)
+
+  let trie = initHexaryTrie(
+    state.db.getStorageUpdatedCache(), accState.account.storageRoot, isPruning = false
+  )
+
+  try:
+    for key in trie.keys():
+      if key.len() == 0:
+        continue # skip the empty node created on initialization
+      yield (KeccakHash.fromBytes(key), trie.getBranch(key))
+  except RlpError as e:
+    raiseAssert(e.msg) # should never happen unless the database is corrupted
+
+proc getUpdatedBytecode*(
+    state: WorldStateRef, address: EthAddress
+): seq[byte] {.inline.} =
+  state.db.getBytecodeUpdatedCache().get(toAccountKey(address).data)

fluffy/tools/portal_bridge/state_bridge/world_state_helper.nim

@@ -11,6 +11,7 @@ import
   chronicles,
   stint,
   results,
+  stew/byteutils,
   eth/common/[eth_types, eth_types_rlp],
   ../../../../nimbus/common/chain_config,
   ./[state_diff, world_state]
@@ -31,12 +32,14 @@ proc applyGenesisAccounts*(worldState: WorldStateRef, alloc: GenesisAlloc) =
 
     worldState.setAccount(address, accState)
 
-proc applyStateDiff*(worldState: WorldStateRef, stateDiff: StateDiffRef) =
+proc applyStateDiff*(worldState: WorldStateRef, txDiff: TransactionDiff) =
-  for address, balanceDiff in stateDiff.balances:
+  for accountDiff in txDiff:
     let
-      nonceDiff = stateDiff.nonces.getOrDefault(address)
+      address = accountDiff.address
-      codeDiff = stateDiff.code.getOrDefault(address)
+      balanceDiff = accountDiff.balanceDiff
-      storageDiff = stateDiff.storage.getOrDefault(address)
+      nonceDiff = accountDiff.nonceDiff
+      codeDiff = accountDiff.codeDiff
+      storageDiff = accountDiff.storageDiff
 
     var
       deleteAccount = false
@@ -59,13 +62,13 @@ proc applyStateDiff*(worldState: WorldStateRef, stateDiff: StateDiffRef) =
     elif codeDiff.kind == delete:
       doAssert deleteAccount == true
 
-    for slotKey, slotDiff in storageDiff:
+    for (slotKey, slotValueDiff) in storageDiff:
-      if slotDiff.kind == create or slotDiff.kind == update:
+      if slotValueDiff.kind == create or slotValueDiff.kind == update:
-        if slotDiff.after == 0:
+        if slotValueDiff.after == 0:
           accState.deleteStorage(slotKey)
         else:
-          accState.setStorage(slotKey, slotDiff.after)
+          accState.setStorage(slotKey, slotValueDiff.after)
-      elif slotDiff.kind == delete:
+      elif slotValueDiff.kind == delete:
         accState.deleteStorage(slotKey)
 
     if deleteAccount:
@@ -75,8 +78,8 @@ proc applyStateDiff*(worldState: WorldStateRef, stateDiff: StateDiffRef) =
 
 proc applyBlockRewards*(
     worldState: WorldStateRef,
-    minerData: tuple[miner: EthAddress, number: uint64],
+    minerData: tuple[miner: EthAddress, blockNumber: uint64],
-    uncleMinersData: openArray[tuple[miner: EthAddress, number: uint64]],
+    uncleMinersData: openArray[tuple[miner: EthAddress, blockNumber: uint64]],
 ) =
   const baseReward = u256(5) * pow(u256(10), 18)
 
@@ -95,7 +98,145 @@ proc applyBlockRewards*(
     let
       uncleMinerAddress = EthAddress(uncleMinerData.miner)
       uncleReward =
-        (u256(8 + uncleMinerData.number - minerData.number) * baseReward) shr 3
+        (u256(8 + uncleMinerData.blockNumber - minerData.blockNumber) * baseReward) shr 3
     var accState = worldState.getAccount(uncleMinerAddress)
     accState.addBalance(uncleReward)
     worldState.setAccount(uncleMinerAddress, accState)
+
+const
+  DAORefundContract: EthAddress =
+    hexToByteArray[20]("0xbf4ed7b27f1d666546e30d74d50d173d20bca754")
+
+  DAODrainList = [
+    hexToByteArray[20]("0xd4fe7bc31cedb7bfb8a345f31e668033056b2728"),
+    hexToByteArray[20]("0xb3fb0e5aba0e20e5c49d252dfd30e102b171a425"),
+    hexToByteArray[20]("0x2c19c7f9ae8b751e37aeb2d93a699722395ae18f"),
+    hexToByteArray[20]("0xecd135fa4f61a655311e86238c92adcd779555d2"),
+    hexToByteArray[20]("0x1975bd06d486162d5dc297798dfc41edd5d160a7"),
+    hexToByteArray[20]("0xa3acf3a1e16b1d7c315e23510fdd7847b48234f6"),
+    hexToByteArray[20]("0x319f70bab6845585f412ec7724b744fec6095c85"),
+    hexToByteArray[20]("0x06706dd3f2c9abf0a21ddcc6941d9b86f0596936"),
+    hexToByteArray[20]("0x5c8536898fbb74fc7445814902fd08422eac56d0"),
+    hexToByteArray[20]("0x6966ab0d485353095148a2155858910e0965b6f9"),
+    hexToByteArray[20]("0x779543a0491a837ca36ce8c635d6154e3c4911a6"),
+    hexToByteArray[20]("0x2a5ed960395e2a49b1c758cef4aa15213cfd874c"),
+    hexToByteArray[20]("0x5c6e67ccd5849c0d29219c4f95f1a7a93b3f5dc5"),
+    hexToByteArray[20]("0x9c50426be05db97f5d64fc54bf89eff947f0a321"),
+    hexToByteArray[20]("0x200450f06520bdd6c527622a273333384d870efb"),
+    hexToByteArray[20]("0xbe8539bfe837b67d1282b2b1d61c3f723966f049"),
+    hexToByteArray[20]("0x6b0c4d41ba9ab8d8cfb5d379c69a612f2ced8ecb"),
+    hexToByteArray[20]("0xf1385fb24aad0cd7432824085e42aff90886fef5"),
+    hexToByteArray[20]("0xd1ac8b1ef1b69ff51d1d401a476e7e612414f091"),
+    hexToByteArray[20]("0x8163e7fb499e90f8544ea62bbf80d21cd26d9efd"),
+    hexToByteArray[20]("0x51e0ddd9998364a2eb38588679f0d2c42653e4a6"),
+    hexToByteArray[20]("0x627a0a960c079c21c34f7612d5d230e01b4ad4c7"),
+    hexToByteArray[20]("0xf0b1aa0eb660754448a7937c022e30aa692fe0c5"),
+    hexToByteArray[20]("0x24c4d950dfd4dd1902bbed3508144a54542bba94"),
+    hexToByteArray[20]("0x9f27daea7aca0aa0446220b98d028715e3bc803d"),
+    hexToByteArray[20]("0xa5dc5acd6a7968a4554d89d65e59b7fd3bff0f90"),
+    hexToByteArray[20]("0xd9aef3a1e38a39c16b31d1ace71bca8ef58d315b"),
+    hexToByteArray[20]("0x63ed5a272de2f6d968408b4acb9024f4cc208ebf"),
+    hexToByteArray[20]("0x6f6704e5a10332af6672e50b3d9754dc460dfa4d"),
+    hexToByteArray[20]("0x77ca7b50b6cd7e2f3fa008e24ab793fd56cb15f6"),
+    hexToByteArray[20]("0x492ea3bb0f3315521c31f273e565b868fc090f17"),
+    hexToByteArray[20]("0x0ff30d6de14a8224aa97b78aea5388d1c51c1f00"),
+    hexToByteArray[20]("0x9ea779f907f0b315b364b0cfc39a0fde5b02a416"),
+    hexToByteArray[20]("0xceaeb481747ca6c540a000c1f3641f8cef161fa7"),
+    hexToByteArray[20]("0xcc34673c6c40e791051898567a1222daf90be287"),
+    hexToByteArray[20]("0x579a80d909f346fbfb1189493f521d7f48d52238"),
+    hexToByteArray[20]("0xe308bd1ac5fda103967359b2712dd89deffb7973"),
+    hexToByteArray[20]("0x4cb31628079fb14e4bc3cd5e30c2f7489b00960c"),
+    hexToByteArray[20]("0xac1ecab32727358dba8962a0f3b261731aad9723"),
+    hexToByteArray[20]("0x4fd6ace747f06ece9c49699c7cabc62d02211f75"),
+    hexToByteArray[20]("0x440c59b325d2997a134c2c7c60a8c61611212bad"),
+    hexToByteArray[20]("0x4486a3d68fac6967006d7a517b889fd3f98c102b"),
+    hexToByteArray[20]("0x9c15b54878ba618f494b38f0ae7443db6af648ba"),
+    hexToByteArray[20]("0x27b137a85656544b1ccb5a0f2e561a5703c6a68f"),
+    hexToByteArray[20]("0x21c7fdb9ed8d291d79ffd82eb2c4356ec0d81241"),
+    hexToByteArray[20]("0x23b75c2f6791eef49c69684db4c6c1f93bf49a50"),
+    hexToByteArray[20]("0x1ca6abd14d30affe533b24d7a21bff4c2d5e1f3b"),
+    hexToByteArray[20]("0xb9637156d330c0d605a791f1c31ba5890582fe1c"),
+    hexToByteArray[20]("0x6131c42fa982e56929107413a9d526fd99405560"),
+    hexToByteArray[20]("0x1591fc0f688c81fbeb17f5426a162a7024d430c2"),
+    hexToByteArray[20]("0x542a9515200d14b68e934e9830d91645a980dd7a"),
+    hexToByteArray[20]("0xc4bbd073882dd2add2424cf47d35213405b01324"),
+    hexToByteArray[20]("0x782495b7b3355efb2833d56ecb34dc22ad7dfcc4"),
+    hexToByteArray[20]("0x58b95c9a9d5d26825e70a82b6adb139d3fd829eb"),
+    hexToByteArray[20]("0x3ba4d81db016dc2890c81f3acec2454bff5aada5"),
+    hexToByteArray[20]("0xb52042c8ca3f8aa246fa79c3feaa3d959347c0ab"),
+    hexToByteArray[20]("0xe4ae1efdfc53b73893af49113d8694a057b9c0d1"),
+    hexToByteArray[20]("0x3c02a7bc0391e86d91b7d144e61c2c01a25a79c5"),
+    hexToByteArray[20]("0x0737a6b837f97f46ebade41b9bc3e1c509c85c53"),
+    hexToByteArray[20]("0x97f43a37f595ab5dd318fb46e7a155eae057317a"),
+    hexToByteArray[20]("0x52c5317c848ba20c7504cb2c8052abd1fde29d03"),
+    hexToByteArray[20]("0x4863226780fe7c0356454236d3b1c8792785748d"),
+    hexToByteArray[20]("0x5d2b2e6fcbe3b11d26b525e085ff818dae332479"),
+    hexToByteArray[20]("0x5f9f3392e9f62f63b8eac0beb55541fc8627f42c"),
+    hexToByteArray[20]("0x057b56736d32b86616a10f619859c6cd6f59092a"),
+    hexToByteArray[20]("0x9aa008f65de0b923a2a4f02012ad034a5e2e2192"),
+    hexToByteArray[20]("0x304a554a310c7e546dfe434669c62820b7d83490"),
+    hexToByteArray[20]("0x914d1b8b43e92723e64fd0a06f5bdb8dd9b10c79"),
+    hexToByteArray[20]("0x4deb0033bb26bc534b197e61d19e0733e5679784"),
+    hexToByteArray[20]("0x07f5c1e1bc2c93e0402f23341973a0e043f7bf8a"),
+    hexToByteArray[20]("0x35a051a0010aba705c9008d7a7eff6fb88f6ea7b"),
+    hexToByteArray[20]("0x4fa802324e929786dbda3b8820dc7834e9134a2a"),
+    hexToByteArray[20]("0x9da397b9e80755301a3b32173283a91c0ef6c87e"),
+    hexToByteArray[20]("0x8d9edb3054ce5c5774a420ac37ebae0ac02343c6"),
+    hexToByteArray[20]("0x0101f3be8ebb4bbd39a2e3b9a3639d4259832fd9"),
+    hexToByteArray[20]("0x5dc28b15dffed94048d73806ce4b7a4612a1d48f"),
+    hexToByteArray[20]("0xbcf899e6c7d9d5a215ab1e3444c86806fa854c76"),
+    hexToByteArray[20]("0x12e626b0eebfe86a56d633b9864e389b45dcb260"),
+    hexToByteArray[20]("0xa2f1ccba9395d7fcb155bba8bc92db9bafaeade7"),
+    hexToByteArray[20]("0xec8e57756626fdc07c63ad2eafbd28d08e7b0ca5"),
+    hexToByteArray[20]("0xd164b088bd9108b60d0ca3751da4bceb207b0782"),
+    hexToByteArray[20]("0x6231b6d0d5e77fe001c2a460bd9584fee60d409b"),
+    hexToByteArray[20]("0x1cba23d343a983e9b5cfd19496b9a9701ada385f"),
+    hexToByteArray[20]("0xa82f360a8d3455c5c41366975bde739c37bfeb8a"),
+    hexToByteArray[20]("0x9fcd2deaff372a39cc679d5c5e4de7bafb0b1339"),
+    hexToByteArray[20]("0x005f5cee7a43331d5a3d3eec71305925a62f34b6"),
+    hexToByteArray[20]("0x0e0da70933f4c7849fc0d203f5d1d43b9ae4532d"),
+    hexToByteArray[20]("0xd131637d5275fd1a68a3200f4ad25c71a2a9522e"),
+    hexToByteArray[20]("0xbc07118b9ac290e4622f5e77a0853539789effbe"),
+    hexToByteArray[20]("0x47e7aa56d6bdf3f36be34619660de61275420af8"),
+    hexToByteArray[20]("0xacd87e28b0c9d1254e868b81cba4cc20d9a32225"),
+    hexToByteArray[20]("0xadf80daec7ba8dcf15392f1ac611fff65d94f880"),
+    hexToByteArray[20]("0x5524c55fb03cf21f549444ccbecb664d0acad706"),
+    hexToByteArray[20]("0x40b803a9abce16f50f36a77ba41180eb90023925"),
+    hexToByteArray[20]("0xfe24cdd8648121a43a7c86d289be4dd2951ed49f"),
+    hexToByteArray[20]("0x17802f43a0137c506ba92291391a8a8f207f487d"),
+    hexToByteArray[20]("0x253488078a4edf4d6f42f113d1e62836a942cf1a"),
+    hexToByteArray[20]("0x86af3e9626fce1957c82e88cbf04ddf3a2ed7915"),
+    hexToByteArray[20]("0xb136707642a4ea12fb4bae820f03d2562ebff487"),
+    hexToByteArray[20]("0xdbe9b615a3ae8709af8b93336ce9b477e4ac0940"),
+    hexToByteArray[20]("0xf14c14075d6c4ed84b86798af0956deef67365b5"),
+    hexToByteArray[20]("0xca544e5c4687d109611d0f8f928b53a25af72448"),
+    hexToByteArray[20]("0xaeeb8ff27288bdabc0fa5ebb731b6f409507516c"),
+    hexToByteArray[20]("0xcbb9d3703e651b0d496cdefb8b92c25aeb2171f7"),
+    hexToByteArray[20]("0x6d87578288b6cb5549d5076a207456a1f6a63dc0"),
+    hexToByteArray[20]("0xb2c6f0dfbb716ac562e2d85d6cb2f8d5ee87603e"),
+    hexToByteArray[20]("0xaccc230e8a6e5be9160b8cdf2864dd2a001c28b6"),
+    hexToByteArray[20]("0x2b3455ec7fedf16e646268bf88846bd7a2319bb2"),
+    hexToByteArray[20]("0x4613f3bca5c44ea06337a9e439fbc6d42e501d0a"),
+    hexToByteArray[20]("0xd343b217de44030afaa275f54d31a9317c7f441e"),
+    hexToByteArray[20]("0x84ef4b2357079cd7a7c69fd7a37cd0609a679106"),
+    hexToByteArray[20]("0xda2fef9e4a3230988ff17df2165440f37e8b1708"),
+    hexToByteArray[20]("0xf4c64518ea10f995918a454158c6b61407ea345c"),
+    hexToByteArray[20]("0x7602b46df5390e432ef1c307d4f2c9ff6d65cc97"),
+    hexToByteArray[20]("0xbb9bc244d798123fde783fcc1c72d3bb8c189413"),
+    hexToByteArray[20]("0x807640a13483f8ac783c557fcdf27be11ea4ac7a"),
+  ]
+
+# ApplyDAOHardFork modifies the state database according to the DAO hard-fork
+# rules, transferring all balances of a set of DAO accounts to a single refund
+# contract.
+proc applyDAOHardFork*(worldState: WorldStateRef) =
+  # Move every DAO account and extra-balance account funds into the refund contract
+  var toAccount = worldState.getAccount(DAORefundContract)
+
+  for address in DAODrainList:
+    var fromAccount = worldState.getAccount(address)
+    toAccount.addBalance(fromAccount.getBalance())
+    fromAccount.setBalance(0.u256)
+    worldState.setAccount(address, fromAccount)
+
+  worldState.setAccount(DAORefundContract, toAccount)