From 947f629903052d51a044f0d650aa3c9036348e8f Mon Sep 17 00:00:00 2001 From: web3-developer <51288821+web3-developer@users.noreply.github.com> Date: Tue, 30 Jul 2024 22:56:21 +0800 Subject: [PATCH] Fluffy State Bridge - State Gossip via Portal JSON-RPC (#2535) * Create block offers queue and collect account preimages. * Implement iterators to return account and storage proofs and bytecode from updatedCaches. * Implement building offers from proofs. * Refactor BlockDataRef type to only include required fields. * Store block data in database. * Improve state diff types. * Implement start state backfill from specific block. * Record last persisted block number in database. * Persist account preimages in db. * Apply state updates for DAO hard fork. * Implement state gossip of block offers via portal JSON RPC. --- fluffy/network/state/content/content_keys.nim | 2 + .../network/state/content/content_values.nim | 6 +- fluffy/network/state/content/nibbles.nim | 4 +- fluffy/network/state/state_gossip.nim | 11 +- fluffy/network/state/state_utils.nim | 14 + .../state_test_helpers.nim | 14 - .../test_state_validation_genesis.nim | 2 +- .../portal_bridge/portal_bridge_conf.nim | 8 +- .../portal_bridge/portal_bridge_state.nim | 328 +++++++++++++----- .../portal_bridge/state_bridge/database.nim | 84 +++-- .../state_bridge/offers_builder.nim | 96 +++++ .../portal_bridge/state_bridge/state_diff.nim | 65 ++-- .../state_bridge/world_state.nim | 119 +++++-- .../state_bridge/world_state_helper.nim | 167 ++++++++- 14 files changed, 722 insertions(+), 198 deletions(-) create mode 100644 fluffy/tools/portal_bridge/state_bridge/offers_builder.nim diff --git a/fluffy/network/state/content/content_keys.nim b/fluffy/network/state/content/content_keys.nim index 3867d767c..acaefbc80 100644 --- a/fluffy/network/state/content/content_keys.nim +++ b/fluffy/network/state/content/content_keys.nim @@ -62,6 +62,8 @@ type of contractCode: contractCodeKey*: ContractCodeKey + ContentKeyType* = AccountTrieNodeKey | ContractTrieNodeKey | ContractCodeKey + func init*(T: type AccountTrieNodeKey, path: Nibbles, nodeHash: NodeHash): T = AccountTrieNodeKey(path: path, nodeHash: nodeHash) diff --git a/fluffy/network/state/content/content_values.nim b/fluffy/network/state/content/content_values.nim index efc5189a3..66d98e9b8 100644 --- a/fluffy/network/state/content/content_values.nim +++ b/fluffy/network/state/content/content_values.nim @@ -15,9 +15,9 @@ import results, eth/common/eth_types, ssz_serialization, ../../../common/common_ export ssz_serialization, common_types, hash, results const - MAX_TRIE_NODE_LEN = 1024 - MAX_TRIE_PROOF_LEN = 65 - MAX_BYTECODE_LEN = 32768 + MAX_TRIE_NODE_LEN* = 1024 + MAX_TRIE_PROOF_LEN* = 65 + MAX_BYTECODE_LEN* = 32768 type TrieNode* = List[byte, MAX_TRIE_NODE_LEN] diff --git a/fluffy/network/state/content/nibbles.nim b/fluffy/network/state/content/nibbles.nim index d84a706d0..af489e386 100644 --- a/fluffy/network/state/content/nibbles.nim +++ b/fluffy/network/state/content/nibbles.nim @@ -102,7 +102,7 @@ func unpackNibbles*(packed: Nibbles): UnpackedNibbles = output.add(first) output.add(second) - move(output) + ensureMove(output) func len(packed: Nibbles): int = let lenExclPrefix = (packed.len() - 1) * 2 @@ -115,4 +115,4 @@ func len(packed: Nibbles): int = func dropN*(unpacked: UnpackedNibbles, num: int): UnpackedNibbles = var nibbles = unpacked nibbles.setLen(nibbles.len() - num) - move(nibbles) + ensureMove(nibbles) diff --git a/fluffy/network/state/state_gossip.nim b/fluffy/network/state/state_gossip.nim index 4dbfbda52..4ad2ddc7d 100644 --- a/fluffy/network/state/state_gossip.nim +++ b/fluffy/network/state/state_gossip.nim @@ -29,17 +29,12 @@ type AccountTrieOfferWithKey* = type ContractTrieOfferWithKey* = tuple[key: ContractTrieNodeKey, offer: ContractTrieNodeOffer] +type ContractCodeOfferWithKey* = tuple[key: ContractCodeKey, offer: ContractCodeOffer] + func withPath(proof: TrieProof, path: Nibbles): ProofWithPath = (path: path, proof: proof) -func withKey*( - offer: AccountTrieNodeOffer, key: AccountTrieNodeKey -): AccountTrieOfferWithKey = - (key: key, offer: offer) - -func withKey*( - offer: ContractTrieNodeOffer, key: ContractTrieNodeKey -): ContractTrieOfferWithKey = +func withKey*(offer: ContentOfferType, key: ContentKeyType): auto = (key: key, offer: offer) func getParent(p: ProofWithPath): ProofWithPath = diff --git a/fluffy/network/state/state_utils.nim b/fluffy/network/state/state_utils.nim index 7c560bc1f..76c7f94b4 100644 --- a/fluffy/network/state/state_utils.nim +++ b/fluffy/network/state/state_utils.nim @@ -70,6 +70,20 @@ func toSlot*(storageProof: TrieProof): Result[UInt256, string] {.inline.} = rlpDecodeContractTrieNode(storageProof[^1]) +func removeLeafKeyEndNibbles*( + nibbles: Nibbles, leafNode: TrieNode +): Nibbles {.raises: RlpError.} = + let nodeRlp = rlpFromBytes(leafNode.asSeq()) + doAssert(nodeRlp.listLen() == 2) + let (_, isLeaf, prefix) = decodePrefix(nodeRlp.listElem(0)) + doAssert(isLeaf) + + let leafPrefix = prefix.unpackNibbles() + var unpackedNibbles = nibbles.unpackNibbles() + doAssert(unpackedNibbles[^leafPrefix.len() .. ^1] == leafPrefix) + + unpackedNibbles.dropN(leafPrefix.len()).packNibbles() + func toPath*(hash: KeccakHash): Nibbles {.inline.} = Nibbles.init(hash.data, isEven = true) diff --git a/fluffy/tests/state_network_tests/state_test_helpers.nim b/fluffy/tests/state_network_tests/state_test_helpers.nim index 13ca3c3d2..717f1fd2f 100644 --- a/fluffy/tests/state_network_tests/state_test_helpers.nim +++ b/fluffy/tests/state_network_tests/state_test_helpers.nim @@ -61,20 +61,6 @@ type func asNibbles*(key: openArray[byte], isEven = true): Nibbles = Nibbles.init(key, isEven) -func removeLeafKeyEndNibbles*( - nibbles: Nibbles, leafNode: TrieNode -): Nibbles {.raises: [RlpError].} = - let nodeRlp = rlpFromBytes(leafNode.asSeq()) - doAssert(nodeRlp.listLen() == 2) - let (_, isLeaf, prefix) = decodePrefix(nodeRlp.listElem(0)) - doAssert(isLeaf) - - let leafPrefix = prefix.unpackNibbles() - var unpackedNibbles = nibbles.unpackNibbles() - doAssert(unpackedNibbles[^leafPrefix.len() .. ^1] == leafPrefix) - - unpackedNibbles.dropN(leafPrefix.len()).packNibbles() - func asTrieProof*(branch: openArray[seq[byte]]): TrieProof = TrieProof.init(branch.map(node => TrieNode.init(node))) diff --git a/fluffy/tests/state_network_tests/test_state_validation_genesis.nim b/fluffy/tests/state_network_tests/test_state_validation_genesis.nim index 8e80807ed..e6150a018 100644 --- a/fluffy/tests/state_network_tests/test_state_validation_genesis.nim +++ b/fluffy/tests/state_network_tests/test_state_validation_genesis.nim @@ -13,7 +13,7 @@ import results, eth/[common, trie, trie/trie_defs], ../../../nimbus/common/chain_config, - ../../network/state/[state_content, state_validation], + ../../network/state/[state_content, state_validation, state_utils], ./state_test_helpers template checkValidProofsForExistingLeafs( diff --git a/fluffy/tools/portal_bridge/portal_bridge_conf.nim b/fluffy/tools/portal_bridge/portal_bridge_conf.nim index d3bcdef94..10f84307c 100644 --- a/fluffy/tools/portal_bridge/portal_bridge_conf.nim +++ b/fluffy/tools/portal_bridge/portal_bridge_conf.nim @@ -141,10 +141,10 @@ type name: "state-dir" .}: InputDir - # TODO: support starting from a specific block. Currently this is not possible using the existing HexaryTrie library. - # startBlockNumber* {. - # desc: "The block number to start from", defaultValue: 1, name: "start-block" - # .}: uint64 + startBlockNumber* {. + desc: "The block number to start from", defaultValue: 1, name: "start-block" + .}: uint64 + verifyState* {. desc: "Verify the fetched state before gossiping it into the network", defaultValue: true, diff --git a/fluffy/tools/portal_bridge/portal_bridge_state.nim b/fluffy/tools/portal_bridge/portal_bridge_state.nim index 157d5a94f..66a5dcf9c 100644 --- a/fluffy/tools/portal_bridge/portal_bridge_state.nim +++ b/fluffy/tools/portal_bridge/portal_bridge_state.nim @@ -12,100 +12,169 @@ import chronicles, chronos, stint, + stew/byteutils, web3/[eth_api, eth_api_types], results, eth/common/[eth_types, eth_types_rlp], ../../../nimbus/common/chain_config, + ../../common/common_utils, ../../rpc/rpc_calls/rpc_trace_calls, - ./state_bridge/[database, state_diff, world_state_helper], + ../../rpc/portal_rpc_client, + ../../network/state/[state_content, state_gossip], + ./state_bridge/[database, state_diff, world_state_helper, offers_builder], ./[portal_bridge_conf, portal_bridge_common] type BlockData = object blockNumber: uint64 - blockObject: BlockObject - stateDiffs: seq[StateDiffRef] - uncleBlocks: seq[BlockObject] + blockHash: KeccakHash + miner: EthAddress + uncles: seq[tuple[miner: EthAddress, blockNumber: uint64]] + parentStateRoot: KeccakHash + stateRoot: KeccakHash + stateDiffs: seq[TransactionDiff] + +type BlockOffersRef = ref object + blockNumber: uint64 + accountTrieOffers: seq[AccountTrieOfferWithKey] + contractTrieOffers: seq[ContractTrieOfferWithKey] + contractCodeOffers: seq[ContractCodeOfferWithKey] + +proc getBlockData(db: DatabaseRef, blockNumber: uint64): Opt[BlockData] = + let blockDataBytes = db.get(rlp.encode(blockNumber)) + if blockDataBytes.len() == 0: + return Opt.none(BlockData) + + try: + Opt.some(rlp.decode(blockDataBytes, BlockData)) + except RlpError as e: + raiseAssert(e.msg) # Should never happen + +proc putBlockData( + db: DatabaseRef, blockNumber: uint64, blockData: BlockData +) {.inline.} = + db.put(rlp.encode(blockNumber), rlp.encode(blockData)) + +proc getLastPersistedBlockNumber(db: DatabaseRef): Opt[uint64] = + let blockNumberBytes = db.get(rlp.encode("lastPersistedBlockNumber")) + if blockNumberBytes.len() == 0: + return Opt.none(uint64) + + try: + Opt.some(rlp.decode(blockNumberBytes, uint64)) + except RlpError as e: + raiseAssert(e.msg) # Should never happen + +proc putLastPersistedBlockNumber(db: DatabaseRef, blockNumber: uint64) {.inline.} = + db.put(rlp.encode("lastPersistedBlockNumber"), rlp.encode(blockNumber)) proc runBackfillCollectBlockDataLoop( + db: DatabaseRef, blockDataQueue: AsyncQueue[BlockData], web3Client: RpcClient, startBlockNumber: uint64, ) {.async: (raises: [CancelledError]).} = - debug "Starting state backfill collect block data loop" + info "Starting state backfill collect block data loop" - var currentBlockNumber = startBlockNumber + let parentBlock = ( + await web3Client.getBlockByNumber(blockId(startBlockNumber - 1.uint64), false) + ).valueOr: + raiseAssert("Failed to get parent block") + + var + parentStateRoot = parentBlock.stateRoot + currentBlockNumber = startBlockNumber while true: if currentBlockNumber mod 10000 == 0: info "Collecting block data for block number: ", blockNumber = currentBlockNumber - let - blockId = blockId(currentBlockNumber) - blockRequest = web3Client.getBlockByNumber(blockId, false) - stateDiffsRequest = web3Client.getStateDiffsByBlockNumber(blockId) - - blockObject = (await blockRequest).valueOr: - error "Failed to get block", error - await sleepAsync(1.seconds) - continue - - var uncleBlockRequests: seq[Future[Result[BlockObject, string]]] - for i in 0 .. blockObject.uncles.high: - uncleBlockRequests.add( - web3Client.getUncleByBlockNumberAndIndex(blockId, i.Quantity) - ) - - let stateDiffs = (await stateDiffsRequest).valueOr: - error "Failed to get state diffs", error - await sleepAsync(1.seconds) - continue - - var uncleBlocks: seq[BlockObject] - for uncleBlockRequest in uncleBlockRequests: - try: - let uncleBlock = (await uncleBlockRequest).valueOr: - error "Failed to get uncle blocks", error + let blockData = db.getBlockData(currentBlockNumber).valueOr: + # block data doesn't exist in db so we fetch it via RPC + let + blockId = blockId(currentBlockNumber) + blockObject = (await web3Client.getBlockByNumber(blockId, false)).valueOr: + error "Failed to get block", error await sleepAsync(1.seconds) - break + continue + stateDiffs = (await web3Client.getStateDiffsByBlockNumber(blockId)).valueOr: + error "Failed to get state diffs", error + await sleepAsync(1.seconds) + continue + + var uncleBlocks: seq[BlockObject] + for i in 0 .. blockObject.uncles.high: + let uncleBlock = ( + await web3Client.getUncleByBlockNumberAndIndex(blockId, i.Quantity) + ).valueOr: + error "Failed to get uncle block", error + await sleepAsync(1.seconds) + continue uncleBlocks.add(uncleBlock) - except CatchableError as e: - error "Failed to get uncleBlockRequest", error = e.msg - break - if uncleBlocks.len() < uncleBlockRequests.len(): - continue + let blockData = BlockData( + blockNumber: currentBlockNumber, + blockHash: KeccakHash.fromBytes(blockObject.hash.bytes()), + miner: blockObject.miner.EthAddress, + uncles: uncleBlocks.mapIt((it.miner.EthAddress, it.number.uint64)), + parentStateRoot: KeccakHash.fromBytes(parentStateRoot.bytes()), + stateRoot: KeccakHash.fromBytes(blockObject.stateRoot.bytes()), + stateDiffs: stateDiffs, + ) + db.putBlockData(currentBlockNumber, blockData) + + parentStateRoot = blockObject.stateRoot + blockData - let blockData = BlockData( - blockNumber: currentBlockNumber, - blockObject: blockObject, - stateDiffs: stateDiffs, - uncleBlocks: uncleBlocks, - ) await blockDataQueue.addLast(blockData) - inc currentBlockNumber -proc runBackfillBuildStateLoop( - blockDataQueue: AsyncQueue[BlockData], stateDir: string +proc runBackfillBuildBlockOffersLoop( + db: DatabaseRef, + blockDataQueue: AsyncQueue[BlockData], + blockOffersQueue: AsyncQueue[BlockOffersRef], ) {.async: (raises: [CancelledError]).} = - debug "Starting state backfill build state loop" + info "Starting state backfill build block offers loop" - let db = DatabaseRef.init(stateDir).get() - defer: - db.close() + # wait for the first block data to be put on the queue + # so that we can access the first block once available + while blockDataQueue.empty(): + await sleepAsync(100.milliseconds) + # peek but don't remove it so that it can be processed later + let firstBlock = blockDataQueue[0] - let worldState = db.withTransaction: - let + # Only apply genesis accounts if starting from block 1 + if firstBlock.blockNumber == 1: + info "Building state for genesis" + + db.withTransaction: # Requires an active transaction because it writes an emptyRlp node # to the accounts HexaryTrie on initialization - ws = WorldStateRef.init(db) - genesisAccounts = - try: - genesisBlockForNetwork(MainNet).alloc - except CatchableError as e: - raiseAssert(e.msg) # Should never happen - ws.applyGenesisAccounts(genesisAccounts) - ws + let + ws = WorldStateRef.init(db) + genesisAccounts = + try: + genesisBlockForNetwork(MainNet).alloc + except CatchableError as e: + raiseAssert(e.msg) # Should never happen + ws.applyGenesisAccounts(genesisAccounts) + + let genesisBlockHash = KeccakHash.fromHex( + "d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" + ) + var builder = OffersBuilderRef.init(ws, genesisBlockHash) + builder.buildBlockOffers() + + await blockOffersQueue.addLast( + BlockOffersRef( + blockNumber: 0.uint64, + accountTrieOffers: builder.getAccountTrieOffers(), + contractTrieOffers: builder.getContractTrieOffers(), + contractCodeOffers: builder.getContractCodeOffers(), + ) + ) + + # Load the world state using the parent state root + let worldState = WorldStateRef.init(db, firstBlock.parentStateRoot) while true: let blockData = await blockDataQueue.popFirst() @@ -113,33 +182,112 @@ proc runBackfillBuildStateLoop( if blockData.blockNumber mod 10000 == 0: info "Building state for block number: ", blockNumber = blockData.blockNumber + # For now all WorldStateRef functions need to be inside a transaction + # because the DatabaseRef backends currently only supports reading and + # writing to/from a single active transaction. db.withTransaction: for stateDiff in blockData.stateDiffs: worldState.applyStateDiff(stateDiff) - let - minerData = - (EthAddress(blockData.blockObject.miner), blockData.blockObject.number.uint64) - uncleMinersData = - blockData.uncleBlocks.mapIt((EthAddress(it.miner), it.number.uint64)) - worldState.applyBlockRewards(minerData, uncleMinersData) - doAssert(blockData.blockObject.stateRoot.bytes() == worldState.stateRoot.data) - trace "State diffs successfully applied to block number:", - blockNumber = blockData.blockNumber + worldState.applyBlockRewards( + (blockData.miner, blockData.blockNumber), blockData.uncles + ) + + if blockData.blockNumber == 1_920_000: + info "Applying state updates for DAO hard fork" + worldState.applyDAOHardFork() + + doAssert( + worldState.stateRoot == blockData.stateRoot, + "State root mismatch at block number: " & $blockData.blockNumber, + ) + trace "State diffs successfully applied to block number:", + blockNumber = blockData.blockNumber + + # worldState.verifyProofs(blockData.parentStateRoot, blockData.stateRoot) + + var builder = OffersBuilderRef.init(worldState, blockData.blockHash) + builder.buildBlockOffers() + + await blockOffersQueue.addLast( + BlockOffersRef( + blockNumber: blockData.blockNumber, + accountTrieOffers: builder.getAccountTrieOffers(), + contractTrieOffers: builder.getContractTrieOffers(), + contractCodeOffers: builder.getContractCodeOffers(), + ) + ) + + # After commit of the above db transaction which stores the updated account state + # then we store the last persisted block number in the database so that we can use it + # to enable restarting from this block if needed + db.putLastPersistedBlockNumber(blockData.blockNumber) + +proc gossipOffer( + portalClient: RpcClient, + offerWithKey: + AccountTrieOfferWithKey | ContractTrieOfferWithKey | ContractCodeOfferWithKey, +) {.async: (raises: [CancelledError]).} = + let + keyBytes = offerWithKey.key.toContentKey().encode().asSeq() + offerBytes = offerWithKey.offer.encode() + try: + let numPeers = + await portalClient.portal_stateGossip(keyBytes.to0xHex(), offerBytes.to0xHex()) + debug "Gossiping offer to peers: ", offerKey = keyBytes.to0xHex(), numPeers + except CatchableError as e: + raiseAssert(e.msg) # Should never happen + +proc recursiveGossipOffer( + portalClient: RpcClient, + offerWithKey: AccountTrieOfferWithKey | ContractTrieOfferWithKey, +) {.async: (raises: [CancelledError]).} = + await portalClient.gossipOffer(offerWithKey) + + # root node, recursive gossip is finished + if offerWithKey.key.path.unpackNibbles().len() == 0: + return + + # continue the recursive gossip by sharing the parent offer with peers + await portalClient.recursiveGossipOffer(offerWithKey.getParent()) + +proc runBackfillGossipBlockOffersLoop( + blockOffersQueue: AsyncQueue[BlockOffersRef], portalClient: RpcClient +) {.async: (raises: [CancelledError]).} = + info "Starting state backfill gossip block offers loop" + + while true: + let blockOffers = await blockOffersQueue.popFirst() + + for offerWithKey in blockOffers.accountTrieOffers: + await portalClient.recursiveGossipOffer(offerWithKey) + + for offerWithKey in blockOffers.contractTrieOffers: + await portalClient.recursiveGossipOffer(offerWithKey) + + for offerWithKey in blockOffers.contractCodeOffers: + await portalClient.gossipOffer(offerWithKey) proc runBackfillMetricsLoop( - blockDataQueue: AsyncQueue[BlockData] + blockDataQueue: AsyncQueue[BlockData], blockOffersQueue: AsyncQueue[BlockOffersRef] ) {.async: (raises: [CancelledError]).} = - debug "Starting state backfill metrics loop" + info "Starting state backfill metrics loop" while true: await sleepAsync(10.seconds) - info "Block data queue length: ", queueLen = blockDataQueue.len() + info "Block data queue length: ", blockDataQueueLen = blockDataQueue.len() + info "Block offers queue length: ", blockOffersQueueLen = blockOffersQueue.len() proc runState*(config: PortalBridgeConf) = let - #portalClient = newRpcClientConnect(config.portalRpcUrl) + portalClient = newRpcClientConnect(config.portalRpcUrl) web3Client = newRpcClientConnect(config.web3UrlState) + db = DatabaseRef.init(config.stateDir.string).get() + defer: + db.close() + + if web3Client of RpcHttpClient: + warn "Using a WebSocket connection to the JSON-RPC API is recommended to improve performance" # TODO: # Here we'd want to implement initially a loop that backfills the state @@ -154,20 +302,34 @@ proc runState*(config: PortalBridgeConf) = # inside the bridge, and getting the blocks from era1 files. if config.backfillState: - const startBlockNumber = 1 - # This will become a parameter in the config once we can support it - info "Starting state backfill from block number: ", startBlockNumber + let maybeLastPersistedBlock = db.getLastPersistedBlockNumber() + if maybeLastPersistedBlock.isSome(): + info "Last persisted block found in the database: ", + lastPersistedBlock = maybeLastPersistedBlock.get() + if config.startBlockNumber < 1 or + config.startBlockNumber > maybeLastPersistedBlock.get(): + warn "Start block must be set to a value between 1 and the last persisted block" + quit QuitFailure + else: + info "No last persisted block found in the database" + if config.startBlockNumber != 1: + warn "Start block must be set to 1" + quit QuitFailure + + info "Starting state backfill from block number: ", + startBlockNumber = config.startBlockNumber const bufferSize = 1000 # Should we make this configurable? - let blockDataQueue = newAsyncQueue[BlockData](bufferSize) + let + blockDataQueue = newAsyncQueue[BlockData](bufferSize) + blockOffersQueue = newAsyncQueue[BlockOffersRef](bufferSize) asyncSpawn runBackfillCollectBlockDataLoop( - blockDataQueue, web3Client, startBlockNumber + db, blockDataQueue, web3Client, config.startBlockNumber ) - - asyncSpawn runBackfillBuildStateLoop(blockDataQueue, config.stateDir.string) - - asyncSpawn runBackfillMetricsLoop(blockDataQueue) + asyncSpawn runBackfillBuildBlockOffersLoop(db, blockDataQueue, blockOffersQueue) + asyncSpawn runBackfillGossipBlockOffersLoop(blockOffersQueue, portalClient) + asyncSpawn runBackfillMetricsLoop(blockDataQueue, blockOffersQueue) while true: poll() diff --git a/fluffy/tools/portal_bridge/state_bridge/database.nim b/fluffy/tools/portal_bridge/state_bridge/database.nim index 8d7e73775..a0c29ecbb 100644 --- a/fluffy/tools/portal_bridge/state_bridge/database.nim +++ b/fluffy/tools/portal_bridge/state_bridge/database.nim @@ -14,27 +14,36 @@ export results, db const COL_FAMILY_NAME_ACCOUNTS = "A" const COL_FAMILY_NAME_STORAGE = "S" const COL_FAMILY_NAME_BYTECODE = "B" +const COL_FAMILY_NAME_PREIMAGES = "P" -const COL_FAMILY_NAMES = - [COL_FAMILY_NAME_ACCOUNTS, COL_FAMILY_NAME_STORAGE, COL_FAMILY_NAME_BYTECODE] +const COL_FAMILY_NAMES = [ + COL_FAMILY_NAME_ACCOUNTS, COL_FAMILY_NAME_STORAGE, COL_FAMILY_NAME_BYTECODE, + COL_FAMILY_NAME_PREIMAGES, +] type AccountsBackendRef = ref object of RootObj cfHandle: ColFamilyHandleRef tx: TransactionRef - updatedCache: TableRef[seq[byte], seq[byte]] + updatedCache: TrieDatabaseRef StorageBackendRef = ref object of RootObj cfHandle: ColFamilyHandleRef tx: TransactionRef - updatedCache: TableRef[seq[byte], seq[byte]] + updatedCache: TrieDatabaseRef BytecodeBackendRef = ref object of RootObj cfHandle: ColFamilyHandleRef tx: TransactionRef - updatedCache: TableRef[seq[byte], seq[byte]] + updatedCache: TrieDatabaseRef - DatabaseBackendRef = AccountsBackendRef | StorageBackendRef | BytecodeBackendRef + PreimagesBackendRef = ref object of RootObj + cfHandle: ColFamilyHandleRef + tx: TransactionRef + updatedCache: TrieDatabaseRef + + DatabaseBackendRef = + AccountsBackendRef | StorageBackendRef | BytecodeBackendRef | PreimagesBackendRef DatabaseRef* = ref object rocksDb: OptimisticTxDbRef @@ -42,6 +51,7 @@ type accountsBackend: AccountsBackendRef storageBackend: StorageBackendRef bytecodeBackend: BytecodeBackendRef + preimagesBackend: PreimagesBackendRef proc init*(T: type DatabaseRef, baseDir: string): Result[T, string] = let dbPath = baseDir / "db" @@ -69,6 +79,9 @@ proc init*(T: type DatabaseRef, baseDir: string): Result[T, string] = bytecodeBackend = BytecodeBackendRef( cfHandle: db.getColFamilyHandle(COL_FAMILY_NAME_BYTECODE).get() ) + preimagesBackend = PreimagesBackendRef( + cfHandle: db.getColFamilyHandle(COL_FAMILY_NAME_PREIMAGES).get() + ) ok( T( @@ -77,6 +90,7 @@ proc init*(T: type DatabaseRef, baseDir: string): Result[T, string] = accountsBackend: accountsBackend, storageBackend: storageBackend, bytecodeBackend: bytecodeBackend, + preimagesBackend: preimagesBackend, ) ) @@ -92,7 +106,8 @@ proc put( dbBackend: DatabaseBackendRef, key, val: openArray[byte] ) {.gcsafe, raises: [].} = doAssert dbBackend.tx.put(key, val, dbBackend.cfHandle).isOk() - dbBackend.updatedCache[@key] = @val + if not dbBackend.updatedCache.isNil(): + dbBackend.updatedCache.put(key, val) proc get( dbBackend: DatabaseBackendRef, key: openArray[byte] @@ -111,15 +126,47 @@ proc del( else: false -proc getAccountsBackend*(db: DatabaseRef): TrieDatabaseRef = +proc getAccountsBackend*(db: DatabaseRef): TrieDatabaseRef {.inline.} = trieDB(db.accountsBackend) -proc getStorageBackend*(db: DatabaseRef): TrieDatabaseRef = +proc getStorageBackend*(db: DatabaseRef): TrieDatabaseRef {.inline.} = trieDB(db.storageBackend) -proc getBytecodeBackend*(db: DatabaseRef): TrieDatabaseRef = +proc getBytecodeBackend*(db: DatabaseRef): TrieDatabaseRef {.inline.} = trieDB(db.bytecodeBackend) +proc getPreimagesBackend*(db: DatabaseRef): TrieDatabaseRef {.inline.} = + trieDB(db.preimagesBackend) + +proc getAccountsUpdatedCache*(db: DatabaseRef): TrieDatabaseRef {.inline.} = + db.accountsBackend.updatedCache + +proc getStorageUpdatedCache*(db: DatabaseRef): TrieDatabaseRef {.inline.} = + db.storageBackend.updatedCache + +proc getBytecodeUpdatedCache*(db: DatabaseRef): TrieDatabaseRef {.inline.} = + db.bytecodeBackend.updatedCache + +proc put*(db: DatabaseRef, key, val: openArray[byte]) = + let tx = db.rocksDb.beginTransaction() + defer: + tx.close() + + # using default column family + doAssert tx.put(key, val).isOk() + doAssert tx.commit().isOk() + +proc get*(db: DatabaseRef, key: openArray[byte]): seq[byte] = + let tx = db.rocksDb.beginTransaction() + defer: + tx.close() + + # using default column family + if tx.get(key, onData).get(): + tx.get(key).get() + else: + @[] + proc beginTransaction*(db: DatabaseRef): Result[void, string] = if not db.pendingTransaction.isNil(): return err("DatabaseRef: Pending transaction already in progress") @@ -129,10 +176,12 @@ proc beginTransaction*(db: DatabaseRef): Result[void, string] = db.accountsBackend.tx = tx db.storageBackend.tx = tx db.bytecodeBackend.tx = tx + db.preimagesBackend.tx = tx - db.accountsBackend.updatedCache = newTable[seq[byte], seq[byte]]() - db.storageBackend.updatedCache = newTable[seq[byte], seq[byte]]() - db.bytecodeBackend.updatedCache = newTable[seq[byte], seq[byte]]() + db.accountsBackend.updatedCache = newMemoryDB() + db.storageBackend.updatedCache = newMemoryDB() + db.bytecodeBackend.updatedCache = newMemoryDB() + db.preimagesBackend.updatedCache = nil # not used ok() @@ -165,15 +214,6 @@ template withTransaction*(db: DatabaseRef, body: untyped): auto = finally: db.commitTransaction().expect("Transaction should be commited") -template accountsBackendUpdatedCache*(db: DatabaseRef): TableRef[seq[byte], seq[byte]] = - db.accountsBackend.updatedCache - -template storageBackendUpdatedCache*(db: DatabaseRef): TableRef[seq[byte], seq[byte]] = - db.storageBackend.updatedCache - -template bytecodeBackendUpdatedCache*(db: DatabaseRef): TableRef[seq[byte], seq[byte]] = - db.bytecodeBackend.updatedCache - proc close*(db: DatabaseRef) = if not db.pendingTransaction.isNil(): discard db.rollbackTransaction() diff --git a/fluffy/tools/portal_bridge/state_bridge/offers_builder.nim b/fluffy/tools/portal_bridge/state_bridge/offers_builder.nim new file mode 100644 index 000000000..7a19a3eeb --- /dev/null +++ b/fluffy/tools/portal_bridge/state_bridge/offers_builder.nim @@ -0,0 +1,96 @@ +# Fluffy +# Copyright (c) 2024 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [].} + +import + std/[sequtils, sugar], + eth/common, + ../../../network/state/[state_content, state_utils, state_gossip], + ./world_state + +type OffersBuilderRef* = ref object + worldState: WorldStateRef + blockHash: BlockHash + accountTrieOffers: seq[AccountTrieOfferWithKey] + contractTrieOffers: seq[ContractTrieOfferWithKey] + contractCodeOffers: seq[ContractCodeOfferWithKey] + +proc init*( + T: type OffersBuilderRef, worldState: WorldStateRef, blockHash: BlockHash +): T = + T(worldState: worldState, blockHash: blockHash) + +proc toTrieProof(proof: seq[seq[byte]]): TrieProof = + TrieProof.init(proof.map((node) => TrieNode.init(node))) + +proc buildAccountTrieNodeOffer( + builder: var OffersBuilderRef, address: EthAddress, proof: TrieProof +) = + try: + let + path = removeLeafKeyEndNibbles( + Nibbles.init(worldState.toAccountKey(address).data, isEven = true), proof[^1] + ) + offerKey = AccountTrieNodeKey.init(path, keccakHash(proof[^1].asSeq())) + offerValue = AccountTrieNodeOffer.init(proof, builder.blockHash) + + builder.accountTrieOffers.add(offerValue.withKey(offerKey)) + except RlpError as e: + raiseAssert(e.msg) # Should never happen + +proc buildContractTrieNodeOffer( + builder: var OffersBuilderRef, + address: EthAddress, + slotHash: SlotKeyHash, + storageProof: TrieProof, + accountProof: TrieProof, +) = + let + path = Nibbles.init(slotHash.data, isEven = true) + offerKey = + ContractTrieNodeKey.init(address, path, keccakHash(storageProof[^1].asSeq())) + offerValue = + ContractTrieNodeOffer.init(storageProof, accountProof, builder.blockHash) + + builder.contractTrieOffers.add(offerValue.withKey(offerKey)) + +proc buildContractCodeOffer( + builder: var OffersBuilderRef, + address: EthAddress, + code: seq[byte], + accountProof: TrieProof, +) = + let + #bytecode = Bytelist.init(code) # This fails to compile for some reason + bytecode = List[byte, MAX_BYTECODE_LEN](code) + offerKey = ContractCodeKey.init(address, keccakHash(code)) + offerValue = ContractCodeOffer.init(bytecode, accountProof, builder.blockHash) + + builder.contractCodeOffers.add(offerValue.withKey(offerKey)) + +proc buildBlockOffers*(builder: var OffersBuilderRef) = + for address, proof in builder.worldState.updatedAccountProofs(): + let accountProof = toTrieProof(proof) + builder.buildAccountTrieNodeOffer(address, accountProof) + + for slotHash, sProof in builder.worldState.updatedStorageProofs(address): + let storageProof = toTrieProof(sProof) + builder.buildContractTrieNodeOffer(address, slotHash, storageProof, accountProof) + + let code = builder.worldState.getUpdatedBytecode(address) + if code.len() > 0: + builder.buildContractCodeOffer(address, code, accountProof) + +proc getAccountTrieOffers*(builder: OffersBuilderRef): seq[AccountTrieOfferWithKey] = + builder.accountTrieOffers + +proc getContractTrieOffers*(builder: OffersBuilderRef): seq[ContractTrieOfferWithKey] = + builder.contractTrieOffers + +proc getContractCodeOffers*(builder: OffersBuilderRef): seq[ContractCodeOfferWithKey] = + builder.contractCodeOffers diff --git a/fluffy/tools/portal_bridge/state_bridge/state_diff.nim b/fluffy/tools/portal_bridge/state_bridge/state_diff.nim index 46562186e..3585dfc5d 100644 --- a/fluffy/tools/portal_bridge/state_bridge/state_diff.nim +++ b/fluffy/tools/portal_bridge/state_bridge/state_diff.nim @@ -30,11 +30,16 @@ type before*: StateValue after*: StateValue - StateDiffRef* = ref object - balances*: Table[EthAddress, StateValueDiff[UInt256]] - nonces*: Table[EthAddress, StateValueDiff[AccountNonce]] - storage*: Table[EthAddress, Table[UInt256, StateValueDiff[UInt256]]] - code*: Table[EthAddress, StateValueDiff[Code]] + SlotDiff* = tuple[slotKey: UInt256, slotValueDiff: StateValueDiff[UInt256]] + + AccountDiff* = object + address*: EthAddress + balanceDiff*: StateValueDiff[UInt256] + nonceDiff*: StateValueDiff[AccountNonce] + storageDiff*: seq[SlotDiff] + codeDiff*: StateValueDiff[Code] + + TransactionDiff* = seq[AccountDiff] proc toStateValue(T: type UInt256, hex: string): T {.raises: [ValueError].} = UInt256.fromHex(hex) @@ -68,45 +73,49 @@ proc toStateValueDiff( else: doAssert false # unreachable -proc toStateDiff(stateDiffJson: JsonNode): StateDiffRef {.raises: [ValueError].} = - let stateDiff = StateDiffRef() +proc toTransactionDiff( + stateDiffJson: JsonNode +): TransactionDiff {.raises: [ValueError].} = + var txDiff = newSeqOfCap[AccountDiff](stateDiffJson.len()) - for addrJson, accJson in stateDiffJson.pairs: - let address = EthAddress.fromHex(addrJson) + for addrJson, accJson in stateDiffJson: + let storageDiffJson = accJson["storage"] + var storageDiff = newSeqOfCap[SlotDiff](storageDiffJson.len()) - stateDiff.balances[address] = toStateValueDiff(accJson["balance"], UInt256) - stateDiff.nonces[address] = toStateValueDiff(accJson["nonce"], AccountNonce) - stateDiff.code[address] = toStateValueDiff(accJson["code"], Code) + for slotKeyJson, slotValueJson in storageDiffJson: + storageDiff.add( + (UInt256.fromHex(slotKeyJson), toStateValueDiff(slotValueJson, UInt256)) + ) - let storageDiff = accJson["storage"] - var accountStorage: Table[UInt256, StateValueDiff[UInt256]] + let accountDiff = AccountDiff( + address: EthAddress.fromHex(addrJson), + balanceDiff: toStateValueDiff(accJson["balance"], UInt256), + nonceDiff: toStateValueDiff(accJson["nonce"], AccountNonce), + storageDiff: storageDiff, + codeDiff: toStateValueDiff(accJson["code"], Code), + ) + txDiff.add(accountDiff) - for slotKeyJson, slotValueJson in storageDiff.pairs: - let slotKey = UInt256.fromHex(slotKeyJson) - accountStorage[slotKey] = toStateValueDiff(slotValueJson, UInt256) + txDiff - stateDiff.storage[address] = ensureMove(accountStorage) - - stateDiff - -proc toStateDiffs( +proc toTransactionDiffs( blockTraceJson: JsonNode -): seq[StateDiffRef] {.raises: [ValueError].} = - var stateDiffs = newSeqOfCap[StateDiffRef](blockTraceJson.len()) +): seq[TransactionDiff] {.raises: [ValueError].} = + var txDiffs = newSeqOfCap[TransactionDiff](blockTraceJson.len()) for blockTrace in blockTraceJson: - stateDiffs.add(blockTrace["stateDiff"].toStateDiff()) + txDiffs.add(blockTrace["stateDiff"].toTransactionDiff()) - stateDiffs + txDiffs proc getStateDiffsByBlockNumber*( client: RpcClient, blockId: BlockIdentifier -): Future[Result[seq[StateDiffRef], string]] {.async: (raises: []).} = +): Future[Result[seq[TransactionDiff], string]] {.async: (raises: []).} = const traceOpts = @["stateDiff"] try: let blockTraceJson = await client.trace_replayBlockTransactions(blockId, traceOpts) if blockTraceJson.isNil: return err("EL failed to provide requested state diff") - ok(blockTraceJson.toStateDiffs()) + ok(blockTraceJson.toTransactionDiffs()) except CatchableError as e: return err("EL JSON-RPC trace_replayBlockTransactions failed: " & e.msg) diff --git a/fluffy/tools/portal_bridge/state_bridge/world_state.nim b/fluffy/tools/portal_bridge/state_bridge/world_state.nim index 0d705ab27..56c5d482d 100644 --- a/fluffy/tools/portal_bridge/state_bridge/world_state.nim +++ b/fluffy/tools/portal_bridge/state_bridge/world_state.nim @@ -9,9 +9,9 @@ import stint, - eth/[common, trie, trie/db], + eth/[common, trie, trie/db, trie/trie_defs], eth/common/[eth_types, eth_types_rlp], - ../../../common/common_types, + ../../../common/[common_types, common_utils], ./database # Account State definition @@ -22,22 +22,30 @@ type AccountState* = ref object code: seq[byte] codeUpdated: bool -proc init(T: type AccountState, account = newAccount()): T = +proc init(T: type AccountState, account = newAccount()): T {.inline.} = T(account: account, codeUpdated: false) -proc setBalance*(accState: var AccountState, balance: UInt256) = +proc getBalance*(accState: AccountState): UInt256 {.inline.} = + accState.account.balance + +proc getNonce*(accState: AccountState): AccountNonce {.inline.} = + accState.account.nonce + +proc setBalance*(accState: var AccountState, balance: UInt256) {.inline.} = accState.account.balance = balance -proc addBalance*(accState: var AccountState, balance: UInt256) = +proc addBalance*(accState: var AccountState, balance: UInt256) {.inline.} = accState.account.balance += balance -proc setNonce*(accState: var AccountState, nonce: AccountNonce) = +proc setNonce*(accState: var AccountState, nonce: AccountNonce) {.inline.} = accState.account.nonce = nonce -proc setStorage*(accState: var AccountState, slotKey: UInt256, slotValue: UInt256) = +proc setStorage*( + accState: var AccountState, slotKey: UInt256, slotValue: UInt256 +) {.inline.} = accState.storageUpdates[slotKey] = slotValue -proc deleteStorage*(accState: var AccountState, slotKey: UInt256) = +proc deleteStorage*(accState: var AccountState, slotKey: UInt256) {.inline.} = # setting to zero has the effect of deleting the slot accState.setStorage(slotKey, 0.u256) @@ -48,32 +56,59 @@ proc setCode*(accState: var AccountState, code: seq[byte]) = # World State definition type - AccountHash = KeccakHash - SlotKeyHash = KeccakHash + AddressHash* = KeccakHash + SlotKeyHash* = KeccakHash WorldStateRef* = ref object + db: DatabaseRef accountsTrie: HexaryTrie - storageTries: TableRef[AccountHash, HexaryTrie] + storageTries: TableRef[AddressHash, HexaryTrie] storageDb: TrieDatabaseRef - bytecodeDb: TrieDatabaseRef # maps AccountHash -> seq[byte] + bytecodeDb: TrieDatabaseRef # maps AddressHash -> seq[byte] + preimagesDb: TrieDatabaseRef # maps AddressHash -> EthAddress -proc init*(T: type WorldStateRef, db: DatabaseRef): T = +proc init*( + T: type WorldStateRef, db: DatabaseRef, accountsTrieRoot: KeccakHash = emptyRlpHash +): T = WorldStateRef( - accountsTrie: initHexaryTrie(db.getAccountsBackend(), isPruning = false), - storageTries: newTable[AccountHash, HexaryTrie](), + db: db, + accountsTrie: + if accountsTrieRoot == emptyRlpHash: + initHexaryTrie(db.getAccountsBackend(), isPruning = false) + else: + initHexaryTrie(db.getAccountsBackend(), accountsTrieRoot, isPruning = false), + storageTries: newTable[AddressHash, HexaryTrie](), storageDb: db.getStorageBackend(), bytecodeDb: db.getBytecodeBackend(), + preimagesDb: db.getPreimagesBackend(), ) -template stateRoot*(state: WorldStateRef): KeccakHash = +proc stateRoot*(state: WorldStateRef): KeccakHash {.inline.} = state.accountsTrie.rootHash() -template toAccountKey(address: EthAddress): AccountHash = +proc toAccountKey*(address: EthAddress): AddressHash {.inline.} = keccakHash(address) -template toStorageKey(slotKey: UInt256): SlotKeyHash = +proc toStorageKey*(slotKey: UInt256): SlotKeyHash {.inline.} = keccakHash(toBytesBE(slotKey)) +proc getAccountPreimage(state: WorldStateRef, accountKey: AddressHash): EthAddress = + doAssert( + state.preimagesDb.contains(rlp.encode(accountKey)), + "No account preimage with address hash: " & $accountKey, + ) + let addressBytes = state.preimagesDb.get(rlp.encode(accountKey)) + + try: + rlp.decode(addressBytes, EthAddress) + except RlpError as e: + raiseAssert(e.msg) # Should never happen + +proc setAccountPreimage( + state: WorldStateRef, accountKey: AddressHash, address: EthAddress +) = + state.preimagesDb.put(rlp.encode(accountKey), rlp.encode(address)) + proc getAccount*(state: WorldStateRef, address: EthAddress): AccountState = let accountKey = toAccountKey(address) @@ -88,11 +123,17 @@ proc getAccount*(state: WorldStateRef, address: EthAddress): AccountState = proc setAccount*(state: WorldStateRef, address: EthAddress, accState: AccountState) = let accountKey = toAccountKey(address) + state.setAccountPreimage(accountKey, address) try: if not state.storageTries.contains(accountKey): - state.storageTries[accountKey] = - initHexaryTrie(state.storageDb, isPruning = false) + if accState.account.storageRoot == EMPTY_ROOT_HASH: + state.storageTries[accountKey] = + initHexaryTrie(state.storageDb, isPruning = false) + else: + state.storageTries[accountKey] = initHexaryTrie( + state.storageDb, accState.account.storageRoot, isPruning = false + ) var storageTrie = state.storageTries.getOrDefault(accountKey) for k, v in accState.storageUpdates: @@ -123,3 +164,41 @@ proc deleteAccount*(state: WorldStateRef, address: EthAddress) = state.bytecodeDb.del(accountKey.data) except RlpError as e: raiseAssert(e.msg) # should never happen unless the database is corrupted + +# Returns the account proofs for all the updated accounts from the last transaction +iterator updatedAccountProofs*(state: WorldStateRef): (EthAddress, seq[seq[byte]]) = + let trie = initHexaryTrie( + state.db.getAccountsUpdatedCache(), state.stateRoot(), isPruning = false + ) + + try: + for key in trie.keys(): + if key.len() == 0: + continue # skip the empty node created on initialization + let address = state.getAccountPreimage(KeccakHash.fromBytes(key)) + yield (address, trie.getBranch(key)) + except RlpError as e: + raiseAssert(e.msg) # should never happen unless the database is corrupted + +# Returns the storage proofs for the updated slots for the given account from the last transaction +iterator updatedStorageProofs*( + state: WorldStateRef, address: EthAddress +): (SlotKeyHash, seq[seq[byte]]) = + let accState = state.getAccount(address) + + let trie = initHexaryTrie( + state.db.getStorageUpdatedCache(), accState.account.storageRoot, isPruning = false + ) + + try: + for key in trie.keys(): + if key.len() == 0: + continue # skip the empty node created on initialization + yield (KeccakHash.fromBytes(key), trie.getBranch(key)) + except RlpError as e: + raiseAssert(e.msg) # should never happen unless the database is corrupted + +proc getUpdatedBytecode*( + state: WorldStateRef, address: EthAddress +): seq[byte] {.inline.} = + state.db.getBytecodeUpdatedCache().get(toAccountKey(address).data) diff --git a/fluffy/tools/portal_bridge/state_bridge/world_state_helper.nim b/fluffy/tools/portal_bridge/state_bridge/world_state_helper.nim index 1fd7c366e..a0ba6754c 100644 --- a/fluffy/tools/portal_bridge/state_bridge/world_state_helper.nim +++ b/fluffy/tools/portal_bridge/state_bridge/world_state_helper.nim @@ -11,6 +11,7 @@ import chronicles, stint, results, + stew/byteutils, eth/common/[eth_types, eth_types_rlp], ../../../../nimbus/common/chain_config, ./[state_diff, world_state] @@ -31,12 +32,14 @@ proc applyGenesisAccounts*(worldState: WorldStateRef, alloc: GenesisAlloc) = worldState.setAccount(address, accState) -proc applyStateDiff*(worldState: WorldStateRef, stateDiff: StateDiffRef) = - for address, balanceDiff in stateDiff.balances: +proc applyStateDiff*(worldState: WorldStateRef, txDiff: TransactionDiff) = + for accountDiff in txDiff: let - nonceDiff = stateDiff.nonces.getOrDefault(address) - codeDiff = stateDiff.code.getOrDefault(address) - storageDiff = stateDiff.storage.getOrDefault(address) + address = accountDiff.address + balanceDiff = accountDiff.balanceDiff + nonceDiff = accountDiff.nonceDiff + codeDiff = accountDiff.codeDiff + storageDiff = accountDiff.storageDiff var deleteAccount = false @@ -59,13 +62,13 @@ proc applyStateDiff*(worldState: WorldStateRef, stateDiff: StateDiffRef) = elif codeDiff.kind == delete: doAssert deleteAccount == true - for slotKey, slotDiff in storageDiff: - if slotDiff.kind == create or slotDiff.kind == update: - if slotDiff.after == 0: + for (slotKey, slotValueDiff) in storageDiff: + if slotValueDiff.kind == create or slotValueDiff.kind == update: + if slotValueDiff.after == 0: accState.deleteStorage(slotKey) else: - accState.setStorage(slotKey, slotDiff.after) - elif slotDiff.kind == delete: + accState.setStorage(slotKey, slotValueDiff.after) + elif slotValueDiff.kind == delete: accState.deleteStorage(slotKey) if deleteAccount: @@ -75,8 +78,8 @@ proc applyStateDiff*(worldState: WorldStateRef, stateDiff: StateDiffRef) = proc applyBlockRewards*( worldState: WorldStateRef, - minerData: tuple[miner: EthAddress, number: uint64], - uncleMinersData: openArray[tuple[miner: EthAddress, number: uint64]], + minerData: tuple[miner: EthAddress, blockNumber: uint64], + uncleMinersData: openArray[tuple[miner: EthAddress, blockNumber: uint64]], ) = const baseReward = u256(5) * pow(u256(10), 18) @@ -95,7 +98,145 @@ proc applyBlockRewards*( let uncleMinerAddress = EthAddress(uncleMinerData.miner) uncleReward = - (u256(8 + uncleMinerData.number - minerData.number) * baseReward) shr 3 + (u256(8 + uncleMinerData.blockNumber - minerData.blockNumber) * baseReward) shr 3 var accState = worldState.getAccount(uncleMinerAddress) accState.addBalance(uncleReward) worldState.setAccount(uncleMinerAddress, accState) + +const + DAORefundContract: EthAddress = + hexToByteArray[20]("0xbf4ed7b27f1d666546e30d74d50d173d20bca754") + + DAODrainList = [ + hexToByteArray[20]("0xd4fe7bc31cedb7bfb8a345f31e668033056b2728"), + hexToByteArray[20]("0xb3fb0e5aba0e20e5c49d252dfd30e102b171a425"), + hexToByteArray[20]("0x2c19c7f9ae8b751e37aeb2d93a699722395ae18f"), + hexToByteArray[20]("0xecd135fa4f61a655311e86238c92adcd779555d2"), + hexToByteArray[20]("0x1975bd06d486162d5dc297798dfc41edd5d160a7"), + hexToByteArray[20]("0xa3acf3a1e16b1d7c315e23510fdd7847b48234f6"), + hexToByteArray[20]("0x319f70bab6845585f412ec7724b744fec6095c85"), + hexToByteArray[20]("0x06706dd3f2c9abf0a21ddcc6941d9b86f0596936"), + hexToByteArray[20]("0x5c8536898fbb74fc7445814902fd08422eac56d0"), + hexToByteArray[20]("0x6966ab0d485353095148a2155858910e0965b6f9"), + hexToByteArray[20]("0x779543a0491a837ca36ce8c635d6154e3c4911a6"), + hexToByteArray[20]("0x2a5ed960395e2a49b1c758cef4aa15213cfd874c"), + hexToByteArray[20]("0x5c6e67ccd5849c0d29219c4f95f1a7a93b3f5dc5"), + hexToByteArray[20]("0x9c50426be05db97f5d64fc54bf89eff947f0a321"), + hexToByteArray[20]("0x200450f06520bdd6c527622a273333384d870efb"), + hexToByteArray[20]("0xbe8539bfe837b67d1282b2b1d61c3f723966f049"), + hexToByteArray[20]("0x6b0c4d41ba9ab8d8cfb5d379c69a612f2ced8ecb"), + hexToByteArray[20]("0xf1385fb24aad0cd7432824085e42aff90886fef5"), + hexToByteArray[20]("0xd1ac8b1ef1b69ff51d1d401a476e7e612414f091"), + hexToByteArray[20]("0x8163e7fb499e90f8544ea62bbf80d21cd26d9efd"), + hexToByteArray[20]("0x51e0ddd9998364a2eb38588679f0d2c42653e4a6"), + hexToByteArray[20]("0x627a0a960c079c21c34f7612d5d230e01b4ad4c7"), + hexToByteArray[20]("0xf0b1aa0eb660754448a7937c022e30aa692fe0c5"), + hexToByteArray[20]("0x24c4d950dfd4dd1902bbed3508144a54542bba94"), + hexToByteArray[20]("0x9f27daea7aca0aa0446220b98d028715e3bc803d"), + hexToByteArray[20]("0xa5dc5acd6a7968a4554d89d65e59b7fd3bff0f90"), + hexToByteArray[20]("0xd9aef3a1e38a39c16b31d1ace71bca8ef58d315b"), + hexToByteArray[20]("0x63ed5a272de2f6d968408b4acb9024f4cc208ebf"), + hexToByteArray[20]("0x6f6704e5a10332af6672e50b3d9754dc460dfa4d"), + hexToByteArray[20]("0x77ca7b50b6cd7e2f3fa008e24ab793fd56cb15f6"), + hexToByteArray[20]("0x492ea3bb0f3315521c31f273e565b868fc090f17"), + hexToByteArray[20]("0x0ff30d6de14a8224aa97b78aea5388d1c51c1f00"), + hexToByteArray[20]("0x9ea779f907f0b315b364b0cfc39a0fde5b02a416"), + hexToByteArray[20]("0xceaeb481747ca6c540a000c1f3641f8cef161fa7"), + hexToByteArray[20]("0xcc34673c6c40e791051898567a1222daf90be287"), + hexToByteArray[20]("0x579a80d909f346fbfb1189493f521d7f48d52238"), + hexToByteArray[20]("0xe308bd1ac5fda103967359b2712dd89deffb7973"), + hexToByteArray[20]("0x4cb31628079fb14e4bc3cd5e30c2f7489b00960c"), + hexToByteArray[20]("0xac1ecab32727358dba8962a0f3b261731aad9723"), + hexToByteArray[20]("0x4fd6ace747f06ece9c49699c7cabc62d02211f75"), + hexToByteArray[20]("0x440c59b325d2997a134c2c7c60a8c61611212bad"), + hexToByteArray[20]("0x4486a3d68fac6967006d7a517b889fd3f98c102b"), + hexToByteArray[20]("0x9c15b54878ba618f494b38f0ae7443db6af648ba"), + hexToByteArray[20]("0x27b137a85656544b1ccb5a0f2e561a5703c6a68f"), + hexToByteArray[20]("0x21c7fdb9ed8d291d79ffd82eb2c4356ec0d81241"), + hexToByteArray[20]("0x23b75c2f6791eef49c69684db4c6c1f93bf49a50"), + hexToByteArray[20]("0x1ca6abd14d30affe533b24d7a21bff4c2d5e1f3b"), + hexToByteArray[20]("0xb9637156d330c0d605a791f1c31ba5890582fe1c"), + hexToByteArray[20]("0x6131c42fa982e56929107413a9d526fd99405560"), + hexToByteArray[20]("0x1591fc0f688c81fbeb17f5426a162a7024d430c2"), + hexToByteArray[20]("0x542a9515200d14b68e934e9830d91645a980dd7a"), + hexToByteArray[20]("0xc4bbd073882dd2add2424cf47d35213405b01324"), + hexToByteArray[20]("0x782495b7b3355efb2833d56ecb34dc22ad7dfcc4"), + hexToByteArray[20]("0x58b95c9a9d5d26825e70a82b6adb139d3fd829eb"), + hexToByteArray[20]("0x3ba4d81db016dc2890c81f3acec2454bff5aada5"), + hexToByteArray[20]("0xb52042c8ca3f8aa246fa79c3feaa3d959347c0ab"), + hexToByteArray[20]("0xe4ae1efdfc53b73893af49113d8694a057b9c0d1"), + hexToByteArray[20]("0x3c02a7bc0391e86d91b7d144e61c2c01a25a79c5"), + hexToByteArray[20]("0x0737a6b837f97f46ebade41b9bc3e1c509c85c53"), + hexToByteArray[20]("0x97f43a37f595ab5dd318fb46e7a155eae057317a"), + hexToByteArray[20]("0x52c5317c848ba20c7504cb2c8052abd1fde29d03"), + hexToByteArray[20]("0x4863226780fe7c0356454236d3b1c8792785748d"), + hexToByteArray[20]("0x5d2b2e6fcbe3b11d26b525e085ff818dae332479"), + hexToByteArray[20]("0x5f9f3392e9f62f63b8eac0beb55541fc8627f42c"), + hexToByteArray[20]("0x057b56736d32b86616a10f619859c6cd6f59092a"), + hexToByteArray[20]("0x9aa008f65de0b923a2a4f02012ad034a5e2e2192"), + hexToByteArray[20]("0x304a554a310c7e546dfe434669c62820b7d83490"), + hexToByteArray[20]("0x914d1b8b43e92723e64fd0a06f5bdb8dd9b10c79"), + hexToByteArray[20]("0x4deb0033bb26bc534b197e61d19e0733e5679784"), + hexToByteArray[20]("0x07f5c1e1bc2c93e0402f23341973a0e043f7bf8a"), + hexToByteArray[20]("0x35a051a0010aba705c9008d7a7eff6fb88f6ea7b"), + hexToByteArray[20]("0x4fa802324e929786dbda3b8820dc7834e9134a2a"), + hexToByteArray[20]("0x9da397b9e80755301a3b32173283a91c0ef6c87e"), + hexToByteArray[20]("0x8d9edb3054ce5c5774a420ac37ebae0ac02343c6"), + hexToByteArray[20]("0x0101f3be8ebb4bbd39a2e3b9a3639d4259832fd9"), + hexToByteArray[20]("0x5dc28b15dffed94048d73806ce4b7a4612a1d48f"), + hexToByteArray[20]("0xbcf899e6c7d9d5a215ab1e3444c86806fa854c76"), + hexToByteArray[20]("0x12e626b0eebfe86a56d633b9864e389b45dcb260"), + hexToByteArray[20]("0xa2f1ccba9395d7fcb155bba8bc92db9bafaeade7"), + hexToByteArray[20]("0xec8e57756626fdc07c63ad2eafbd28d08e7b0ca5"), + hexToByteArray[20]("0xd164b088bd9108b60d0ca3751da4bceb207b0782"), + hexToByteArray[20]("0x6231b6d0d5e77fe001c2a460bd9584fee60d409b"), + hexToByteArray[20]("0x1cba23d343a983e9b5cfd19496b9a9701ada385f"), + hexToByteArray[20]("0xa82f360a8d3455c5c41366975bde739c37bfeb8a"), + hexToByteArray[20]("0x9fcd2deaff372a39cc679d5c5e4de7bafb0b1339"), + hexToByteArray[20]("0x005f5cee7a43331d5a3d3eec71305925a62f34b6"), + hexToByteArray[20]("0x0e0da70933f4c7849fc0d203f5d1d43b9ae4532d"), + hexToByteArray[20]("0xd131637d5275fd1a68a3200f4ad25c71a2a9522e"), + hexToByteArray[20]("0xbc07118b9ac290e4622f5e77a0853539789effbe"), + hexToByteArray[20]("0x47e7aa56d6bdf3f36be34619660de61275420af8"), + hexToByteArray[20]("0xacd87e28b0c9d1254e868b81cba4cc20d9a32225"), + hexToByteArray[20]("0xadf80daec7ba8dcf15392f1ac611fff65d94f880"), + hexToByteArray[20]("0x5524c55fb03cf21f549444ccbecb664d0acad706"), + hexToByteArray[20]("0x40b803a9abce16f50f36a77ba41180eb90023925"), + hexToByteArray[20]("0xfe24cdd8648121a43a7c86d289be4dd2951ed49f"), + hexToByteArray[20]("0x17802f43a0137c506ba92291391a8a8f207f487d"), + hexToByteArray[20]("0x253488078a4edf4d6f42f113d1e62836a942cf1a"), + hexToByteArray[20]("0x86af3e9626fce1957c82e88cbf04ddf3a2ed7915"), + hexToByteArray[20]("0xb136707642a4ea12fb4bae820f03d2562ebff487"), + hexToByteArray[20]("0xdbe9b615a3ae8709af8b93336ce9b477e4ac0940"), + hexToByteArray[20]("0xf14c14075d6c4ed84b86798af0956deef67365b5"), + hexToByteArray[20]("0xca544e5c4687d109611d0f8f928b53a25af72448"), + hexToByteArray[20]("0xaeeb8ff27288bdabc0fa5ebb731b6f409507516c"), + hexToByteArray[20]("0xcbb9d3703e651b0d496cdefb8b92c25aeb2171f7"), + hexToByteArray[20]("0x6d87578288b6cb5549d5076a207456a1f6a63dc0"), + hexToByteArray[20]("0xb2c6f0dfbb716ac562e2d85d6cb2f8d5ee87603e"), + hexToByteArray[20]("0xaccc230e8a6e5be9160b8cdf2864dd2a001c28b6"), + hexToByteArray[20]("0x2b3455ec7fedf16e646268bf88846bd7a2319bb2"), + hexToByteArray[20]("0x4613f3bca5c44ea06337a9e439fbc6d42e501d0a"), + hexToByteArray[20]("0xd343b217de44030afaa275f54d31a9317c7f441e"), + hexToByteArray[20]("0x84ef4b2357079cd7a7c69fd7a37cd0609a679106"), + hexToByteArray[20]("0xda2fef9e4a3230988ff17df2165440f37e8b1708"), + hexToByteArray[20]("0xf4c64518ea10f995918a454158c6b61407ea345c"), + hexToByteArray[20]("0x7602b46df5390e432ef1c307d4f2c9ff6d65cc97"), + hexToByteArray[20]("0xbb9bc244d798123fde783fcc1c72d3bb8c189413"), + hexToByteArray[20]("0x807640a13483f8ac783c557fcdf27be11ea4ac7a"), + ] + +# ApplyDAOHardFork modifies the state database according to the DAO hard-fork +# rules, transferring all balances of a set of DAO accounts to a single refund +# contract. +proc applyDAOHardFork*(worldState: WorldStateRef) = + # Move every DAO account and extra-balance account funds into the refund contract + var toAccount = worldState.getAccount(DAORefundContract) + + for address in DAODrainList: + var fromAccount = worldState.getAccount(address) + toAccount.addBalance(fromAccount.getBalance()) + fromAccount.setBalance(0.u256) + worldState.setAccount(address, fromAccount) + + worldState.setAccount(DAORefundContract, toAccount)