diff --git a/nimbus/db/accounts_cache.nim b/nimbus/db/accounts_cache.nim index 14d716d18..e2176bb57 100644 --- a/nimbus/db/accounts_cache.nim +++ b/nimbus/db/accounts_cache.nim @@ -64,6 +64,13 @@ const proc beginSavepoint*(ac: var AccountsCache): SavePoint {.gcsafe.} + +# FIXME-Adam: this is only necessary because of my sanity checks on the latest rootHash; +# take this out once those are gone. +proc rawTrie*(ac: AccountsCache): AccountsTrie = ac.trie +proc rawDb*(ac: AccountsCache): TrieDatabaseRef = ac.trie.db + + # The AccountsCache is modeled after TrieDatabase for it's transaction style proc init*(x: typedesc[AccountsCache], db: TrieDatabaseRef, root: KeccakHash, pruneTrie: bool = true): AccountsCache = diff --git a/nimbus/evm/async/data_sources.nim b/nimbus/evm/async/data_sources.nim new file mode 100644 index 000000000..ea90b4d8a --- /dev/null +++ b/nimbus/evm/async/data_sources.nim @@ -0,0 +1,50 @@ +import + options, + chronos, + stint, + eth/common, + eth/trie/db, + ../../db/db_chain + +type + AsyncDataSource* = ref object of RootObj + ifNecessaryGetSlots*: proc(db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, slots: seq[UInt256], newStateRootForSanityChecking: Hash256): Future[void] {.gcsafe.} + ifNecessaryGetCode*: proc(db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, newStateRootForSanityChecking: Hash256): Future[void] {.gcsafe.} + ifNecessaryGetAccount*: proc(db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, newStateRootForSanityChecking: Hash256): Future[void] {.gcsafe.} + ifNecessaryGetBlockHeaderByNumber*: proc(chainDB: ChainDBRef, blockNumber: BlockNumber): Future[void] {.gcsafe.} + # FIXME-Adam: Later. + #fetchNodes*: proc(stateRoot: Hash256, paths: seq[seq[seq[byte]]], nodeHashes: seq[Hash256]): Future[seq[seq[byte]]] {.gcsafe.} + fetchBlockHeaderWithHash*: proc(h: Hash256): Future[BlockHeader] {.gcsafe.} + fetchBlockHeaderWithNumber*: proc(n: BlockNumber): Future[BlockHeader] {.gcsafe.} + fetchBlockHeaderAndBodyWithHash*: proc(h: Hash256): Future[(BlockHeader, BlockBody)] {.gcsafe.} + fetchBlockHeaderAndBodyWithNumber*: proc(n: BlockNumber): Future[(BlockHeader, BlockBody)] {.gcsafe.} + + # FIXME-Adam: maybe rename this? + AsyncOperationFactory* = ref object of RootObj + maybeDataSource*: Option[AsyncDataSource] + + +# FIXME-Adam: Can I make a singleton? +proc asyncFactoryWithNoDataSource*(): AsyncOperationFactory = + AsyncOperationFactory(maybeDataSource: none[AsyncDataSource]()) + + +# FIXME-Adam: Ugly but straightforward; can this be cleaned up using some combination of: +# - an ifSome/map operation on Options +# - some kind of "what are we fetching" tuple, so that this is just one thing + +proc ifNecessaryGetSlots*(asyncFactory: AsyncOperationFactory, db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, slots: seq[UInt256], newStateRootForSanityChecking: Hash256): Future[void] {.async.} = + if asyncFactory.maybeDataSource.isSome: + await asyncFactory.maybeDataSource.get.ifNecessaryGetSlots(db, blockNumber, stateRoot, address, slots, newStateRootForSanityChecking) + +proc ifNecessaryGetCode*(asyncFactory: AsyncOperationFactory, db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, newStateRootForSanityChecking: Hash256): Future[void] {.async.} = + if asyncFactory.maybeDataSource.isSome: + await asyncFactory.maybeDataSource.get.ifNecessaryGetCode(db, blockNumber, stateRoot, address, newStateRootForSanityChecking) + +proc ifNecessaryGetAccount*(asyncFactory: AsyncOperationFactory, db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, newStateRootForSanityChecking: Hash256): Future[void] {.async.} = + if asyncFactory.maybeDataSource.isSome: + await asyncFactory.maybeDataSource.get.ifNecessaryGetAccount(db, blockNumber, stateRoot, address, newStateRootForSanityChecking) + +proc ifNecessaryGetBlockHeaderByNumber*(asyncFactory: AsyncOperationFactory, chainDB: ChainDBRef, blockNumber: BlockNumber): Future[void] {.async.} = + if asyncFactory.maybeDataSource.isSome: + await asyncFactory.maybeDataSource.get.ifNecessaryGetBlockHeaderByNumber(chainDB, blockNumber) diff --git a/nimbus/evm/async/data_sources/json_rpc_data_source.nim b/nimbus/evm/async/data_sources/json_rpc_data_source.nim new file mode 100644 index 000000000..466152282 --- /dev/null +++ b/nimbus/evm/async/data_sources/json_rpc_data_source.nim @@ -0,0 +1,260 @@ +import + std/[sequtils, typetraits, options], + chronicles, + chronos, + nimcrypto, + stint, + stew/byteutils, + json_rpc/rpcclient, + eth/common, + eth/rlp, + eth/trie/[db, hexary_proof_verification], + eth/p2p, + eth/p2p/rlpx, + eth/p2p/private/p2p_types, + ../../../sync/protocol, + ../../../sync/protocol/eth66 as proto_eth66, + ../../../db/[db_chain, distinct_tries, incomplete_db, storage_types], + ../data_fetching, + ../data_sources, + ../rpc_api + + +export AsyncOperationFactory, AsyncDataSource + + +proc verifyFetchedAccount(stateRoot: Hash256, address: EthAddress, acc: Account, accProof: seq[seq[byte]]): Result[void, string] = + let accKey = toSeq(keccakHash(address).data) + let accEncoded = rlp.encode(acc) + let accProofResult = verifyMptProof(accProof, stateRoot, accKey, accEncoded) + case accProofResult.kind + of ValidProof: + return ok() + of MissingKey: + # For an account that doesn't exist yet, which is fine. + return ok() + of InvalidProof: + return err(accProofResult.errorMsg) + +type + CodeFetchingInfo = tuple[blockNumber: BlockNumber, address: EthAddress] + +proc fetchCode(client: RpcClient, p: CodeFetchingInfo): Future[seq[byte]] {.async.} = + let (blockNumber, address) = p + let fetchedCode = await fetchCode(client, blockNumber, address) + return fetchedCode + +proc verifyFetchedCode(fetchedCode: seq[byte], desiredCodeHash: Hash256): Result[void, Hash256] = + let fetchedCodeHash = keccakHash(fetchedCode) + if (desiredCodeHash == fetchedCodeHash): + ok[void]() + else: + err(fetchedCodeHash) + +proc fetchAndVerifyCode(client: RpcClient, p: CodeFetchingInfo, desiredCodeHash: Hash256): Future[seq[byte]] {.async.} = + let fetchedCode: seq[byte] = await fetchCode(client, p) + let verificationRes = verifyFetchedCode(fetchedCode, desiredCodeHash) + if verificationRes.isOk(): + return fetchedCode + else: + let fetchedCodeHash = verificationRes.error + error("code hash values do not match", p=p, desiredCodeHash=desiredCodeHash, fetchedCodeHash=fetchedCodeHash) + raise newException(CatchableError, "async code received code for " & $(p.address) & " whose hash (" & $(fetchedCodeHash) & ") does not match the desired hash (" & $(desiredCodeHash) & ")") + +proc storeCode(trie: AccountsTrie, p: CodeFetchingInfo, desiredCodeHash: Hash256, fetchedCode: seq[byte]) = + trie.putCode(desiredCodeHash, fetchedCode) + +proc assertThatWeHaveStoredCode(trie: AccountsTrie, p: CodeFetchingInfo, codeHash: Hash256) = + # FIXME-Adam: this is a bit wrong because we're not checking it against the blockNumber, only the address. (That is, + # if the code for this address has *changed* (which is unlikely), this check isn't the right thing to do.) + let maybeFoundCode = trie.maybeGetCode(p.address) + if maybeFoundCode.isNone: + error("code didn't get put into the db", p=p, codeHash=codeHash) + doAssert false, "code didn't get put into the db" + else: + let foundCode = maybeFoundCode.get + let foundCodeHash = keccakHash(foundCode) + if foundCodeHash != codeHash: + error("code does not have the right hash", p=p, codeHash=codeHash, foundCode=foundCode) + doAssert false, "code does not have the right hash" + + +proc assertThatWeHaveStoredAccount(trie: AccountsTrie, address: EthAddress, fetchedAcc: Account, isForTheNewTrie: bool = false) = + let foundAcc = ifNodesExistGetAccount(trie, address).get + if fetchedAcc != foundAcc: + error "account didn't come out the same", address=address, fetchedAcc=fetchedAcc, foundAcc=foundAcc, isForTheNewTrie=isForTheNewTrie + doAssert false, "account didn't come out the same" + doAssert(trie.hasAllNodesForAccount(address), "Can I check the account this way, too?") + + +proc verifyFetchedSlot(accountStorageRoot: Hash256, slot: UInt256, fetchedVal: UInt256, storageMptNodes: seq[seq[byte]]): Result[void, string] = + if storageMptNodes.len == 0: + # I think an empty storage proof is okay; I see lots of these + # where the account is empty and the value is zero. + return ok() + else: + let storageKey = toSeq(keccakHash(toBytesBE(slot)).data) + let storageValueEncoded = rlp.encode(fetchedVal) + let storageProofResult = verifyMptProof(storageMptNodes, accountStorageRoot, storageKey, storageValueEncoded) + case storageProofResult.kind + of ValidProof: + return ok() + of MissingKey: + # This is for a slot that doesn't have anything stored at it, but that's fine. + return ok() + of InvalidProof: + return err(storageProofResult.errorMsg) + + +proc assertThatWeHaveStoredSlot(trie: AccountsTrie, address: EthAddress, acc: Account, slot: UInt256, fetchedVal: UInt256, isForTheNewTrie: bool = false) = + if acc.storageRoot == EMPTY_ROOT_HASH and fetchedVal.isZero: + # I believe this is okay. + discard + else: + let foundVal = ifNodesExistGetStorage(trie, address, slot).get + if (fetchedVal != foundVal): + error("slot didn't come out the same", address=address, slot=slot, fetchedVal=fetchedVal, foundVal=foundVal, isForTheNewTrie=isForTheNewTrie) + doAssert false, "slot didn't come out the same" + + +proc verifyFetchedBlockHeader(fetchedHeader: BlockHeader, desiredBlockNumber: BlockNumber): Result[void, BlockNumber] = + # *Can* we do anything to verify this header, given that all we know + # is the desiredBlockNumber and we want to run statelessly so we don't + # know what block hash we want? + ok[void]() + +proc storeBlockHeader(chainDB: ChainDBRef, header: BlockHeader) = + chainDB.persistHeaderToDbWithoutSetHeadOrScore(header) + +proc assertThatWeHaveStoredBlockHeader(chainDB: ChainDBRef, blockNumber: BlockNumber, header: BlockHeader) = + let h = chainDB.getBlockHash(blockNumber) + doAssert(h == header.blockHash, "stored the block header for block " & $(blockNumber)) + +template raiseExceptionIfError[E](whatAreWeVerifying: untyped, r: Result[void, E]) = + if r.isErr: + error("async code failed to verify", whatAreWeVerifying=whatAreWeVerifying, err=r.error) + raise newException(CatchableError, "async code failed to verify: " & $(whatAreWeVerifying) & ", error is: " & $(r.error)) + + +const shouldDoUnnecessarySanityChecks = true + +# This proc fetches both the account and also optionally some of its slots, because that's what eth_getProof can do. +proc ifNecessaryGetAccountAndSlots*(client: RpcClient, db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, slots: seq[UInt256], justCheckingAccount: bool, justCheckingSlots: bool, newStateRootForSanityChecking: Hash256): Future[void] {.async.} = + let trie = initAccountsTrie(db, stateRoot, false) # important for sanity checks + let trie2 = initAccountsTrie(db, newStateRootForSanityChecking, false) # important for sanity checks + let doesAccountActuallyNeedToBeFetched = not trie.hasAllNodesForAccount(address) + let slotsToActuallyFetch = slots.filter(proc(slot: UInt256): bool = not (trie.hasAllNodesForStorageSlot(address, slot))) + if (not doesAccountActuallyNeedToBeFetched) and (slotsToActuallyFetch.len == 0): + # Already have them, no need to fetch either the account or the slots + discard + else: + let (acc, accProof, storageProofs) = await fetchAccountAndSlots(client, address, slotsToActuallyFetch, blockNumber) + + # We need to verify the proof even if we already had this account, + # to make sure the data is valid. + let accountVerificationRes = verifyFetchedAccount(stateRoot, address, acc, accProof) + let whatAreWeVerifying = ("account proof", address, acc) + raiseExceptionIfError(whatAreWeVerifying, accountVerificationRes) + + if not doesAccountActuallyNeedToBeFetched: + # We already had the account, no need to populate the DB with it again. + discard + else: + if not justCheckingAccount: + populateDbWithBranch(db, accProof) + if shouldDoUnnecessarySanityChecks: + assertThatWeHaveStoredAccount(trie, address, acc, false) + if doesAccountActuallyNeedToBeFetched: # this second check makes no sense if it's not the first time + assertThatWeHaveStoredAccount(trie2, address, acc, true) + + doAssert(slotsToActuallyFetch.len == storageProofs.len, "We should get back the same number of storage proofs as slots that we asked for. I think.") + + for storageProof in storageProofs: + let slot = UInt256.fromHex(string(storageProof.key)) + let fetchedVal = UInt256.fromHex(string(storageProof.value)) + let storageMptNodes = storageProof.proof.mapIt(hexToSeqByte(string(it))) + let storageVerificationRes = verifyFetchedSlot(acc.storageRoot, slot, fetchedVal, storageMptNodes) + let whatAreWeVerifying = ("storage proof", address, acc, slot, fetchedVal) + raiseExceptionIfError(whatAreWeVerifying, storageVerificationRes) + + if not justCheckingSlots: + populateDbWithBranch(db, storageMptNodes) + + # I believe this is done so that we can iterate over the slots. See + # persistStorage in accounts_cache.nim. + let slotAsKey = createTrieKeyFromSlot(slot) + let slotHash = keccakHash(slotAsKey) + let slotEncoded = rlp.encode(slot) + db.put(slotHashToSlotKey(slotHash.data).toOpenArray, slotEncoded) + + if shouldDoUnnecessarySanityChecks: + assertThatWeHaveStoredSlot(trie, address, acc, slot, fetchedVal, false) + assertThatWeHaveStoredSlot(trie2, address, acc, slot, fetchedVal, true) + +proc ifNecessaryGetCode*(client: RpcClient, db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, justChecking: bool, newStateRootForSanityChecking: Hash256): Future[void] {.async.} = + await ifNecessaryGetAccountAndSlots(client, db, blockNumber, stateRoot, address, @[], false, false, newStateRootForSanityChecking) # to make sure we've got the codeHash + let trie = initAccountsTrie(db, stateRoot, false) # important for sanity checks + + let acc = ifNodesExistGetAccount(trie, address).get + let desiredCodeHash = acc.codeHash + + let p = (blockNumber, address) + if not(trie.hasAllNodesForCode(address)): + let fetchedCode = await fetchAndVerifyCode(client, p, desiredCodeHash) + + if not justChecking: + storeCode(trie, p, desiredCodeHash, fetchedCode) + if shouldDoUnnecessarySanityChecks: + assertThatWeHaveStoredCode(trie, p, desiredCodeHash) + +proc ifNecessaryGetBlockHeaderByNumber*(client: RpcClient, chainDB: ChainDBRef, blockNumber: BlockNumber, justChecking: bool): Future[void] {.async.} = + let maybeHeaderAndHash = chainDB.getBlockHeaderWithHash(blockNumber) + if maybeHeaderAndHash.isNone: + let fetchedHeader = await fetchBlockHeaderWithNumber(client, blockNumber) + let headerVerificationRes = verifyFetchedBlockHeader(fetchedHeader, blockNumber) + let whatAreWeVerifying = ("block header by number", blockNumber, fetchedHeader) + raiseExceptionIfError(whatAreWeVerifying, headerVerificationRes) + + if not justChecking: + storeBlockHeader(chainDB, fetchedHeader) + if shouldDoUnnecessarySanityChecks: + assertThatWeHaveStoredBlockHeader(chainDB, blockNumber, fetchedHeader) + + + + + +# Used in asynchronous on-demand-data-fetching mode. +proc realAsyncDataSource*(peerPool: PeerPool, client: RpcClient, justChecking: bool): AsyncDataSource = + AsyncDataSource( + ifNecessaryGetAccount: (proc(db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, newStateRootForSanityChecking: Hash256): Future[void] {.async.} = + await ifNecessaryGetAccountAndSlots(client, db, blockNumber, stateRoot, address, @[], false, false, newStateRootForSanityChecking) + ), + ifNecessaryGetSlots: (proc(db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, slots: seq[UInt256], newStateRootForSanityChecking: Hash256): Future[void] {.async.} = + await ifNecessaryGetAccountAndSlots(client, db, blockNumber, stateRoot, address, slots, false, false, newStateRootForSanityChecking) + ), + ifNecessaryGetCode: (proc(db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, newStateRootForSanityChecking: Hash256): Future[void] {.async.} = + await ifNecessaryGetCode(client, db, blockNumber, stateRoot, address, justChecking, newStateRootForSanityChecking) + ), + ifNecessaryGetBlockHeaderByNumber: (proc(chainDB: ChainDBRef, blockNumber: BlockNumber): Future[void] {.async.} = + await ifNecessaryGetBlockHeaderByNumber(client, chainDB, blockNumber, justChecking) + ), + + # FIXME-Adam: This will be needed later, but for now let's just get the basic methods in place. + #fetchNodes: (proc(stateRoot: Hash256, paths: seq[seq[seq[byte]]], nodeHashes: seq[Hash256]): Future[seq[seq[byte]]] {.async.} = + # return await fetchNodes(peerPool, stateRoot, paths, nodeHashes) + #), + + fetchBlockHeaderWithHash: (proc(h: Hash256): Future[BlockHeader] {.async.} = + return await fetchBlockHeaderWithHash(client, h) + ), + fetchBlockHeaderWithNumber: (proc(n: BlockNumber): Future[BlockHeader] {.async.} = + return await fetchBlockHeaderWithNumber(client, n) + ), + fetchBlockHeaderAndBodyWithHash: (proc(h: Hash256): Future[(BlockHeader, BlockBody)] {.async.} = + return await fetchBlockHeaderAndBodyWithHash(client, h) + ), + fetchBlockHeaderAndBodyWithNumber: (proc(n: BlockNumber): Future[(BlockHeader, BlockBody)] {.async.} = + return await fetchBlockHeaderAndBodyWithNumber(client, n) + ) + ) diff --git a/nimbus/evm/async/operations.nim b/nimbus/evm/async/operations.nim new file mode 100644 index 000000000..b4c87215f --- /dev/null +++ b/nimbus/evm/async/operations.nim @@ -0,0 +1,49 @@ +import + chronicles, + chronos, + stint, + eth/common/eth_types, + ../../common, + ../../db/distinct_tries, + ../../db/accounts_cache, + #../../db/incomplete_db, + ../types, + ./data_sources + + + +proc ifNecessaryGetAccount*(vmState: BaseVMState, address: EthAddress): Future[void] {.async.} = + await vmState.asyncFactory.ifNecessaryGetAccount(vmState.com.db.db, vmState.parent.blockNumber, vmState.parent.stateRoot, address, vmState.stateDB.rawTrie.rootHash) + +proc ifNecessaryGetCode*(vmState: BaseVMState, address: EthAddress): Future[void] {.async.} = + await vmState.asyncFactory.ifNecessaryGetCode(vmState.com.db.db, vmState.parent.blockNumber, vmState.parent.stateRoot, address, vmState.stateDB.rawTrie.rootHash) + +proc ifNecessaryGetSlots*(vmState: BaseVMState, address: EthAddress, slots: seq[UInt256]): Future[void] {.async.} = + await vmState.asyncFactory.ifNecessaryGetSlots(vmState.com.db.db, vmState.parent.blockNumber, vmState.parent.stateRoot, address, slots, vmState.stateDB.rawTrie.rootHash) + +proc ifNecessaryGetSlot*(vmState: BaseVMState, address: EthAddress, slot: UInt256): Future[void] {.async.} = + await ifNecessaryGetSlots(vmState, address, @[slot]) + +proc ifNecessaryGetBlockHeaderByNumber*(vmState: BaseVMState, blockNumber: BlockNumber): Future[void] {.async.} = + await vmState.asyncFactory.ifNecessaryGetBlockHeaderByNumber(vmState.com.db, blockNumber) + +#[ +FIXME-Adam: This is for later. +proc fetchAndPopulateNodes*(vmState: BaseVMState, paths: seq[seq[seq[byte]]], nodeHashes: seq[Hash256]): Future[void] {.async.} = + if vmState.asyncFactory.maybeDataSource.isSome: + # let stateRoot = vmState.stateDB.rawTrie.rootHash # FIXME-Adam: this might not be right, huh? the peer might expect the parent block's final stateRoot, not this weirdo intermediate one + let stateRoot = vmState.parent.stateRoot + let nodes = await vmState.asyncFactory.maybeDataSource.get.fetchNodes(stateRoot, paths, nodeHashes) + populateDbWithNodes(vmState.stateDB.rawDb, nodes) +]# + + +# Sometimes it's convenient to be able to do multiple at once. + +proc ifNecessaryGetAccounts*(vmState: BaseVMState, addresses: seq[EthAddress]): Future[void] {.async.} = + for address in addresses: + await ifNecessaryGetAccount(vmState, address) + +proc ifNecessaryGetCodeForAccounts*(vmState: BaseVMState, addresses: seq[EthAddress]): Future[void] {.async.} = + for address in addresses: + await ifNecessaryGetCode(vmState, address) diff --git a/nimbus/evm/async_operations.nim b/nimbus/evm/async_operations.nim deleted file mode 100644 index 087c35d80..000000000 --- a/nimbus/evm/async_operations.nim +++ /dev/null @@ -1,104 +0,0 @@ -# Nimbus -# Copyright (c) 2022-2023 Status Research & Development GmbH -# Licensed under either of -# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or -# http://www.apache.org/licenses/LICENSE-2.0) -# * MIT license ([LICENSE-MIT](LICENSE-MIT) or -# http://opensource.org/licenses/MIT) -# at your option. This file may not be copied, modified, or distributed except -# according to those terms. - -import - chronos, - stint, - json_rpc/rpcclient, - web3, - ./computation, - ./state, - ./types, - ../db/accounts_cache - - - -# Used in synchronous mode. -proc noLazyDataSource*(): LazyDataSource = - LazyDataSource( - ifNecessaryGetStorage: (proc(c: Computation, slot: UInt256): Future[void] {.async.} = - discard - ) - ) - -# Will be used in asynchronous on-demand-data-fetching mode, once -# that is implemented. -proc realLazyDataSource*(client: RpcClient): LazyDataSource = - LazyDataSource( - ifNecessaryGetStorage: (proc(c: Computation, slot: UInt256): Future[void] {.async.} = - # TODO: find some way to check whether we already have it. - # This is WRONG, but good enough for now, considering this - # code is unused except in a few tests. I'm working on - # doing this properly. - if not c.getStorage(slot).isZero: return - - # FIXME-onDemandStorageNotImplementedYet - # (I sketched in this code, but haven't actually tried running it yet.) - echo("Attempting to for-real fetch slot " & $(slot)) - # ethAddressStr("0xfff33a3bd36abdbd412707b8e310d6011454a7ae") - # check hexDataStr(0.u256).string == res.string - let ethAddress = c.msg.contractAddress - let address: Address = Address(ethAddress) - let quantity: int = slot.truncate(int) # this is probably wrong; what's the right way to convert this? - let blockId: BlockIdentifier = blockId(c.vmState.parent.blockNumber.truncate(uint64)) # ditto - let res = await client.eth_getStorageAt(address, quantity, blockId) - echo("Fetched slot " & $(slot) & ", result is " & $(res)) - # let v = res # will res be the actual value, or do I need to convert or something? - - # Before implementing this, see the note from Zahary here: - # https://github.com/status-im/nimbus-eth1/pull/1260#discussion_r999669139 - # - # c.vmState.mutateStateDB: - # db.setStorage(c.msg.contractAddress, slot, UInt256.fromBytesBE(v)) - ) - ) - -# Used for unit testing. Contains some prepopulated data. -proc fakeLazyDataSource*(fakePairs: seq[tuple[key, val: array[32, byte]]]): LazyDataSource = - LazyDataSource( - ifNecessaryGetStorage: (proc(c: Computation, slot: UInt256): Future[void] {.async.} = - # See the comment above. - if not c.getStorage(slot).isZero: return - - # FIXME-writeAutomatedTestsToShowThatItCanRunConcurrently - - # For now, until I've implemented some more automated way to - # capture and verify the fact that this can run concurrently, - # this is useful just to see in the console that the echo - # statements from multiple Computations can run at the same - # time and be interleaved. - # echo("Attempting to fake-fetch slot " & $(slot)) - # await sleepAsync(2.seconds) - - let slotBytes = toBytesBE(slot) - # The linear search is obviously slow, but doesn't matter - # for tests with only a few initialStorage entries. Fix - # this if we ever want to write tests with more. - for (k, v) in fakePairs: - if slotBytes == k: - c.vmState.mutateStateDB: - db.setStorage(c.msg.contractAddress, slot, UInt256.fromBytesBE(v)) - break - - # echo("Finished fake-fetch of slot " & $(slot)) - ) - ) - - - - -# Gotta find the place where we're creating a Computation without setting -# its asyncFactory in the first place, but this is fine for now. -proc asyncFactory*(c: Computation): AsyncOperationFactory = - # Does Nim have an "ifNil" macro/template? - if isNil(c.vmState.asyncFactory): - AsyncOperationFactory(lazyDataSource: noLazyDataSource()) # AARDVARK - can I make a singleton? - else: - c.vmState.asyncFactory diff --git a/nimbus/evm/interpreter/op_handlers/oph_memory.nim b/nimbus/evm/interpreter/op_handlers/oph_memory.nim index 88b6e8303..f51afc58c 100644 --- a/nimbus/evm/interpreter/op_handlers/oph_memory.nim +++ b/nimbus/evm/interpreter/op_handlers/oph_memory.nim @@ -14,7 +14,7 @@ import ../../../errors, - ../../async_operations, + ../../async/operations, ../../code_stream, ../../computation, ../../memory, @@ -168,7 +168,7 @@ const ## 0x54, Load word from storage. let cpt = k.cpt # so it can safely be captured by the asyncChainTo closure below let (slot) = cpt.stack.popInt(1) - cpt.asyncChainTo(asyncFactory(cpt).lazyDataSource.ifNecessaryGetStorage(cpt, slot)): + cpt.asyncChainTo(ifNecessaryGetSlot(cpt.vmState, cpt.msg.contractAddress, slot)): cpt.stack.push: cpt.getStorage(slot) diff --git a/nimbus/evm/state.nim b/nimbus/evm/state.nim index 1635fe93e..8a38f94e3 100644 --- a/nimbus/evm/state.nim +++ b/nimbus/evm/state.nim @@ -16,6 +16,7 @@ import ../../stateless/[witness_from_tree, witness_types], ../db/accounts_cache, ../common/[common, evmforks], + ./async/data_sources, ./transaction_tracer, ./types @@ -46,6 +47,7 @@ proc init( self.stateDB = ac self.touchedAccounts = initHashSet[EthAddress]() self.minerAddress = miner + self.asyncFactory = AsyncOperationFactory(maybeDataSource: none[AsyncDataSource]()) proc init( self: BaseVMState; diff --git a/nimbus/evm/types.nim b/nimbus/evm/types.nim index 35b0dd832..45bbceb5a 100644 --- a/nimbus/evm/types.nim +++ b/nimbus/evm/types.nim @@ -14,6 +14,7 @@ import json_rpc/rpcclient, "."/[stack, memory, code_stream], ./interpreter/[gas_costs, op_codes], + ./async/data_sources, ../db/accounts_cache, ../common/[common, evmforks] @@ -132,11 +133,3 @@ type value*: UInt256 data*: seq[byte] flags*: MsgFlags - - LazyDataSource* = ref object of RootObj - ifNecessaryGetStorage*: - proc(c: Computation, slot: UInt256): Future[void] - {.gcsafe, raises: [CatchableError].} - - AsyncOperationFactory* = ref object of RootObj - lazyDataSource*: LazyDataSource diff --git a/tests/evm_tests.nim b/tests/evm_tests.nim index a872b95ac..2e8514792 100644 --- a/tests/evm_tests.nim +++ b/tests/evm_tests.nim @@ -13,8 +13,6 @@ import ../test_macro # place. I want to be able to gradually add to this test suite. # --Adam -# FIXME-asyncAndEvmc -# The test_op_memory_lazy test fails under EVMC. when not defined(evmc_enabled): cliBuilder: import ./test_op_arith, @@ -23,5 +21,4 @@ when not defined(evmc_enabled): ./test_op_memory, ./test_op_misc, ./test_op_custom, - ./test_tracer_json, - ./test_op_memory_lazy + ./test_tracer_json diff --git a/tests/macro_assembler.nim b/tests/macro_assembler.nim index d64af994e..9fc391f35 100644 --- a/tests/macro_assembler.nim +++ b/tests/macro_assembler.nim @@ -6,7 +6,7 @@ import import eth/trie/hexary, ../nimbus/db/[accounts_cache, distinct_tries], - ../nimbus/evm/[async_operations, types], + ../nimbus/evm/types, ../nimbus/vm_internals, ../nimbus/transaction/[call_common, call_evm], ../nimbus/[vm_types, vm_state], @@ -241,46 +241,23 @@ proc parseAssemblers(list: NimNode): seq[Assembler] = let body = callSection[1] result.add parseAssembler(body) -proc parseConcurrencyTest(list: NimNode): ConcurrencyTest = - list.expectKind nnkStmtList - for callSection in list: - callSection.expectKind(nnkCall) - let label = callSection[0].strVal - let body = callSection[1] - case label.normalize - of "title": result.title = parseStringLiteral(body) - of "assemblers": result.assemblers = parseAssemblers(body) - else: error("unknown section '" & label & "'", callSection[0]) - type VMProxy = tuple[sym: NimNode, pr: NimNode] -proc generateVMProxy(boa: Assembler, shouldBeAsync: bool): VMProxy = +proc generateVMProxy(boa: Assembler): VMProxy = let - vmProxySym = genSym(nskProc, "asyncVMProxy") + vmProxySym = genSym(nskProc, "vmProxy") chainDB = ident(if boa.chainDBIdentName == "": "chainDB" else: boa.chainDBIdentName) vmState = ident(if boa.vmStateIdentName == "": "vmState" else: boa.vmStateIdentName) body = newLitFixed(boa) - returnType = if shouldBeAsync: - quote do: Future[bool] - else: - quote do: bool - runVMProcName = ident(if shouldBeAsync: "asyncRunVM" else: "runVM") vmProxyProc = quote do: - proc `vmProxySym`(): `returnType` = + proc `vmProxySym`(): bool = let boa = `body` - let asyncFactory = - AsyncOperationFactory( - lazyDataSource: - if len(boa.initialStorage) == 0: - noLazyDataSource() - else: - fakeLazyDataSource(boa.initialStorage)) - `runVMProcName`(`vmState`, `chainDB`, boa, asyncFactory) + runVM(`vmState`, `chainDB`, boa) (vmProxySym, vmProxyProc) proc generateAssemblerTest(boa: Assembler): NimNode = let - (vmProxySym, vmProxyProc) = generateVMProxy(boa, false) + (vmProxySym, vmProxyProc) = generateVMProxy(boa) title: string = boa.title result = quote do: @@ -292,31 +269,6 @@ proc generateAssemblerTest(boa: Assembler): NimNode = when defined(macro_assembler_debug): echo result.toStrLit.strVal -type - AsyncVMProxyTestProc* = proc(): Future[bool] - -proc generateConcurrencyTest(t: ConcurrencyTest): NimNode = - let - vmProxies: seq[VMProxy] = t.assemblers.map(proc(boa: Assembler): VMProxy = generateVMProxy(boa, true)) - vmProxyProcs: seq[NimNode] = vmProxies.map(proc(x: VMProxy): NimNode = x.pr) - vmProxySyms: seq[NimNode] = vmProxies.map(proc(x: VMProxy): NimNode = x.sym) - title: string = t.title - - let runVMProxy = quote do: - {.gcsafe.}: - let procs: seq[AsyncVMProxyTestProc] = @(`vmProxySyms`) - let futures: seq[Future[bool]] = procs.map(proc(s: AsyncVMProxyTestProc): Future[bool] = s()) - waitFor(allFutures(futures)) - - # Is there a way to use "quote" (or something like it) to splice - # in a statement list? - let stmtList = newStmtList(vmProxyProcs) - stmtList.add(runVMProxy) - result = newCall("test", newStrLitNode(title), stmtList) - - when defined(macro_assembler_debug): - echo result.toStrLit.strVal - proc initDatabase*(networkId = MainNet): (BaseVMState, CommonRef) = let com = CommonRef.new(newMemoryDB(), false, networkId, networkParams(networkId)) com.initializeEmptyDb() @@ -446,8 +398,7 @@ proc createSignedTx(boaData: Blob, chainId: ChainId): Transaction = ) signTransaction(unsignedTx, privateKey, chainId, false) -proc runVM*(vmState: BaseVMState, com: CommonRef, boa: Assembler, asyncFactory: AsyncOperationFactory): bool = - vmState.asyncFactory = asyncFactory +proc runVM*(vmState: BaseVMState, com: CommonRef, boa: Assembler): bool = vmState.mutateStateDB: db.setCode(codeAddress, boa.code) db.setBalance(codeAddress, 1_000_000.u256) @@ -455,22 +406,9 @@ proc runVM*(vmState: BaseVMState, com: CommonRef, boa: Assembler, asyncFactory: let asmResult = testCallEvm(tx, tx.getSender, vmState, boa.fork) verifyAsmResult(vmState, com, boa, asmResult) -# FIXME-duplicatedForAsync -proc asyncRunVM*(vmState: BaseVMState, com: CommonRef, boa: Assembler, asyncFactory: AsyncOperationFactory): Future[bool] {.async.} = - vmState.asyncFactory = asyncFactory - vmState.mutateStateDB: - db.setCode(codeAddress, boa.code) - db.setBalance(codeAddress, 1_000_000.u256) - let tx = createSignedTx(boa.data, com.chainId) - let asmResult = await asyncTestCallEvm(tx, tx.getSender, vmState, boa.fork) - return verifyAsmResult(vmState, com, boa, asmResult) - macro assembler*(list: untyped): untyped = result = parseAssembler(list).generateAssemblerTest() -macro concurrentAssemblers*(list: untyped): untyped = - result = parseConcurrencyTest(list).generateConcurrencyTest() - macro evmByteCode*(list: untyped): untyped = list.expectKind nnkStmtList var code = parseCode(list) diff --git a/tests/test_op_memory_lazy.nim b/tests/test_op_memory_lazy.nim deleted file mode 100644 index 60989897a..000000000 --- a/tests/test_op_memory_lazy.nim +++ /dev/null @@ -1,74 +0,0 @@ -import macro_assembler, unittest2, macros, strutils - -proc opMemoryLazyMain*() = - suite "Lazy Loading With Memory Opcodes": - let (vmState, chainDB) = initDatabase() - - assembler: # SLOAD OP with (fake) lazy data fetching - title: "LAZY_SLOAD_1" - initialStorage: - "0xAA": "0x42" - code: - PUSH1 "0xAA" - SLOAD - PUSH1 "0x01" - ADD - PUSH1 "0xAA" - SSTORE - PUSH1 "0xAA" - SLOAD - storage: - "0xAA": "0x43" - stack: - "0x0000000000000000000000000000000000000000000000000000000000000043" - - let (vmState1, chainDB1) = initDatabase() - let (vmState2, chainDB2) = initDatabase() - concurrentAssemblers: - title: "Concurrent Assemblers" - assemblers: - asm1: - title: "asm1" - vmState: vmState1 - chainDB: chainDB1 - initialStorage: - "0xBB": "0x42" - "0xCC": "0x20" - code: - PUSH1 "0xBB" - SLOAD - PUSH1 "0xCC" - SLOAD - ADD - PUSH1 "0xBB" - SSTORE - PUSH1 "0xBB" - SLOAD - storage: - "0xBB": "0x62" - "0xCC": "0x20" - stack: "0x0000000000000000000000000000000000000000000000000000000000000062" - asm2: - title: "asm2" - vmState: vmState2 - chainDB: chainDB2 - initialStorage: - "0xDD": "0x30" - "0xEE": "0x20" - code: - PUSH1 "0xDD" - SLOAD - PUSH1 "0xEE" - SLOAD - ADD - PUSH1 "0xEE" - SSTORE - PUSH1 "0xEE" - SLOAD - storage: - "0xDD": "0x30" - "0xEE": "0x50" - stack: "0x0000000000000000000000000000000000000000000000000000000000000050" - -when isMainModule: - opMemoryLazyMain()