Recreating some of the old stateless code that had bitrotted. (#1493)
Note that we're not really using it yet; I'm just refactoring and adding new code, working in small steps.
This commit is contained in:
Adam Spitz
GitHub
parent
7ed3982929
commit
11771f8444
|
|
@ -64,6 +64,13 @@ const
|
|||
|
||||
proc beginSavepoint*(ac: var AccountsCache): SavePoint {.gcsafe.}
|
||||
|
||||
|
||||
# FIXME-Adam: this is only necessary because of my sanity checks on the latest rootHash;
|
||||
# take this out once those are gone.
|
||||
proc rawTrie*(ac: AccountsCache): AccountsTrie = ac.trie
|
||||
proc rawDb*(ac: AccountsCache): TrieDatabaseRef = ac.trie.db
|
||||
|
||||
|
||||
# The AccountsCache is modeled after TrieDatabase for it's transaction style
|
||||
proc init*(x: typedesc[AccountsCache], db: TrieDatabaseRef,
|
||||
root: KeccakHash, pruneTrie: bool = true): AccountsCache =
|
||||
|
|
|
|||
|
|
@ -0,0 +1,50 @@
|
|||
import
|
||||
options,
|
||||
chronos,
|
||||
stint,
|
||||
eth/common,
|
||||
eth/trie/db,
|
||||
../../db/db_chain
|
||||
|
||||
type
|
||||
AsyncDataSource* = ref object of RootObj
|
||||
ifNecessaryGetSlots*: proc(db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, slots: seq[UInt256], newStateRootForSanityChecking: Hash256): Future[void] {.gcsafe.}
|
||||
ifNecessaryGetCode*: proc(db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, newStateRootForSanityChecking: Hash256): Future[void] {.gcsafe.}
|
||||
ifNecessaryGetAccount*: proc(db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, newStateRootForSanityChecking: Hash256): Future[void] {.gcsafe.}
|
||||
ifNecessaryGetBlockHeaderByNumber*: proc(chainDB: ChainDBRef, blockNumber: BlockNumber): Future[void] {.gcsafe.}
|
||||
# FIXME-Adam: Later.
|
||||
#fetchNodes*: proc(stateRoot: Hash256, paths: seq[seq[seq[byte]]], nodeHashes: seq[Hash256]): Future[seq[seq[byte]]] {.gcsafe.}
|
||||
fetchBlockHeaderWithHash*: proc(h: Hash256): Future[BlockHeader] {.gcsafe.}
|
||||
fetchBlockHeaderWithNumber*: proc(n: BlockNumber): Future[BlockHeader] {.gcsafe.}
|
||||
fetchBlockHeaderAndBodyWithHash*: proc(h: Hash256): Future[(BlockHeader, BlockBody)] {.gcsafe.}
|
||||
fetchBlockHeaderAndBodyWithNumber*: proc(n: BlockNumber): Future[(BlockHeader, BlockBody)] {.gcsafe.}
|
||||
|
||||
# FIXME-Adam: maybe rename this?
|
||||
AsyncOperationFactory* = ref object of RootObj
|
||||
maybeDataSource*: Option[AsyncDataSource]
|
||||
|
||||
|
||||
# FIXME-Adam: Can I make a singleton?
|
||||
proc asyncFactoryWithNoDataSource*(): AsyncOperationFactory =
|
||||
AsyncOperationFactory(maybeDataSource: none[AsyncDataSource]())
|
||||
|
||||
|
||||
# FIXME-Adam: Ugly but straightforward; can this be cleaned up using some combination of:
|
||||
# - an ifSome/map operation on Options
|
||||
# - some kind of "what are we fetching" tuple, so that this is just one thing
|
||||
|
||||
proc ifNecessaryGetSlots*(asyncFactory: AsyncOperationFactory, db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, slots: seq[UInt256], newStateRootForSanityChecking: Hash256): Future[void] {.async.} =
|
||||
if asyncFactory.maybeDataSource.isSome:
|
||||
await asyncFactory.maybeDataSource.get.ifNecessaryGetSlots(db, blockNumber, stateRoot, address, slots, newStateRootForSanityChecking)
|
||||
|
||||
proc ifNecessaryGetCode*(asyncFactory: AsyncOperationFactory, db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, newStateRootForSanityChecking: Hash256): Future[void] {.async.} =
|
||||
if asyncFactory.maybeDataSource.isSome:
|
||||
await asyncFactory.maybeDataSource.get.ifNecessaryGetCode(db, blockNumber, stateRoot, address, newStateRootForSanityChecking)
|
||||
|
||||
proc ifNecessaryGetAccount*(asyncFactory: AsyncOperationFactory, db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, newStateRootForSanityChecking: Hash256): Future[void] {.async.} =
|
||||
if asyncFactory.maybeDataSource.isSome:
|
||||
await asyncFactory.maybeDataSource.get.ifNecessaryGetAccount(db, blockNumber, stateRoot, address, newStateRootForSanityChecking)
|
||||
|
||||
proc ifNecessaryGetBlockHeaderByNumber*(asyncFactory: AsyncOperationFactory, chainDB: ChainDBRef, blockNumber: BlockNumber): Future[void] {.async.} =
|
||||
if asyncFactory.maybeDataSource.isSome:
|
||||
await asyncFactory.maybeDataSource.get.ifNecessaryGetBlockHeaderByNumber(chainDB, blockNumber)
|
||||
|
|
@ -0,0 +1,260 @@
|
|||
import
|
||||
std/[sequtils, typetraits, options],
|
||||
chronicles,
|
||||
chronos,
|
||||
nimcrypto,
|
||||
stint,
|
||||
stew/byteutils,
|
||||
json_rpc/rpcclient,
|
||||
eth/common,
|
||||
eth/rlp,
|
||||
eth/trie/[db, hexary_proof_verification],
|
||||
eth/p2p,
|
||||
eth/p2p/rlpx,
|
||||
eth/p2p/private/p2p_types,
|
||||
../../../sync/protocol,
|
||||
../../../sync/protocol/eth66 as proto_eth66,
|
||||
../../../db/[db_chain, distinct_tries, incomplete_db, storage_types],
|
||||
../data_fetching,
|
||||
../data_sources,
|
||||
../rpc_api
|
||||
|
||||
|
||||
export AsyncOperationFactory, AsyncDataSource
|
||||
|
||||
|
||||
proc verifyFetchedAccount(stateRoot: Hash256, address: EthAddress, acc: Account, accProof: seq[seq[byte]]): Result[void, string] =
|
||||
let accKey = toSeq(keccakHash(address).data)
|
||||
let accEncoded = rlp.encode(acc)
|
||||
let accProofResult = verifyMptProof(accProof, stateRoot, accKey, accEncoded)
|
||||
case accProofResult.kind
|
||||
of ValidProof:
|
||||
return ok()
|
||||
of MissingKey:
|
||||
# For an account that doesn't exist yet, which is fine.
|
||||
return ok()
|
||||
of InvalidProof:
|
||||
return err(accProofResult.errorMsg)
|
||||
|
||||
type
|
||||
CodeFetchingInfo = tuple[blockNumber: BlockNumber, address: EthAddress]
|
||||
|
||||
proc fetchCode(client: RpcClient, p: CodeFetchingInfo): Future[seq[byte]] {.async.} =
|
||||
let (blockNumber, address) = p
|
||||
let fetchedCode = await fetchCode(client, blockNumber, address)
|
||||
return fetchedCode
|
||||
|
||||
proc verifyFetchedCode(fetchedCode: seq[byte], desiredCodeHash: Hash256): Result[void, Hash256] =
|
||||
let fetchedCodeHash = keccakHash(fetchedCode)
|
||||
if (desiredCodeHash == fetchedCodeHash):
|
||||
ok[void]()
|
||||
else:
|
||||
err(fetchedCodeHash)
|
||||
|
||||
proc fetchAndVerifyCode(client: RpcClient, p: CodeFetchingInfo, desiredCodeHash: Hash256): Future[seq[byte]] {.async.} =
|
||||
let fetchedCode: seq[byte] = await fetchCode(client, p)
|
||||
let verificationRes = verifyFetchedCode(fetchedCode, desiredCodeHash)
|
||||
if verificationRes.isOk():
|
||||
return fetchedCode
|
||||
else:
|
||||
let fetchedCodeHash = verificationRes.error
|
||||
error("code hash values do not match", p=p, desiredCodeHash=desiredCodeHash, fetchedCodeHash=fetchedCodeHash)
|
||||
raise newException(CatchableError, "async code received code for " & $(p.address) & " whose hash (" & $(fetchedCodeHash) & ") does not match the desired hash (" & $(desiredCodeHash) & ")")
|
||||
|
||||
proc storeCode(trie: AccountsTrie, p: CodeFetchingInfo, desiredCodeHash: Hash256, fetchedCode: seq[byte]) =
|
||||
trie.putCode(desiredCodeHash, fetchedCode)
|
||||
|
||||
proc assertThatWeHaveStoredCode(trie: AccountsTrie, p: CodeFetchingInfo, codeHash: Hash256) =
|
||||
# FIXME-Adam: this is a bit wrong because we're not checking it against the blockNumber, only the address. (That is,
|
||||
# if the code for this address has *changed* (which is unlikely), this check isn't the right thing to do.)
|
||||
let maybeFoundCode = trie.maybeGetCode(p.address)
|
||||
if maybeFoundCode.isNone:
|
||||
error("code didn't get put into the db", p=p, codeHash=codeHash)
|
||||
doAssert false, "code didn't get put into the db"
|
||||
else:
|
||||
let foundCode = maybeFoundCode.get
|
||||
let foundCodeHash = keccakHash(foundCode)
|
||||
if foundCodeHash != codeHash:
|
||||
error("code does not have the right hash", p=p, codeHash=codeHash, foundCode=foundCode)
|
||||
doAssert false, "code does not have the right hash"
|
||||
|
||||
|
||||
proc assertThatWeHaveStoredAccount(trie: AccountsTrie, address: EthAddress, fetchedAcc: Account, isForTheNewTrie: bool = false) =
|
||||
let foundAcc = ifNodesExistGetAccount(trie, address).get
|
||||
if fetchedAcc != foundAcc:
|
||||
error "account didn't come out the same", address=address, fetchedAcc=fetchedAcc, foundAcc=foundAcc, isForTheNewTrie=isForTheNewTrie
|
||||
doAssert false, "account didn't come out the same"
|
||||
doAssert(trie.hasAllNodesForAccount(address), "Can I check the account this way, too?")
|
||||
|
||||
|
||||
proc verifyFetchedSlot(accountStorageRoot: Hash256, slot: UInt256, fetchedVal: UInt256, storageMptNodes: seq[seq[byte]]): Result[void, string] =
|
||||
if storageMptNodes.len == 0:
|
||||
# I think an empty storage proof is okay; I see lots of these
|
||||
# where the account is empty and the value is zero.
|
||||
return ok()
|
||||
else:
|
||||
let storageKey = toSeq(keccakHash(toBytesBE(slot)).data)
|
||||
let storageValueEncoded = rlp.encode(fetchedVal)
|
||||
let storageProofResult = verifyMptProof(storageMptNodes, accountStorageRoot, storageKey, storageValueEncoded)
|
||||
case storageProofResult.kind
|
||||
of ValidProof:
|
||||
return ok()
|
||||
of MissingKey:
|
||||
# This is for a slot that doesn't have anything stored at it, but that's fine.
|
||||
return ok()
|
||||
of InvalidProof:
|
||||
return err(storageProofResult.errorMsg)
|
||||
|
||||
|
||||
proc assertThatWeHaveStoredSlot(trie: AccountsTrie, address: EthAddress, acc: Account, slot: UInt256, fetchedVal: UInt256, isForTheNewTrie: bool = false) =
|
||||
if acc.storageRoot == EMPTY_ROOT_HASH and fetchedVal.isZero:
|
||||
# I believe this is okay.
|
||||
discard
|
||||
else:
|
||||
let foundVal = ifNodesExistGetStorage(trie, address, slot).get
|
||||
if (fetchedVal != foundVal):
|
||||
error("slot didn't come out the same", address=address, slot=slot, fetchedVal=fetchedVal, foundVal=foundVal, isForTheNewTrie=isForTheNewTrie)
|
||||
doAssert false, "slot didn't come out the same"
|
||||
|
||||
|
||||
proc verifyFetchedBlockHeader(fetchedHeader: BlockHeader, desiredBlockNumber: BlockNumber): Result[void, BlockNumber] =
|
||||
# *Can* we do anything to verify this header, given that all we know
|
||||
# is the desiredBlockNumber and we want to run statelessly so we don't
|
||||
# know what block hash we want?
|
||||
ok[void]()
|
||||
|
||||
proc storeBlockHeader(chainDB: ChainDBRef, header: BlockHeader) =
|
||||
chainDB.persistHeaderToDbWithoutSetHeadOrScore(header)
|
||||
|
||||
proc assertThatWeHaveStoredBlockHeader(chainDB: ChainDBRef, blockNumber: BlockNumber, header: BlockHeader) =
|
||||
let h = chainDB.getBlockHash(blockNumber)
|
||||
doAssert(h == header.blockHash, "stored the block header for block " & $(blockNumber))
|
||||
|
||||
template raiseExceptionIfError[E](whatAreWeVerifying: untyped, r: Result[void, E]) =
|
||||
if r.isErr:
|
||||
error("async code failed to verify", whatAreWeVerifying=whatAreWeVerifying, err=r.error)
|
||||
raise newException(CatchableError, "async code failed to verify: " & $(whatAreWeVerifying) & ", error is: " & $(r.error))
|
||||
|
||||
|
||||
const shouldDoUnnecessarySanityChecks = true
|
||||
|
||||
# This proc fetches both the account and also optionally some of its slots, because that's what eth_getProof can do.
|
||||
proc ifNecessaryGetAccountAndSlots*(client: RpcClient, db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, slots: seq[UInt256], justCheckingAccount: bool, justCheckingSlots: bool, newStateRootForSanityChecking: Hash256): Future[void] {.async.} =
|
||||
let trie = initAccountsTrie(db, stateRoot, false) # important for sanity checks
|
||||
let trie2 = initAccountsTrie(db, newStateRootForSanityChecking, false) # important for sanity checks
|
||||
let doesAccountActuallyNeedToBeFetched = not trie.hasAllNodesForAccount(address)
|
||||
let slotsToActuallyFetch = slots.filter(proc(slot: UInt256): bool = not (trie.hasAllNodesForStorageSlot(address, slot)))
|
||||
if (not doesAccountActuallyNeedToBeFetched) and (slotsToActuallyFetch.len == 0):
|
||||
# Already have them, no need to fetch either the account or the slots
|
||||
discard
|
||||
else:
|
||||
let (acc, accProof, storageProofs) = await fetchAccountAndSlots(client, address, slotsToActuallyFetch, blockNumber)
|
||||
|
||||
# We need to verify the proof even if we already had this account,
|
||||
# to make sure the data is valid.
|
||||
let accountVerificationRes = verifyFetchedAccount(stateRoot, address, acc, accProof)
|
||||
let whatAreWeVerifying = ("account proof", address, acc)
|
||||
raiseExceptionIfError(whatAreWeVerifying, accountVerificationRes)
|
||||
|
||||
if not doesAccountActuallyNeedToBeFetched:
|
||||
# We already had the account, no need to populate the DB with it again.
|
||||
discard
|
||||
else:
|
||||
if not justCheckingAccount:
|
||||
populateDbWithBranch(db, accProof)
|
||||
if shouldDoUnnecessarySanityChecks:
|
||||
assertThatWeHaveStoredAccount(trie, address, acc, false)
|
||||
if doesAccountActuallyNeedToBeFetched: # this second check makes no sense if it's not the first time
|
||||
assertThatWeHaveStoredAccount(trie2, address, acc, true)
|
||||
|
||||
doAssert(slotsToActuallyFetch.len == storageProofs.len, "We should get back the same number of storage proofs as slots that we asked for. I think.")
|
||||
|
||||
for storageProof in storageProofs:
|
||||
let slot = UInt256.fromHex(string(storageProof.key))
|
||||
let fetchedVal = UInt256.fromHex(string(storageProof.value))
|
||||
let storageMptNodes = storageProof.proof.mapIt(hexToSeqByte(string(it)))
|
||||
let storageVerificationRes = verifyFetchedSlot(acc.storageRoot, slot, fetchedVal, storageMptNodes)
|
||||
let whatAreWeVerifying = ("storage proof", address, acc, slot, fetchedVal)
|
||||
raiseExceptionIfError(whatAreWeVerifying, storageVerificationRes)
|
||||
|
||||
if not justCheckingSlots:
|
||||
populateDbWithBranch(db, storageMptNodes)
|
||||
|
||||
# I believe this is done so that we can iterate over the slots. See
|
||||
# persistStorage in accounts_cache.nim.
|
||||
let slotAsKey = createTrieKeyFromSlot(slot)
|
||||
let slotHash = keccakHash(slotAsKey)
|
||||
let slotEncoded = rlp.encode(slot)
|
||||
db.put(slotHashToSlotKey(slotHash.data).toOpenArray, slotEncoded)
|
||||
|
||||
if shouldDoUnnecessarySanityChecks:
|
||||
assertThatWeHaveStoredSlot(trie, address, acc, slot, fetchedVal, false)
|
||||
assertThatWeHaveStoredSlot(trie2, address, acc, slot, fetchedVal, true)
|
||||
|
||||
proc ifNecessaryGetCode*(client: RpcClient, db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, justChecking: bool, newStateRootForSanityChecking: Hash256): Future[void] {.async.} =
|
||||
await ifNecessaryGetAccountAndSlots(client, db, blockNumber, stateRoot, address, @[], false, false, newStateRootForSanityChecking) # to make sure we've got the codeHash
|
||||
let trie = initAccountsTrie(db, stateRoot, false) # important for sanity checks
|
||||
|
||||
let acc = ifNodesExistGetAccount(trie, address).get
|
||||
let desiredCodeHash = acc.codeHash
|
||||
|
||||
let p = (blockNumber, address)
|
||||
if not(trie.hasAllNodesForCode(address)):
|
||||
let fetchedCode = await fetchAndVerifyCode(client, p, desiredCodeHash)
|
||||
|
||||
if not justChecking:
|
||||
storeCode(trie, p, desiredCodeHash, fetchedCode)
|
||||
if shouldDoUnnecessarySanityChecks:
|
||||
assertThatWeHaveStoredCode(trie, p, desiredCodeHash)
|
||||
|
||||
proc ifNecessaryGetBlockHeaderByNumber*(client: RpcClient, chainDB: ChainDBRef, blockNumber: BlockNumber, justChecking: bool): Future[void] {.async.} =
|
||||
let maybeHeaderAndHash = chainDB.getBlockHeaderWithHash(blockNumber)
|
||||
if maybeHeaderAndHash.isNone:
|
||||
let fetchedHeader = await fetchBlockHeaderWithNumber(client, blockNumber)
|
||||
let headerVerificationRes = verifyFetchedBlockHeader(fetchedHeader, blockNumber)
|
||||
let whatAreWeVerifying = ("block header by number", blockNumber, fetchedHeader)
|
||||
raiseExceptionIfError(whatAreWeVerifying, headerVerificationRes)
|
||||
|
||||
if not justChecking:
|
||||
storeBlockHeader(chainDB, fetchedHeader)
|
||||
if shouldDoUnnecessarySanityChecks:
|
||||
assertThatWeHaveStoredBlockHeader(chainDB, blockNumber, fetchedHeader)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# Used in asynchronous on-demand-data-fetching mode.
|
||||
proc realAsyncDataSource*(peerPool: PeerPool, client: RpcClient, justChecking: bool): AsyncDataSource =
|
||||
AsyncDataSource(
|
||||
ifNecessaryGetAccount: (proc(db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, newStateRootForSanityChecking: Hash256): Future[void] {.async.} =
|
||||
await ifNecessaryGetAccountAndSlots(client, db, blockNumber, stateRoot, address, @[], false, false, newStateRootForSanityChecking)
|
||||
),
|
||||
ifNecessaryGetSlots: (proc(db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, slots: seq[UInt256], newStateRootForSanityChecking: Hash256): Future[void] {.async.} =
|
||||
await ifNecessaryGetAccountAndSlots(client, db, blockNumber, stateRoot, address, slots, false, false, newStateRootForSanityChecking)
|
||||
),
|
||||
ifNecessaryGetCode: (proc(db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, newStateRootForSanityChecking: Hash256): Future[void] {.async.} =
|
||||
await ifNecessaryGetCode(client, db, blockNumber, stateRoot, address, justChecking, newStateRootForSanityChecking)
|
||||
),
|
||||
ifNecessaryGetBlockHeaderByNumber: (proc(chainDB: ChainDBRef, blockNumber: BlockNumber): Future[void] {.async.} =
|
||||
await ifNecessaryGetBlockHeaderByNumber(client, chainDB, blockNumber, justChecking)
|
||||
),
|
||||
|
||||
# FIXME-Adam: This will be needed later, but for now let's just get the basic methods in place.
|
||||
#fetchNodes: (proc(stateRoot: Hash256, paths: seq[seq[seq[byte]]], nodeHashes: seq[Hash256]): Future[seq[seq[byte]]] {.async.} =
|
||||
# return await fetchNodes(peerPool, stateRoot, paths, nodeHashes)
|
||||
#),
|
||||
|
||||
fetchBlockHeaderWithHash: (proc(h: Hash256): Future[BlockHeader] {.async.} =
|
||||
return await fetchBlockHeaderWithHash(client, h)
|
||||
),
|
||||
fetchBlockHeaderWithNumber: (proc(n: BlockNumber): Future[BlockHeader] {.async.} =
|
||||
return await fetchBlockHeaderWithNumber(client, n)
|
||||
),
|
||||
fetchBlockHeaderAndBodyWithHash: (proc(h: Hash256): Future[(BlockHeader, BlockBody)] {.async.} =
|
||||
return await fetchBlockHeaderAndBodyWithHash(client, h)
|
||||
),
|
||||
fetchBlockHeaderAndBodyWithNumber: (proc(n: BlockNumber): Future[(BlockHeader, BlockBody)] {.async.} =
|
||||
return await fetchBlockHeaderAndBodyWithNumber(client, n)
|
||||
)
|
||||
)
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
import
|
||||
chronicles,
|
||||
chronos,
|
||||
stint,
|
||||
eth/common/eth_types,
|
||||
../../common,
|
||||
../../db/distinct_tries,
|
||||
../../db/accounts_cache,
|
||||
#../../db/incomplete_db,
|
||||
../types,
|
||||
./data_sources
|
||||
|
||||
|
||||
|
||||
proc ifNecessaryGetAccount*(vmState: BaseVMState, address: EthAddress): Future[void] {.async.} =
|
||||
await vmState.asyncFactory.ifNecessaryGetAccount(vmState.com.db.db, vmState.parent.blockNumber, vmState.parent.stateRoot, address, vmState.stateDB.rawTrie.rootHash)
|
||||
|
||||
proc ifNecessaryGetCode*(vmState: BaseVMState, address: EthAddress): Future[void] {.async.} =
|
||||
await vmState.asyncFactory.ifNecessaryGetCode(vmState.com.db.db, vmState.parent.blockNumber, vmState.parent.stateRoot, address, vmState.stateDB.rawTrie.rootHash)
|
||||
|
||||
proc ifNecessaryGetSlots*(vmState: BaseVMState, address: EthAddress, slots: seq[UInt256]): Future[void] {.async.} =
|
||||
await vmState.asyncFactory.ifNecessaryGetSlots(vmState.com.db.db, vmState.parent.blockNumber, vmState.parent.stateRoot, address, slots, vmState.stateDB.rawTrie.rootHash)
|
||||
|
||||
proc ifNecessaryGetSlot*(vmState: BaseVMState, address: EthAddress, slot: UInt256): Future[void] {.async.} =
|
||||
await ifNecessaryGetSlots(vmState, address, @[slot])
|
||||
|
||||
proc ifNecessaryGetBlockHeaderByNumber*(vmState: BaseVMState, blockNumber: BlockNumber): Future[void] {.async.} =
|
||||
await vmState.asyncFactory.ifNecessaryGetBlockHeaderByNumber(vmState.com.db, blockNumber)
|
||||
|
||||
#[
|
||||
FIXME-Adam: This is for later.
|
||||
proc fetchAndPopulateNodes*(vmState: BaseVMState, paths: seq[seq[seq[byte]]], nodeHashes: seq[Hash256]): Future[void] {.async.} =
|
||||
if vmState.asyncFactory.maybeDataSource.isSome:
|
||||
# let stateRoot = vmState.stateDB.rawTrie.rootHash # FIXME-Adam: this might not be right, huh? the peer might expect the parent block's final stateRoot, not this weirdo intermediate one
|
||||
let stateRoot = vmState.parent.stateRoot
|
||||
let nodes = await vmState.asyncFactory.maybeDataSource.get.fetchNodes(stateRoot, paths, nodeHashes)
|
||||
populateDbWithNodes(vmState.stateDB.rawDb, nodes)
|
||||
]#
|
||||
|
||||
|
||||
# Sometimes it's convenient to be able to do multiple at once.
|
||||
|
||||
proc ifNecessaryGetAccounts*(vmState: BaseVMState, addresses: seq[EthAddress]): Future[void] {.async.} =
|
||||
for address in addresses:
|
||||
await ifNecessaryGetAccount(vmState, address)
|
||||
|
||||
proc ifNecessaryGetCodeForAccounts*(vmState: BaseVMState, addresses: seq[EthAddress]): Future[void] {.async.} =
|
||||
for address in addresses:
|
||||
await ifNecessaryGetCode(vmState, address)
|
||||
|
|
@ -1,104 +0,0 @@
|
|||
# Nimbus
|
||||
# Copyright (c) 2022-2023 Status Research & Development GmbH
|
||||
# Licensed under either of
|
||||
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||
# http://opensource.org/licenses/MIT)
|
||||
# at your option. This file may not be copied, modified, or distributed except
|
||||
# according to those terms.
|
||||
|
||||
import
|
||||
chronos,
|
||||
stint,
|
||||
json_rpc/rpcclient,
|
||||
web3,
|
||||
./computation,
|
||||
./state,
|
||||
./types,
|
||||
../db/accounts_cache
|
||||
|
||||
|
||||
|
||||
# Used in synchronous mode.
|
||||
proc noLazyDataSource*(): LazyDataSource =
|
||||
LazyDataSource(
|
||||
ifNecessaryGetStorage: (proc(c: Computation, slot: UInt256): Future[void] {.async.} =
|
||||
discard
|
||||
)
|
||||
)
|
||||
|
||||
# Will be used in asynchronous on-demand-data-fetching mode, once
|
||||
# that is implemented.
|
||||
proc realLazyDataSource*(client: RpcClient): LazyDataSource =
|
||||
LazyDataSource(
|
||||
ifNecessaryGetStorage: (proc(c: Computation, slot: UInt256): Future[void] {.async.} =
|
||||
# TODO: find some way to check whether we already have it.
|
||||
# This is WRONG, but good enough for now, considering this
|
||||
# code is unused except in a few tests. I'm working on
|
||||
# doing this properly.
|
||||
if not c.getStorage(slot).isZero: return
|
||||
|
||||
# FIXME-onDemandStorageNotImplementedYet
|
||||
# (I sketched in this code, but haven't actually tried running it yet.)
|
||||
echo("Attempting to for-real fetch slot " & $(slot))
|
||||
# ethAddressStr("0xfff33a3bd36abdbd412707b8e310d6011454a7ae")
|
||||
# check hexDataStr(0.u256).string == res.string
|
||||
let ethAddress = c.msg.contractAddress
|
||||
let address: Address = Address(ethAddress)
|
||||
let quantity: int = slot.truncate(int) # this is probably wrong; what's the right way to convert this?
|
||||
let blockId: BlockIdentifier = blockId(c.vmState.parent.blockNumber.truncate(uint64)) # ditto
|
||||
let res = await client.eth_getStorageAt(address, quantity, blockId)
|
||||
echo("Fetched slot " & $(slot) & ", result is " & $(res))
|
||||
# let v = res # will res be the actual value, or do I need to convert or something?
|
||||
|
||||
# Before implementing this, see the note from Zahary here:
|
||||
# https://github.com/status-im/nimbus-eth1/pull/1260#discussion_r999669139
|
||||
#
|
||||
# c.vmState.mutateStateDB:
|
||||
# db.setStorage(c.msg.contractAddress, slot, UInt256.fromBytesBE(v))
|
||||
)
|
||||
)
|
||||
|
||||
# Used for unit testing. Contains some prepopulated data.
|
||||
proc fakeLazyDataSource*(fakePairs: seq[tuple[key, val: array[32, byte]]]): LazyDataSource =
|
||||
LazyDataSource(
|
||||
ifNecessaryGetStorage: (proc(c: Computation, slot: UInt256): Future[void] {.async.} =
|
||||
# See the comment above.
|
||||
if not c.getStorage(slot).isZero: return
|
||||
|
||||
# FIXME-writeAutomatedTestsToShowThatItCanRunConcurrently
|
||||
|
||||
# For now, until I've implemented some more automated way to
|
||||
# capture and verify the fact that this can run concurrently,
|
||||
# this is useful just to see in the console that the echo
|
||||
# statements from multiple Computations can run at the same
|
||||
# time and be interleaved.
|
||||
# echo("Attempting to fake-fetch slot " & $(slot))
|
||||
# await sleepAsync(2.seconds)
|
||||
|
||||
let slotBytes = toBytesBE(slot)
|
||||
# The linear search is obviously slow, but doesn't matter
|
||||
# for tests with only a few initialStorage entries. Fix
|
||||
# this if we ever want to write tests with more.
|
||||
for (k, v) in fakePairs:
|
||||
if slotBytes == k:
|
||||
c.vmState.mutateStateDB:
|
||||
db.setStorage(c.msg.contractAddress, slot, UInt256.fromBytesBE(v))
|
||||
break
|
||||
|
||||
# echo("Finished fake-fetch of slot " & $(slot))
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
# Gotta find the place where we're creating a Computation without setting
|
||||
# its asyncFactory in the first place, but this is fine for now.
|
||||
proc asyncFactory*(c: Computation): AsyncOperationFactory =
|
||||
# Does Nim have an "ifNil" macro/template?
|
||||
if isNil(c.vmState.asyncFactory):
|
||||
AsyncOperationFactory(lazyDataSource: noLazyDataSource()) # AARDVARK - can I make a singleton?
|
||||
else:
|
||||
c.vmState.asyncFactory
|
||||
|
|
@ -14,7 +14,7 @@
|
|||
|
||||
import
|
||||
../../../errors,
|
||||
../../async_operations,
|
||||
../../async/operations,
|
||||
../../code_stream,
|
||||
../../computation,
|
||||
../../memory,
|
||||
|
|
@ -168,7 +168,7 @@ const
|
|||
## 0x54, Load word from storage.
|
||||
let cpt = k.cpt # so it can safely be captured by the asyncChainTo closure below
|
||||
let (slot) = cpt.stack.popInt(1)
|
||||
cpt.asyncChainTo(asyncFactory(cpt).lazyDataSource.ifNecessaryGetStorage(cpt, slot)):
|
||||
cpt.asyncChainTo(ifNecessaryGetSlot(cpt.vmState, cpt.msg.contractAddress, slot)):
|
||||
cpt.stack.push:
|
||||
cpt.getStorage(slot)
|
||||
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ import
|
|||
../../stateless/[witness_from_tree, witness_types],
|
||||
../db/accounts_cache,
|
||||
../common/[common, evmforks],
|
||||
./async/data_sources,
|
||||
./transaction_tracer,
|
||||
./types
|
||||
|
||||
|
|
@ -46,6 +47,7 @@ proc init(
|
|||
self.stateDB = ac
|
||||
self.touchedAccounts = initHashSet[EthAddress]()
|
||||
self.minerAddress = miner
|
||||
self.asyncFactory = AsyncOperationFactory(maybeDataSource: none[AsyncDataSource]())
|
||||
|
||||
proc init(
|
||||
self: BaseVMState;
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import
|
|||
json_rpc/rpcclient,
|
||||
"."/[stack, memory, code_stream],
|
||||
./interpreter/[gas_costs, op_codes],
|
||||
./async/data_sources,
|
||||
../db/accounts_cache,
|
||||
../common/[common, evmforks]
|
||||
|
||||
|
|
@ -132,11 +133,3 @@ type
|
|||
value*: UInt256
|
||||
data*: seq[byte]
|
||||
flags*: MsgFlags
|
||||
|
||||
LazyDataSource* = ref object of RootObj
|
||||
ifNecessaryGetStorage*:
|
||||
proc(c: Computation, slot: UInt256): Future[void]
|
||||
{.gcsafe, raises: [CatchableError].}
|
||||
|
||||
AsyncOperationFactory* = ref object of RootObj
|
||||
lazyDataSource*: LazyDataSource
|
||||
|
|
|
|||
|
|
@ -13,8 +13,6 @@ import ../test_macro
|
|||
# place. I want to be able to gradually add to this test suite.
|
||||
# --Adam
|
||||
|
||||
# FIXME-asyncAndEvmc
|
||||
# The test_op_memory_lazy test fails under EVMC.
|
||||
when not defined(evmc_enabled):
|
||||
cliBuilder:
|
||||
import ./test_op_arith,
|
||||
|
|
@ -23,5 +21,4 @@ when not defined(evmc_enabled):
|
|||
./test_op_memory,
|
||||
./test_op_misc,
|
||||
./test_op_custom,
|
||||
./test_tracer_json,
|
||||
./test_op_memory_lazy
|
||||
./test_tracer_json
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import
|
|||
import
|
||||
eth/trie/hexary,
|
||||
../nimbus/db/[accounts_cache, distinct_tries],
|
||||
../nimbus/evm/[async_operations, types],
|
||||
../nimbus/evm/types,
|
||||
../nimbus/vm_internals,
|
||||
../nimbus/transaction/[call_common, call_evm],
|
||||
../nimbus/[vm_types, vm_state],
|
||||
|
|
@ -241,46 +241,23 @@ proc parseAssemblers(list: NimNode): seq[Assembler] =
|
|||
let body = callSection[1]
|
||||
result.add parseAssembler(body)
|
||||
|
||||
proc parseConcurrencyTest(list: NimNode): ConcurrencyTest =
|
||||
list.expectKind nnkStmtList
|
||||
for callSection in list:
|
||||
callSection.expectKind(nnkCall)
|
||||
let label = callSection[0].strVal
|
||||
let body = callSection[1]
|
||||
case label.normalize
|
||||
of "title": result.title = parseStringLiteral(body)
|
||||
of "assemblers": result.assemblers = parseAssemblers(body)
|
||||
else: error("unknown section '" & label & "'", callSection[0])
|
||||
|
||||
type VMProxy = tuple[sym: NimNode, pr: NimNode]
|
||||
|
||||
proc generateVMProxy(boa: Assembler, shouldBeAsync: bool): VMProxy =
|
||||
proc generateVMProxy(boa: Assembler): VMProxy =
|
||||
let
|
||||
vmProxySym = genSym(nskProc, "asyncVMProxy")
|
||||
vmProxySym = genSym(nskProc, "vmProxy")
|
||||
chainDB = ident(if boa.chainDBIdentName == "": "chainDB" else: boa.chainDBIdentName)
|
||||
vmState = ident(if boa.vmStateIdentName == "": "vmState" else: boa.vmStateIdentName)
|
||||
body = newLitFixed(boa)
|
||||
returnType = if shouldBeAsync:
|
||||
quote do: Future[bool]
|
||||
else:
|
||||
quote do: bool
|
||||
runVMProcName = ident(if shouldBeAsync: "asyncRunVM" else: "runVM")
|
||||
vmProxyProc = quote do:
|
||||
proc `vmProxySym`(): `returnType` =
|
||||
proc `vmProxySym`(): bool =
|
||||
let boa = `body`
|
||||
let asyncFactory =
|
||||
AsyncOperationFactory(
|
||||
lazyDataSource:
|
||||
if len(boa.initialStorage) == 0:
|
||||
noLazyDataSource()
|
||||
else:
|
||||
fakeLazyDataSource(boa.initialStorage))
|
||||
`runVMProcName`(`vmState`, `chainDB`, boa, asyncFactory)
|
||||
runVM(`vmState`, `chainDB`, boa)
|
||||
(vmProxySym, vmProxyProc)
|
||||
|
||||
proc generateAssemblerTest(boa: Assembler): NimNode =
|
||||
let
|
||||
(vmProxySym, vmProxyProc) = generateVMProxy(boa, false)
|
||||
(vmProxySym, vmProxyProc) = generateVMProxy(boa)
|
||||
title: string = boa.title
|
||||
|
||||
result = quote do:
|
||||
|
|
@ -292,31 +269,6 @@ proc generateAssemblerTest(boa: Assembler): NimNode =
|
|||
when defined(macro_assembler_debug):
|
||||
echo result.toStrLit.strVal
|
||||
|
||||
type
|
||||
AsyncVMProxyTestProc* = proc(): Future[bool]
|
||||
|
||||
proc generateConcurrencyTest(t: ConcurrencyTest): NimNode =
|
||||
let
|
||||
vmProxies: seq[VMProxy] = t.assemblers.map(proc(boa: Assembler): VMProxy = generateVMProxy(boa, true))
|
||||
vmProxyProcs: seq[NimNode] = vmProxies.map(proc(x: VMProxy): NimNode = x.pr)
|
||||
vmProxySyms: seq[NimNode] = vmProxies.map(proc(x: VMProxy): NimNode = x.sym)
|
||||
title: string = t.title
|
||||
|
||||
let runVMProxy = quote do:
|
||||
{.gcsafe.}:
|
||||
let procs: seq[AsyncVMProxyTestProc] = @(`vmProxySyms`)
|
||||
let futures: seq[Future[bool]] = procs.map(proc(s: AsyncVMProxyTestProc): Future[bool] = s())
|
||||
waitFor(allFutures(futures))
|
||||
|
||||
# Is there a way to use "quote" (or something like it) to splice
|
||||
# in a statement list?
|
||||
let stmtList = newStmtList(vmProxyProcs)
|
||||
stmtList.add(runVMProxy)
|
||||
result = newCall("test", newStrLitNode(title), stmtList)
|
||||
|
||||
when defined(macro_assembler_debug):
|
||||
echo result.toStrLit.strVal
|
||||
|
||||
proc initDatabase*(networkId = MainNet): (BaseVMState, CommonRef) =
|
||||
let com = CommonRef.new(newMemoryDB(), false, networkId, networkParams(networkId))
|
||||
com.initializeEmptyDb()
|
||||
|
|
@ -446,8 +398,7 @@ proc createSignedTx(boaData: Blob, chainId: ChainId): Transaction =
|
|||
)
|
||||
signTransaction(unsignedTx, privateKey, chainId, false)
|
||||
|
||||
proc runVM*(vmState: BaseVMState, com: CommonRef, boa: Assembler, asyncFactory: AsyncOperationFactory): bool =
|
||||
vmState.asyncFactory = asyncFactory
|
||||
proc runVM*(vmState: BaseVMState, com: CommonRef, boa: Assembler): bool =
|
||||
vmState.mutateStateDB:
|
||||
db.setCode(codeAddress, boa.code)
|
||||
db.setBalance(codeAddress, 1_000_000.u256)
|
||||
|
|
@ -455,22 +406,9 @@ proc runVM*(vmState: BaseVMState, com: CommonRef, boa: Assembler, asyncFactory:
|
|||
let asmResult = testCallEvm(tx, tx.getSender, vmState, boa.fork)
|
||||
verifyAsmResult(vmState, com, boa, asmResult)
|
||||
|
||||
# FIXME-duplicatedForAsync
|
||||
proc asyncRunVM*(vmState: BaseVMState, com: CommonRef, boa: Assembler, asyncFactory: AsyncOperationFactory): Future[bool] {.async.} =
|
||||
vmState.asyncFactory = asyncFactory
|
||||
vmState.mutateStateDB:
|
||||
db.setCode(codeAddress, boa.code)
|
||||
db.setBalance(codeAddress, 1_000_000.u256)
|
||||
let tx = createSignedTx(boa.data, com.chainId)
|
||||
let asmResult = await asyncTestCallEvm(tx, tx.getSender, vmState, boa.fork)
|
||||
return verifyAsmResult(vmState, com, boa, asmResult)
|
||||
|
||||
macro assembler*(list: untyped): untyped =
|
||||
result = parseAssembler(list).generateAssemblerTest()
|
||||
|
||||
macro concurrentAssemblers*(list: untyped): untyped =
|
||||
result = parseConcurrencyTest(list).generateConcurrencyTest()
|
||||
|
||||
macro evmByteCode*(list: untyped): untyped =
|
||||
list.expectKind nnkStmtList
|
||||
var code = parseCode(list)
|
||||
|
|
|
|||
|
|
@ -1,74 +0,0 @@
|
|||
import macro_assembler, unittest2, macros, strutils
|
||||
|
||||
proc opMemoryLazyMain*() =
|
||||
suite "Lazy Loading With Memory Opcodes":
|
||||
let (vmState, chainDB) = initDatabase()
|
||||
|
||||
assembler: # SLOAD OP with (fake) lazy data fetching
|
||||
title: "LAZY_SLOAD_1"
|
||||
initialStorage:
|
||||
"0xAA": "0x42"
|
||||
code:
|
||||
PUSH1 "0xAA"
|
||||
SLOAD
|
||||
PUSH1 "0x01"
|
||||
ADD
|
||||
PUSH1 "0xAA"
|
||||
SSTORE
|
||||
PUSH1 "0xAA"
|
||||
SLOAD
|
||||
storage:
|
||||
"0xAA": "0x43"
|
||||
stack:
|
||||
"0x0000000000000000000000000000000000000000000000000000000000000043"
|
||||
|
||||
let (vmState1, chainDB1) = initDatabase()
|
||||
let (vmState2, chainDB2) = initDatabase()
|
||||
concurrentAssemblers:
|
||||
title: "Concurrent Assemblers"
|
||||
assemblers:
|
||||
asm1:
|
||||
title: "asm1"
|
||||
vmState: vmState1
|
||||
chainDB: chainDB1
|
||||
initialStorage:
|
||||
"0xBB": "0x42"
|
||||
"0xCC": "0x20"
|
||||
code:
|
||||
PUSH1 "0xBB"
|
||||
SLOAD
|
||||
PUSH1 "0xCC"
|
||||
SLOAD
|
||||
ADD
|
||||
PUSH1 "0xBB"
|
||||
SSTORE
|
||||
PUSH1 "0xBB"
|
||||
SLOAD
|
||||
storage:
|
||||
"0xBB": "0x62"
|
||||
"0xCC": "0x20"
|
||||
stack: "0x0000000000000000000000000000000000000000000000000000000000000062"
|
||||
asm2:
|
||||
title: "asm2"
|
||||
vmState: vmState2
|
||||
chainDB: chainDB2
|
||||
initialStorage:
|
||||
"0xDD": "0x30"
|
||||
"0xEE": "0x20"
|
||||
code:
|
||||
PUSH1 "0xDD"
|
||||
SLOAD
|
||||
PUSH1 "0xEE"
|
||||
SLOAD
|
||||
ADD
|
||||
PUSH1 "0xEE"
|
||||
SSTORE
|
||||
PUSH1 "0xEE"
|
||||
SLOAD
|
||||
storage:
|
||||
"0xDD": "0x30"
|
||||
"0xEE": "0x50"
|
||||
stack: "0x0000000000000000000000000000000000000000000000000000000000000050"
|
||||
|
||||
when isMainModule:
|
||||
opMemoryLazyMain()
|
||||
Loading…
Reference in New Issue