import std/[sequtils, typetraits, options], chronicles, chronos, nimcrypto, stint, stew/byteutils, json_rpc/rpcclient, eth/common, eth/rlp, eth/trie/[db, hexary_proof_verification], eth/p2p, eth/p2p/rlpx, eth/p2p/private/p2p_types, ../../../sync/protocol, ../../../sync/protocol/eth66 as proto_eth66, ../../../db/[db_chain, distinct_tries, incomplete_db, storage_types], ../data_fetching, ../data_sources, ../rpc_api export AsyncOperationFactory, AsyncDataSource proc verifyFetchedAccount(stateRoot: Hash256, address: EthAddress, acc: Account, accProof: seq[seq[byte]]): Result[void, string] = let accKey = toSeq(keccakHash(address).data) let accEncoded = rlp.encode(acc) let accProofResult = verifyMptProof(accProof, stateRoot, accKey, accEncoded) case accProofResult.kind of ValidProof: return ok() of MissingKey: # For an account that doesn't exist yet, which is fine. return ok() of InvalidProof: return err(accProofResult.errorMsg) type CodeFetchingInfo = tuple[blockNumber: BlockNumber, address: EthAddress] proc fetchCode(client: RpcClient, p: CodeFetchingInfo): Future[seq[byte]] {.async.} = let (blockNumber, address) = p let fetchedCode = await fetchCode(client, blockNumber, address) return fetchedCode proc verifyFetchedCode(fetchedCode: seq[byte], desiredCodeHash: Hash256): Result[void, Hash256] = let fetchedCodeHash = keccakHash(fetchedCode) if (desiredCodeHash == fetchedCodeHash): ok[void]() else: err(fetchedCodeHash) proc fetchAndVerifyCode(client: RpcClient, p: CodeFetchingInfo, desiredCodeHash: Hash256): Future[seq[byte]] {.async.} = let fetchedCode: seq[byte] = await fetchCode(client, p) let verificationRes = verifyFetchedCode(fetchedCode, desiredCodeHash) if verificationRes.isOk(): return fetchedCode else: let fetchedCodeHash = verificationRes.error error("code hash values do not match", p=p, desiredCodeHash=desiredCodeHash, fetchedCodeHash=fetchedCodeHash) raise newException(CatchableError, "async code received code for " & $(p.address) & " whose hash (" & $(fetchedCodeHash) & ") does not match the desired hash (" & $(desiredCodeHash) & ")") proc storeCode(trie: AccountsTrie, p: CodeFetchingInfo, desiredCodeHash: Hash256, fetchedCode: seq[byte]) = trie.putCode(desiredCodeHash, fetchedCode) proc assertThatWeHaveStoredCode(trie: AccountsTrie, p: CodeFetchingInfo, codeHash: Hash256) = # FIXME-Adam: this is a bit wrong because we're not checking it against the blockNumber, only the address. (That is, # if the code for this address has *changed* (which is unlikely), this check isn't the right thing to do.) let maybeFoundCode = trie.maybeGetCode(p.address) if maybeFoundCode.isNone: error("code didn't get put into the db", p=p, codeHash=codeHash) doAssert false, "code didn't get put into the db" else: let foundCode = maybeFoundCode.get let foundCodeHash = keccakHash(foundCode) if foundCodeHash != codeHash: error("code does not have the right hash", p=p, codeHash=codeHash, foundCode=foundCode) doAssert false, "code does not have the right hash" proc assertThatWeHaveStoredAccount(trie: AccountsTrie, address: EthAddress, fetchedAcc: Account, isForTheNewTrie: bool = false) = let foundAcc = ifNodesExistGetAccount(trie, address).get if fetchedAcc != foundAcc: error "account didn't come out the same", address=address, fetchedAcc=fetchedAcc, foundAcc=foundAcc, isForTheNewTrie=isForTheNewTrie doAssert false, "account didn't come out the same" doAssert(trie.hasAllNodesForAccount(address), "Can I check the account this way, too?") proc verifyFetchedSlot(accountStorageRoot: Hash256, slot: UInt256, fetchedVal: UInt256, storageMptNodes: seq[seq[byte]]): Result[void, string] = if storageMptNodes.len == 0: # I think an empty storage proof is okay; I see lots of these # where the account is empty and the value is zero. return ok() else: let storageKey = toSeq(keccakHash(toBytesBE(slot)).data) let storageValueEncoded = rlp.encode(fetchedVal) let storageProofResult = verifyMptProof(storageMptNodes, accountStorageRoot, storageKey, storageValueEncoded) case storageProofResult.kind of ValidProof: return ok() of MissingKey: # This is for a slot that doesn't have anything stored at it, but that's fine. return ok() of InvalidProof: return err(storageProofResult.errorMsg) proc assertThatWeHaveStoredSlot(trie: AccountsTrie, address: EthAddress, acc: Account, slot: UInt256, fetchedVal: UInt256, isForTheNewTrie: bool = false) = if acc.storageRoot == EMPTY_ROOT_HASH and fetchedVal.isZero: # I believe this is okay. discard else: let foundVal = ifNodesExistGetStorage(trie, address, slot).get if (fetchedVal != foundVal): error("slot didn't come out the same", address=address, slot=slot, fetchedVal=fetchedVal, foundVal=foundVal, isForTheNewTrie=isForTheNewTrie) doAssert false, "slot didn't come out the same" proc verifyFetchedBlockHeader(fetchedHeader: BlockHeader, desiredBlockNumber: BlockNumber): Result[void, BlockNumber] = # *Can* we do anything to verify this header, given that all we know # is the desiredBlockNumber and we want to run statelessly so we don't # know what block hash we want? ok[void]() proc storeBlockHeader(chainDB: ChainDBRef, header: BlockHeader) = chainDB.persistHeaderToDbWithoutSetHeadOrScore(header) proc assertThatWeHaveStoredBlockHeader(chainDB: ChainDBRef, blockNumber: BlockNumber, header: BlockHeader) = let h = chainDB.getBlockHash(blockNumber) doAssert(h == header.blockHash, "stored the block header for block " & $(blockNumber)) template raiseExceptionIfError[E](whatAreWeVerifying: untyped, r: Result[void, E]) = if r.isErr: error("async code failed to verify", whatAreWeVerifying=whatAreWeVerifying, err=r.error) raise newException(CatchableError, "async code failed to verify: " & $(whatAreWeVerifying) & ", error is: " & $(r.error)) const shouldDoUnnecessarySanityChecks = true # This proc fetches both the account and also optionally some of its slots, because that's what eth_getProof can do. proc ifNecessaryGetAccountAndSlots*(client: RpcClient, db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, slots: seq[UInt256], justCheckingAccount: bool, justCheckingSlots: bool, newStateRootForSanityChecking: Hash256): Future[void] {.async.} = let trie = initAccountsTrie(db, stateRoot, false) # important for sanity checks let trie2 = initAccountsTrie(db, newStateRootForSanityChecking, false) # important for sanity checks let doesAccountActuallyNeedToBeFetched = not trie.hasAllNodesForAccount(address) let slotsToActuallyFetch = slots.filter(proc(slot: UInt256): bool = not (trie.hasAllNodesForStorageSlot(address, slot))) if (not doesAccountActuallyNeedToBeFetched) and (slotsToActuallyFetch.len == 0): # Already have them, no need to fetch either the account or the slots discard else: let (acc, accProof, storageProofs) = await fetchAccountAndSlots(client, address, slotsToActuallyFetch, blockNumber) # We need to verify the proof even if we already had this account, # to make sure the data is valid. let accountVerificationRes = verifyFetchedAccount(stateRoot, address, acc, accProof) let whatAreWeVerifying = ("account proof", address, acc) raiseExceptionIfError(whatAreWeVerifying, accountVerificationRes) if not doesAccountActuallyNeedToBeFetched: # We already had the account, no need to populate the DB with it again. discard else: if not justCheckingAccount: populateDbWithBranch(db, accProof) if shouldDoUnnecessarySanityChecks: assertThatWeHaveStoredAccount(trie, address, acc, false) if doesAccountActuallyNeedToBeFetched: # this second check makes no sense if it's not the first time assertThatWeHaveStoredAccount(trie2, address, acc, true) doAssert(slotsToActuallyFetch.len == storageProofs.len, "We should get back the same number of storage proofs as slots that we asked for. I think.") for storageProof in storageProofs: let slot = UInt256.fromHex(string(storageProof.key)) let fetchedVal = UInt256.fromHex(string(storageProof.value)) let storageMptNodes = storageProof.proof.mapIt(hexToSeqByte(string(it))) let storageVerificationRes = verifyFetchedSlot(acc.storageRoot, slot, fetchedVal, storageMptNodes) let whatAreWeVerifying = ("storage proof", address, acc, slot, fetchedVal) raiseExceptionIfError(whatAreWeVerifying, storageVerificationRes) if not justCheckingSlots: populateDbWithBranch(db, storageMptNodes) # I believe this is done so that we can iterate over the slots. See # persistStorage in accounts_cache.nim. let slotAsKey = createTrieKeyFromSlot(slot) let slotHash = keccakHash(slotAsKey) let slotEncoded = rlp.encode(slot) db.put(slotHashToSlotKey(slotHash.data).toOpenArray, slotEncoded) if shouldDoUnnecessarySanityChecks: assertThatWeHaveStoredSlot(trie, address, acc, slot, fetchedVal, false) assertThatWeHaveStoredSlot(trie2, address, acc, slot, fetchedVal, true) proc ifNecessaryGetCode*(client: RpcClient, db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, justChecking: bool, newStateRootForSanityChecking: Hash256): Future[void] {.async.} = await ifNecessaryGetAccountAndSlots(client, db, blockNumber, stateRoot, address, @[], false, false, newStateRootForSanityChecking) # to make sure we've got the codeHash let trie = initAccountsTrie(db, stateRoot, false) # important for sanity checks let acc = ifNodesExistGetAccount(trie, address).get let desiredCodeHash = acc.codeHash let p = (blockNumber, address) if not(trie.hasAllNodesForCode(address)): let fetchedCode = await fetchAndVerifyCode(client, p, desiredCodeHash) if not justChecking: storeCode(trie, p, desiredCodeHash, fetchedCode) if shouldDoUnnecessarySanityChecks: assertThatWeHaveStoredCode(trie, p, desiredCodeHash) proc ifNecessaryGetBlockHeaderByNumber*(client: RpcClient, chainDB: ChainDBRef, blockNumber: BlockNumber, justChecking: bool): Future[void] {.async.} = let maybeHeaderAndHash = chainDB.getBlockHeaderWithHash(blockNumber) if maybeHeaderAndHash.isNone: let fetchedHeader = await fetchBlockHeaderWithNumber(client, blockNumber) let headerVerificationRes = verifyFetchedBlockHeader(fetchedHeader, blockNumber) let whatAreWeVerifying = ("block header by number", blockNumber, fetchedHeader) raiseExceptionIfError(whatAreWeVerifying, headerVerificationRes) if not justChecking: storeBlockHeader(chainDB, fetchedHeader) if shouldDoUnnecessarySanityChecks: assertThatWeHaveStoredBlockHeader(chainDB, blockNumber, fetchedHeader) # Used in asynchronous on-demand-data-fetching mode. proc realAsyncDataSource*(peerPool: PeerPool, client: RpcClient, justChecking: bool): AsyncDataSource = AsyncDataSource( ifNecessaryGetAccount: (proc(db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, newStateRootForSanityChecking: Hash256): Future[void] {.async.} = await ifNecessaryGetAccountAndSlots(client, db, blockNumber, stateRoot, address, @[], false, false, newStateRootForSanityChecking) ), ifNecessaryGetSlots: (proc(db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, slots: seq[UInt256], newStateRootForSanityChecking: Hash256): Future[void] {.async.} = await ifNecessaryGetAccountAndSlots(client, db, blockNumber, stateRoot, address, slots, false, false, newStateRootForSanityChecking) ), ifNecessaryGetCode: (proc(db: TrieDatabaseRef, blockNumber: BlockNumber, stateRoot: Hash256, address: EthAddress, newStateRootForSanityChecking: Hash256): Future[void] {.async.} = await ifNecessaryGetCode(client, db, blockNumber, stateRoot, address, justChecking, newStateRootForSanityChecking) ), ifNecessaryGetBlockHeaderByNumber: (proc(chainDB: ChainDBRef, blockNumber: BlockNumber): Future[void] {.async.} = await ifNecessaryGetBlockHeaderByNumber(client, chainDB, blockNumber, justChecking) ), # FIXME-Adam: This will be needed later, but for now let's just get the basic methods in place. #fetchNodes: (proc(stateRoot: Hash256, paths: seq[seq[seq[byte]]], nodeHashes: seq[Hash256]): Future[seq[seq[byte]]] {.async.} = # return await fetchNodes(peerPool, stateRoot, paths, nodeHashes) #), fetchBlockHeaderWithHash: (proc(h: Hash256): Future[BlockHeader] {.async.} = return await fetchBlockHeaderWithHash(client, h) ), fetchBlockHeaderWithNumber: (proc(n: BlockNumber): Future[BlockHeader] {.async.} = return await fetchBlockHeaderWithNumber(client, n) ), fetchBlockHeaderAndBodyWithHash: (proc(h: Hash256): Future[(BlockHeader, BlockBody)] {.async.} = return await fetchBlockHeaderAndBodyWithHash(client, h) ), fetchBlockHeaderAndBodyWithNumber: (proc(n: BlockNumber): Future[(BlockHeader, BlockBody)] {.async.} = return await fetchBlockHeaderAndBodyWithNumber(client, n) ) )