diff --git a/GeneralStateTests.md b/GeneralStateTests.md index 07cb5696f..42c2f2af3 100644 --- a/GeneralStateTests.md +++ b/GeneralStateTests.md @@ -423,7 +423,7 @@ OK: 0/30 Fail: 22/30 Skip: 8/30 - callcodeOutput3partialFail.json Fail + callcodeWithHighValueAndGasOOG.json OK - deleagateCallAfterValueTransfer.json Fail -- delegatecallAndOOGatTxLevel.json Fail ++ delegatecallAndOOGatTxLevel.json OK - delegatecallBasic.json Fail - delegatecallEmptycontract.json Fail - delegatecallInInitcodeToEmptyContract.json Fail @@ -435,7 +435,7 @@ OK: 0/30 Fail: 22/30 Skip: 8/30 - delegatecodeDynamicCode.json Fail - delegatecodeDynamicCode2SelfCall.json Fail ``` -OK: 2/34 Fail: 31/34 Skip: 1/34 +OK: 3/34 Fail: 30/34 Skip: 1/34 ## stEIP150Specific ```diff CallAndCallcodeConsumeMoreGasThenTransactionHas.json Skip @@ -457,21 +457,21 @@ OK: 0/13 Fail: 1/13 Skip: 12/13 ```diff + RawBalanceGas.json OK - RawCallCodeGas.json Fail -- RawCallCodeGasAsk.json Fail ++ RawCallCodeGasAsk.json OK - RawCallCodeGasMemory.json Fail -- RawCallCodeGasMemoryAsk.json Fail ++ RawCallCodeGasMemoryAsk.json OK - RawCallCodeGasValueTransfer.json Fail + RawCallCodeGasValueTransferAsk.json OK - RawCallCodeGasValueTransferMemory.json Fail + RawCallCodeGasValueTransferMemoryAsk.json OK - RawCallGas.json Fail -- RawCallGasAsk.json Fail ++ RawCallGasAsk.json OK - RawCallGasValueTransfer.json Fail + RawCallGasValueTransferAsk.json OK - RawCallGasValueTransferMemory.json Fail + RawCallGasValueTransferMemoryAsk.json OK - RawCallMemoryGas.json Fail -- RawCallMemoryGasAsk.json Fail ++ RawCallMemoryGasAsk.json OK - RawCreateFailGasValueTransfer.json Fail - RawCreateFailGasValueTransfer2.json Fail - RawCreateGas.json Fail @@ -479,14 +479,14 @@ OK: 0/13 Fail: 1/13 Skip: 12/13 - RawCreateGasValueTransfer.json Fail - RawCreateGasValueTransferMemory.json Fail - RawDelegateCallGas.json Fail -- RawDelegateCallGasAsk.json Fail ++ RawDelegateCallGasAsk.json OK - RawDelegateCallGasMemory.json Fail -- RawDelegateCallGasMemoryAsk.json Fail ++ RawDelegateCallGasMemoryAsk.json OK + RawExtCodeCopyGas.json OK + RawExtCodeCopyMemoryGas.json OK + RawExtCodeSizeGas.json OK ``` -OK: 8/30 Fail: 22/30 Skip: 0/30 +OK: 14/30 Fail: 16/30 Skip: 0/30 ## stEIP158Specific ```diff CALL_OneVCallSuicide.json Skip @@ -585,16 +585,16 @@ OK: 1/18 Fail: 17/18 Skip: 0/18 OK: 0/46 Fail: 46/46 Skip: 0/46 ## stMemExpandingEIP150Calls ```diff -- CallAndCallcodeConsumeMoreGasThenTransactionHasWithMemExpandingCalls.jsonFail ++ CallAndCallcodeConsumeMoreGasThenTransactionHasWithMemExpandingCalls.jsonOK - CallAskMoreGasOnDepth2ThenTransactionHasWithMemExpandingCalls.jsonFail -- CallGoesOOGOnSecondLevel2WithMemExpandingCalls.json Fail -- CallGoesOOGOnSecondLevelWithMemExpandingCalls.json Fail ++ CallGoesOOGOnSecondLevel2WithMemExpandingCalls.json OK ++ CallGoesOOGOnSecondLevelWithMemExpandingCalls.json OK - CreateAndGasInsideCreateWithMemExpandingCalls.json Fail -- DelegateCallOnEIPWithMemExpandingCalls.json Fail -- ExecuteCallThatAskMoreGasThenTransactionHasWithMemExpandingCalls.jsonFail ++ DelegateCallOnEIPWithMemExpandingCalls.json OK ++ ExecuteCallThatAskMoreGasThenTransactionHasWithMemExpandingCalls.jsonOK - NewGasPriceForCodesWithMemExpandingCalls.json Fail ``` -OK: 0/8 Fail: 8/8 Skip: 0/8 +OK: 5/8 Fail: 3/8 Skip: 0/8 ## stMemoryStressTest ```diff CALLCODE_Bounds.json Skip @@ -623,7 +623,7 @@ OK: 0/8 Fail: 8/8 Skip: 0/8 MSTORE_Bounds2.json Skip MSTORE_Bounds2a.json Skip POP_Bounds.json Skip -- RETURN_Bounds.json Fail ++ RETURN_Bounds.json OK SLOAD_Bounds.json Skip SSTORE_Bounds.json Skip mload32bitBound.json Skip @@ -636,7 +636,7 @@ OK: 0/8 Fail: 8/8 Skip: 0/8 static_CALL_Bounds2a.json Skip static_CALL_Bounds3.json Skip ``` -OK: 4/38 Fail: 1/38 Skip: 33/38 +OK: 5/38 Fail: 0/38 Skip: 33/38 ## stMemoryTest ```diff - callDataCopyOffset.json Fail @@ -1548,9 +1548,9 @@ OK: 11/19 Fail: 8/19 Skip: 0/19 ## stReturnDataTest ```diff + call_ecrec_success_empty_then_returndatasize.json OK -- call_outsize_then_create_successful_then_returndatasize.json Fail ++ call_outsize_then_create_successful_then_returndatasize.json OK + call_then_call_value_fail_then_returndatasize.json OK -- call_then_create_successful_then_returndatasize.json Fail ++ call_then_create_successful_then_returndatasize.json OK - create_callprecompile_returndatasize.json Fail modexp_modsize0_returndatasize.json Skip - returndatacopy_0_0_following_successful_create.json Fail @@ -1585,17 +1585,17 @@ OK: 11/19 Fail: 8/19 Skip: 0/19 + returndatasize_initial.json OK + returndatasize_initial_zero_read.json OK ``` -OK: 26/37 Fail: 8/37 Skip: 3/37 +OK: 28/37 Fail: 6/37 Skip: 3/37 ## stRevertTest ```diff -- LoopCallsDepthThenRevert.json Fail ++ LoopCallsDepthThenRevert.json OK LoopCallsDepthThenRevert2.json Skip LoopCallsDepthThenRevert3.json Skip - LoopCallsThenRevert.json Fail -- LoopDelegateCallsDepthThenRevert.json Fail ++ LoopDelegateCallsDepthThenRevert.json OK - NashatyrevSuicideRevert.json Fail + PythonRevertTestTue201814-1430.json OK -- RevertDepth2.json Fail ++ RevertDepth2.json OK - RevertDepthCreateAddressCollision.json Fail - RevertDepthCreateOOG.json Fail + RevertInCallCode.json OK @@ -1628,11 +1628,11 @@ OK: 26/37 Fail: 8/37 Skip: 3/37 - RevertRemoteSubCallStorageOOG2.json Fail + RevertSubCallStorageOOG.json OK + RevertSubCallStorageOOG2.json OK -- TouchToEmptyAccountRevert.json Fail ++ TouchToEmptyAccountRevert.json OK - TouchToEmptyAccountRevert2.json Fail - TouchToEmptyAccountRevert3.json Fail ``` -OK: 10/43 Fail: 28/43 Skip: 5/43 +OK: 14/43 Fail: 24/43 Skip: 5/43 ## stShift ```diff sar00.json Skip @@ -2026,7 +2026,7 @@ OK: 0/284 Fail: 0/284 Skip: 284/284 - CallRecursiveBomb0_OOG_atMaxCallDepth.json Fail - CallRecursiveBomb1.json Fail - CallRecursiveBomb2.json Fail -- CallRecursiveBomb3.json Fail ++ CallRecursiveBomb3.json OK - CallRecursiveBombLog.json Fail - CallRecursiveBombLog2.json Fail - CallToNameRegistrator0.json Fail @@ -2083,7 +2083,7 @@ OK: 0/284 Fail: 0/284 Skip: 284/284 + suicideSendEtherToMe.json OK - testRandomTest.json Fail ``` -OK: 25/67 Fail: 40/67 Skip: 2/67 +OK: 26/67 Fail: 39/67 Skip: 2/67 ## stTransactionTest ```diff + ContractStoreClearsOOG.json OK diff --git a/nimbus/db/db_chain.nim b/nimbus/db/db_chain.nim index 405e5c6f8..089710658 100644 --- a/nimbus/db/db_chain.nim +++ b/nimbus/db/db_chain.nim @@ -12,7 +12,11 @@ import type BaseChainDB* = ref object - db*: TrieDatabaseRef + db*: TrieDatabaseRef + # XXX: intentionally simple stand-in for one use of full JournalDB + # Doesn't handle CREATE+revert, etc. But also creates minimal tech + # debt while setting a CI baseline from which to improve/replace. + accountCodes*: TableRef[Hash256, ByteRange] # TODO db*: JournalDB KeyType = enum @@ -26,6 +30,7 @@ type proc newBaseChainDB*(db: TrieDatabaseRef): BaseChainDB = new(result) result.db = db + result.accountCodes = newTable[Hash256, ByteRange]() proc `$`*(db: BaseChainDB): string = result = "BaseChainDB" @@ -251,7 +256,7 @@ proc persistBlockToDb*(self: BaseChainDB; blk: Block) = proc getStateDb*(self: BaseChainDB; stateRoot: Hash256; readOnly: bool = false): AccountStateDB = # TODO: readOnly is not used. - result = newAccountStateDB(self.db, stateRoot) + result = newAccountStateDB(self.db, stateRoot, readOnly, self.accountCodes) # Deprecated: diff --git a/nimbus/db/state_db.nim b/nimbus/db/state_db.nim index 88d9e55d0..d373588a0 100644 --- a/nimbus/db/state_db.nim +++ b/nimbus/db/state_db.nim @@ -15,7 +15,8 @@ logScope: type AccountStateDB* = ref object - trie: SecureHexaryTrie + trie: SecureHexaryTrie + accountCodes: TableRef[Hash256, ByteRange] proc rootHash*(accountDb: AccountStateDB): KeccakHash = accountDb.trie.rootHash @@ -24,9 +25,11 @@ proc rootHash*(accountDb: AccountStateDB): KeccakHash = # TODO: self.Trie.rootHash = value proc newAccountStateDB*(backingStore: TrieDatabaseRef, - root: KeccakHash, readOnly: bool = false): AccountStateDB = + root: KeccakHash, readOnly: bool = false, + accountCodes = newTable[Hash256, ByteRange]()): AccountStateDB = result.new() result.trie = initSecureHexaryTrie(backingStore, root) + result.accountCodes = accountCodes template createRangeFromAddress(address: EthAddress): ByteRange = ## XXX: The name of this proc is intentionally long, because it @@ -129,24 +132,18 @@ proc getNonce*(db: AccountStateDB, address: EthAddress): AccountNonce = let account = db.getAccount(address) account.nonce -proc toByteRange_Unnecessary*(h: KeccakHash): ByteRange = - ## XXX: Another proc used to mark unnecessary conversions it the code - var s = @(h.data) - return s.toRange - proc setCode*(db: var AccountStateDB, address: EthAddress, code: ByteRange) = var account = db.getAccount(address) let newCodeHash = keccak256.digest code.toOpenArray if newCodeHash != account.codeHash: account.codeHash = newCodeHash + db.accountCodes[newCodeHash] = code # XXX: this uses the journaldb in py-evm - # Breaks state hash root calculations # db.trie.put(account.codeHash.toByteRange_Unnecessary, code) db.setAccount(address, account) proc getCode*(db: AccountStateDB, address: EthAddress): ByteRange = - let codeHash = db.getCodeHash(address) - result = db.trie.get(codeHash.toByteRange_Unnecessary) + db.accountCodes.getOrDefault(db.getCodeHash(address)) proc hasCodeOrNonce*(account: AccountStateDB, address: EthAddress): bool {.inline.} = account.getNonce(address) != 0 or account.getCodeHash(address) != EMPTY_SHA3 diff --git a/nimbus/vm/interpreter/opcodes_impl.nim b/nimbus/vm/interpreter/opcodes_impl.nim index 885a54b50..c95035424 100644 --- a/nimbus/vm/interpreter/opcodes_impl.nim +++ b/nimbus/vm/interpreter/opcodes_impl.nim @@ -225,6 +225,7 @@ proc writePaddedResult(mem: var Memory, # Don't duplicate zero-padding of mem.extend let paddingOffset = memPos + sourceBytes.len + # TODO: avoid unnecessary memory allocation mem.write(paddingOffset, repeat(paddingValue, max(prevLen - paddingOffset, 0))) op address, inline = true: diff --git a/tests/test_generalstate_failing.nim b/tests/test_generalstate_failing.nim index 06082a297..4cb058906 100644 --- a/tests/test_generalstate_failing.nim +++ b/tests/test_generalstate_failing.nim @@ -163,7 +163,6 @@ func allowedFailingGeneralStateTest*(folder, name: string): bool = "callcodeOutput3partial.json", "callcodeOutput3partialFail.json", "deleagateCallAfterValueTransfer.json", - "delegatecallAndOOGatTxLevel.json", "delegatecallBasic.json", "delegatecallEmptycontract.json", "delegatecallInInitcodeToEmptyContract.json", @@ -176,17 +175,13 @@ func allowedFailingGeneralStateTest*(folder, name: string): bool = "delegatecodeDynamicCode2SelfCall.json", "NewGasPriceForCodes.json", "RawCallCodeGas.json", - "RawCallCodeGasAsk.json", "RawCallCodeGasMemory.json", - "RawCallCodeGasMemoryAsk.json", "RawCallCodeGasValueTransfer.json", "RawCallCodeGasValueTransferMemory.json", "RawCallGas.json", - "RawCallGasAsk.json", "RawCallGasValueTransfer.json", "RawCallGasValueTransferMemory.json", "RawCallMemoryGas.json", - "RawCallMemoryGasAsk.json", "RawCreateFailGasValueTransfer.json", "RawCreateFailGasValueTransfer2.json", "RawCreateGas.json", @@ -194,9 +189,7 @@ func allowedFailingGeneralStateTest*(folder, name: string): bool = "RawCreateGasValueTransfer.json", "RawCreateGasValueTransferMemory.json", "RawDelegateCallGas.json", - "RawDelegateCallGasAsk.json", "RawDelegateCallGasMemory.json", - "RawDelegateCallGasMemoryAsk.json", "contractCreationOOGdontLeaveEmptyContract.json", "contractCreationOOGdontLeaveEmptyContractViaTransaction.json", "createContractViaContract.json", @@ -265,15 +258,9 @@ func allowedFailingGeneralStateTest*(folder, name: string): bool = "log4_nonEmptyMem_logMemSize1.json", "log4_nonEmptyMem_logMemSize1_logMemStart31.json", "logInOOG_Call.json", - "CallAndCallcodeConsumeMoreGasThenTransactionHasWithMemExpandingCalls.json", "CallAskMoreGasOnDepth2ThenTransactionHasWithMemExpandingCalls.json", - "CallGoesOOGOnSecondLevel2WithMemExpandingCalls.json", - "CallGoesOOGOnSecondLevelWithMemExpandingCalls.json", "CreateAndGasInsideCreateWithMemExpandingCalls.json", - "DelegateCallOnEIPWithMemExpandingCalls.json", - "ExecuteCallThatAskMoreGasThenTransactionHasWithMemExpandingCalls.json", "NewGasPriceForCodesWithMemExpandingCalls.json", - "RETURN_Bounds.json", "callDataCopyOffset.json", "codeCopyOffset.json", "NonZeroValue_CALL.json", @@ -420,19 +407,14 @@ func allowedFailingGeneralStateTest*(folder, name: string): bool = "refund_CallToSuicideTwice.json", "refund_multimpleSuicide.json", "refund_singleSuicide.json", - "call_outsize_then_create_successful_then_returndatasize.json", - "call_then_create_successful_then_returndatasize.json", "create_callprecompile_returndatasize.json", "returndatacopy_0_0_following_successful_create.json", "returndatacopy_following_create.json", "returndatacopy_following_revert_in_create.json", "returndatacopy_following_successful_create.json", "returndatasize_following_successful_create.json", - "LoopCallsDepthThenRevert.json", "LoopCallsThenRevert.json", - "LoopDelegateCallsDepthThenRevert.json", "NashatyrevSuicideRevert.json", - "RevertDepth2.json", "RevertDepthCreateAddressCollision.json", "RevertDepthCreateOOG.json", "RevertInCreateInInit.json", @@ -453,7 +435,6 @@ func allowedFailingGeneralStateTest*(folder, name: string): bool = "RevertPrefoundOOG.json", "RevertRemoteSubCallStorageOOG.json", "RevertRemoteSubCallStorageOOG2.json", - "TouchToEmptyAccountRevert.json", "TouchToEmptyAccountRevert2.json", "TouchToEmptyAccountRevert3.json", "CallLowLevelCreatesSolidity.json", @@ -482,7 +463,6 @@ func allowedFailingGeneralStateTest*(folder, name: string): bool = "CallRecursiveBomb0_OOG_atMaxCallDepth.json", "CallRecursiveBomb1.json", "CallRecursiveBomb2.json", - "CallRecursiveBomb3.json", "CallRecursiveBombLog.json", "CallRecursiveBombLog2.json", "CallToNameRegistrator0.json", diff --git a/tests/test_generalstate_json.nim b/tests/test_generalstate_json.nim index ff376826b..07cd92924 100644 --- a/tests/test_generalstate_json.nim +++ b/tests/test_generalstate_json.nim @@ -36,7 +36,7 @@ proc validateTransaction(vmState: BaseVMState, transaction: Transaction, sender: transaction.accountNonce == readOnlyDB.getNonce(sender) and readOnlyDB.getBalance(sender) >= gas_cost -proc setupComputation(header: BlockHeader, vmState: var BaseVMState, transaction: Transaction, sender: EthAddress, code: seq[byte]) : BaseComputation = +proc setupComputation(header: BlockHeader, vmState: var BaseVMState, transaction: Transaction, sender: EthAddress) : BaseComputation = let message = newMessage( gas = transaction.gasLimit - transaction.getFixtureIntrinsicGas, gasPrice = transaction.gasPrice, @@ -44,7 +44,7 @@ proc setupComputation(header: BlockHeader, vmState: var BaseVMState, transaction sender = sender, value = transaction.value, data = transaction.payload, - code = code, + code = vmState.readOnlyStateDB.getCode(transaction.to).toSeq, options = newMessageOptions(origin = sender, createAddress = transaction.to)) @@ -87,14 +87,11 @@ proc testFixtureIndexes(header: BlockHeader, pre: JsonNode, transaction: Transac let gas_cost = transaction.gasLimit.u256 * transaction.gasPrice.u256 vmState.mutateStateDB: - db.setBalance(sender, db.getBalance(sender) - gas_cost) db.setNonce(sender, db.getNonce(sender) + 1) db.addBalance(transaction.to, transaction.value) - db.setBalance(sender, db.getBalance(sender) - transaction.value) - - var computation = setupComputation(header, vmState, transaction, sender, - pre.getFixtureCode(transaction.to)) + db.subBalance(sender, transaction.value + gas_cost) + var computation = setupComputation(header, vmState, transaction, sender) if execComputation(computation, vmState): let gasRemaining = computation.gasMeter.gasRemaining.u256 @@ -105,14 +102,14 @@ proc testFixtureIndexes(header: BlockHeader, pre: JsonNode, transaction: Transac vmState.mutateStateDB: if header.coinbase notin computation.getAccountsForDeletion: - db.setBalance(header.coinbase, db.getBalance(header.coinbase) - gasRefundAmount) + db.subBalance(header.coinbase, gasRefundAmount) db.addBalance(header.coinbase, gas_cost) db.addBalance(sender, gasRefundAmount) # TODO: only here does one commit, with some nuance/caveat else: vmState.mutateStateDB: # XXX: the coinbase has to be committed; the rest are basically reverts - db.setBalance(transaction.to, db.getBalance(transaction.to) - transaction.value) + db.subBalance(transaction.to, transaction.value) db.addBalance(sender, transaction.value) db.setStorageRoot(transaction.to, storageRoot) db.addBalance(header.coinbase, gas_cost) @@ -144,5 +141,4 @@ proc testFixture(fixtures: JsonNode, testStatusIMPL: var TestStatus) = valueIndex = indexes["value"].getInt let transaction = ftrans.getFixtureTransaction(dataIndex, gasIndex, valueIndex) let sender = ftrans.getFixtureTransactionSender - echo "testing fixture indexes dataIndex = ", dataIndex, ", gasIndex = ", gasIndex, ", and valueIndex = ", valueIndex testFixtureIndexes(header, fixture["pre"], transaction, sender, expectedHash) diff --git a/tests/test_helpers.nim b/tests/test_helpers.nim index a81719555..c84b08e9b 100644 --- a/tests/test_helpers.nim +++ b/tests/test_helpers.nim @@ -50,6 +50,8 @@ func failIn32Bits(folder, name: string): bool = "sha3_dejavu.json", "HighGasLimit.json", "OverflowGasRequire2.json", + + # TODO: obvious theme; check returndatasize/returndatacopy "call_ecrec_success_empty_then_returndatasize.json", "call_then_call_value_fail_then_returndatasize.json", "returndatacopy_after_failing_callcode.json", @@ -75,7 +77,9 @@ func failIn32Bits(folder, name: string): bool = "returndatasize_after_successful_staticcall.json", "returndatasize_bug.json", "returndatasize_initial.json", - "returndatasize_initial_zero_read.json"] + "returndatasize_initial_zero_read.json", + "call_then_create_successful_then_returndatasize.json", + "call_outsize_then_create_successful_then_returndatasize.json"] func validTest*(folder: string, name: string): bool = # tests we want to skip or which segfault will be skipped here @@ -206,9 +210,7 @@ proc verifyStateDB*(wantedState: JsonNode, stateDB: AccountStateDB) = actualBalance = stateDB.getBalance(account) actualNonce = stateDB.getNonce(account) - # XXX: actualCode is sourced from wrong location currently, incompatible with - # state hash root. Can/should be fixed, but blocks further progress as-is. - # doAssert wantedCode == actualCode, &"{wantedCode} != {actualCode}" + doAssert wantedCode == actualCode, &"{wantedCode} != {actualCode}" doAssert wantedBalance == actualBalance, &"{wantedBalance.toHex} != {actualBalance.toHex}" doAssert wantedNonce == actualNonce, &"{wantedNonce.toHex} != {actualNonce.toHex}" @@ -246,14 +248,6 @@ proc getFixtureTransactionSender*(j: JsonNode): EthAddress = # XXX: appropriate failure mode; probably raise something discard -func getFixtureCode*(pre: JsonNode, targetAccount: EthAddress) : seq[byte] = - # XXX: Workaround for broken setCode/getCode. Remove when feasible. - for ac, preState in pre: - if ethAddressFromHex(ac) == targetAccount: - return preState["code"].getStr.safeHexToSeqByte - - # Fail loudly if it falls off the end (by default) - proc getFixtureIntrinsicGas*(transaction: Transaction) : auto = # Py-EVM has _get_homestead_intrinsic_gas and _get_frontier_intrinsic_gas # Using former.