From 50816f2ebb3fe179f8e8e2cf9d9c08ff48f5d172 Mon Sep 17 00:00:00 2001 From: jangko Date: Wed, 3 Jun 2020 20:50:13 +0700 Subject: [PATCH 1/3] implement block witness data collection in accounts_cache --- nimbus/db/accounts_cache.nim | 70 ++++++++++++++++++++++++++---------- stateless/multi_keys.nim | 15 ++++++++ 2 files changed, 67 insertions(+), 18 deletions(-) diff --git a/nimbus/db/accounts_cache.nim b/nimbus/db/accounts_cache.nim index 6d92e719e..157f4e977 100644 --- a/nimbus/db/accounts_cache.nim +++ b/nimbus/db/accounts_cache.nim @@ -1,7 +1,8 @@ import tables, hashes, sets, eth/[common, rlp], eth/trie/[hexary, db, trie_defs], - ../constants, ../utils, storage_types + ../constants, ../utils, storage_types, + ../../stateless/[witness_types, multi_keys] type AccountFlag = enum @@ -23,11 +24,15 @@ type originalStorage: TableRef[UInt256, UInt256] overlayStorage: Table[UInt256, UInt256] + WitnessData* = object + storageKeys*: HashSet[UInt256] + codeTouched*: bool + AccountsCache* = ref object db: TrieDatabaseRef trie: SecureHexaryTrie savePoint: SavePoint - unrevertablyTouched: HashSet[EthAddress] + witnessCache: Table[EthAddress, WitnessData] ReadOnlyStateDB* = distinct AccountsCache @@ -51,7 +56,7 @@ proc init*(x: typedesc[AccountsCache], db: TrieDatabaseRef, new result result.db = db result.trie = initSecureHexaryTrie(db, root, pruneTrie) - result.unrevertablyTouched = initHashSet[EthAddress]() + result.witnessCache = initTable[EthAddress, WitnessData]() discard result.beginSavepoint proc init*(x: typedesc[AccountsCache], db: TrieDatabaseRef, pruneTrie: bool = true): AccountsCache = @@ -373,21 +378,6 @@ proc clearStorage*(ac: var AccountsCache, address: EthAddress) = # there is no point to clone the storage since we want to remove it ac.makeDirty(address, cloneStorage = false).account.storageRoot = emptyRlpHash -proc unrevertableTouch*(ac: var AccountsCache, address: EthAddress) = - ac.unrevertablyTouched.incl address - -proc removeEmptyAccounts*(ac: var AccountsCache) = - # make sure all savepoints already committed - doAssert(ac.savePoint.parentSavePoint.isNil) - for _, acc in ac.savePoint.cache: - if IsTouched in acc.flags and acc.isEmpty: - acc.kill() - - for address in ac.unrevertablyTouched: - var acc = ac.getAccount(address) - if acc.isEmpty: - acc.kill() - proc deleteAccount*(ac: var AccountsCache, address: EthAddress) = # make sure all savepoints already committed doAssert(ac.savePoint.parentSavePoint.isNil) @@ -433,6 +423,50 @@ proc getStorageRoot*(ac: AccountsCache, address: EthAddress): Hash256 = if acc.isNil: emptyAcc.storageRoot else: acc.account.storageRoot +func update(wd: var WitnessData, acc: RefAccount) = + wd.codeTouched = CodeChanged in acc.flags + + if not acc.originalStorage.isNil: + for k, v in acc.originalStorage: + if v == 0: continue + wd.storageKeys.incl k + + for k, v in acc.overlayStorage: + if v == 0 and k notin wd.storageKeys: + continue + if v == 0 and k in wd.storageKeys: + wd.storageKeys.excl k + continue + wd.storageKeys.incl k + +func witnessData(acc: RefAccount): WitnessData = + result.storageKeys = initHashSet[UInt256]() + update(result, acc) + +proc collectWitnessData*(ac: var AccountsCache) = + # make sure all savepoint already committed + doAssert(ac.savePoint.parentSavePoint.isNil) + # usually witness data is collected before we call persist() + for address, acc in ac.savePoint.cache: + ac.witnessCache.withValue(address, val) do: + update(val[], acc) + do: + ac.witnessCache[address] = witnessData(acc) + +func multiKeys(slots: HashSet[UInt256]): MultikeysRef = + if slots.len == 0: return + new result + for x in slots: + result.add x.toBytesBE + result.sort() + +proc makeMultiKeys*(ac: AccountsCache): MultikeysRef = + # this proc is called after we done executing a block + new result + for k, v in ac.witnessCache: + result.add(k, v.codeTouched, multiKeys(v.storageKeys)) + result.sort() + proc rootHash*(db: ReadOnlyStateDB): KeccakHash {.borrow.} proc getCodeHash*(db: ReadOnlyStateDB, address: EthAddress): Hash256 {.borrow.} proc getStorageRoot*(db: ReadOnlyStateDB, address: EthAddress): Hash256 {.borrow.} diff --git a/stateless/multi_keys.nim b/stateless/multi_keys.nim index 1b799e2fd..263547836 100644 --- a/stateless/multi_keys.nim +++ b/stateless/multi_keys.nim @@ -76,6 +76,21 @@ proc newMultiKeys*(keys: openArray[StorageSlot]): MultikeysRef = result.keys[i] = KeyData(storageMode: true, hash: keccak(a).data, storageSlot: a) result.keys.sort(cmpHash) +# never mix storageMode! +proc add*(m: MultikeysRef, address: EthAddress, codeTouched: bool, storageKeys = MultikeysRef(nil)) = + m.keys.add KeyData( + storageMode: false, + hash: keccak(address).data, + address: address, + codeTouched: codeTouched, + storageKeys: storageKeys) + +proc add*(m: MultikeysRef, slot: StorageSlot) = + m.keys.add KeyData(storageMode: true, hash: keccak(slot).data, storageSlot: slot) + +proc sort*(m: MultikeysRef) = + m.keys.sort(cmpHash) + func initGroup*(m: MultikeysRef): Group = type T = type result.last result = Group(first: 0.T, last: (m.keys.len - 1).T) From 7c026e1b486ada306e0a5696536c0d90d06c7603 Mon Sep 17 00:00:00 2001 From: jangko Date: Sat, 6 Jun 2020 10:05:11 +0700 Subject: [PATCH 2/3] modify VMState to enable witness data collection --- nimbus/p2p/executor.nim | 5 ++++- nimbus/vm/computation.nim | 12 ++++++------ nimbus/vm_state.nim | 27 ++++++++++++++++++++------- nimbus/vm_types.nim | 7 +++++-- 4 files changed, 35 insertions(+), 16 deletions(-) diff --git a/nimbus/p2p/executor.nim b/nimbus/p2p/executor.nim index 29705b317..ea1179f9b 100644 --- a/nimbus/p2p/executor.nim +++ b/nimbus/p2p/executor.nim @@ -42,7 +42,8 @@ proc processTransaction*(tx: Transaction, sender: EthAddress, vmState: BaseVMSta debug "state clearing", account db.deleteAccount(account) - #vmState.accountDb.updateOriginalRoot() + if vmState.generateWitness: + vmState.accountDb.collectWitnessData() vmState.accountDb.persist() type @@ -143,6 +144,8 @@ proc processBlock*(chainDB: BaseChainDB, header: BlockHeader, body: BlockBody, v # Reward beneficiary vmState.mutateStateDB: db.addBalance(header.coinbase, mainReward) + if vmState.generateWitness: + db.collectWitnessData() db.persist() let stateDb = vmState.accountDb diff --git a/nimbus/vm/computation.nim b/nimbus/vm/computation.nim index e1507ae9c..befc75c79 100644 --- a/nimbus/vm/computation.nim +++ b/nimbus/vm/computation.nim @@ -322,19 +322,19 @@ proc refundSelfDestruct*(c: Computation) = let cost = gasFees[c.fork][RefundSelfDestruct] c.gasMeter.refundGas(cost * c.suicides.len) -proc tracingEnabled*(c: Computation): bool = - c.vmState.tracingEnabled +proc tracingEnabled*(c: Computation): bool {.inline.} = + EnableTracing in c.vmState.tracer.flags -proc traceOpCodeStarted*(c: Computation, op: Op): int = +proc traceOpCodeStarted*(c: Computation, op: Op): int {.inline.} = c.vmState.tracer.traceOpCodeStarted(c, op) -proc traceOpCodeEnded*(c: Computation, op: Op, lastIndex: int) = +proc traceOpCodeEnded*(c: Computation, op: Op, lastIndex: int) {.inline.} = c.vmState.tracer.traceOpCodeEnded(c, op, lastIndex) -proc traceError*(c: Computation) = +proc traceError*(c: Computation) {.inline.} = c.vmState.tracer.traceError(c) -proc prepareTracer*(c: Computation) = +proc prepareTracer*(c: Computation) {.inline.} = c.vmState.tracer.prepare(c.msg.depth) include interpreter_dispatch diff --git a/nimbus/vm_state.nim b/nimbus/vm_state.nim index ddb765c3d..eabc63976 100644 --- a/nimbus/vm_state.nim +++ b/nimbus/vm_state.nim @@ -34,7 +34,6 @@ proc init*(self: BaseVMState, prevStateRoot: Hash256, header: BlockHeader, self.blockHeader = header self.chaindb = chainDB self.tracer.initTracer(tracerFlags) - self.tracingEnabled = TracerFlags.EnableTracing in tracerFlags self.logEntries = @[] self.accountDb = AccountsCache.init(chainDB.db, prevStateRoot, chainDB.pruneTrie) self.touchedAccounts = initHashSet[EthAddress]() @@ -103,19 +102,19 @@ template mutateStateDB*(vmState: BaseVMState, body: untyped) = var db {.inject.} = vmState.accountDb body -proc getTracingResult*(vmState: BaseVMState): JsonNode = - doAssert(vmState.tracingEnabled) +proc getTracingResult*(vmState: BaseVMState): JsonNode {.inline.} = + doAssert(EnableTracing in vmState.tracer.flags) vmState.tracer.trace proc getAndClearLogEntries*(vmState: BaseVMState): seq[Log] = shallowCopy(result, vmState.logEntries) vmState.logEntries = @[] -proc enableTracing*(vmState: BaseVMState) = - vmState.tracingEnabled = true +proc enableTracing*(vmState: BaseVMState) {.inline.} = + vmState.tracer.flags.incl EnableTracing -proc disableTracing*(vmState: BaseVMState) = - vmState.tracingEnabled = false +proc disableTracing*(vmState: BaseVMState) {.inline.} = + vmState.tracer.flags.excl EnableTracing iterator tracedAccounts*(vmState: BaseVMState): EthAddress = for acc in vmState.tracer.accounts: @@ -130,3 +129,17 @@ iterator tracedAccountsPairs*(vmState: BaseVMState): (int, EthAddress) = proc removeTracedAccounts*(vmState: BaseVMState, accounts: varargs[EthAddress]) = for acc in accounts: vmState.tracer.accounts.excl acc + +proc status*(vmState: BaseVMState): bool {.inline.} = + ExecutionOK in vmState.flags + +proc `status=`*(vmState: BaseVMState, status: bool) = + if status: vmState.flags.incl ExecutionOK + else: vmState.flags.excl ExecutionOK + +proc generateWitness*(vmState: BaseVMState): bool {.inline.} = + GenerateWitness in vmState.flags + +proc `generateWitness=`*(vmState: BaseVMState, status: bool) = + if status: vmState.flags.incl GenerateWitness + else: vmState.flags.excl GenerateWitness diff --git a/nimbus/vm_types.nim b/nimbus/vm_types.nim index 2233421c4..3ce3c602f 100644 --- a/nimbus/vm_types.nim +++ b/nimbus/vm_types.nim @@ -16,13 +16,17 @@ when defined(evmc_enabled): import ./vm/evmc_api type + VMFlag* = enum + ExecutionOK + GenerateWitness + BaseVMState* = ref object of RootObj prevHeaders* : seq[BlockHeader] chaindb* : BaseChainDB accessLogs* : AccessLogs blockHeader* : BlockHeader name* : string - tracingEnabled*: bool + flags* : set[VMFlag] tracer* : TransactionTracer logEntries* : seq[Log] receipts* : seq[Receipt] @@ -30,7 +34,6 @@ type cumulativeGasUsed*: GasInt touchedAccounts*: HashSet[EthAddress] suicides* : HashSet[EthAddress] - status* : bool txOrigin* : EthAddress txGasPrice* : GasInt gasCosts* : GasCosts From 3947e9a8530b419e76a26e77f3f338dda6641dba Mon Sep 17 00:00:00 2001 From: jangko Date: Sat, 6 Jun 2020 10:26:36 +0700 Subject: [PATCH 3/3] piggyback generate block witness test on test_blockchain_json --- tests/test_blockchain_json.nim | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/tests/test_blockchain_json.nim b/tests/test_blockchain_json.nim index aaba592b0..599cfa6b6 100644 --- a/tests/test_blockchain_json.nim +++ b/tests/test_blockchain_json.nim @@ -17,7 +17,8 @@ import ../nimbus/db/[db_chain, accounts_cache], ../nimbus/utils/header, ../nimbus/p2p/[executor, dao], - ../nimbus/config + ../nimbus/config, + ../stateless/[multi_keys, tree_from_witness, witness_from_tree, witness_types] type SealEngine = enum @@ -280,6 +281,28 @@ proc parseTester(fixture: JsonNode, testStatusIMPL: var TestStatus): Tester = #if result.network in ["HomesteadToDaoAt5"]: #result.good = false +proc blockWitness(vmState: BaseVMState, fork: Fork, chainDB: BaseChainDB) = + let rootHash = vmState.accountDb.rootHash + let mkeys = vmState.accountDb.makeMultiKeys() + let flags = if fork >= FKSpurious: {wfEIP170} else: {} + + # build witness from tree + var wb = initWitnessBuilder(chainDB.db, rootHash, flags) + let witness = wb.buildWitness(mkeys) + + # build tree from witness + var db = newMemoryDB() + when defined(useInputStream): + var input = memoryInput(witness) + var tb = initTreeBuilder(input, db, flags) + else: + var tb = initTreeBuilder(witness, db, flags) + let root = tb.buildTree() + + # compare the result + if root != rootHash: + raise newException(ValidationError, "Invalid trie generated from block witness") + proc assignBlockRewards(minedBlock: PlainBlock, vmState: BaseVMState, fork: Fork, chainDB: BaseChainDB) = let blockReward = blockRewards[fork] var mainReward = blockReward @@ -299,6 +322,8 @@ proc assignBlockRewards(minedBlock: PlainBlock, vmState: BaseVMState, fork: Fork # Reward beneficiary vmState.mutateStateDB: db.addBalance(minedBlock.header.coinbase, mainReward) + if vmState.generateWitness: + db.collectWitnessData() db.persist() let stateDb = vmState.accountDb @@ -317,6 +342,9 @@ proc assignBlockRewards(minedBlock: PlainBlock, vmState: BaseVMState, fork: Fork if minedBlock.header.txRoot != txRoot: raise newException(ValidationError, "wrong txRoot") + if vmState.generateWitness: + blockWitness(vmState, fork, chainDB) + proc processBlock(chainDB: BaseChainDB, vmState: BaseVMState, minedBlock: PlainBlock, fork: Fork) = var dbTx = chainDB.db.beginTransaction() defer: dbTx.dispose() @@ -697,6 +725,8 @@ proc testFixture(node: JsonNode, testStatusIMPL: var TestStatus, debugMode = fal var vmState = newBaseVMState(emptyRlpHash, tester.genesisBlockHeader, chainDB) + vmState.generateWitness = true + vmState.mutateStateDB: setupStateDB(fixture["pre"], db) db.persist()