From 08caeedd842526373d30a929e63101a5fe7fda55 Mon Sep 17 00:00:00 2001
From: Jeffrey Wilcke <geffobscura@gmail.com>
Date: Tue, 30 Jun 2015 23:49:34 +0200
Subject: [PATCH 1/4] core, core/state: only write necessary state. Skip
 intermediate

---
 core/block_processor.go    |  2 +-
 core/state/state_object.go |  8 --------
 core/state/statedb.go      | 22 ++++++++++++++++++++--
 3 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/core/block_processor.go b/core/block_processor.go
index e8014ec22..5f745e491 100644
--- a/core/block_processor.go
+++ b/core/block_processor.go
@@ -243,7 +243,7 @@ func (sm *BlockProcessor) processWithParent(block, parent *types.Block) (logs st
 
 	// Commit state objects/accounts to a temporary trie (does not save)
 	// used to calculate the state root.
-	state.Update()
+	state.CleanUpdate()
 	if header.Root != state.Root() {
 		err = fmt.Errorf("invalid merkle root. received=%x got=%x", header.Root, state.Root())
 		return
diff --git a/core/state/state_object.go b/core/state/state_object.go
index a31c182d2..e40aeda82 100644
--- a/core/state/state_object.go
+++ b/core/state/state_object.go
@@ -57,8 +57,6 @@ type StateObject struct {
 	initCode Code
 	// Cached storage (flushed when updated)
 	storage Storage
-	// Temporary prepaid gas, reward after transition
-	prepaid *big.Int
 
 	// Total gas pool is the total amount of gas currently
 	// left if this object is the coinbase. Gas is directly
@@ -77,14 +75,10 @@ func (self *StateObject) Reset() {
 }
 
 func NewStateObject(address common.Address, db common.Database) *StateObject {
-	// This to ensure that it has 20 bytes (and not 0 bytes), thus left or right pad doesn't matter.
-	//address := common.ToAddress(addr)
-
 	object := &StateObject{db: db, address: address, balance: new(big.Int), gasPool: new(big.Int), dirty: true}
 	object.trie = trie.NewSecure((common.Hash{}).Bytes(), db)
 	object.storage = make(Storage)
 	object.gasPool = new(big.Int)
-	object.prepaid = new(big.Int)
 
 	return object
 }
@@ -110,7 +104,6 @@ func NewStateObjectFromBytes(address common.Address, data []byte, db common.Data
 	object.trie = trie.NewSecure(extobject.Root[:], db)
 	object.storage = make(map[string]common.Hash)
 	object.gasPool = new(big.Int)
-	object.prepaid = new(big.Int)
 	object.code, _ = db.Get(extobject.CodeHash)
 
 	return object
@@ -172,7 +165,6 @@ func (self *StateObject) Update() {
 
 		self.setAddr([]byte(key), value)
 	}
-	self.storage = make(Storage)
 }
 
 func (c *StateObject) GetInstr(pc *big.Int) *common.Value {
diff --git a/core/state/statedb.go b/core/state/statedb.go
index f6f63f329..bd4ff6ff3 100644
--- a/core/state/statedb.go
+++ b/core/state/statedb.go
@@ -18,6 +18,7 @@ import (
 type StateDB struct {
 	db   common.Database
 	trie *trie.SecureTrie
+	root common.Hash
 
 	stateObjects map[string]*StateObject
 
@@ -31,7 +32,7 @@ type StateDB struct {
 // Create a new state from a given trie
 func New(root common.Hash, db common.Database) *StateDB {
 	trie := trie.NewSecure(root[:], db)
-	return &StateDB{db: db, trie: trie, stateObjects: make(map[string]*StateObject), refund: new(big.Int), logs: make(map[common.Hash]Logs)}
+	return &StateDB{root: root, db: db, trie: trie, stateObjects: make(map[string]*StateObject), refund: new(big.Int), logs: make(map[common.Hash]Logs)}
 }
 
 func (self *StateDB) PrintRoot() {
@@ -185,7 +186,7 @@ func (self *StateDB) DeleteStateObject(stateObject *StateObject) {
 	addr := stateObject.Address()
 	self.trie.Delete(addr[:])
 
-	delete(self.stateObjects, addr.Str())
+	//delete(self.stateObjects, addr.Str())
 }
 
 // Retrieve a state object given my the address. Nil if not found
@@ -340,6 +341,23 @@ func (self *StateDB) Update() {
 	}
 }
 
+func (self *StateDB) CleanUpdate() {
+	self.trie = trie.NewSecure(self.root[:], self.db)
+
+	self.refund = new(big.Int)
+
+	for _, stateObject := range self.stateObjects {
+		if stateObject.remove {
+			self.DeleteStateObject(stateObject)
+		} else {
+			stateObject.Update()
+
+			self.UpdateStateObject(stateObject)
+		}
+		stateObject.dirty = false
+	}
+}
+
 // Debug stuff
 func (self *StateDB) CreateOutputForDiff() {
 	for _, stateObject := range self.stateObjects {

From ab16ce70fc68d9ab1b7d8cda57c180b4785cab6a Mon Sep 17 00:00:00 2001
From: Jeffrey Wilcke <geffobscura@gmail.com>
Date: Wed, 1 Jul 2015 12:07:14 +0200
Subject: [PATCH 2/4] core, miner, tests: renamed state methods

* Update => SyncIntermediate
* Added SyncObjects

SyncIntermediate only updates whatever has changed, but, as a side
effect, requires much more disk space.

SyncObjects will only sync whatever is required for a block and will not
save intermediate state to disk. As drawback this requires more time
when more txs come in.
---
 core/block_processor.go  | 4 ++--
 core/chain_makers.go     | 4 ++--
 core/genesis.go          | 2 +-
 core/state/state_test.go | 2 +-
 core/state/statedb.go    | 6 ++++--
 miner/worker.go          | 2 +-
 tests/block_test_util.go | 2 +-
 tests/state_test_util.go | 2 +-
 8 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/core/block_processor.go b/core/block_processor.go
index 5f745e491..e7ad059c3 100644
--- a/core/block_processor.go
+++ b/core/block_processor.go
@@ -77,7 +77,7 @@ func (self *BlockProcessor) ApplyTransaction(coinbase *state.StateObject, stated
 	}
 
 	// Update the state with pending changes
-	statedb.Update()
+	statedb.SyncIntermediate()
 
 	usedGas.Add(usedGas, gas)
 	receipt := types.NewReceipt(statedb.Root().Bytes(), usedGas)
@@ -243,7 +243,7 @@ func (sm *BlockProcessor) processWithParent(block, parent *types.Block) (logs st
 
 	// Commit state objects/accounts to a temporary trie (does not save)
 	// used to calculate the state root.
-	state.CleanUpdate()
+	state.SyncObjects()
 	if header.Root != state.Root() {
 		err = fmt.Errorf("invalid merkle root. received=%x got=%x", header.Root, state.Root())
 		return
diff --git a/core/chain_makers.go b/core/chain_makers.go
index 37475e0ae..c46f627f8 100644
--- a/core/chain_makers.go
+++ b/core/chain_makers.go
@@ -77,7 +77,7 @@ func (b *BlockGen) AddTx(tx *types.Transaction) {
 	if err != nil {
 		panic(err)
 	}
-	b.statedb.Update()
+	b.statedb.SyncIntermediate()
 	b.header.GasUsed.Add(b.header.GasUsed, gas)
 	receipt := types.NewReceipt(b.statedb.Root().Bytes(), b.header.GasUsed)
 	logs := b.statedb.GetLogs(tx.Hash())
@@ -135,7 +135,7 @@ func GenerateChain(parent *types.Block, db common.Database, n int, gen func(int,
 			gen(i, b)
 		}
 		AccumulateRewards(statedb, h, b.uncles)
-		statedb.Update()
+		statedb.SyncIntermediate()
 		h.Root = statedb.Root()
 		return types.NewBlock(h, b.txs, b.uncles, b.receipts)
 	}
diff --git a/core/genesis.go b/core/genesis.go
index df13466ec..d27e7097b 100644
--- a/core/genesis.go
+++ b/core/genesis.go
@@ -64,7 +64,7 @@ func GenesisBlockForTesting(db common.Database, addr common.Address, balance *bi
 	statedb := state.New(common.Hash{}, db)
 	obj := statedb.GetOrNewStateObject(addr)
 	obj.SetBalance(balance)
-	statedb.Update()
+	statedb.SyncObjects()
 	statedb.Sync()
 	block := types.NewBlock(&types.Header{
 		Difficulty: params.GenesisDifficulty,
diff --git a/core/state/state_test.go b/core/state/state_test.go
index 00e133dab..b63b8ae9b 100644
--- a/core/state/state_test.go
+++ b/core/state/state_test.go
@@ -72,7 +72,7 @@ func TestNull(t *testing.T) {
 	//value := common.FromHex("0x823140710bf13990e4500136726d8b55")
 	var value common.Hash
 	state.SetState(address, common.Hash{}, value)
-	state.Update()
+	state.SyncIntermediate()
 	state.Sync()
 	value = state.GetState(address, common.Hash{})
 	if !common.EmptyHash(value) {
diff --git a/core/state/statedb.go b/core/state/statedb.go
index bd4ff6ff3..4ccda1fc7 100644
--- a/core/state/statedb.go
+++ b/core/state/statedb.go
@@ -324,7 +324,8 @@ func (self *StateDB) Refunds() *big.Int {
 	return self.refund
 }
 
-func (self *StateDB) Update() {
+// SyncIntermediate updates the intermediate state and all mid steps
+func (self *StateDB) SyncIntermediate() {
 	self.refund = new(big.Int)
 
 	for _, stateObject := range self.stateObjects {
@@ -341,7 +342,8 @@ func (self *StateDB) Update() {
 	}
 }
 
-func (self *StateDB) CleanUpdate() {
+// SyncObjects syncs the changed objects to the trie
+func (self *StateDB) SyncObjects() {
 	self.trie = trie.NewSecure(self.root[:], self.db)
 
 	self.refund = new(big.Int)
diff --git a/miner/worker.go b/miner/worker.go
index dd004da6e..1615ff84b 100644
--- a/miner/worker.go
+++ b/miner/worker.go
@@ -453,7 +453,7 @@ func (self *worker) commitNewWork() {
 	if atomic.LoadInt32(&self.mining) == 1 {
 		// commit state root after all state transitions.
 		core.AccumulateRewards(self.current.state, header, uncles)
-		current.state.Update()
+		current.state.SyncObjects()
 		self.current.state.Sync()
 		header.Root = current.state.Root()
 	}
diff --git a/tests/block_test_util.go b/tests/block_test_util.go
index 67f6a1d18..3b20da492 100644
--- a/tests/block_test_util.go
+++ b/tests/block_test_util.go
@@ -215,7 +215,7 @@ func (t *BlockTest) InsertPreState(ethereum *eth.Ethereum) (*state.StateDB, erro
 		}
 	}
 	// sync objects to trie
-	statedb.Update()
+	statedb.SyncObjects()
 	// sync trie to disk
 	statedb.Sync()
 
diff --git a/tests/state_test_util.go b/tests/state_test_util.go
index 2f3d497be..7f1a22ac0 100644
--- a/tests/state_test_util.go
+++ b/tests/state_test_util.go
@@ -175,7 +175,7 @@ func RunState(statedb *state.StateDB, env, tx map[string]string) ([]byte, state.
 	if core.IsNonceErr(err) || core.IsInvalidTxErr(err) || state.IsGasLimitErr(err) {
 		statedb.Set(snapshot)
 	}
-	statedb.Update()
+	statedb.SyncObjects()
 
 	return ret, vmenv.state.Logs(), vmenv.Gas, err
 }

From 0a1ff68c11706f60355b392cb16a681630260ac3 Mon Sep 17 00:00:00 2001
From: Jeffrey Wilcke <geffobscura@gmail.com>
Date: Wed, 1 Jul 2015 15:38:32 +0200
Subject: [PATCH 3/4] trie: dirty tracking

---
 trie/fullnode.go  | 15 ++++++++-------
 trie/hashnode.go  | 11 ++++++++---
 trie/node.go      |  1 +
 trie/shortnode.go | 12 +++++++++---
 trie/trie.go      | 37 +++++++++++++++++++++++++++++--------
 trie/trie_test.go |  2 +-
 trie/valuenode.go | 23 +++++++++++++++++------
 7 files changed, 73 insertions(+), 28 deletions(-)

diff --git a/trie/fullnode.go b/trie/fullnode.go
index 522fdb373..95c8eb4be 100644
--- a/trie/fullnode.go
+++ b/trie/fullnode.go
@@ -1,17 +1,16 @@
 package trie
 
-import "fmt"
-
 type FullNode struct {
 	trie  *Trie
 	nodes [17]Node
+	dirty bool
 }
 
 func NewFullNode(t *Trie) *FullNode {
 	return &FullNode{trie: t}
 }
 
-func (self *FullNode) Dirty() bool { return true }
+func (self *FullNode) Dirty() bool { return self.dirty }
 func (self *FullNode) Value() Node {
 	self.nodes[16] = self.trie.trans(self.nodes[16])
 	return self.nodes[16]
@@ -27,6 +26,7 @@ func (self *FullNode) Copy(t *Trie) Node {
 			nnode.nodes[i] = node.Copy(t)
 		}
 	}
+	nnode.dirty = true
 
 	return nnode
 }
@@ -60,11 +60,8 @@ func (self *FullNode) RlpData() interface{} {
 }
 
 func (self *FullNode) set(k byte, value Node) {
-	if _, ok := value.(*ValueNode); ok && k != 16 {
-		fmt.Println(value, k)
-	}
-
 	self.nodes[int(k)] = value
+	self.dirty = true
 }
 
 func (self *FullNode) branch(i byte) Node {
@@ -75,3 +72,7 @@ func (self *FullNode) branch(i byte) Node {
 	}
 	return nil
 }
+
+func (self *FullNode) setDirty(dirty bool) {
+	self.dirty = dirty
+}
diff --git a/trie/hashnode.go b/trie/hashnode.go
index 8125cc3c9..e82ab8069 100644
--- a/trie/hashnode.go
+++ b/trie/hashnode.go
@@ -3,12 +3,13 @@ package trie
 import "github.com/ethereum/go-ethereum/common"
 
 type HashNode struct {
-	key  []byte
-	trie *Trie
+	key   []byte
+	trie  *Trie
+	dirty bool
 }
 
 func NewHash(key []byte, trie *Trie) *HashNode {
-	return &HashNode{key, trie}
+	return &HashNode{key, trie, false}
 }
 
 func (self *HashNode) RlpData() interface{} {
@@ -19,6 +20,10 @@ func (self *HashNode) Hash() interface{} {
 	return self.key
 }
 
+func (self *HashNode) setDirty(dirty bool) {
+	self.dirty = dirty
+}
+
 // These methods will never be called but we have to satisfy Node interface
 func (self *HashNode) Value() Node       { return nil }
 func (self *HashNode) Dirty() bool       { return true }
diff --git a/trie/node.go b/trie/node.go
index 0d8a7cff9..dccbc64a3 100644
--- a/trie/node.go
+++ b/trie/node.go
@@ -11,6 +11,7 @@ type Node interface {
 	fstring(string) string
 	Hash() interface{}
 	RlpData() interface{}
+	setDirty(dirty bool)
 }
 
 // Value node
diff --git a/trie/shortnode.go b/trie/shortnode.go
index edd490b4d..c86e50096 100644
--- a/trie/shortnode.go
+++ b/trie/shortnode.go
@@ -6,20 +6,22 @@ type ShortNode struct {
 	trie  *Trie
 	key   []byte
 	value Node
+	dirty bool
 }
 
 func NewShortNode(t *Trie, key []byte, value Node) *ShortNode {
-	return &ShortNode{t, []byte(CompactEncode(key)), value}
+	return &ShortNode{t, []byte(CompactEncode(key)), value, false}
 }
 func (self *ShortNode) Value() Node {
 	self.value = self.trie.trans(self.value)
 
 	return self.value
 }
-func (self *ShortNode) Dirty() bool { return true }
+func (self *ShortNode) Dirty() bool { return self.dirty }
 func (self *ShortNode) Copy(t *Trie) Node {
-	node := &ShortNode{t, nil, self.value.Copy(t)}
+	node := &ShortNode{t, nil, self.value.Copy(t), self.dirty}
 	node.key = common.CopyBytes(self.key)
+	node.dirty = true
 	return node
 }
 
@@ -33,3 +35,7 @@ func (self *ShortNode) Hash() interface{} {
 func (self *ShortNode) Key() []byte {
 	return CompactDecode(string(self.key))
 }
+
+func (self *ShortNode) setDirty(dirty bool) {
+	self.dirty = dirty
+}
diff --git a/trie/trie.go b/trie/trie.go
index d990338ee..7e17baa2f 100644
--- a/trie/trie.go
+++ b/trie/trie.go
@@ -117,7 +117,9 @@ func (self *Trie) Update(key, value []byte) Node {
 	k := CompactHexDecode(string(key))
 
 	if len(value) != 0 {
-		self.root = self.insert(self.root, k, &ValueNode{self, value})
+		node := NewValueNode(self, value)
+		node.dirty = true
+		self.root = self.insert(self.root, k, node)
 	} else {
 		self.root = self.delete(self.root, k)
 	}
@@ -157,7 +159,9 @@ func (self *Trie) insert(node Node, key []byte, value Node) Node {
 	}
 
 	if node == nil {
-		return NewShortNode(self, key, value)
+		node := NewShortNode(self, key, value)
+		node.dirty = true
+		return node
 	}
 
 	switch node := node.(type) {
@@ -165,7 +169,10 @@ func (self *Trie) insert(node Node, key []byte, value Node) Node {
 		k := node.Key()
 		cnode := node.Value()
 		if bytes.Equal(k, key) {
-			return NewShortNode(self, key, value)
+			node := NewShortNode(self, key, value)
+			node.dirty = true
+			return node
+
 		}
 
 		var n Node
@@ -176,6 +183,7 @@ func (self *Trie) insert(node Node, key []byte, value Node) Node {
 			pnode := self.insert(nil, k[matchlength+1:], cnode)
 			nnode := self.insert(nil, key[matchlength+1:], value)
 			fulln := NewFullNode(self)
+			fulln.dirty = true
 			fulln.set(k[matchlength], pnode)
 			fulln.set(key[matchlength], nnode)
 			n = fulln
@@ -184,11 +192,14 @@ func (self *Trie) insert(node Node, key []byte, value Node) Node {
 			return n
 		}
 
-		return NewShortNode(self, key[:matchlength], n)
+		snode := NewShortNode(self, key[:matchlength], n)
+		snode.dirty = true
+		return snode
 
 	case *FullNode:
 		cpy := node.Copy(self).(*FullNode)
 		cpy.set(key[0], self.insert(node.branch(key[0]), key[1:], value))
+		cpy.dirty = true
 
 		return cpy
 
@@ -242,8 +253,10 @@ func (self *Trie) delete(node Node, key []byte) Node {
 			case *ShortNode:
 				nkey := append(k, child.Key()...)
 				n = NewShortNode(self, nkey, child.Value())
+				n.(*ShortNode).dirty = true
 			case *FullNode:
 				sn := NewShortNode(self, node.Key(), child)
+				sn.dirty = true
 				sn.key = node.key
 				n = sn
 			}
@@ -256,6 +269,7 @@ func (self *Trie) delete(node Node, key []byte) Node {
 	case *FullNode:
 		n := node.Copy(self).(*FullNode)
 		n.set(key[0], self.delete(n.branch(key[0]), key[1:]))
+		n.dirty = true
 
 		pos := -1
 		for i := 0; i < 17; i++ {
@@ -271,6 +285,7 @@ func (self *Trie) delete(node Node, key []byte) Node {
 		var nnode Node
 		if pos == 16 {
 			nnode = NewShortNode(self, []byte{16}, n.branch(byte(pos)))
+			nnode.(*ShortNode).dirty = true
 		} else if pos >= 0 {
 			cnode := n.branch(byte(pos))
 			switch cnode := cnode.(type) {
@@ -278,8 +293,10 @@ func (self *Trie) delete(node Node, key []byte) Node {
 				// Stitch keys
 				k := append([]byte{byte(pos)}, cnode.Key()...)
 				nnode = NewShortNode(self, k, cnode.Value())
+				nnode.(*ShortNode).dirty = true
 			case *FullNode:
 				nnode = NewShortNode(self, []byte{byte(pos)}, n.branch(byte(pos)))
+				nnode.(*ShortNode).dirty = true
 			}
 		} else {
 			nnode = n
@@ -304,7 +321,7 @@ func (self *Trie) mknode(value *common.Value) Node {
 		if value.Get(0).Len() != 0 {
 			key := CompactDecode(string(value.Get(0).Bytes()))
 			if key[len(key)-1] == 16 {
-				return NewShortNode(self, key, &ValueNode{self, value.Get(1).Bytes()})
+				return NewShortNode(self, key, NewValueNode(self, value.Get(1).Bytes()))
 			} else {
 				return NewShortNode(self, key, self.mknode(value.Get(1)))
 			}
@@ -318,10 +335,10 @@ func (self *Trie) mknode(value *common.Value) Node {
 			return fnode
 		}
 	case 32:
-		return &HashNode{value.Bytes(), self}
+		return NewHash(value.Bytes(), self)
 	}
 
-	return &ValueNode{self, value.Bytes()}
+	return NewValueNode(self, value.Bytes())
 }
 
 func (self *Trie) trans(node Node) Node {
@@ -338,7 +355,11 @@ func (self *Trie) store(node Node) interface{} {
 	data := common.Encode(node)
 	if len(data) >= 32 {
 		key := crypto.Sha3(data)
-		self.cache.Put(key, data)
+		if node.Dirty() {
+			//fmt.Println("save", node)
+			//fmt.Println()
+			self.cache.Put(key, data)
+		}
 
 		return key
 	}
diff --git a/trie/trie_test.go b/trie/trie_test.go
index 9a58958d8..60f0873a8 100644
--- a/trie/trie_test.go
+++ b/trie/trie_test.go
@@ -152,7 +152,7 @@ func TestReplication(t *testing.T) {
 	}
 	trie.Commit()
 
-	trie2 := New(trie.roothash, trie.cache.backend)
+	trie2 := New(trie.Root(), trie.cache.backend)
 	if string(trie2.GetString("horse")) != "stallion" {
 		t.Error("expected to have horse => stallion")
 	}
diff --git a/trie/valuenode.go b/trie/valuenode.go
index 7bf8ff06e..6adb59652 100644
--- a/trie/valuenode.go
+++ b/trie/valuenode.go
@@ -3,13 +3,24 @@ package trie
 import "github.com/ethereum/go-ethereum/common"
 
 type ValueNode struct {
-	trie *Trie
-	data []byte
+	trie  *Trie
+	data  []byte
+	dirty bool
 }
 
-func (self *ValueNode) Value() Node          { return self } // Best not to call :-)
-func (self *ValueNode) Val() []byte          { return self.data }
-func (self *ValueNode) Dirty() bool          { return true }
-func (self *ValueNode) Copy(t *Trie) Node    { return &ValueNode{t, common.CopyBytes(self.data)} }
+func NewValueNode(trie *Trie, data []byte) *ValueNode {
+	return &ValueNode{trie, data, false}
+}
+
+func (self *ValueNode) Value() Node { return self } // Best not to call :-)
+func (self *ValueNode) Val() []byte { return self.data }
+func (self *ValueNode) Dirty() bool { return self.dirty }
+func (self *ValueNode) Copy(t *Trie) Node {
+	return &ValueNode{t, common.CopyBytes(self.data), self.dirty}
+}
 func (self *ValueNode) RlpData() interface{} { return self.data }
 func (self *ValueNode) Hash() interface{}    { return self.data }
+
+func (self *ValueNode) setDirty(dirty bool) {
+	self.dirty = dirty
+}

From 47460b3b4af51e08518c781680897cf2986415cc Mon Sep 17 00:00:00 2001
From: Jeffrey Wilcke <geffobscura@gmail.com>
Date: Thu, 2 Jul 2015 18:21:14 +0200
Subject: [PATCH 4/4] trie: removed shallow copies (thanks to @fjl)

---
 trie/fullnode.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/trie/fullnode.go b/trie/fullnode.go
index 95c8eb4be..1bfdcd5bf 100644
--- a/trie/fullnode.go
+++ b/trie/fullnode.go
@@ -23,7 +23,7 @@ func (self *FullNode) Copy(t *Trie) Node {
 	nnode := NewFullNode(t)
 	for i, node := range self.nodes {
 		if node != nil {
-			nnode.nodes[i] = node.Copy(t)
+			nnode.nodes[i] = node
 		}
 	}
 	nnode.dirty = true