eth/downloader: speed up tests by generating chain only once (#17916)

* core: speed up GenerateChain

Use a mock implementation of ChainReader instead of creating
and destroying a BlockChain object for each generated block.

* eth/downloader: speed up tests by generating chain only once

This change reworks the downloader tests so they share a common test
blockchain instead of generating a chain in every test. The tests are
roughly twice as fast now.

core/chain_makers.go

@@ -33,12 +33,11 @@ import (
 // BlockGen creates blocks for testing.
 // See GenerateChain for a detailed explanation.
 type BlockGen struct {
-	i           int
-	parent      *types.Block
-	chain       []*types.Block
-	chainReader consensus.ChainReader
-	header      *types.Header
-	statedb     *state.StateDB
+	i       int
+	parent  *types.Block
+	chain   []*types.Block
+	header  *types.Header
+	statedb *state.StateDB
 
 	gasPool  *GasPool
 	txs      []*types.Transaction
@@ -138,7 +137,7 @@ func (b *BlockGen) AddUncle(h *types.Header) {
 // For index -1, PrevBlock returns the parent block given to GenerateChain.
 func (b *BlockGen) PrevBlock(index int) *types.Block {
 	if index >= b.i {
-		panic("block index out of range")
+		panic(fmt.Errorf("block index %d out of range (%d,%d)", index, -1, b.i))
 	}
 	if index == -1 {
 		return b.parent
@@ -154,7 +153,8 @@ func (b *BlockGen) OffsetTime(seconds int64) {
 	if b.header.Time.Cmp(b.parent.Header().Time) <= 0 {
 		panic("block time out of range")
 	}
-	b.header.Difficulty = b.engine.CalcDifficulty(b.chainReader, b.header.Time.Uint64(), b.parent.Header())
+	chainreader := &fakeChainReader{config: b.config}
+	b.header.Difficulty = b.engine.CalcDifficulty(chainreader, b.header.Time.Uint64(), b.parent.Header())
 }
 
 // GenerateChain creates a chain of n blocks. The first block's
@@ -174,14 +174,10 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 		config = params.TestChainConfig
 	}
 	blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n)
+	chainreader := &fakeChainReader{config: config}
 	genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts) {
-		// TODO(karalabe): This is needed for clique, which depends on multiple blocks.
-		// It's nonetheless ugly to spin up a blockchain here. Get rid of this somehow.
-		blockchain, _ := NewBlockChain(db, nil, config, engine, vm.Config{}, nil)
-		defer blockchain.Stop()
-
-		b := &BlockGen{i: i, parent: parent, chain: blocks, chainReader: blockchain, statedb: statedb, config: config, engine: engine}
-		b.header = makeHeader(b.chainReader, parent, statedb, b.engine)
+		b := &BlockGen{i: i, chain: blocks, parent: parent, statedb: statedb, config: config, engine: engine}
+		b.header = makeHeader(chainreader, parent, statedb, b.engine)
 
 		// Mutate the state and block according to any hard-fork specs
 		if daoBlock := config.DAOForkBlock; daoBlock != nil {
@@ -201,7 +197,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 		}
 		if b.engine != nil {
 			// Finalize and seal the block
-			block, _ := b.engine.Finalize(b.chainReader, b.header, statedb, b.txs, b.uncles, b.receipts)
+			block, _ := b.engine.Finalize(chainreader, b.header, statedb, b.txs, b.uncles, b.receipts)
 
 			// Write state changes to db
 			root, err := statedb.Commit(config.IsEIP158(b.header.Number))
@@ -269,3 +265,19 @@ func makeBlockChain(parent *types.Block, n int, engine consensus.Engine, db ethd
 	})
 	return blocks
 }
+
+type fakeChainReader struct {
+	config  *params.ChainConfig
+	genesis *types.Block
+}
+
+// Config returns the chain configuration.
+func (cr *fakeChainReader) Config() *params.ChainConfig {
+	return cr.config
+}
+
+func (cr *fakeChainReader) CurrentHeader() *types.Header                            { return nil }
+func (cr *fakeChainReader) GetHeaderByNumber(number uint64) *types.Header           { return nil }
+func (cr *fakeChainReader) GetHeaderByHash(hash common.Hash) *types.Header          { return nil }
+func (cr *fakeChainReader) GetHeader(hash common.Hash, number uint64) *types.Header { return nil }
+func (cr *fakeChainReader) GetBlock(hash common.Hash, number uint64) *types.Block   { return nil }

eth/downloader/testchain_test.go

@@ -0,0 +1,221 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package downloader
+
+import (
+	"fmt"
+	"math/big"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/consensus/ethash"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/params"
+)
+
+// Test chain parameters.
+var (
+	testKey, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+	testAddress = crypto.PubkeyToAddress(testKey.PublicKey)
+	testDB      = ethdb.NewMemDatabase()
+	testGenesis = core.GenesisBlockForTesting(testDB, testAddress, big.NewInt(1000000000))
+)
+
+// The common prefix of all test chains:
+var testChainBase = newTestChain(blockCacheItems+200, testGenesis)
+
+// Different forks on top of the base chain:
+var testChainForkLightA, testChainForkLightB, testChainForkHeavy *testChain
+
+func init() {
+	var forkLen = int(MaxForkAncestry + 50)
+	var wg sync.WaitGroup
+	wg.Add(3)
+	go func() { testChainForkLightA = testChainBase.makeFork(forkLen, false, 1); wg.Done() }()
+	go func() { testChainForkLightB = testChainBase.makeFork(forkLen, false, 2); wg.Done() }()
+	go func() { testChainForkHeavy = testChainBase.makeFork(forkLen, true, 3); wg.Done() }()
+	wg.Wait()
+}
+
+type testChain struct {
+	genesis  *types.Block
+	chain    []common.Hash
+	headerm  map[common.Hash]*types.Header
+	blockm   map[common.Hash]*types.Block
+	receiptm map[common.Hash][]*types.Receipt
+	tdm      map[common.Hash]*big.Int
+}
+
+// newTestChain creates a blockchain of the given length.
+func newTestChain(length int, genesis *types.Block) *testChain {
+	tc := new(testChain).copy(length)
+	tc.genesis = genesis
+	tc.chain = append(tc.chain, genesis.Hash())
+	tc.headerm[tc.genesis.Hash()] = tc.genesis.Header()
+	tc.tdm[tc.genesis.Hash()] = tc.genesis.Difficulty()
+	tc.blockm[tc.genesis.Hash()] = tc.genesis
+	tc.generate(length-1, 0, genesis, false)
+	return tc
+}
+
+// makeFork creates a fork on top of the test chain.
+func (tc *testChain) makeFork(length int, heavy bool, seed byte) *testChain {
+	fork := tc.copy(tc.len() + length)
+	fork.generate(length, seed, tc.headBlock(), heavy)
+	return fork
+}
+
+// shorten creates a copy of the chain with the given length. It panics if the
+// length is longer than the number of available blocks.
+func (tc *testChain) shorten(length int) *testChain {
+	if length > tc.len() {
+		panic(fmt.Errorf("can't shorten test chain to %d blocks, it's only %d blocks long", length, tc.len()))
+	}
+	return tc.copy(length)
+}
+
+func (tc *testChain) copy(newlen int) *testChain {
+	cpy := &testChain{
+		genesis:  tc.genesis,
+		headerm:  make(map[common.Hash]*types.Header, newlen),
+		blockm:   make(map[common.Hash]*types.Block, newlen),
+		receiptm: make(map[common.Hash][]*types.Receipt, newlen),
+		tdm:      make(map[common.Hash]*big.Int, newlen),
+	}
+	for i := 0; i < len(tc.chain) && i < newlen; i++ {
+		hash := tc.chain[i]
+		cpy.chain = append(cpy.chain, tc.chain[i])
+		cpy.tdm[hash] = tc.tdm[hash]
+		cpy.blockm[hash] = tc.blockm[hash]
+		cpy.headerm[hash] = tc.headerm[hash]
+		cpy.receiptm[hash] = tc.receiptm[hash]
+	}
+	return cpy
+}
+
+// generate creates a chain of n blocks starting at and including parent.
+// the returned hash chain is ordered head->parent. In addition, every 22th block
+// contains a transaction and every 5th an uncle to allow testing correct block
+// reassembly.
+func (tc *testChain) generate(n int, seed byte, parent *types.Block, heavy bool) {
+	// start := time.Now()
+	// defer func() { fmt.Printf("test chain generated in %v\n", time.Since(start)) }()
+
+	blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testDB, n, func(i int, block *core.BlockGen) {
+		block.SetCoinbase(common.Address{seed})
+		// If a heavy chain is requested, delay blocks to raise difficulty
+		if heavy {
+			block.OffsetTime(-1)
+		}
+		// Include transactions to the miner to make blocks more interesting.
+		if parent == tc.genesis && i%22 == 0 {
+			signer := types.MakeSigner(params.TestChainConfig, block.Number())
+			tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, nil, nil), signer, testKey)
+			if err != nil {
+				panic(err)
+			}
+			block.AddTx(tx)
+		}
+		// if the block number is a multiple of 5, add a bonus uncle to the block
+		if i > 0 && i%5 == 0 {
+			block.AddUncle(&types.Header{
+				ParentHash: block.PrevBlock(i - 1).Hash(),
+				Number:     big.NewInt(block.Number().Int64() - 1),
+			})
+		}
+	})
+
+	// Convert the block-chain into a hash-chain and header/block maps
+	td := new(big.Int).Set(tc.td(parent.Hash()))
+	for i, b := range blocks {
+		td := td.Add(td, b.Difficulty())
+		hash := b.Hash()
+		tc.chain = append(tc.chain, hash)
+		tc.blockm[hash] = b
+		tc.headerm[hash] = b.Header()
+		tc.receiptm[hash] = receipts[i]
+		tc.tdm[hash] = new(big.Int).Set(td)
+	}
+}
+
+// len returns the total number of blocks in the chain.
+func (tc *testChain) len() int {
+	return len(tc.chain)
+}
+
+// headBlock returns the head of the chain.
+func (tc *testChain) headBlock() *types.Block {
+	return tc.blockm[tc.chain[len(tc.chain)-1]]
+}
+
+// td returns the total difficulty of the given block.
+func (tc *testChain) td(hash common.Hash) *big.Int {
+	return tc.tdm[hash]
+}
+
+// headersByHash returns headers in ascending order from the given hash.
+func (tc *testChain) headersByHash(origin common.Hash, amount int, skip int) []*types.Header {
+	num, _ := tc.hashToNumber(origin)
+	return tc.headersByNumber(num, amount, skip)
+}
+
+// headersByNumber returns headers in ascending order from the given number.
+func (tc *testChain) headersByNumber(origin uint64, amount int, skip int) []*types.Header {
+	result := make([]*types.Header, 0, amount)
+	for num := origin; num < uint64(len(tc.chain)) && len(result) < amount; num += uint64(skip) + 1 {
+		if header, ok := tc.headerm[tc.chain[int(num)]]; ok {
+			result = append(result, header)
+		}
+	}
+	return result
+}
+
+// receipts returns the receipts of the given block hashes.
+func (tc *testChain) receipts(hashes []common.Hash) [][]*types.Receipt {
+	results := make([][]*types.Receipt, 0, len(hashes))
+	for _, hash := range hashes {
+		if receipt, ok := tc.receiptm[hash]; ok {
+			results = append(results, receipt)
+		}
+	}
+	return results
+}
+
+// bodies returns the block bodies of the given block hashes.
+func (tc *testChain) bodies(hashes []common.Hash) ([][]*types.Transaction, [][]*types.Header) {
+	transactions := make([][]*types.Transaction, 0, len(hashes))
+	uncles := make([][]*types.Header, 0, len(hashes))
+	for _, hash := range hashes {
+		if block, ok := tc.blockm[hash]; ok {
+			transactions = append(transactions, block.Transactions())
+			uncles = append(uncles, block.Uncles())
+		}
+	}
+	return transactions, uncles
+}
+
+func (tc *testChain) hashToNumber(target common.Hash) (uint64, bool) {
+	for num, hash := range tc.chain {
+		if hash == target {
+			return uint64(num), true
+		}
+	}
+	return 0, false
+}