From 7fd0ccaa68cbc8e3f4fc59d3b99ba5067ba7c73a Mon Sep 17 00:00:00 2001 From: gary rong Date: Thu, 21 Feb 2019 21:14:35 +0800 Subject: [PATCH] core: remove unnecessary fields in logs, receipts and tx lookups (#17106) * core: remove unnecessary fields in log * core: bump blockchain database version * core, les: remove unnecessary fields in txlookup * eth: print db version explicitly * core/rawdb: drop txlookup entry struct wrapper --- core/blockchain.go | 8 +++- core/rawdb/accessors_chain.go | 10 +++++ core/rawdb/accessors_chain_test.go | 2 + core/rawdb/accessors_indexes.go | 65 ++++++++++++++++------------ core/rawdb/accessors_indexes_test.go | 23 ++++++++++ core/rawdb/schema.go | 6 +-- core/types/gen_log_json.go | 2 + core/types/log.go | 41 ++++++++++-------- core/types/receipt.go | 27 ++++++++++-- eth/backend.go | 14 +++--- les/handler.go | 4 +- les/handler_test.go | 4 +- les/protocol.go | 2 +- 13 files changed, 147 insertions(+), 61 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index a3121cfd3..d6dad2799 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -65,7 +65,13 @@ const ( triesInMemory = 128 // BlockChainVersion ensures that an incompatible database forces a resync from scratch. - BlockChainVersion uint64 = 3 + // + // During the process of upgrading the database version from 3 to 4, + // the following incompatible database changes were added. + // * the `BlockNumber`, `TxHash`, `TxIndex`, `BlockHash` and `Index` fields of log are deleted + // * the `Bloom` field of receipt is deleted + // * the `BlockIndex` and `TxIndex` fields of txlookup are deleted + BlockChainVersion uint64 = 4 ) // CacheConfig contains the configuration values for the trie caching/pruning diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 491a125c6..8a95dafe9 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -294,7 +294,17 @@ func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Rece return nil } receipts := make(types.Receipts, len(storageReceipts)) + logIndex := uint(0) for i, receipt := range storageReceipts { + // Assemble deriving fields for log. + for _, log := range receipt.Logs { + log.TxHash = receipt.TxHash + log.BlockHash = hash + log.BlockNumber = number + log.TxIndex = uint(i) + log.Index = logIndex + logIndex += 1 + } receipts[i] = (*types.Receipt)(receipt) } return receipts diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index fcc36dc2b..37e0d4fda 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -279,6 +279,7 @@ func TestBlockReceiptStorage(t *testing.T) { ContractAddress: common.BytesToAddress([]byte{0x01, 0x11, 0x11}), GasUsed: 111111, } + receipt1.Bloom = types.CreateBloom(types.Receipts{receipt1}) receipt2 := &types.Receipt{ PostState: common.Hash{2}.Bytes(), CumulativeGasUsed: 2, @@ -290,6 +291,7 @@ func TestBlockReceiptStorage(t *testing.T) { ContractAddress: common.BytesToAddress([]byte{0x02, 0x22, 0x22}), GasUsed: 222222, } + receipt2.Bloom = types.CreateBloom(types.Receipts{receipt2}) receipts := []*types.Receipt{receipt1, receipt2} // Check that no receipt entries are in a pristine database diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index 4ff7e5bd3..e6f7782a1 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -25,33 +25,28 @@ import ( // ReadTxLookupEntry retrieves the positional metadata associated with a transaction // hash to allow retrieving the transaction or receipt by hash. -func ReadTxLookupEntry(db DatabaseReader, hash common.Hash) (common.Hash, uint64, uint64) { +func ReadTxLookupEntry(db DatabaseReader, hash common.Hash) common.Hash { data, _ := db.Get(txLookupKey(hash)) if len(data) == 0 { - return common.Hash{}, 0, 0 + return common.Hash{} } - var entry TxLookupEntry + if len(data) == common.HashLength { + return common.BytesToHash(data) + } + // Probably it's legacy txlookup entry data, try to decode it. + var entry LegacyTxLookupEntry if err := rlp.DecodeBytes(data, &entry); err != nil { - log.Error("Invalid transaction lookup entry RLP", "hash", hash, "err", err) - return common.Hash{}, 0, 0 + log.Error("Invalid transaction lookup entry RLP", "hash", hash, "blob", data, "err", err) + return common.Hash{} } - return entry.BlockHash, entry.BlockIndex, entry.Index + return entry.BlockHash } // WriteTxLookupEntries stores a positional metadata for every transaction from // a block, enabling hash based transaction and receipt lookups. func WriteTxLookupEntries(db DatabaseWriter, block *types.Block) { - for i, tx := range block.Transactions() { - entry := TxLookupEntry{ - BlockHash: block.Hash(), - BlockIndex: block.NumberU64(), - Index: uint64(i), - } - data, err := rlp.EncodeToBytes(entry) - if err != nil { - log.Crit("Failed to encode transaction lookup entry", "err", err) - } - if err := db.Put(txLookupKey(tx.Hash()), data); err != nil { + for _, tx := range block.Transactions() { + if err := db.Put(txLookupKey(tx.Hash()), block.Hash().Bytes()); err != nil { log.Crit("Failed to store transaction lookup entry", "err", err) } } @@ -65,31 +60,47 @@ func DeleteTxLookupEntry(db DatabaseDeleter, hash common.Hash) { // ReadTransaction retrieves a specific transaction from the database, along with // its added positional metadata. func ReadTransaction(db DatabaseReader, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) { - blockHash, blockNumber, txIndex := ReadTxLookupEntry(db, hash) + blockHash := ReadTxLookupEntry(db, hash) if blockHash == (common.Hash{}) { return nil, common.Hash{}, 0, 0 } - body := ReadBody(db, blockHash, blockNumber) - if body == nil || len(body.Transactions) <= int(txIndex) { - log.Error("Transaction referenced missing", "number", blockNumber, "hash", blockHash, "index", txIndex) + blockNumber := ReadHeaderNumber(db, blockHash) + if blockNumber == nil { return nil, common.Hash{}, 0, 0 } - return body.Transactions[txIndex], blockHash, blockNumber, txIndex + body := ReadBody(db, blockHash, *blockNumber) + if body == nil { + log.Error("Transaction referenced missing", "number", blockNumber, "hash", blockHash) + return nil, common.Hash{}, 0, 0 + } + for txIndex, tx := range body.Transactions { + if tx.Hash() == hash { + return tx, blockHash, *blockNumber, uint64(txIndex) + } + } + log.Error("Transaction not found", "number", blockNumber, "hash", blockHash, "txhash", hash) + return nil, common.Hash{}, 0, 0 } // ReadReceipt retrieves a specific transaction receipt from the database, along with // its added positional metadata. func ReadReceipt(db DatabaseReader, hash common.Hash) (*types.Receipt, common.Hash, uint64, uint64) { - blockHash, blockNumber, receiptIndex := ReadTxLookupEntry(db, hash) + blockHash := ReadTxLookupEntry(db, hash) if blockHash == (common.Hash{}) { return nil, common.Hash{}, 0, 0 } - receipts := ReadReceipts(db, blockHash, blockNumber) - if len(receipts) <= int(receiptIndex) { - log.Error("Receipt refereced missing", "number", blockNumber, "hash", blockHash, "index", receiptIndex) + blockNumber := ReadHeaderNumber(db, blockHash) + if blockNumber == nil { return nil, common.Hash{}, 0, 0 } - return receipts[receiptIndex], blockHash, blockNumber, receiptIndex + receipts := ReadReceipts(db, blockHash, *blockNumber) + for receiptIndex, receipt := range receipts { + if receipt.TxHash == hash { + return receipt, blockHash, *blockNumber, uint64(receiptIndex) + } + } + log.Error("Receipt not found", "number", blockNumber, "hash", blockHash, "txhash", hash) + return nil, common.Hash{}, 0, 0 } // ReadBloomBits retrieves the compressed bloom bit vector belonging to the given diff --git a/core/rawdb/accessors_indexes_test.go b/core/rawdb/accessors_indexes_test.go index d9c10e149..bed03a5e6 100644 --- a/core/rawdb/accessors_indexes_test.go +++ b/core/rawdb/accessors_indexes_test.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/rlp" ) // Tests that positional lookup metadata can be stored and retrieved. @@ -65,4 +66,26 @@ func TestLookupStorage(t *testing.T) { t.Fatalf("tx #%d [%x]: deleted transaction returned: %v", i, tx.Hash(), txn) } } + // Insert legacy txlookup and verify the data retrieval + for index, tx := range block.Transactions() { + entry := LegacyTxLookupEntry{ + BlockHash: block.Hash(), + BlockIndex: block.NumberU64(), + Index: uint64(index), + } + data, _ := rlp.EncodeToBytes(entry) + db.Put(txLookupKey(tx.Hash()), data) + } + for i, tx := range txs { + if txn, hash, number, index := ReadTransaction(db, tx.Hash()); txn == nil { + t.Fatalf("tx #%d [%x]: transaction not found", i, tx.Hash()) + } else { + if hash != block.Hash() || number != block.NumberU64() || index != uint64(i) { + t.Fatalf("tx #%d [%x]: positional metadata mismatch: have %x/%d/%d, want %x/%v/%v", i, tx.Hash(), hash, number, index, block.Hash(), block.NumberU64(), i) + } + if tx.Hash() != txn.Hash() { + t.Fatalf("tx #%d [%x]: transaction mismatch: have %v, want %v", i, tx.Hash(), txn, tx) + } + } + } } diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 8a9921ef4..3bb86e7ff 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -63,9 +63,9 @@ var ( preimageHitCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil) ) -// TxLookupEntry is a positional metadata to help looking up the data content of -// a transaction or receipt given only its hash. -type TxLookupEntry struct { +// LegacyTxLookupEntry is the legacy TxLookupEntry definition with some unnecessary +// fields. +type LegacyTxLookupEntry struct { BlockHash common.Hash BlockIndex uint64 Index uint64 diff --git a/core/types/gen_log_json.go b/core/types/gen_log_json.go index 1b5ae3c65..6e9433947 100644 --- a/core/types/gen_log_json.go +++ b/core/types/gen_log_json.go @@ -12,6 +12,7 @@ import ( var _ = (*logMarshaling)(nil) +// MarshalJSON marshals as JSON. func (l Log) MarshalJSON() ([]byte, error) { type Log struct { Address common.Address `json:"address" gencodec:"required"` @@ -37,6 +38,7 @@ func (l Log) MarshalJSON() ([]byte, error) { return json.Marshal(&enc) } +// UnmarshalJSON unmarshals from JSON. func (l *Log) UnmarshalJSON(input []byte) error { type Log struct { Address *common.Address `json:"address" gencodec:"required"` diff --git a/core/types/log.go b/core/types/log.go index 717cd2e5a..a395d5a67 100644 --- a/core/types/log.go +++ b/core/types/log.go @@ -68,7 +68,11 @@ type rlpLog struct { Data []byte } -type rlpStorageLog struct { +// rlpStorageLog is the storage encoding of a log. +type rlpStorageLog rlpLog + +// LegacyRlpStorageLog is the previous storage encoding of a log including some redundant fields. +type LegacyRlpStorageLog struct { Address common.Address Topics []common.Hash Data []byte @@ -101,31 +105,34 @@ type LogForStorage Log // EncodeRLP implements rlp.Encoder. func (l *LogForStorage) EncodeRLP(w io.Writer) error { return rlp.Encode(w, rlpStorageLog{ - Address: l.Address, - Topics: l.Topics, - Data: l.Data, - BlockNumber: l.BlockNumber, - TxHash: l.TxHash, - TxIndex: l.TxIndex, - BlockHash: l.BlockHash, - Index: l.Index, + Address: l.Address, + Topics: l.Topics, + Data: l.Data, }) } // DecodeRLP implements rlp.Decoder. +// +// Note some redundant fields(e.g. block number, tx hash etc) will be assembled later. func (l *LogForStorage) DecodeRLP(s *rlp.Stream) error { var dec rlpStorageLog err := s.Decode(&dec) if err == nil { *l = LogForStorage{ - Address: dec.Address, - Topics: dec.Topics, - Data: dec.Data, - BlockNumber: dec.BlockNumber, - TxHash: dec.TxHash, - TxIndex: dec.TxIndex, - BlockHash: dec.BlockHash, - Index: dec.Index, + Address: dec.Address, + Topics: dec.Topics, + Data: dec.Data, + } + } else { + // Try to decode log with previous definition. + var dec LegacyRlpStorageLog + err = s.Decode(&dec) + if err == nil { + *l = LogForStorage{ + Address: dec.Address, + Topics: dec.Topics, + Data: dec.Data, + } } } return err diff --git a/core/types/receipt.go b/core/types/receipt.go index 3d1fc95aa..ac1ebe349 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -72,7 +72,18 @@ type receiptRLP struct { Logs []*Log } +// receiptStorageRLP is the storage encoding of a receipt. type receiptStorageRLP struct { + PostStateOrStatus []byte + CumulativeGasUsed uint64 + TxHash common.Hash + ContractAddress common.Address + Logs []*LogForStorage + GasUsed uint64 +} + +// LegacyReceiptStorageRLP is the previous storage encoding of a receipt including some unnecessary fields. +type LegacyReceiptStorageRLP struct { PostStateOrStatus []byte CumulativeGasUsed uint64 Bloom Bloom @@ -159,7 +170,6 @@ func (r *ReceiptForStorage) EncodeRLP(w io.Writer) error { enc := &receiptStorageRLP{ PostStateOrStatus: (*Receipt)(r).statusEncoding(), CumulativeGasUsed: r.CumulativeGasUsed, - Bloom: r.Bloom, TxHash: r.TxHash, ContractAddress: r.ContractAddress, Logs: make([]*LogForStorage, len(r.Logs)), @@ -176,17 +186,28 @@ func (r *ReceiptForStorage) EncodeRLP(w io.Writer) error { func (r *ReceiptForStorage) DecodeRLP(s *rlp.Stream) error { var dec receiptStorageRLP if err := s.Decode(&dec); err != nil { - return err + var sdec LegacyReceiptStorageRLP + if err := s.Decode(&sdec); err != nil { + return err + } + dec.PostStateOrStatus = common.CopyBytes(sdec.PostStateOrStatus) + dec.CumulativeGasUsed = sdec.CumulativeGasUsed + dec.TxHash = sdec.TxHash + dec.ContractAddress = sdec.ContractAddress + dec.Logs = sdec.Logs + dec.GasUsed = sdec.GasUsed } if err := (*Receipt)(r).setStatus(dec.PostStateOrStatus); err != nil { return err } // Assign the consensus fields - r.CumulativeGasUsed, r.Bloom = dec.CumulativeGasUsed, dec.Bloom + r.CumulativeGasUsed = dec.CumulativeGasUsed r.Logs = make([]*Log, len(dec.Logs)) for i, log := range dec.Logs { r.Logs[i] = (*Log)(log) } + + r.Bloom = CreateBloom(Receipts{(*Receipt)(r)}) // Assign the implementation fields r.TxHash, r.ContractAddress, r.GasUsed = dec.TxHash, dec.ContractAddress, dec.GasUsed return nil diff --git a/eth/backend.go b/eth/backend.go index 6a136182a..6710e4513 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -145,16 +145,20 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { bloomIndexer: NewBloomIndexer(chainDb, params.BloomBitsBlocks, params.BloomConfirms), } - log.Info("Initialising Ethereum protocol", "versions", ProtocolVersions, "network", config.NetworkId) + bcVersion := rawdb.ReadDatabaseVersion(chainDb) + var dbVer = "" + if bcVersion != nil { + dbVer = fmt.Sprintf("%d", *bcVersion) + } + log.Info("Initialising Ethereum protocol", "versions", ProtocolVersions, "network", config.NetworkId, "dbversion", dbVer) if !config.SkipBcVersionCheck { - bcVersion := rawdb.ReadDatabaseVersion(chainDb) if bcVersion != nil && *bcVersion > core.BlockChainVersion { return nil, fmt.Errorf("database version is v%d, Geth %s only supports v%d", *bcVersion, params.VersionWithMeta, core.BlockChainVersion) - } else if bcVersion != nil && *bcVersion < core.BlockChainVersion { - log.Warn("Upgrade blockchain database version", "from", *bcVersion, "to", core.BlockChainVersion) + } else if bcVersion == nil || *bcVersion < core.BlockChainVersion { + log.Warn("Upgrade blockchain database version", "from", dbVer, "to", core.BlockChainVersion) + rawdb.WriteDatabaseVersion(chainDb, core.BlockChainVersion) } - rawdb.WriteDatabaseVersion(chainDb, core.BlockChainVersion) } var ( vmConfig = vm.Config{ diff --git a/les/handler.go b/les/handler.go index 46a1ed2d7..680e115b0 100644 --- a/les/handler.go +++ b/les/handler.go @@ -1193,9 +1193,9 @@ func (pm *ProtocolManager) txStatus(hashes []common.Hash) []txStatus { // If the transaction is unknown to the pool, try looking it up locally if stat == core.TxStatusUnknown { - if block, number, index := rawdb.ReadTxLookupEntry(pm.chainDb, hashes[i]); block != (common.Hash{}) { + if tx, blockHash, blockNumber, txIndex := rawdb.ReadTransaction(pm.chainDb, hashes[i]); tx != nil { stats[i].Status = core.TxStatusIncluded - stats[i].Lookup = &rawdb.TxLookupEntry{BlockHash: block, BlockIndex: number, Index: index} + stats[i].Lookup = &rawdb.LegacyTxLookupEntry{BlockHash: blockHash, BlockIndex: blockNumber, Index: txIndex} } } } diff --git a/les/handler_test.go b/les/handler_test.go index 72ba266b3..e9033729e 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -559,8 +559,8 @@ func TestTransactionStatusLes2(t *testing.T) { // check if their status is included now block1hash := rawdb.ReadCanonicalHash(db, 1) - test(tx1, false, txStatus{Status: core.TxStatusIncluded, Lookup: &rawdb.TxLookupEntry{BlockHash: block1hash, BlockIndex: 1, Index: 0}}) - test(tx2, false, txStatus{Status: core.TxStatusIncluded, Lookup: &rawdb.TxLookupEntry{BlockHash: block1hash, BlockIndex: 1, Index: 1}}) + test(tx1, false, txStatus{Status: core.TxStatusIncluded, Lookup: &rawdb.LegacyTxLookupEntry{BlockHash: block1hash, BlockIndex: 1, Index: 0}}) + test(tx2, false, txStatus{Status: core.TxStatusIncluded, Lookup: &rawdb.LegacyTxLookupEntry{BlockHash: block1hash, BlockIndex: 1, Index: 1}}) // create a reorg that rolls them back gchain, _ = core.GenerateChain(params.TestChainConfig, chain.GetBlockByNumber(0), ethash.NewFaker(), db, 2, func(i int, block *core.BlockGen) {}) diff --git a/les/protocol.go b/les/protocol.go index 0b24f5aed..b75f92bf7 100644 --- a/les/protocol.go +++ b/les/protocol.go @@ -221,6 +221,6 @@ type proofsData [][]rlp.RawValue type txStatus struct { Status core.TxStatus - Lookup *rawdb.TxLookupEntry `rlp:"nil"` + Lookup *rawdb.LegacyTxLookupEntry `rlp:"nil"` Error string }