This commit is contained in:
Dario Gabriel Lipicar 2024-10-04 12:44:15 -03:00
parent e8bd4d5685
commit 90251aa93a
No known key found for this signature in database
GPG Key ID: 9625E9494309D203
9 changed files with 691 additions and 33 deletions

348
rpc/chain/ethclient/db.go Normal file
View File

@ -0,0 +1,348 @@
package ethclient
import (
"database/sql"
"encoding/json"
"errors"
"math/big"
sq "github.com/Masterminds/squirrel"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/status-im/status-go/services/wallet/bigint"
"github.com/status-im/status-go/sqlite"
)
var ErrNotFound = errors.New("not found")
type EthClientStorageReader interface {
GetBlockJSONByNumber(chainID uint64, blockNumber *big.Int, withTransactionDetails bool) (json.RawMessage, error)
GetBlockJSONByHash(chainID uint64, blockHash common.Hash, withTransactionDetails bool) (json.RawMessage, error)
GetBlockUncleJSONByHashAndIndex(chainID uint64, blockHash common.Hash, index uint64) (json.RawMessage, error)
GetTransactionJSONByHash(chainID uint64, transactionHash common.Hash) (json.RawMessage, error)
GetTransactionReceiptJSONByHash(chainID uint64, transactionHash common.Hash) (json.RawMessage, error)
}
type EthClientStorageWriter interface {
PutBlockJSON(chainID uint64, blkJSON json.RawMessage, transactionDetailsFlag bool) error
PutBlockUnclesJSON(chainID uint64, blockHash common.Hash, unclesJSON []json.RawMessage) error
PutTransactionsJSON(chainID uint64, transactionsJSON []json.RawMessage) error
PutTransactionReceiptsJSON(chainID uint64, receiptsJSON []json.RawMessage) error
}
type EthClientStorage interface {
EthClientStorageReader
EthClientStorageWriter
}
type DB struct {
db *sql.DB
}
func NewDB(db *sql.DB) *DB {
return &DB{db: db}
}
func (b *DB) GetBlockJSONByNumber(chainID uint64, blockNumber *big.Int, withTransactionDetails bool) (json.RawMessage, error) {
q := sq.Select("block_json").
From("blockchain_data_blocks").
Where(sq.Eq{"chain_id": chainID, "block_number": (*bigint.SQLBigInt)(blockNumber), "with_transaction_details": withTransactionDetails})
query, args, err := q.ToSql()
if err != nil {
return nil, err
}
blockJSON := json.RawMessage{}
err = b.db.QueryRow(query, args...).Scan(&blockJSON)
if err != nil {
return nil, err
}
return blockJSON, nil
}
func (b *DB) GetBlockJSONByHash(chainID uint64, blockHash common.Hash, withTransactionDetails bool) (json.RawMessage, error) {
q := sq.Select("block_json").
From("blockchain_data_blocks").
Where(sq.Eq{"chain_id": chainID, "block_hash": blockHash, "with_transaction_details": withTransactionDetails})
query, args, err := q.ToSql()
if err != nil {
return nil, err
}
blockJSON := json.RawMessage{}
err = b.db.QueryRow(query, args...).Scan(&blockJSON)
if err != nil {
return nil, err
}
return blockJSON, nil
}
func (b *DB) GetBlockUncleJSONByHashAndIndex(chainID uint64, blockHash common.Hash, index uint64) (json.RawMessage, error) {
q := sq.Select("block_uncle_json").
From("blockchain_data_block_uncles").
Where(sq.Eq{"chain_id": chainID, "block_hash": blockHash, "uncle_index": index})
query, args, err := q.ToSql()
if err != nil {
return nil, err
}
uncleJSON := json.RawMessage{}
err = b.db.QueryRow(query, args...).Scan(&uncleJSON)
if err != nil {
return nil, err
}
return uncleJSON, nil
}
func (b *DB) GetTransactionJSONByHash(chainID uint64, transactionHash common.Hash) (json.RawMessage, error) {
q := sq.Select("transaction_json").
From("blockchain_data_transactions").
Where(sq.Eq{"chain_id": chainID, "transaction_hash": transactionHash})
query, args, err := q.ToSql()
if err != nil {
return nil, err
}
transactionJSON := json.RawMessage{}
err = b.db.QueryRow(query, args...).Scan(&transactionJSON)
if err != nil {
return nil, err
}
return transactionJSON, nil
}
func (b *DB) GetTransactionReceiptJSONByHash(chainID uint64, transactionHash common.Hash) (json.RawMessage, error) {
q := sq.Select("receipt_json").
From("blockchain_data_receipts").
Where(sq.Eq{"chain_id": chainID, "transaction_hash": transactionHash})
query, args, err := q.ToSql()
if err != nil {
return nil, err
}
receiptJSON := json.RawMessage{}
err = b.db.QueryRow(query, args...).Scan(&receiptJSON)
if err != nil {
return nil, err
}
return receiptJSON, nil
}
func (b *DB) PutBlockJSON(chainID uint64, blkJSON json.RawMessage, transactionDetailsFlag bool) (err error) {
var tx *sql.Tx
tx, err = b.db.Begin()
if err != nil {
return err
}
defer func() {
if err == nil {
err = tx.Commit()
return
}
_ = tx.Rollback()
}()
err = putBlockJSON(tx, chainID, blkJSON, transactionDetailsFlag)
return
}
func (b *DB) PutBlockUnclesJSON(chainID uint64, blockHash common.Hash, unclesJSON []json.RawMessage) (err error) {
var tx *sql.Tx
tx, err = b.db.Begin()
if err != nil {
return err
}
defer func() {
if err == nil {
err = tx.Commit()
return
}
_ = tx.Rollback()
}()
for index, uncleJSON := range unclesJSON {
err = putBlockUncleJSON(tx, chainID, blockHash, uint64(index), uncleJSON)
if err != nil {
return
}
}
return
}
func (b *DB) PutTransactionsJSON(chainID uint64, transactionsJSON []json.RawMessage) (err error) {
var tx *sql.Tx
tx, err = b.db.Begin()
if err != nil {
return
}
defer func() {
if err == nil {
err = tx.Commit()
return
}
_ = tx.Rollback()
}()
for _, transactionJSON := range transactionsJSON {
err = putTransactionJSON(tx, chainID, transactionJSON)
if err != nil {
return
}
}
return
}
func (b *DB) PutTransactionReceiptsJSON(chainID uint64, receiptsJSON []json.RawMessage) (err error) {
var tx *sql.Tx
tx, err = b.db.Begin()
if err != nil {
return
}
defer func() {
if err == nil {
err = tx.Commit()
return
}
_ = tx.Rollback()
}()
for _, receiptJSON := range receiptsJSON {
err = putReceiptJSON(tx, chainID, receiptJSON)
if err != nil {
return
}
}
return
}
func putBlockJSON(creator sqlite.StatementCreator, chainID uint64, blkJSON json.RawMessage, transactionDetailsFlag bool) error {
var rpcBlock rpcBlock
if err := json.Unmarshal(blkJSON, &rpcBlock); err != nil {
return err
}
if rpcBlock.Number == nil {
// Pending block, don't store
return nil
}
q := sq.Replace("blockchain_data_blocks").
SetMap(sq.Eq{"chain_id": chainID, "block_number": (*bigint.SQLBigInt)(rpcBlock.Number.ToInt()), "block_hash": rpcBlock.Hash, "with_transaction_details": transactionDetailsFlag,
"block_json": blkJSON,
})
query, args, err := q.ToSql()
if err != nil {
return err
}
stmt, err := creator.Prepare(query)
if err != nil {
return err
}
defer stmt.Close()
_, err = stmt.Exec(args...)
return err
}
func putBlockUncleJSON(creator sqlite.StatementCreator, chainID uint64, blockHash common.Hash, index uint64, uncleJSON json.RawMessage) error {
q := sq.Replace("blockchain_data_block_uncles").
SetMap(sq.Eq{"chain_id": chainID, "block_hash": blockHash, "uncle_index": index,
"block_uncle_json": uncleJSON,
})
query, args, err := q.ToSql()
if err != nil {
return err
}
stmt, err := creator.Prepare(query)
if err != nil {
return err
}
defer stmt.Close()
_, err = stmt.Exec(args...)
return err
}
func putTransactionJSON(creator sqlite.StatementCreator, chainID uint64, txJSON json.RawMessage) error {
var rpcTransaction rpcTransaction
if err := json.Unmarshal(txJSON, &rpcTransaction); err != nil {
return err
}
if rpcTransaction.BlockNumber == nil {
// Pending transaction, don't store
return nil
}
q := sq.Replace("blockchain_data_transactions").
SetMap(sq.Eq{"chain_id": chainID, "transaction_hash": rpcTransaction.tx.Hash(),
"transaction_json": txJSON,
})
query, args, err := q.ToSql()
if err != nil {
return err
}
stmt, err := creator.Prepare(query)
if err != nil {
return err
}
defer stmt.Close()
_, err = stmt.Exec(args...)
return err
}
func putReceiptJSON(creator sqlite.StatementCreator, chainID uint64, receiptJSON json.RawMessage) error {
var receipt types.Receipt
if err := json.Unmarshal(receiptJSON, &receipt); err != nil {
return err
}
q := sq.Replace("blockchain_data_receipts").
SetMap(sq.Eq{"chain_id": chainID, "transaction_hash": receipt.TxHash,
"receipt_json": receiptJSON,
})
query, args, err := q.ToSql()
if err != nil {
return err
}
stmt, err := creator.Prepare(query)
if err != nil {
return err
}
defer stmt.Close()
_, err = stmt.Exec(args...)
return err
}

View File

@ -0,0 +1,76 @@
package ethclient
import (
"encoding/json"
"math/big"
"github.com/ethereum/go-ethereum/common"
)
type EthClientChainStorageReader interface {
GetBlockJSONByNumber(blockNumber *big.Int, withTransactionDetails bool) (json.RawMessage, error)
GetBlockJSONByHash(blockHash common.Hash, withTransactionDetails bool) (json.RawMessage, error)
GetBlockUncleJSONByHashAndIndex(blockHash common.Hash, index uint64) (json.RawMessage, error)
GetTransactionJSONByHash(transactionHash common.Hash) (json.RawMessage, error)
GetTransactionReceiptJSONByHash(transactionHash common.Hash) (json.RawMessage, error)
}
type EthClientChainStorageWriter interface {
PutBlockJSON(blkJSON json.RawMessage, transactionDetailsFlag bool) error
PutBlockUnclesJSON(blockHash common.Hash, unclesJSON []json.RawMessage) error
PutTransactionsJSON(transactionsJSON []json.RawMessage) error
PutTransactionReceiptsJSON(receiptsJSON []json.RawMessage) error
}
type EthClientChainStorage interface {
EthClientChainStorageReader
EthClientChainStorageWriter
}
type DBChain struct {
s EthClientStorage
chainID uint64
}
func NewDBChain(s EthClientStorage, chainID uint64) *DBChain {
return &DBChain{
s: s,
chainID: chainID,
}
}
func (b *DBChain) GetBlockJSONByNumber(blockNumber *big.Int, withTransactionDetails bool) (json.RawMessage, error) {
return b.s.GetBlockJSONByNumber(b.chainID, blockNumber, withTransactionDetails)
}
func (b *DBChain) GetBlockJSONByHash(blockHash common.Hash, withTransactionDetails bool) (json.RawMessage, error) {
return b.s.GetBlockJSONByHash(b.chainID, blockHash, withTransactionDetails)
}
func (b *DBChain) GetBlockUncleJSONByHashAndIndex(blockHash common.Hash, index uint64) (json.RawMessage, error) {
return b.s.GetBlockUncleJSONByHashAndIndex(b.chainID, blockHash, index)
}
func (b *DBChain) GetTransactionJSONByHash(transactionHash common.Hash) (json.RawMessage, error) {
return b.s.GetTransactionJSONByHash(b.chainID, transactionHash)
}
func (b *DBChain) GetTransactionReceiptJSONByHash(transactionHash common.Hash) (json.RawMessage, error) {
return b.s.GetTransactionReceiptJSONByHash(b.chainID, transactionHash)
}
func (b *DBChain) PutBlockJSON(blkJSON json.RawMessage, transactionDetailsFlag bool) error {
return b.s.PutBlockJSON(b.chainID, blkJSON, transactionDetailsFlag)
}
func (b *DBChain) PutBlockUnclesJSON(blockHash common.Hash, unclesJSON []json.RawMessage) error {
return b.s.PutBlockUnclesJSON(b.chainID, blockHash, unclesJSON)
}
func (b *DBChain) PutTransactionsJSON(transactionsJSON []json.RawMessage) error {
return b.s.PutTransactionsJSON(b.chainID, transactionsJSON)
}
func (b *DBChain) PutTransactionReceiptsJSON(receiptsJSON []json.RawMessage) error {
return b.s.PutTransactionReceiptsJSON(b.chainID, receiptsJSON)
}

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,92 @@
package ethclient_test
import (
"encoding/json"
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/status-im/status-go/rpc/chain/ethclient"
"github.com/status-im/status-go/t/helpers"
"github.com/status-im/status-go/walletdatabase"
"github.com/stretchr/testify/require"
)
func setupDBTest(t *testing.T) (*ethclient.DBChain, func()) {
db, err := helpers.SetupTestMemorySQLDB(walletdatabase.DbInitializer{})
require.NoError(t, err)
return ethclient.NewDBChain(ethclient.NewDB(db), 1), func() {
require.NoError(t, db.Close())
}
}
func TestPutBlock(t *testing.T) {
db, cleanup := setupDBTest(t)
defer cleanup()
blkJSON, blkNumber, blkHash := getTestBlockJSONWithoutTxDetails()
err := db.PutBlockJSON(blkJSON, false)
require.NoError(t, err)
retrievedBlkJSON, err := db.GetBlockJSONByNumber(blkNumber, false)
require.NoError(t, err)
require.Equal(t, blkJSON, retrievedBlkJSON)
retrievedBlkJSON, err = db.GetBlockJSONByHash(blkHash, false)
require.NoError(t, err)
require.Equal(t, blkJSON, retrievedBlkJSON)
blkJSON, blkNumber, blkHash = getTestBlockJSONWithTxDetails()
err = db.PutBlockJSON(blkJSON, true)
require.NoError(t, err)
retrievedBlkJSON, err = db.GetBlockJSONByNumber(blkNumber, true)
require.NoError(t, err)
require.Equal(t, blkJSON, retrievedBlkJSON)
retrievedBlkJSON, err = db.GetBlockJSONByHash(blkHash, true)
require.NoError(t, err)
require.Equal(t, blkJSON, retrievedBlkJSON)
}
func TestPutBlockUncles(t *testing.T) {
db, cleanup := setupDBTest(t)
defer cleanup()
blkHash := common.HexToHash("0x1234")
uncles := getTestBlockUnclesJSON()
err := db.PutBlockUnclesJSON(blkHash, uncles)
require.NoError(t, err)
retrievedUncles, err := db.GetBlockUncleJSONByHashAndIndex(blkHash, 0)
require.NoError(t, err)
require.Equal(t, uncles[0], retrievedUncles)
}
func TestPutTransactions(t *testing.T) {
db, cleanup := setupDBTest(t)
defer cleanup()
txJSON, txHash := getTestTransactionJSON()
err := db.PutTransactionsJSON([]json.RawMessage{txJSON})
require.NoError(t, err)
retrievedTxJSON, err := db.GetTransactionJSONByHash(txHash)
require.NoError(t, err)
require.Equal(t, txJSON, retrievedTxJSON)
}
func TestPutTransactionReceipts(t *testing.T) {
db, cleanup := setupDBTest(t)
defer cleanup()
receiptJSON, txHash := getTestReceiptJSON()
err := db.PutTransactionReceiptsJSON([]json.RawMessage{receiptJSON})
require.NoError(t, err)
retrievedReceiptJSON, err := db.GetTransactionReceiptJSONByHash(txHash)
require.NoError(t, err)
require.Equal(t, receiptJSON, retrievedReceiptJSON)
}

File diff suppressed because one or more lines are too long

View File

@ -1,4 +1,4 @@
package ethclient
package ethclient_test
import (
"testing"
@ -7,7 +7,7 @@ import (
)
func TestGeth_HeaderHash(t *testing.T) {
number, hash, header := getTestBlockHeader()
header, number, hash := getTestBlockHeader()
require.Equal(t, number.String(), header.Number.String())
require.Equal(t, hash, header.Hash())
}

View File

@ -0,0 +1,33 @@
package ethclient
import (
"encoding/json"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core/types"
)
type rpcBlock struct {
Hash common.Hash `json:"hash"`
Number *hexutil.Big `json:"number,omitempty"`
UncleHashes []common.Hash `json:"uncles"`
}
type rpcTransaction struct {
tx *types.Transaction
txExtraInfo
}
func (tx *rpcTransaction) UnmarshalJSON(msg []byte) error {
if err := json.Unmarshal(msg, &tx.tx); err != nil {
return err
}
return json.Unmarshal(msg, &tx.txExtraInfo)
}
type txExtraInfo struct {
BlockNumber *string `json:"blockNumber,omitempty"`
BlockHash *common.Hash `json:"blockHash,omitempty"`
From *common.Address `json:"from,omitempty"`
}

View File

@ -16,6 +16,7 @@ import (
"sync"
"time"
eth_common "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
gethrpc "github.com/ethereum/go-ethereum/rpc"
@ -110,6 +111,8 @@ type Client struct {
walletNotifier func(chainID uint64, message string)
providerConfigs []params.ProviderConfig
db *sql.DB
}
// Is initialized in a build-tag-dependent module
@ -142,6 +145,7 @@ func NewClient(client *gethrpc.Client, upstreamChainID uint64, upstream params.U
limiterPerProvider: make(map[string]*rpclimiter.RPCRpsLimiter),
log: log,
providerConfigs: providerConfigs,
db: db,
}
var opts []gethrpc.ClientOption
@ -172,7 +176,7 @@ func NewClient(client *gethrpc.Client, upstreamChainID uint64, upstream params.U
rpcName := fmt.Sprintf("%s-chain-id-%d", hostPortUpstream, upstreamChainID)
ethClients := []ethclient.RPSLimitedEthClientInterface{
ethclient.NewRPSLimitedEthClient(upstreamClient, limiter, rpcName),
buildEthClient(upstreamClient, limiter, rpcName, c.UpstreamChainID, c.db),
}
c.upstream = chain.NewClient(ethClients, upstreamChainID)
}
@ -321,7 +325,7 @@ func (c *Client) getEthClients(network *params.Network) []ethclient.RPSLimitedEt
c.log.Error("get RPC limiter "+key, "error", err)
}
ethClients = append(ethClients, ethclient.NewRPSLimitedEthClient(rpcClient, rpcLimiter, circuitKey))
ethClients = append(ethClients, buildEthClient(rpcClient, rpcLimiter, circuitKey, network.ChainID, c.db))
}
}
@ -389,7 +393,7 @@ func (c *Client) UpdateUpstreamURL(url string) error {
}
ethClients := []ethclient.RPSLimitedEthClientInterface{
ethclient.NewRPSLimitedEthClient(rpcClient, rpsLimiter, hostPortUpstream),
buildEthClient(rpcClient, rpsLimiter, hostPortUpstream, c.UpstreamChainID, c.db),
}
c.upstream = chain.NewClient(ethClients, c.UpstreamChainID)
c.upstreamURL = url
@ -541,3 +545,16 @@ func setResultFromRPCResponse(result, response interface{}) (err error) {
return nil
}
func buildEthClient(rpcClient *gethrpc.Client, limiter *rpclimiter.RPCRpsLimiter, name string, chainID uint64, db *sql.DB) ethclient.RPSLimitedEthClientInterface {
ethClient := ethclient.NewRPSLimitedEthClient(rpcClient, limiter, name)
//return ethclient.NewCachedEthClient(ethClient, ethClientStorage)
// Test
ethClientStorage := ethclient.NewDBChain(ethclient.NewDB(db), chainID)
_, err := ethClientStorage.GetTransactionJSONByHash(eth_common.HexToHash("0x1"))
if err != nil {
log.Error("GetTransactionJSONByHash", "error", err)
}
return ethClient
}

View File

@ -0,0 +1,47 @@
-- store raw blocks
CREATE TABLE IF NOT EXISTS blockchain_data_blocks (
chain_id UNSIGNED BIGINT NOT NULL,
block_number BLOB NOT NULL,
block_hash BLOB NOT NULL,
with_transaction_details BOOLEAN NOT NULL,
block_json JSON NOT NULL,
CONSTRAINT unique_block_per_chain_per_block_number UNIQUE (chain_id,block_number,with_transaction_details) ON CONFLICT REPLACE,
CONSTRAINT unique_block_per_chain_per_block_hash UNIQUE (chain_id,block_hash,with_transaction_details) ON CONFLICT REPLACE
);
CREATE INDEX IF NOT EXISTS idx_blockchain_data_blocks_chain_id_block_number ON blockchain_data_blocks (chain_id, block_number, with_transaction_details);
CREATE INDEX IF NOT EXISTS idx_blockchain_data_blocks_chain_id_block_hash ON blockchain_data_blocks (chain_id, block_hash, with_transaction_details);
-- store raw block uncles
CREATE TABLE IF NOT EXISTS blockchain_data_block_uncles (
chain_id UNSIGNED BIGINT NOT NULL,
block_hash BLOB NOT NULL,
uncle_index UNSIGNED BIGINT NOT NULL,
block_uncle_json JSON,
PRIMARY KEY (chain_id, block_hash, uncle_index),
CONSTRAINT unique_block_uncles_per_chain_per_block_hash_per_index UNIQUE (chain_id,block_hash,uncle_index) ON CONFLICT REPLACE
) WITHOUT ROWID;
CREATE INDEX IF NOT EXISTS idx_blockchain_data_block_uncles_chain_id_block_hash_uncle_index ON blockchain_data_block_uncles (chain_id, block_hash, uncle_index);
-- store raw transactions
CREATE TABLE IF NOT EXISTS blockchain_data_transactions (
chain_id UNSIGNED BIGINT NOT NULL,
transaction_hash BLOB NOT NULL,
transaction_json JSON NOT NULL,
PRIMARY KEY (chain_id, transaction_hash),
CONSTRAINT unique_transaction_per_chain_per_transaction_hash UNIQUE (chain_id, transaction_hash) ON CONFLICT REPLACE
) WITHOUT ROWID;
CREATE INDEX IF NOT EXISTS idx_blockchain_data_transactions_chain_id_transaction_hash ON blockchain_data_transactions (chain_id, transaction_hash);
-- store raw transaction receipts
CREATE TABLE IF NOT EXISTS blockchain_data_receipts (
chain_id UNSIGNED BIGINT NOT NULL,
transaction_hash BLOB NOT NULL,
receipt_json JSON NOT NULL,
PRIMARY KEY (chain_id, transaction_hash),
CONSTRAINT unique_receipt_per_chain_per_transaction_hash UNIQUE (chain_id, transaction_hash) ON CONFLICT REPLACE
) WITHOUT ROWID;
CREATE INDEX IF NOT EXISTS idx_blockchain_data_receipts_chain_id_transaction_hash ON blockchain_data_receipts (chain_id, transaction_hash);