From d82c50b50a6cdd9ca8e97ff10b05bc018f7be1da Mon Sep 17 00:00:00 2001 From: Roman Volosovskyi Date: Thu, 5 Oct 2023 14:35:16 +0200 Subject: [PATCH] [#3930] Prevent repeated eth_getLogs calls (ERC20 history tail) --- .../wallet/transfer/commands_sequential.go | 43 +++++++++++++------ .../transfer/commands_sequential_test.go | 35 +++++++++++++-- 2 files changed, 63 insertions(+), 15 deletions(-) diff --git a/services/wallet/transfer/commands_sequential.go b/services/wallet/transfer/commands_sequential.go index 95b929314..998606e59 100644 --- a/services/wallet/transfer/commands_sequential.go +++ b/services/wallet/transfer/commands_sequential.go @@ -94,11 +94,16 @@ func (c *findBlocksCommand) Command() async.Command { }.Run } -func (c *findBlocksCommand) ERC20ScanByBalance(parent context.Context, fromBlock, toBlock *big.Int, token common.Address) ([]*DBHeader, error) { +type ERC20BlockRange struct { + from *big.Int + to *big.Int +} + +func (c *findBlocksCommand) ERC20ScanByBalance(parent context.Context, fromBlock, toBlock *big.Int, token common.Address) ([]ERC20BlockRange, error) { var err error batchSize := getErc20BatchSize(c.chainClient.NetworkID()) ranges := [][]*big.Int{{fromBlock, toBlock}} - foundHeaders := []*DBHeader{} + foundRanges := []ERC20BlockRange{} cache := map[int64]*big.Int{} for { nextRanges := [][]*big.Int{} @@ -132,12 +137,7 @@ func (c *findBlocksCommand) ERC20ScanByBalance(parent context.Context, fromBlock if fromBalance.Cmp(toBalance) != 0 { diff := new(big.Int).Sub(to, from) if diff.Cmp(batchSize) <= 0 { - headers, err := c.fastIndexErc20(parent, from, to, true) - if err != nil { - return nil, err - } - foundHeaders = append(foundHeaders, headers...) - + foundRanges = append(foundRanges, ERC20BlockRange{from, to}) continue } @@ -156,7 +156,7 @@ func (c *findBlocksCommand) ERC20ScanByBalance(parent context.Context, fromBlock ranges = nextRanges } - return foundHeaders, nil + return foundRanges, nil } func (c *findBlocksCommand) checkERC20Tail(parent context.Context) ([]*DBHeader, error) { @@ -181,7 +181,7 @@ func (c *findBlocksCommand) checkERC20Tail(parent context.Context) ([]*DBHeader, return nil, err } - headers := []*DBHeader{} + foundRanges := []ERC20BlockRange{} for token, balance := range balances[c.chainClient.NetworkID()][c.account] { bigintBalance := big.NewInt(balance.ToInt().Int64()) if bigintBalance.Cmp(big.NewInt(0)) <= 0 { @@ -192,10 +192,29 @@ func (c *findBlocksCommand) checkERC20Tail(parent context.Context) ([]*DBHeader, return nil, err } - headers = append(headers, result...) + foundRanges = append(foundRanges, result...) } - return headers, nil + uniqRanges := []ERC20BlockRange{} + rangesMap := map[string]bool{} + for _, rangeItem := range foundRanges { + key := rangeItem.from.String() + "-" + rangeItem.to.String() + if _, ok := rangesMap[key]; !ok { + rangesMap[key] = true + uniqRanges = append(uniqRanges, rangeItem) + } + } + + foundHeaders := []*DBHeader{} + for _, rangeItem := range uniqRanges { + headers, err := c.fastIndexErc20(parent, rangeItem.from, rangeItem.to, true) + if err != nil { + return nil, err + } + foundHeaders = append(foundHeaders, headers...) + } + + return foundHeaders, nil } func (c *findBlocksCommand) Run(parent context.Context) (err error) { diff --git a/services/wallet/transfer/commands_sequential_test.go b/services/wallet/transfer/commands_sequential_test.go index 35fd32ee7..45f3ed719 100644 --- a/services/wallet/transfer/commands_sequential_test.go +++ b/services/wallet/transfer/commands_sequential_test.go @@ -153,6 +153,9 @@ func (tc *TestClient) BalanceAt(ctx context.Context, account common.Address, blo func (tc *TestClient) tokenBalanceAt(token common.Address, blockNumber *big.Int) *big.Int { balance := tc.tokenBalanceHistory[token][blockNumber.Uint64()] + if balance == nil { + balance = big.NewInt(0) + } if tc.traceAPICalls { tc.t.Log("tokenBalanceAt", token, blockNumber, "result:", balance) @@ -266,7 +269,7 @@ func (tc *TestClient) CallContract(ctx context.Context, call ethereum.CallMsg, b return output, nil } - if *call.To == tokenTXXAddress || *call.To == tokenTXZAddress { + if *call.To == tokenTXXAddress || *call.To == tokenTXYAddress { balance := tc.tokenBalanceAt(*call.To, blockNumber) parsed, err := abi.JSON(strings.NewReader(ierc20.IERC20ABI)) @@ -653,6 +656,23 @@ func getCases() []findBlockCase { }, } + case9emptyHistoryWithERC20Transfers := findBlockCase{ + balanceChanges: [][]int{}, + toBlock: 100, + rangeSize: 20, + // we expect only a single eth_getLogs to be executed here for both erc20 transfers, + // thus only 2 blocks found + expectedBlocksFound: 2, + incomingERC20Transfers: []testERC20Transfer{ + {big.NewInt(7), tokenTXYAddress, big.NewInt(1)}, + {big.NewInt(6), tokenTXXAddress, big.NewInt(1)}, + }, + expectedCalls: map[string]int{ + "FilterLogs": 3, + "CallContract": 5, + }, + } + cases = append(cases, case1) cases = append(cases, case100transfers) cases = append(cases, case3) @@ -662,14 +682,15 @@ func getCases() []findBlockCase { cases = append(cases, case6) cases = append(cases, case7emptyHistoryWithOneERC20Transfer) cases = append(cases, case8emptyHistoryWithERC20Transfers) + cases = append(cases, case9emptyHistoryWithERC20Transfers) - //cases = append([]findBlockCase{}, case1) + //cases = append([]findBlockCase{}, case9emptyHistoryWithERC20Transfers) return cases } var tokenTXXAddress = common.HexToAddress("0x53211") -var tokenTXZAddress = common.HexToAddress("0x73211") +var tokenTXYAddress = common.HexToAddress("0x73211") func TestFindBlocksCommand(t *testing.T) { for idx, testCase := range getCases() { @@ -710,6 +731,14 @@ func TestFindBlocksCommand(t *testing.T) { Name: "Test Token 1", Verified: true, }, + { + Address: tokenTXYAddress, + Symbol: "TXY", + Decimals: 18, + ChainID: tc.NetworkID(), + Name: "Test Token 2", + Verified: true, + }, }) fbc := &findBlocksCommand{ account: common.HexToAddress("0x1234"),