feat: config optimisation

This commit is contained in:
Anthony Laibe 2023-03-28 14:46:46 +02:00 committed by Anthony Laibe
parent 91c6949cd2
commit 60dbc5d44d
5 changed files with 11 additions and 8 deletions

View File

@ -54,7 +54,7 @@ func NewSimpleClient(main *rpc.Client, chainID uint64) *ClientWithFallback {
func NewClient(main, fallback *rpc.Client, chainID uint64) *ClientWithFallback { func NewClient(main, fallback *rpc.Client, chainID uint64) *ClientWithFallback {
hystrix.ConfigureCommand(fmt.Sprintf("ethClient_%d", chainID), hystrix.CommandConfig{ hystrix.ConfigureCommand(fmt.Sprintf("ethClient_%d", chainID), hystrix.CommandConfig{
Timeout: 10000, Timeout: 20000,
MaxConcurrentRequests: 100, MaxConcurrentRequests: 100,
SleepWindow: 300000, SleepWindow: 300000,
ErrorPercentThreshold: 25, ErrorPercentThreshold: 25,

View File

@ -504,7 +504,7 @@ func (tm *Manager) GetBalances(parent context.Context, clients []*chain.ClientWi
if err == nil { if err == nil {
fetchChainBalance := false fetchChainBalance := false
var tokenChunks [][]common.Address var tokenChunks [][]common.Address
chunkSize := 100 chunkSize := 500
for i := 0; i < len(tokens); i += chunkSize { for i := 0; i < len(tokens); i += chunkSize {
end := i + chunkSize end := i + chunkSize
if end > len(tokens) { if end > len(tokens) {
@ -638,7 +638,7 @@ func (tm *Manager) GetBalancesByChain(parent context.Context, clients []*chain.C
if err == nil { if err == nil {
fetchChainBalance := false fetchChainBalance := false
var tokenChunks [][]common.Address var tokenChunks [][]common.Address
chunkSize := 100 chunkSize := 500
for i := 0; i < len(tokens); i += chunkSize { for i := 0; i < len(tokens); i += chunkSize {
end := i + chunkSize end := i + chunkSize
if end > len(tokens) { if end > len(tokens) {

View File

@ -35,9 +35,11 @@ var (
binanceChainMaxInitialRange = big.NewInt(500000) binanceChainMaxInitialRange = big.NewInt(500000)
binanceChainErc20BatchSize = big.NewInt(5000) binanceChainErc20BatchSize = big.NewInt(5000)
goerliErc20BatchSize = big.NewInt(100000) goerliErc20BatchSize = big.NewInt(100000)
goerliErc20ArbitrumBatchSize = big.NewInt(100000)
erc20BatchSize = big.NewInt(500000) erc20BatchSize = big.NewInt(500000)
binancChainID = uint64(56) binancChainID = uint64(56)
goerliChainID = uint64(5) goerliChainID = uint64(5)
goerliArbitrumChainID = uint64(421613)
binanceTestChainID = uint64(97) binanceTestChainID = uint64(97)
numberOfBlocksCheckedPerIteration = 40 numberOfBlocksCheckedPerIteration = 40
) )
@ -122,6 +124,10 @@ func getErc20BatchSize(chainID uint64) *big.Int {
return goerliErc20BatchSize return goerliErc20BatchSize
} }
if chainID == goerliArbitrumChainID {
return goerliErc20ArbitrumBatchSize
}
return erc20BatchSize return erc20BatchSize
} }

View File

@ -451,24 +451,21 @@ func (d *ERC20TransfersDownloader) GetHeadersInRange(parent context.Context, fro
start := time.Now() start := time.Now()
log.Debug("get erc20 transfers in range", "from", from, "to", to) log.Debug("get erc20 transfers in range", "from", from, "to", to)
headers := []*DBHeader{} headers := []*DBHeader{}
ctx := context.Background()
for _, address := range d.accounts { for _, address := range d.accounts {
ctx, cancel := context.WithTimeout(parent, 5*time.Second)
outbound, err := d.client.FilterLogs(ctx, ethereum.FilterQuery{ outbound, err := d.client.FilterLogs(ctx, ethereum.FilterQuery{
FromBlock: from, FromBlock: from,
ToBlock: to, ToBlock: to,
Topics: d.outboundTopics(address), Topics: d.outboundTopics(address),
}) })
cancel()
if err != nil { if err != nil {
return nil, err return nil, err
} }
ctx, cancel = context.WithTimeout(parent, 5*time.Second)
inbound, err := d.client.FilterLogs(ctx, ethereum.FilterQuery{ inbound, err := d.client.FilterLogs(ctx, ethereum.FilterQuery{
FromBlock: from, FromBlock: from,
ToBlock: to, ToBlock: to,
Topics: d.inboundTopics(address), Topics: d.inboundTopics(address),
}) })
cancel()
if err != nil { if err != nil {
return nil, err return nil, err
} }

View File

@ -64,8 +64,8 @@ func (d *IterativeDownloader) Next(parent context.Context) ([]*DBHeader, *big.In
if from.Cmp(d.from) == -1 { if from.Cmp(d.from) == -1 {
from = d.from from = d.from
} }
log.Info("load erc20 transfers in range", "from", from, "to", to, "batchSize", d.batchSize)
headers, err := d.downloader.GetHeadersInRange(parent, from, to) headers, err := d.downloader.GetHeadersInRange(parent, from, to)
log.Info("load erc20 transfers in range", "from", from, "to", to, "batchSize", d.batchSize)
if err != nil { if err != nil {
log.Error("failed to get transfer in between two bloks", "from", from, "to", to, "error", err) log.Error("failed to get transfer in between two bloks", "from", from, "to", to, "error", err)
return nil, nil, nil, err return nil, nil, nil, err