2021-09-09 14:28:54 +00:00
package transfer
2019-06-14 10:16:30 +00:00
import (
"context"
2023-09-06 19:15:07 +00:00
"database/sql"
2019-06-14 10:16:30 +00:00
"math/big"
"time"
2023-11-27 10:08:17 +00:00
"golang.org/x/exp/maps"
2019-06-14 10:16:30 +00:00
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/log"
2023-05-19 15:31:45 +00:00
2023-02-20 09:32:45 +00:00
"github.com/status-im/status-go/rpc/chain"
2021-09-09 14:28:54 +00:00
"github.com/status-im/status-go/services/wallet/async"
2023-09-04 05:34:09 +00:00
"github.com/status-im/status-go/services/wallet/balance"
2023-05-19 15:31:45 +00:00
w_common "github.com/status-im/status-go/services/wallet/common"
2023-06-02 20:08:45 +00:00
"github.com/status-im/status-go/services/wallet/token"
2022-11-29 13:43:18 +00:00
"github.com/status-im/status-go/services/wallet/walletevent"
2023-06-21 14:09:55 +00:00
"github.com/status-im/status-go/transactions"
2022-11-29 13:43:18 +00:00
)
const (
// EventNewTransfers emitted when new block was added to the same canonical chan.
EventNewTransfers walletevent . EventType = "new-transfers"
// EventFetchingRecentHistory emitted when fetching of lastest tx history is started
EventFetchingRecentHistory walletevent . EventType = "recent-history-fetching"
// EventRecentHistoryReady emitted when fetching of lastest tx history is started
EventRecentHistoryReady walletevent . EventType = "recent-history-ready"
// EventFetchingHistoryError emitted when fetching of tx history failed
EventFetchingHistoryError walletevent . EventType = "fetching-history-error"
// EventNonArchivalNodeDetected emitted when a connection to a non archival node is detected
EventNonArchivalNodeDetected walletevent . EventType = "non-archival-node-detected"
2023-05-19 08:19:48 +00:00
2023-11-27 21:28:16 +00:00
// Internal events emitted when different kinds of transfers are detected
EventInternalETHTransferDetected walletevent . EventType = walletevent . InternalEventTypePrefix + "eth-transfer-detected"
EventInternalERC20TransferDetected walletevent . EventType = walletevent . InternalEventTypePrefix + "erc20-transfer-detected"
EventInternalERC721TransferDetected walletevent . EventType = walletevent . InternalEventTypePrefix + "erc721-transfer-detected"
EventInternalERC1155TransferDetected walletevent . EventType = walletevent . InternalEventTypePrefix + "erc1155-transfer-detected"
2023-10-09 12:43:53 +00:00
2023-05-19 08:19:48 +00:00
numberOfBlocksCheckedPerIteration = 40
noBlockLimit = 0
2019-06-14 10:16:30 +00:00
)
2021-11-24 12:59:45 +00:00
var (
// This will work only for binance testnet as mainnet doesn't support
// archival request.
2023-05-19 08:19:48 +00:00
binanceChainErc20BatchSize = big . NewInt ( 5000 )
goerliErc20BatchSize = big . NewInt ( 100000 )
2023-08-01 10:56:52 +00:00
goerliErc20ArbitrumBatchSize = big . NewInt ( 10000 )
goerliErc20OptimismBatchSize = big . NewInt ( 10000 )
2023-05-19 08:19:48 +00:00
erc20BatchSize = big . NewInt ( 500000 )
binancChainID = uint64 ( 56 )
goerliChainID = uint64 ( 5 )
goerliArbitrumChainID = uint64 ( 421613 )
2023-08-01 10:56:52 +00:00
goerliOptimismChainID = uint64 ( 420 )
2023-05-19 08:19:48 +00:00
binanceTestChainID = uint64 ( 97 )
2024-01-19 12:43:25 +00:00
transfersRetryInterval = 5 * time . Second
2021-11-24 12:59:45 +00:00
)
status-im/status-react#9203 Faster tx fetching with less request
*** How it worked before this PR on multiaccount creation:
- On multiacc creation we scanned chain for eth and erc20 transfers. For
each address of a new empty multiaccount this scan required
1. two `eth_getBalance` requests to find out that there is no any
balance change between zero and the last block, for eth transfers
2. and `chain-size/100000` (currently ~100) `eth_getLogs` requests,
for erc20 transfers
- For some reason we scanned an address of the chat account as well, and
also accounts were not deduplicated. So even for an empty multiacc we
scanned chain twice for each chat and main wallet addresses, in result
app had to execute about 400 requests.
- As mentioned above, `eth_getBalance` requests were used to check if
there were any eth transfers, and that caused empty history in case
if user already used all available eth (so that both zero and latest
blocks show 0 eth for an address). There might have been transactions
but we wouldn't fetch/show them.
- There was no upper limit for the number of rpc requests during the
scan, so it could require indefinite number of requests; the scanning
algorithm was written so that we persisted the whole history of
transactions or tried to scan form the beginning again in case of
failure, giving up only after 10 minutes of failures. In result
addresses with sufficient number of transactions would never be fully
scanned and during these 10 minutes app could use gigabytes of
internet data.
- Failures were caused by `eth_getBlockByNumber`/`eth_getBlockByHash`
requests. These requests return significantly bigger responses than
`eth_getBalance`/`eth_transactionsCount` and it is likely that
execution of thousands of them in parallel caused failures for
accounts with hundreds of transactions. Even for an account with 12k
we could successfully determine blocks with transaction in a few
minutes using `eth_getBalance` requests, but `eth_getBlock...`
couldn't be processed for this acc.
- There was no caching for for `eth_getBalance` requests, and this
caused in average 3-4 times more such requests than is needed.
*** How it works now on multiaccount creation:
- On multiacc creation we scan chain for last ~30 eth transactions and
then check erc20 in the range where these eth transactions were found.
For an empty address in multiacc this means:
1. two `eth_getBalance` transactions to determine that there was no
balance change between zero and the last block; two
`eth_transactionsCount` requests to determine there are no outgoing
transactions for this address; total 4 requests for eth transfers
2. 20 `eth_getLogs` for erc20 transfers. This number can be lowered,
but that's not a big deal
- Deduplication of addresses is added and also we don't scan chat
account, so a new multiacc requires ~25 (we also request latest block
number and probably execute a few other calls) request to determine
that multiacc is empty (comparing to ~400 before)
- In case if address contains transactions we:
1. determine the range which contains 20-25 outgoing eth/erc20
transactions. This usually requires up to 10 `eth_transactionCount`
requests
2. then we scan chain for eth transfers using `eth_getBalance` and
`eth_transactionCount` (for double checking zero balances)
3. we make sure that we do not scan db for more than 30 blocks with
transfers. That's important for accounts with mostly incoming
transactions, because the range found on the first step might
contain any number of incoming transfers, but only 20-25 outgoing
transactions
4. when we found ~30 blocks in a given range, we update initial
range `from` block using the oldest found block
5. and now we scan db for erc20transfers using `eth_getLogs`
`oldest-found-eth-block`-`latest-block`, we make not more than 20 calls
6. when all blocks which contain incoming/outgoing transfers for a
given address are found, we save these blocks to db and mark that
transfers from these blocks are still to be fetched
7. Then we select latest ~30 (the number can be adjusted) blocks from
these which were found and fetch transfers, this requires 3-4
requests per transfer.
8. we persist scanned range so that we know were to start next time
9. we dispatch an event which tells client that transactions are found
10. client fetches latest 20 transfers
- when user presses "fetch more" button we check if app's db contains next
20 transfers, if not we scan chain again and return transfers after
small fixes
2019-12-18 11:01:46 +00:00
2019-06-14 10:16:30 +00:00
type ethHistoricalCommand struct {
2023-09-04 05:34:09 +00:00
address common . Address
2023-09-19 11:17:36 +00:00
chainClient chain . ClientInterface
2023-09-04 05:34:09 +00:00
balanceCacher balance . Cacher
feed * event . Feed
foundHeaders [ ] * DBHeader
error error
noLimit bool
2019-06-14 10:16:30 +00:00
2023-05-19 08:19:48 +00:00
from * Block
to , resultingFrom , startBlock * big . Int
threadLimit uint32
2019-06-14 10:16:30 +00:00
}
2023-06-13 14:06:36 +00:00
type Transaction [ ] * Transfer
2021-09-09 14:28:54 +00:00
func ( c * ethHistoricalCommand ) Command ( ) async . Command {
return async . FiniteCommand {
2019-06-14 10:16:30 +00:00
Interval : 5 * time . Second ,
Runable : c . Run ,
} . Run
}
func ( c * ethHistoricalCommand ) Run ( ctx context . Context ) ( err error ) {
2023-09-20 08:41:23 +00:00
log . Debug ( "eth historical downloader start" , "chainID" , c . chainClient . NetworkID ( ) , "address" , c . address ,
2023-06-01 13:09:50 +00:00
"from" , c . from . Number , "to" , c . to , "noLimit" , c . noLimit )
2023-05-08 06:02:00 +00:00
2019-06-14 10:16:30 +00:00
start := time . Now ( )
2021-02-19 13:34:24 +00:00
if c . from . Number != nil && c . from . Balance != nil {
2023-09-19 11:17:36 +00:00
c . balanceCacher . Cache ( ) . AddBalance ( c . address , c . chainClient . NetworkID ( ) , c . from . Number , c . from . Balance )
2021-02-19 13:34:24 +00:00
}
if c . from . Number != nil && c . from . Nonce != nil {
2023-09-19 11:17:36 +00:00
c . balanceCacher . Cache ( ) . AddNonce ( c . address , c . chainClient . NetworkID ( ) , c . from . Number , c . from . Nonce )
2021-02-19 13:34:24 +00:00
}
2023-05-19 11:46:54 +00:00
from , headers , startBlock , err := findBlocksWithEthTransfers ( ctx , c . chainClient ,
2023-09-04 05:34:09 +00:00
c . balanceCacher , c . address , c . from . Number , c . to , c . noLimit , c . threadLimit )
status-im/status-react#9203 Faster tx fetching with less request
*** How it worked before this PR on multiaccount creation:
- On multiacc creation we scanned chain for eth and erc20 transfers. For
each address of a new empty multiaccount this scan required
1. two `eth_getBalance` requests to find out that there is no any
balance change between zero and the last block, for eth transfers
2. and `chain-size/100000` (currently ~100) `eth_getLogs` requests,
for erc20 transfers
- For some reason we scanned an address of the chat account as well, and
also accounts were not deduplicated. So even for an empty multiacc we
scanned chain twice for each chat and main wallet addresses, in result
app had to execute about 400 requests.
- As mentioned above, `eth_getBalance` requests were used to check if
there were any eth transfers, and that caused empty history in case
if user already used all available eth (so that both zero and latest
blocks show 0 eth for an address). There might have been transactions
but we wouldn't fetch/show them.
- There was no upper limit for the number of rpc requests during the
scan, so it could require indefinite number of requests; the scanning
algorithm was written so that we persisted the whole history of
transactions or tried to scan form the beginning again in case of
failure, giving up only after 10 minutes of failures. In result
addresses with sufficient number of transactions would never be fully
scanned and during these 10 minutes app could use gigabytes of
internet data.
- Failures were caused by `eth_getBlockByNumber`/`eth_getBlockByHash`
requests. These requests return significantly bigger responses than
`eth_getBalance`/`eth_transactionsCount` and it is likely that
execution of thousands of them in parallel caused failures for
accounts with hundreds of transactions. Even for an account with 12k
we could successfully determine blocks with transaction in a few
minutes using `eth_getBalance` requests, but `eth_getBlock...`
couldn't be processed for this acc.
- There was no caching for for `eth_getBalance` requests, and this
caused in average 3-4 times more such requests than is needed.
*** How it works now on multiaccount creation:
- On multiacc creation we scan chain for last ~30 eth transactions and
then check erc20 in the range where these eth transactions were found.
For an empty address in multiacc this means:
1. two `eth_getBalance` transactions to determine that there was no
balance change between zero and the last block; two
`eth_transactionsCount` requests to determine there are no outgoing
transactions for this address; total 4 requests for eth transfers
2. 20 `eth_getLogs` for erc20 transfers. This number can be lowered,
but that's not a big deal
- Deduplication of addresses is added and also we don't scan chat
account, so a new multiacc requires ~25 (we also request latest block
number and probably execute a few other calls) request to determine
that multiacc is empty (comparing to ~400 before)
- In case if address contains transactions we:
1. determine the range which contains 20-25 outgoing eth/erc20
transactions. This usually requires up to 10 `eth_transactionCount`
requests
2. then we scan chain for eth transfers using `eth_getBalance` and
`eth_transactionCount` (for double checking zero balances)
3. we make sure that we do not scan db for more than 30 blocks with
transfers. That's important for accounts with mostly incoming
transactions, because the range found on the first step might
contain any number of incoming transfers, but only 20-25 outgoing
transactions
4. when we found ~30 blocks in a given range, we update initial
range `from` block using the oldest found block
5. and now we scan db for erc20transfers using `eth_getLogs`
`oldest-found-eth-block`-`latest-block`, we make not more than 20 calls
6. when all blocks which contain incoming/outgoing transfers for a
given address are found, we save these blocks to db and mark that
transfers from these blocks are still to be fetched
7. Then we select latest ~30 (the number can be adjusted) blocks from
these which were found and fetch transfers, this requires 3-4
requests per transfer.
8. we persist scanned range so that we know were to start next time
9. we dispatch an event which tells client that transactions are found
10. client fetches latest 20 transfers
- when user presses "fetch more" button we check if app's db contains next
20 transfers, if not we scan chain again and return transfers after
small fixes
2019-12-18 11:01:46 +00:00
2019-06-14 10:16:30 +00:00
if err != nil {
2020-12-30 15:46:47 +00:00
c . error = err
2023-09-19 11:17:36 +00:00
log . Error ( "failed to find blocks with transfers" , "error" , err , "chainID" , c . chainClient . NetworkID ( ) ,
2023-06-01 13:09:50 +00:00
"address" , c . address , "from" , c . from . Number , "to" , c . to )
2020-12-30 15:46:47 +00:00
return nil
2019-06-14 10:16:30 +00:00
}
status-im/status-react#9203 Faster tx fetching with less request
*** How it worked before this PR on multiaccount creation:
- On multiacc creation we scanned chain for eth and erc20 transfers. For
each address of a new empty multiaccount this scan required
1. two `eth_getBalance` requests to find out that there is no any
balance change between zero and the last block, for eth transfers
2. and `chain-size/100000` (currently ~100) `eth_getLogs` requests,
for erc20 transfers
- For some reason we scanned an address of the chat account as well, and
also accounts were not deduplicated. So even for an empty multiacc we
scanned chain twice for each chat and main wallet addresses, in result
app had to execute about 400 requests.
- As mentioned above, `eth_getBalance` requests were used to check if
there were any eth transfers, and that caused empty history in case
if user already used all available eth (so that both zero and latest
blocks show 0 eth for an address). There might have been transactions
but we wouldn't fetch/show them.
- There was no upper limit for the number of rpc requests during the
scan, so it could require indefinite number of requests; the scanning
algorithm was written so that we persisted the whole history of
transactions or tried to scan form the beginning again in case of
failure, giving up only after 10 minutes of failures. In result
addresses with sufficient number of transactions would never be fully
scanned and during these 10 minutes app could use gigabytes of
internet data.
- Failures were caused by `eth_getBlockByNumber`/`eth_getBlockByHash`
requests. These requests return significantly bigger responses than
`eth_getBalance`/`eth_transactionsCount` and it is likely that
execution of thousands of them in parallel caused failures for
accounts with hundreds of transactions. Even for an account with 12k
we could successfully determine blocks with transaction in a few
minutes using `eth_getBalance` requests, but `eth_getBlock...`
couldn't be processed for this acc.
- There was no caching for for `eth_getBalance` requests, and this
caused in average 3-4 times more such requests than is needed.
*** How it works now on multiaccount creation:
- On multiacc creation we scan chain for last ~30 eth transactions and
then check erc20 in the range where these eth transactions were found.
For an empty address in multiacc this means:
1. two `eth_getBalance` transactions to determine that there was no
balance change between zero and the last block; two
`eth_transactionsCount` requests to determine there are no outgoing
transactions for this address; total 4 requests for eth transfers
2. 20 `eth_getLogs` for erc20 transfers. This number can be lowered,
but that's not a big deal
- Deduplication of addresses is added and also we don't scan chat
account, so a new multiacc requires ~25 (we also request latest block
number and probably execute a few other calls) request to determine
that multiacc is empty (comparing to ~400 before)
- In case if address contains transactions we:
1. determine the range which contains 20-25 outgoing eth/erc20
transactions. This usually requires up to 10 `eth_transactionCount`
requests
2. then we scan chain for eth transfers using `eth_getBalance` and
`eth_transactionCount` (for double checking zero balances)
3. we make sure that we do not scan db for more than 30 blocks with
transfers. That's important for accounts with mostly incoming
transactions, because the range found on the first step might
contain any number of incoming transfers, but only 20-25 outgoing
transactions
4. when we found ~30 blocks in a given range, we update initial
range `from` block using the oldest found block
5. and now we scan db for erc20transfers using `eth_getLogs`
`oldest-found-eth-block`-`latest-block`, we make not more than 20 calls
6. when all blocks which contain incoming/outgoing transfers for a
given address are found, we save these blocks to db and mark that
transfers from these blocks are still to be fetched
7. Then we select latest ~30 (the number can be adjusted) blocks from
these which were found and fetch transfers, this requires 3-4
requests per transfer.
8. we persist scanned range so that we know were to start next time
9. we dispatch an event which tells client that transactions are found
10. client fetches latest 20 transfers
- when user presses "fetch more" button we check if app's db contains next
20 transfers, if not we scan chain again and return transfers after
small fixes
2019-12-18 11:01:46 +00:00
c . foundHeaders = headers
c . resultingFrom = from
2023-05-19 08:19:48 +00:00
c . startBlock = startBlock
status-im/status-react#9203 Faster tx fetching with less request
*** How it worked before this PR on multiaccount creation:
- On multiacc creation we scanned chain for eth and erc20 transfers. For
each address of a new empty multiaccount this scan required
1. two `eth_getBalance` requests to find out that there is no any
balance change between zero and the last block, for eth transfers
2. and `chain-size/100000` (currently ~100) `eth_getLogs` requests,
for erc20 transfers
- For some reason we scanned an address of the chat account as well, and
also accounts were not deduplicated. So even for an empty multiacc we
scanned chain twice for each chat and main wallet addresses, in result
app had to execute about 400 requests.
- As mentioned above, `eth_getBalance` requests were used to check if
there were any eth transfers, and that caused empty history in case
if user already used all available eth (so that both zero and latest
blocks show 0 eth for an address). There might have been transactions
but we wouldn't fetch/show them.
- There was no upper limit for the number of rpc requests during the
scan, so it could require indefinite number of requests; the scanning
algorithm was written so that we persisted the whole history of
transactions or tried to scan form the beginning again in case of
failure, giving up only after 10 minutes of failures. In result
addresses with sufficient number of transactions would never be fully
scanned and during these 10 minutes app could use gigabytes of
internet data.
- Failures were caused by `eth_getBlockByNumber`/`eth_getBlockByHash`
requests. These requests return significantly bigger responses than
`eth_getBalance`/`eth_transactionsCount` and it is likely that
execution of thousands of them in parallel caused failures for
accounts with hundreds of transactions. Even for an account with 12k
we could successfully determine blocks with transaction in a few
minutes using `eth_getBalance` requests, but `eth_getBlock...`
couldn't be processed for this acc.
- There was no caching for for `eth_getBalance` requests, and this
caused in average 3-4 times more such requests than is needed.
*** How it works now on multiaccount creation:
- On multiacc creation we scan chain for last ~30 eth transactions and
then check erc20 in the range where these eth transactions were found.
For an empty address in multiacc this means:
1. two `eth_getBalance` transactions to determine that there was no
balance change between zero and the last block; two
`eth_transactionsCount` requests to determine there are no outgoing
transactions for this address; total 4 requests for eth transfers
2. 20 `eth_getLogs` for erc20 transfers. This number can be lowered,
but that's not a big deal
- Deduplication of addresses is added and also we don't scan chat
account, so a new multiacc requires ~25 (we also request latest block
number and probably execute a few other calls) request to determine
that multiacc is empty (comparing to ~400 before)
- In case if address contains transactions we:
1. determine the range which contains 20-25 outgoing eth/erc20
transactions. This usually requires up to 10 `eth_transactionCount`
requests
2. then we scan chain for eth transfers using `eth_getBalance` and
`eth_transactionCount` (for double checking zero balances)
3. we make sure that we do not scan db for more than 30 blocks with
transfers. That's important for accounts with mostly incoming
transactions, because the range found on the first step might
contain any number of incoming transfers, but only 20-25 outgoing
transactions
4. when we found ~30 blocks in a given range, we update initial
range `from` block using the oldest found block
5. and now we scan db for erc20transfers using `eth_getLogs`
`oldest-found-eth-block`-`latest-block`, we make not more than 20 calls
6. when all blocks which contain incoming/outgoing transfers for a
given address are found, we save these blocks to db and mark that
transfers from these blocks are still to be fetched
7. Then we select latest ~30 (the number can be adjusted) blocks from
these which were found and fetch transfers, this requires 3-4
requests per transfer.
8. we persist scanned range so that we know were to start next time
9. we dispatch an event which tells client that transactions are found
10. client fetches latest 20 transfers
- when user presses "fetch more" button we check if app's db contains next
20 transfers, if not we scan chain again and return transfers after
small fixes
2019-12-18 11:01:46 +00:00
2023-09-20 08:41:23 +00:00
log . Debug ( "eth historical downloader finished successfully" , "chain" , c . chainClient . NetworkID ( ) ,
2023-05-19 08:19:48 +00:00
"address" , c . address , "from" , from , "to" , c . to , "total blocks" , len ( headers ) , "time" , time . Since ( start ) )
status-im/status-react#9203 Faster tx fetching with less request
*** How it worked before this PR on multiaccount creation:
- On multiacc creation we scanned chain for eth and erc20 transfers. For
each address of a new empty multiaccount this scan required
1. two `eth_getBalance` requests to find out that there is no any
balance change between zero and the last block, for eth transfers
2. and `chain-size/100000` (currently ~100) `eth_getLogs` requests,
for erc20 transfers
- For some reason we scanned an address of the chat account as well, and
also accounts were not deduplicated. So even for an empty multiacc we
scanned chain twice for each chat and main wallet addresses, in result
app had to execute about 400 requests.
- As mentioned above, `eth_getBalance` requests were used to check if
there were any eth transfers, and that caused empty history in case
if user already used all available eth (so that both zero and latest
blocks show 0 eth for an address). There might have been transactions
but we wouldn't fetch/show them.
- There was no upper limit for the number of rpc requests during the
scan, so it could require indefinite number of requests; the scanning
algorithm was written so that we persisted the whole history of
transactions or tried to scan form the beginning again in case of
failure, giving up only after 10 minutes of failures. In result
addresses with sufficient number of transactions would never be fully
scanned and during these 10 minutes app could use gigabytes of
internet data.
- Failures were caused by `eth_getBlockByNumber`/`eth_getBlockByHash`
requests. These requests return significantly bigger responses than
`eth_getBalance`/`eth_transactionsCount` and it is likely that
execution of thousands of them in parallel caused failures for
accounts with hundreds of transactions. Even for an account with 12k
we could successfully determine blocks with transaction in a few
minutes using `eth_getBalance` requests, but `eth_getBlock...`
couldn't be processed for this acc.
- There was no caching for for `eth_getBalance` requests, and this
caused in average 3-4 times more such requests than is needed.
*** How it works now on multiaccount creation:
- On multiacc creation we scan chain for last ~30 eth transactions and
then check erc20 in the range where these eth transactions were found.
For an empty address in multiacc this means:
1. two `eth_getBalance` transactions to determine that there was no
balance change between zero and the last block; two
`eth_transactionsCount` requests to determine there are no outgoing
transactions for this address; total 4 requests for eth transfers
2. 20 `eth_getLogs` for erc20 transfers. This number can be lowered,
but that's not a big deal
- Deduplication of addresses is added and also we don't scan chat
account, so a new multiacc requires ~25 (we also request latest block
number and probably execute a few other calls) request to determine
that multiacc is empty (comparing to ~400 before)
- In case if address contains transactions we:
1. determine the range which contains 20-25 outgoing eth/erc20
transactions. This usually requires up to 10 `eth_transactionCount`
requests
2. then we scan chain for eth transfers using `eth_getBalance` and
`eth_transactionCount` (for double checking zero balances)
3. we make sure that we do not scan db for more than 30 blocks with
transfers. That's important for accounts with mostly incoming
transactions, because the range found on the first step might
contain any number of incoming transfers, but only 20-25 outgoing
transactions
4. when we found ~30 blocks in a given range, we update initial
range `from` block using the oldest found block
5. and now we scan db for erc20transfers using `eth_getLogs`
`oldest-found-eth-block`-`latest-block`, we make not more than 20 calls
6. when all blocks which contain incoming/outgoing transfers for a
given address are found, we save these blocks to db and mark that
transfers from these blocks are still to be fetched
7. Then we select latest ~30 (the number can be adjusted) blocks from
these which were found and fetch transfers, this requires 3-4
requests per transfer.
8. we persist scanned range so that we know were to start next time
9. we dispatch an event which tells client that transactions are found
10. client fetches latest 20 transfers
- when user presses "fetch more" button we check if app's db contains next
20 transfers, if not we scan chain again and return transfers after
small fixes
2019-12-18 11:01:46 +00:00
2019-06-14 10:16:30 +00:00
return nil
}
type erc20HistoricalCommand struct {
2021-09-09 14:28:54 +00:00
erc20 BatchDownloader
2023-09-19 11:17:36 +00:00
chainClient chain . ClientInterface
2021-09-09 14:28:54 +00:00
feed * event . Feed
2019-06-14 10:16:30 +00:00
2023-11-02 17:24:23 +00:00
iterator * IterativeDownloader
to * big . Int
from * big . Int
status-im/status-react#9203 Faster tx fetching with less request
*** How it worked before this PR on multiaccount creation:
- On multiacc creation we scanned chain for eth and erc20 transfers. For
each address of a new empty multiaccount this scan required
1. two `eth_getBalance` requests to find out that there is no any
balance change between zero and the last block, for eth transfers
2. and `chain-size/100000` (currently ~100) `eth_getLogs` requests,
for erc20 transfers
- For some reason we scanned an address of the chat account as well, and
also accounts were not deduplicated. So even for an empty multiacc we
scanned chain twice for each chat and main wallet addresses, in result
app had to execute about 400 requests.
- As mentioned above, `eth_getBalance` requests were used to check if
there were any eth transfers, and that caused empty history in case
if user already used all available eth (so that both zero and latest
blocks show 0 eth for an address). There might have been transactions
but we wouldn't fetch/show them.
- There was no upper limit for the number of rpc requests during the
scan, so it could require indefinite number of requests; the scanning
algorithm was written so that we persisted the whole history of
transactions or tried to scan form the beginning again in case of
failure, giving up only after 10 minutes of failures. In result
addresses with sufficient number of transactions would never be fully
scanned and during these 10 minutes app could use gigabytes of
internet data.
- Failures were caused by `eth_getBlockByNumber`/`eth_getBlockByHash`
requests. These requests return significantly bigger responses than
`eth_getBalance`/`eth_transactionsCount` and it is likely that
execution of thousands of them in parallel caused failures for
accounts with hundreds of transactions. Even for an account with 12k
we could successfully determine blocks with transaction in a few
minutes using `eth_getBalance` requests, but `eth_getBlock...`
couldn't be processed for this acc.
- There was no caching for for `eth_getBalance` requests, and this
caused in average 3-4 times more such requests than is needed.
*** How it works now on multiaccount creation:
- On multiacc creation we scan chain for last ~30 eth transactions and
then check erc20 in the range where these eth transactions were found.
For an empty address in multiacc this means:
1. two `eth_getBalance` transactions to determine that there was no
balance change between zero and the last block; two
`eth_transactionsCount` requests to determine there are no outgoing
transactions for this address; total 4 requests for eth transfers
2. 20 `eth_getLogs` for erc20 transfers. This number can be lowered,
but that's not a big deal
- Deduplication of addresses is added and also we don't scan chat
account, so a new multiacc requires ~25 (we also request latest block
number and probably execute a few other calls) request to determine
that multiacc is empty (comparing to ~400 before)
- In case if address contains transactions we:
1. determine the range which contains 20-25 outgoing eth/erc20
transactions. This usually requires up to 10 `eth_transactionCount`
requests
2. then we scan chain for eth transfers using `eth_getBalance` and
`eth_transactionCount` (for double checking zero balances)
3. we make sure that we do not scan db for more than 30 blocks with
transfers. That's important for accounts with mostly incoming
transactions, because the range found on the first step might
contain any number of incoming transfers, but only 20-25 outgoing
transactions
4. when we found ~30 blocks in a given range, we update initial
range `from` block using the oldest found block
5. and now we scan db for erc20transfers using `eth_getLogs`
`oldest-found-eth-block`-`latest-block`, we make not more than 20 calls
6. when all blocks which contain incoming/outgoing transfers for a
given address are found, we save these blocks to db and mark that
transfers from these blocks are still to be fetched
7. Then we select latest ~30 (the number can be adjusted) blocks from
these which were found and fetch transfers, this requires 3-4
requests per transfer.
8. we persist scanned range so that we know were to start next time
9. we dispatch an event which tells client that transactions are found
10. client fetches latest 20 transfers
- when user presses "fetch more" button we check if app's db contains next
20 transfers, if not we scan chain again and return transfers after
small fixes
2019-12-18 11:01:46 +00:00
foundHeaders [ ] * DBHeader
2019-06-14 10:16:30 +00:00
}
2021-09-09 14:28:54 +00:00
func ( c * erc20HistoricalCommand ) Command ( ) async . Command {
return async . FiniteCommand {
2019-06-14 10:16:30 +00:00
Interval : 5 * time . Second ,
Runable : c . Run ,
} . Run
}
2021-11-24 12:59:45 +00:00
func getErc20BatchSize ( chainID uint64 ) * big . Int {
if isBinanceChain ( chainID ) {
return binanceChainErc20BatchSize
}
2022-10-09 15:36:11 +00:00
if chainID == goerliChainID {
return goerliErc20BatchSize
}
2023-08-01 10:56:52 +00:00
if chainID == goerliOptimismChainID {
return goerliErc20OptimismBatchSize
}
2023-03-28 12:46:46 +00:00
if chainID == goerliArbitrumChainID {
return goerliErc20ArbitrumBatchSize
}
2021-11-24 12:59:45 +00:00
return erc20BatchSize
}
2019-06-14 10:16:30 +00:00
func ( c * erc20HistoricalCommand ) Run ( ctx context . Context ) ( err error ) {
2023-11-27 10:08:17 +00:00
log . Debug ( "wallet historical downloader for erc20 transfers start" , "chainID" , c . chainClient . NetworkID ( ) ,
2023-05-08 06:02:00 +00:00
"from" , c . from , "to" , c . to )
status-im/status-react#9203 Faster tx fetching with less request
*** How it worked before this PR on multiaccount creation:
- On multiacc creation we scanned chain for eth and erc20 transfers. For
each address of a new empty multiaccount this scan required
1. two `eth_getBalance` requests to find out that there is no any
balance change between zero and the last block, for eth transfers
2. and `chain-size/100000` (currently ~100) `eth_getLogs` requests,
for erc20 transfers
- For some reason we scanned an address of the chat account as well, and
also accounts were not deduplicated. So even for an empty multiacc we
scanned chain twice for each chat and main wallet addresses, in result
app had to execute about 400 requests.
- As mentioned above, `eth_getBalance` requests were used to check if
there were any eth transfers, and that caused empty history in case
if user already used all available eth (so that both zero and latest
blocks show 0 eth for an address). There might have been transactions
but we wouldn't fetch/show them.
- There was no upper limit for the number of rpc requests during the
scan, so it could require indefinite number of requests; the scanning
algorithm was written so that we persisted the whole history of
transactions or tried to scan form the beginning again in case of
failure, giving up only after 10 minutes of failures. In result
addresses with sufficient number of transactions would never be fully
scanned and during these 10 minutes app could use gigabytes of
internet data.
- Failures were caused by `eth_getBlockByNumber`/`eth_getBlockByHash`
requests. These requests return significantly bigger responses than
`eth_getBalance`/`eth_transactionsCount` and it is likely that
execution of thousands of them in parallel caused failures for
accounts with hundreds of transactions. Even for an account with 12k
we could successfully determine blocks with transaction in a few
minutes using `eth_getBalance` requests, but `eth_getBlock...`
couldn't be processed for this acc.
- There was no caching for for `eth_getBalance` requests, and this
caused in average 3-4 times more such requests than is needed.
*** How it works now on multiaccount creation:
- On multiacc creation we scan chain for last ~30 eth transactions and
then check erc20 in the range where these eth transactions were found.
For an empty address in multiacc this means:
1. two `eth_getBalance` transactions to determine that there was no
balance change between zero and the last block; two
`eth_transactionsCount` requests to determine there are no outgoing
transactions for this address; total 4 requests for eth transfers
2. 20 `eth_getLogs` for erc20 transfers. This number can be lowered,
but that's not a big deal
- Deduplication of addresses is added and also we don't scan chat
account, so a new multiacc requires ~25 (we also request latest block
number and probably execute a few other calls) request to determine
that multiacc is empty (comparing to ~400 before)
- In case if address contains transactions we:
1. determine the range which contains 20-25 outgoing eth/erc20
transactions. This usually requires up to 10 `eth_transactionCount`
requests
2. then we scan chain for eth transfers using `eth_getBalance` and
`eth_transactionCount` (for double checking zero balances)
3. we make sure that we do not scan db for more than 30 blocks with
transfers. That's important for accounts with mostly incoming
transactions, because the range found on the first step might
contain any number of incoming transfers, but only 20-25 outgoing
transactions
4. when we found ~30 blocks in a given range, we update initial
range `from` block using the oldest found block
5. and now we scan db for erc20transfers using `eth_getLogs`
`oldest-found-eth-block`-`latest-block`, we make not more than 20 calls
6. when all blocks which contain incoming/outgoing transfers for a
given address are found, we save these blocks to db and mark that
transfers from these blocks are still to be fetched
7. Then we select latest ~30 (the number can be adjusted) blocks from
these which were found and fetch transfers, this requires 3-4
requests per transfer.
8. we persist scanned range so that we know were to start next time
9. we dispatch an event which tells client that transactions are found
10. client fetches latest 20 transfers
- when user presses "fetch more" button we check if app's db contains next
20 transfers, if not we scan chain again and return transfers after
small fixes
2019-12-18 11:01:46 +00:00
start := time . Now ( )
2019-06-14 10:16:30 +00:00
if c . iterator == nil {
c . iterator , err = SetupIterativeDownloader (
2023-11-27 10:08:17 +00:00
c . chainClient ,
2023-09-19 11:17:36 +00:00
c . erc20 , getErc20BatchSize ( c . chainClient . NetworkID ( ) ) , c . to , c . from )
2019-06-14 10:16:30 +00:00
if err != nil {
log . Error ( "failed to setup historical downloader for erc20" )
return err
}
}
for ! c . iterator . Finished ( ) {
status-im/status-react#9203 Faster tx fetching with less request
*** How it worked before this PR on multiaccount creation:
- On multiacc creation we scanned chain for eth and erc20 transfers. For
each address of a new empty multiaccount this scan required
1. two `eth_getBalance` requests to find out that there is no any
balance change between zero and the last block, for eth transfers
2. and `chain-size/100000` (currently ~100) `eth_getLogs` requests,
for erc20 transfers
- For some reason we scanned an address of the chat account as well, and
also accounts were not deduplicated. So even for an empty multiacc we
scanned chain twice for each chat and main wallet addresses, in result
app had to execute about 400 requests.
- As mentioned above, `eth_getBalance` requests were used to check if
there were any eth transfers, and that caused empty history in case
if user already used all available eth (so that both zero and latest
blocks show 0 eth for an address). There might have been transactions
but we wouldn't fetch/show them.
- There was no upper limit for the number of rpc requests during the
scan, so it could require indefinite number of requests; the scanning
algorithm was written so that we persisted the whole history of
transactions or tried to scan form the beginning again in case of
failure, giving up only after 10 minutes of failures. In result
addresses with sufficient number of transactions would never be fully
scanned and during these 10 minutes app could use gigabytes of
internet data.
- Failures were caused by `eth_getBlockByNumber`/`eth_getBlockByHash`
requests. These requests return significantly bigger responses than
`eth_getBalance`/`eth_transactionsCount` and it is likely that
execution of thousands of them in parallel caused failures for
accounts with hundreds of transactions. Even for an account with 12k
we could successfully determine blocks with transaction in a few
minutes using `eth_getBalance` requests, but `eth_getBlock...`
couldn't be processed for this acc.
- There was no caching for for `eth_getBalance` requests, and this
caused in average 3-4 times more such requests than is needed.
*** How it works now on multiaccount creation:
- On multiacc creation we scan chain for last ~30 eth transactions and
then check erc20 in the range where these eth transactions were found.
For an empty address in multiacc this means:
1. two `eth_getBalance` transactions to determine that there was no
balance change between zero and the last block; two
`eth_transactionsCount` requests to determine there are no outgoing
transactions for this address; total 4 requests for eth transfers
2. 20 `eth_getLogs` for erc20 transfers. This number can be lowered,
but that's not a big deal
- Deduplication of addresses is added and also we don't scan chat
account, so a new multiacc requires ~25 (we also request latest block
number and probably execute a few other calls) request to determine
that multiacc is empty (comparing to ~400 before)
- In case if address contains transactions we:
1. determine the range which contains 20-25 outgoing eth/erc20
transactions. This usually requires up to 10 `eth_transactionCount`
requests
2. then we scan chain for eth transfers using `eth_getBalance` and
`eth_transactionCount` (for double checking zero balances)
3. we make sure that we do not scan db for more than 30 blocks with
transfers. That's important for accounts with mostly incoming
transactions, because the range found on the first step might
contain any number of incoming transfers, but only 20-25 outgoing
transactions
4. when we found ~30 blocks in a given range, we update initial
range `from` block using the oldest found block
5. and now we scan db for erc20transfers using `eth_getLogs`
`oldest-found-eth-block`-`latest-block`, we make not more than 20 calls
6. when all blocks which contain incoming/outgoing transfers for a
given address are found, we save these blocks to db and mark that
transfers from these blocks are still to be fetched
7. Then we select latest ~30 (the number can be adjusted) blocks from
these which were found and fetch transfers, this requires 3-4
requests per transfer.
8. we persist scanned range so that we know were to start next time
9. we dispatch an event which tells client that transactions are found
10. client fetches latest 20 transfers
- when user presses "fetch more" button we check if app's db contains next
20 transfers, if not we scan chain again and return transfers after
small fixes
2019-12-18 11:01:46 +00:00
headers , _ , _ , err := c . iterator . Next ( ctx )
2019-06-14 10:16:30 +00:00
if err != nil {
2023-09-22 16:09:14 +00:00
log . Error ( "failed to get next batch" , "error" , err , "chainID" , c . chainClient . NetworkID ( ) ) // TODO: stop inifinite command in case of an error that we can't fix like missing trie node
2019-07-15 11:16:07 +00:00
return err
2019-06-14 10:16:30 +00:00
}
status-im/status-react#9203 Faster tx fetching with less request
*** How it worked before this PR on multiaccount creation:
- On multiacc creation we scanned chain for eth and erc20 transfers. For
each address of a new empty multiaccount this scan required
1. two `eth_getBalance` requests to find out that there is no any
balance change between zero and the last block, for eth transfers
2. and `chain-size/100000` (currently ~100) `eth_getLogs` requests,
for erc20 transfers
- For some reason we scanned an address of the chat account as well, and
also accounts were not deduplicated. So even for an empty multiacc we
scanned chain twice for each chat and main wallet addresses, in result
app had to execute about 400 requests.
- As mentioned above, `eth_getBalance` requests were used to check if
there were any eth transfers, and that caused empty history in case
if user already used all available eth (so that both zero and latest
blocks show 0 eth for an address). There might have been transactions
but we wouldn't fetch/show them.
- There was no upper limit for the number of rpc requests during the
scan, so it could require indefinite number of requests; the scanning
algorithm was written so that we persisted the whole history of
transactions or tried to scan form the beginning again in case of
failure, giving up only after 10 minutes of failures. In result
addresses with sufficient number of transactions would never be fully
scanned and during these 10 minutes app could use gigabytes of
internet data.
- Failures were caused by `eth_getBlockByNumber`/`eth_getBlockByHash`
requests. These requests return significantly bigger responses than
`eth_getBalance`/`eth_transactionsCount` and it is likely that
execution of thousands of them in parallel caused failures for
accounts with hundreds of transactions. Even for an account with 12k
we could successfully determine blocks with transaction in a few
minutes using `eth_getBalance` requests, but `eth_getBlock...`
couldn't be processed for this acc.
- There was no caching for for `eth_getBalance` requests, and this
caused in average 3-4 times more such requests than is needed.
*** How it works now on multiaccount creation:
- On multiacc creation we scan chain for last ~30 eth transactions and
then check erc20 in the range where these eth transactions were found.
For an empty address in multiacc this means:
1. two `eth_getBalance` transactions to determine that there was no
balance change between zero and the last block; two
`eth_transactionsCount` requests to determine there are no outgoing
transactions for this address; total 4 requests for eth transfers
2. 20 `eth_getLogs` for erc20 transfers. This number can be lowered,
but that's not a big deal
- Deduplication of addresses is added and also we don't scan chat
account, so a new multiacc requires ~25 (we also request latest block
number and probably execute a few other calls) request to determine
that multiacc is empty (comparing to ~400 before)
- In case if address contains transactions we:
1. determine the range which contains 20-25 outgoing eth/erc20
transactions. This usually requires up to 10 `eth_transactionCount`
requests
2. then we scan chain for eth transfers using `eth_getBalance` and
`eth_transactionCount` (for double checking zero balances)
3. we make sure that we do not scan db for more than 30 blocks with
transfers. That's important for accounts with mostly incoming
transactions, because the range found on the first step might
contain any number of incoming transfers, but only 20-25 outgoing
transactions
4. when we found ~30 blocks in a given range, we update initial
range `from` block using the oldest found block
5. and now we scan db for erc20transfers using `eth_getLogs`
`oldest-found-eth-block`-`latest-block`, we make not more than 20 calls
6. when all blocks which contain incoming/outgoing transfers for a
given address are found, we save these blocks to db and mark that
transfers from these blocks are still to be fetched
7. Then we select latest ~30 (the number can be adjusted) blocks from
these which were found and fetch transfers, this requires 3-4
requests per transfer.
8. we persist scanned range so that we know were to start next time
9. we dispatch an event which tells client that transactions are found
10. client fetches latest 20 transfers
- when user presses "fetch more" button we check if app's db contains next
20 transfers, if not we scan chain again and return transfers after
small fixes
2019-12-18 11:01:46 +00:00
c . foundHeaders = append ( c . foundHeaders , headers ... )
2019-06-14 10:16:30 +00:00
}
2023-11-27 10:08:17 +00:00
log . Debug ( "wallet historical downloader for erc20 transfers finished" , "chainID" , c . chainClient . NetworkID ( ) ,
2023-06-14 10:00:56 +00:00
"from" , c . from , "to" , c . to , "time" , time . Since ( start ) , "headers" , len ( c . foundHeaders ) )
2019-06-14 10:16:30 +00:00
return nil
}
status-im/status-react#9203 Faster tx fetching with less request
*** How it worked before this PR on multiaccount creation:
- On multiacc creation we scanned chain for eth and erc20 transfers. For
each address of a new empty multiaccount this scan required
1. two `eth_getBalance` requests to find out that there is no any
balance change between zero and the last block, for eth transfers
2. and `chain-size/100000` (currently ~100) `eth_getLogs` requests,
for erc20 transfers
- For some reason we scanned an address of the chat account as well, and
also accounts were not deduplicated. So even for an empty multiacc we
scanned chain twice for each chat and main wallet addresses, in result
app had to execute about 400 requests.
- As mentioned above, `eth_getBalance` requests were used to check if
there were any eth transfers, and that caused empty history in case
if user already used all available eth (so that both zero and latest
blocks show 0 eth for an address). There might have been transactions
but we wouldn't fetch/show them.
- There was no upper limit for the number of rpc requests during the
scan, so it could require indefinite number of requests; the scanning
algorithm was written so that we persisted the whole history of
transactions or tried to scan form the beginning again in case of
failure, giving up only after 10 minutes of failures. In result
addresses with sufficient number of transactions would never be fully
scanned and during these 10 minutes app could use gigabytes of
internet data.
- Failures were caused by `eth_getBlockByNumber`/`eth_getBlockByHash`
requests. These requests return significantly bigger responses than
`eth_getBalance`/`eth_transactionsCount` and it is likely that
execution of thousands of them in parallel caused failures for
accounts with hundreds of transactions. Even for an account with 12k
we could successfully determine blocks with transaction in a few
minutes using `eth_getBalance` requests, but `eth_getBlock...`
couldn't be processed for this acc.
- There was no caching for for `eth_getBalance` requests, and this
caused in average 3-4 times more such requests than is needed.
*** How it works now on multiaccount creation:
- On multiacc creation we scan chain for last ~30 eth transactions and
then check erc20 in the range where these eth transactions were found.
For an empty address in multiacc this means:
1. two `eth_getBalance` transactions to determine that there was no
balance change between zero and the last block; two
`eth_transactionsCount` requests to determine there are no outgoing
transactions for this address; total 4 requests for eth transfers
2. 20 `eth_getLogs` for erc20 transfers. This number can be lowered,
but that's not a big deal
- Deduplication of addresses is added and also we don't scan chat
account, so a new multiacc requires ~25 (we also request latest block
number and probably execute a few other calls) request to determine
that multiacc is empty (comparing to ~400 before)
- In case if address contains transactions we:
1. determine the range which contains 20-25 outgoing eth/erc20
transactions. This usually requires up to 10 `eth_transactionCount`
requests
2. then we scan chain for eth transfers using `eth_getBalance` and
`eth_transactionCount` (for double checking zero balances)
3. we make sure that we do not scan db for more than 30 blocks with
transfers. That's important for accounts with mostly incoming
transactions, because the range found on the first step might
contain any number of incoming transfers, but only 20-25 outgoing
transactions
4. when we found ~30 blocks in a given range, we update initial
range `from` block using the oldest found block
5. and now we scan db for erc20transfers using `eth_getLogs`
`oldest-found-eth-block`-`latest-block`, we make not more than 20 calls
6. when all blocks which contain incoming/outgoing transfers for a
given address are found, we save these blocks to db and mark that
transfers from these blocks are still to be fetched
7. Then we select latest ~30 (the number can be adjusted) blocks from
these which were found and fetch transfers, this requires 3-4
requests per transfer.
8. we persist scanned range so that we know were to start next time
9. we dispatch an event which tells client that transactions are found
10. client fetches latest 20 transfers
- when user presses "fetch more" button we check if app's db contains next
20 transfers, if not we scan chain again and return transfers after
small fixes
2019-12-18 11:01:46 +00:00
type transfersCommand struct {
2023-02-15 12:28:19 +00:00
db * Database
2023-06-01 13:09:50 +00:00
blockDAO * BlockDAO
2023-02-15 12:28:19 +00:00
eth * ETHDownloader
2023-06-01 13:09:50 +00:00
blockNums [ ] * big . Int
2023-02-15 12:28:19 +00:00
address common . Address
2023-09-20 08:41:23 +00:00
chainClient chain . ClientInterface
2023-06-01 13:09:50 +00:00
blocksLimit int
2023-02-15 12:28:19 +00:00
transactionManager * TransactionManager
2023-08-01 18:50:30 +00:00
pendingTxManager * transactions . PendingTxTracker
2023-06-02 20:08:45 +00:00
tokenManager * token . Manager
2023-06-05 15:05:50 +00:00
feed * event . Feed
2023-06-01 13:09:50 +00:00
// result
fetchedTransfers [ ] Transfer
status-im/status-react#9203 Faster tx fetching with less request
*** How it worked before this PR on multiaccount creation:
- On multiacc creation we scanned chain for eth and erc20 transfers. For
each address of a new empty multiaccount this scan required
1. two `eth_getBalance` requests to find out that there is no any
balance change between zero and the last block, for eth transfers
2. and `chain-size/100000` (currently ~100) `eth_getLogs` requests,
for erc20 transfers
- For some reason we scanned an address of the chat account as well, and
also accounts were not deduplicated. So even for an empty multiacc we
scanned chain twice for each chat and main wallet addresses, in result
app had to execute about 400 requests.
- As mentioned above, `eth_getBalance` requests were used to check if
there were any eth transfers, and that caused empty history in case
if user already used all available eth (so that both zero and latest
blocks show 0 eth for an address). There might have been transactions
but we wouldn't fetch/show them.
- There was no upper limit for the number of rpc requests during the
scan, so it could require indefinite number of requests; the scanning
algorithm was written so that we persisted the whole history of
transactions or tried to scan form the beginning again in case of
failure, giving up only after 10 minutes of failures. In result
addresses with sufficient number of transactions would never be fully
scanned and during these 10 minutes app could use gigabytes of
internet data.
- Failures were caused by `eth_getBlockByNumber`/`eth_getBlockByHash`
requests. These requests return significantly bigger responses than
`eth_getBalance`/`eth_transactionsCount` and it is likely that
execution of thousands of them in parallel caused failures for
accounts with hundreds of transactions. Even for an account with 12k
we could successfully determine blocks with transaction in a few
minutes using `eth_getBalance` requests, but `eth_getBlock...`
couldn't be processed for this acc.
- There was no caching for for `eth_getBalance` requests, and this
caused in average 3-4 times more such requests than is needed.
*** How it works now on multiaccount creation:
- On multiacc creation we scan chain for last ~30 eth transactions and
then check erc20 in the range where these eth transactions were found.
For an empty address in multiacc this means:
1. two `eth_getBalance` transactions to determine that there was no
balance change between zero and the last block; two
`eth_transactionsCount` requests to determine there are no outgoing
transactions for this address; total 4 requests for eth transfers
2. 20 `eth_getLogs` for erc20 transfers. This number can be lowered,
but that's not a big deal
- Deduplication of addresses is added and also we don't scan chat
account, so a new multiacc requires ~25 (we also request latest block
number and probably execute a few other calls) request to determine
that multiacc is empty (comparing to ~400 before)
- In case if address contains transactions we:
1. determine the range which contains 20-25 outgoing eth/erc20
transactions. This usually requires up to 10 `eth_transactionCount`
requests
2. then we scan chain for eth transfers using `eth_getBalance` and
`eth_transactionCount` (for double checking zero balances)
3. we make sure that we do not scan db for more than 30 blocks with
transfers. That's important for accounts with mostly incoming
transactions, because the range found on the first step might
contain any number of incoming transfers, but only 20-25 outgoing
transactions
4. when we found ~30 blocks in a given range, we update initial
range `from` block using the oldest found block
5. and now we scan db for erc20transfers using `eth_getLogs`
`oldest-found-eth-block`-`latest-block`, we make not more than 20 calls
6. when all blocks which contain incoming/outgoing transfers for a
given address are found, we save these blocks to db and mark that
transfers from these blocks are still to be fetched
7. Then we select latest ~30 (the number can be adjusted) blocks from
these which were found and fetch transfers, this requires 3-4
requests per transfer.
8. we persist scanned range so that we know were to start next time
9. we dispatch an event which tells client that transactions are found
10. client fetches latest 20 transfers
- when user presses "fetch more" button we check if app's db contains next
20 transfers, if not we scan chain again and return transfers after
small fixes
2019-12-18 11:01:46 +00:00
}
2024-01-19 12:43:25 +00:00
func ( c * transfersCommand ) Runner ( interval ... time . Duration ) async . Runner {
intvl := transfersRetryInterval
if len ( interval ) > 0 {
intvl = interval [ 0 ]
}
return async . FiniteCommandWithErrorCounter {
FiniteCommand : async . FiniteCommand {
Interval : intvl ,
Runable : c . Run ,
} ,
ErrorCounter : async . NewErrorCounter ( 5 , "transfersCommand" ) ,
}
}
func ( c * transfersCommand ) Command ( interval ... time . Duration ) async . Command {
return c . Runner ( interval ... ) . Run
status-im/status-react#9203 Faster tx fetching with less request
*** How it worked before this PR on multiaccount creation:
- On multiacc creation we scanned chain for eth and erc20 transfers. For
each address of a new empty multiaccount this scan required
1. two `eth_getBalance` requests to find out that there is no any
balance change between zero and the last block, for eth transfers
2. and `chain-size/100000` (currently ~100) `eth_getLogs` requests,
for erc20 transfers
- For some reason we scanned an address of the chat account as well, and
also accounts were not deduplicated. So even for an empty multiacc we
scanned chain twice for each chat and main wallet addresses, in result
app had to execute about 400 requests.
- As mentioned above, `eth_getBalance` requests were used to check if
there were any eth transfers, and that caused empty history in case
if user already used all available eth (so that both zero and latest
blocks show 0 eth for an address). There might have been transactions
but we wouldn't fetch/show them.
- There was no upper limit for the number of rpc requests during the
scan, so it could require indefinite number of requests; the scanning
algorithm was written so that we persisted the whole history of
transactions or tried to scan form the beginning again in case of
failure, giving up only after 10 minutes of failures. In result
addresses with sufficient number of transactions would never be fully
scanned and during these 10 minutes app could use gigabytes of
internet data.
- Failures were caused by `eth_getBlockByNumber`/`eth_getBlockByHash`
requests. These requests return significantly bigger responses than
`eth_getBalance`/`eth_transactionsCount` and it is likely that
execution of thousands of them in parallel caused failures for
accounts with hundreds of transactions. Even for an account with 12k
we could successfully determine blocks with transaction in a few
minutes using `eth_getBalance` requests, but `eth_getBlock...`
couldn't be processed for this acc.
- There was no caching for for `eth_getBalance` requests, and this
caused in average 3-4 times more such requests than is needed.
*** How it works now on multiaccount creation:
- On multiacc creation we scan chain for last ~30 eth transactions and
then check erc20 in the range where these eth transactions were found.
For an empty address in multiacc this means:
1. two `eth_getBalance` transactions to determine that there was no
balance change between zero and the last block; two
`eth_transactionsCount` requests to determine there are no outgoing
transactions for this address; total 4 requests for eth transfers
2. 20 `eth_getLogs` for erc20 transfers. This number can be lowered,
but that's not a big deal
- Deduplication of addresses is added and also we don't scan chat
account, so a new multiacc requires ~25 (we also request latest block
number and probably execute a few other calls) request to determine
that multiacc is empty (comparing to ~400 before)
- In case if address contains transactions we:
1. determine the range which contains 20-25 outgoing eth/erc20
transactions. This usually requires up to 10 `eth_transactionCount`
requests
2. then we scan chain for eth transfers using `eth_getBalance` and
`eth_transactionCount` (for double checking zero balances)
3. we make sure that we do not scan db for more than 30 blocks with
transfers. That's important for accounts with mostly incoming
transactions, because the range found on the first step might
contain any number of incoming transfers, but only 20-25 outgoing
transactions
4. when we found ~30 blocks in a given range, we update initial
range `from` block using the oldest found block
5. and now we scan db for erc20transfers using `eth_getLogs`
`oldest-found-eth-block`-`latest-block`, we make not more than 20 calls
6. when all blocks which contain incoming/outgoing transfers for a
given address are found, we save these blocks to db and mark that
transfers from these blocks are still to be fetched
7. Then we select latest ~30 (the number can be adjusted) blocks from
these which were found and fetch transfers, this requires 3-4
requests per transfer.
8. we persist scanned range so that we know were to start next time
9. we dispatch an event which tells client that transactions are found
10. client fetches latest 20 transfers
- when user presses "fetch more" button we check if app's db contains next
20 transfers, if not we scan chain again and return transfers after
small fixes
2019-12-18 11:01:46 +00:00
}
func ( c * transfersCommand ) Run ( ctx context . Context ) ( err error ) {
2023-06-01 13:09:50 +00:00
// Take blocks from cache if available and disrespect the limit
// If no blocks are available in cache, take blocks from DB respecting the limit
// If no limit is set, take all blocks from DB
2023-11-02 17:24:23 +00:00
log . Debug ( "start transfersCommand" , "chain" , c . chainClient . NetworkID ( ) , "address" , c . address , "blockNums" , c . blockNums )
2023-06-05 15:05:50 +00:00
startTs := time . Now ( )
2023-06-01 13:09:50 +00:00
for {
blocks := c . blockNums
if blocks == nil {
2023-09-19 11:17:36 +00:00
blocks , _ = c . blockDAO . GetBlocksToLoadByAddress ( c . chainClient . NetworkID ( ) , c . address , numberOfBlocksCheckedPerIteration )
2023-06-01 13:09:50 +00:00
}
2023-05-19 08:19:48 +00:00
2023-06-01 13:09:50 +00:00
for _ , blockNum := range blocks {
2023-09-19 11:17:36 +00:00
log . Debug ( "transfersCommand block start" , "chain" , c . chainClient . NetworkID ( ) , "address" , c . address , "block" , blockNum )
2023-06-01 13:09:50 +00:00
allTransfers , err := c . eth . GetTransfersByNumber ( ctx , blockNum )
if err != nil {
log . Error ( "getTransfersByBlocks error" , "error" , err )
return err
}
2023-08-31 07:47:24 +00:00
c . processUnknownErc20CommunityTransactions ( ctx , allTransfers )
2023-06-02 20:08:45 +00:00
err = c . processMultiTransactions ( ctx , allTransfers )
2023-06-01 13:09:50 +00:00
if err != nil {
2023-06-13 14:06:36 +00:00
log . Error ( "processMultiTransactions error" , "error" , err )
2023-06-01 13:09:50 +00:00
return err
}
if len ( allTransfers ) > 0 {
2023-08-01 18:50:30 +00:00
err := c . saveAndConfirmPending ( allTransfers , blockNum )
2023-06-01 13:09:50 +00:00
if err != nil {
2023-08-01 18:50:30 +00:00
log . Error ( "saveAndConfirmPending error" , "error" , err )
2023-06-01 13:09:50 +00:00
return err
}
} else {
// If no transfers found, that is suspecting, because downloader returned this block as containing transfers
2023-09-19 11:17:36 +00:00
log . Error ( "no transfers found in block" , "chain" , c . chainClient . NetworkID ( ) , "address" , c . address , "block" , blockNum )
2023-06-01 13:09:50 +00:00
2023-09-19 11:17:36 +00:00
err = markBlocksAsLoaded ( c . chainClient . NetworkID ( ) , c . db . client , c . address , [ ] * big . Int { blockNum } )
2023-06-01 13:09:50 +00:00
if err != nil {
log . Error ( "Mark blocks loaded error" , "error" , err )
return err
}
}
c . fetchedTransfers = append ( c . fetchedTransfers , allTransfers ... )
2023-10-04 12:00:12 +00:00
c . notifyOfNewTransfers ( blockNum , allTransfers )
2023-11-27 21:28:16 +00:00
c . notifyOfLatestTransfers ( allTransfers , w_common . EthTransfer )
c . notifyOfLatestTransfers ( allTransfers , w_common . Erc20Transfer )
c . notifyOfLatestTransfers ( allTransfers , w_common . Erc721Transfer )
c . notifyOfLatestTransfers ( allTransfers , w_common . Erc1155Transfer )
2023-06-05 15:05:50 +00:00
2023-09-19 11:17:36 +00:00
log . Debug ( "transfersCommand block end" , "chain" , c . chainClient . NetworkID ( ) , "address" , c . address ,
2023-06-05 15:05:50 +00:00
"block" , blockNum , "tranfers.len" , len ( allTransfers ) , "fetchedTransfers.len" , len ( c . fetchedTransfers ) )
2023-06-01 13:09:50 +00:00
}
if c . blockNums != nil || len ( blocks ) == 0 ||
( c . blocksLimit > noBlockLimit && len ( blocks ) >= c . blocksLimit ) {
2023-09-19 11:17:36 +00:00
log . Debug ( "loadTransfers breaking loop on block limits reached or 0 blocks" , "chain" , c . chainClient . NetworkID ( ) ,
2023-06-01 13:09:50 +00:00
"address" , c . address , "limit" , c . blocksLimit , "blocks" , len ( blocks ) )
break
}
status-im/status-react#9203 Faster tx fetching with less request
*** How it worked before this PR on multiaccount creation:
- On multiacc creation we scanned chain for eth and erc20 transfers. For
each address of a new empty multiaccount this scan required
1. two `eth_getBalance` requests to find out that there is no any
balance change between zero and the last block, for eth transfers
2. and `chain-size/100000` (currently ~100) `eth_getLogs` requests,
for erc20 transfers
- For some reason we scanned an address of the chat account as well, and
also accounts were not deduplicated. So even for an empty multiacc we
scanned chain twice for each chat and main wallet addresses, in result
app had to execute about 400 requests.
- As mentioned above, `eth_getBalance` requests were used to check if
there were any eth transfers, and that caused empty history in case
if user already used all available eth (so that both zero and latest
blocks show 0 eth for an address). There might have been transactions
but we wouldn't fetch/show them.
- There was no upper limit for the number of rpc requests during the
scan, so it could require indefinite number of requests; the scanning
algorithm was written so that we persisted the whole history of
transactions or tried to scan form the beginning again in case of
failure, giving up only after 10 minutes of failures. In result
addresses with sufficient number of transactions would never be fully
scanned and during these 10 minutes app could use gigabytes of
internet data.
- Failures were caused by `eth_getBlockByNumber`/`eth_getBlockByHash`
requests. These requests return significantly bigger responses than
`eth_getBalance`/`eth_transactionsCount` and it is likely that
execution of thousands of them in parallel caused failures for
accounts with hundreds of transactions. Even for an account with 12k
we could successfully determine blocks with transaction in a few
minutes using `eth_getBalance` requests, but `eth_getBlock...`
couldn't be processed for this acc.
- There was no caching for for `eth_getBalance` requests, and this
caused in average 3-4 times more such requests than is needed.
*** How it works now on multiaccount creation:
- On multiacc creation we scan chain for last ~30 eth transactions and
then check erc20 in the range where these eth transactions were found.
For an empty address in multiacc this means:
1. two `eth_getBalance` transactions to determine that there was no
balance change between zero and the last block; two
`eth_transactionsCount` requests to determine there are no outgoing
transactions for this address; total 4 requests for eth transfers
2. 20 `eth_getLogs` for erc20 transfers. This number can be lowered,
but that's not a big deal
- Deduplication of addresses is added and also we don't scan chat
account, so a new multiacc requires ~25 (we also request latest block
number and probably execute a few other calls) request to determine
that multiacc is empty (comparing to ~400 before)
- In case if address contains transactions we:
1. determine the range which contains 20-25 outgoing eth/erc20
transactions. This usually requires up to 10 `eth_transactionCount`
requests
2. then we scan chain for eth transfers using `eth_getBalance` and
`eth_transactionCount` (for double checking zero balances)
3. we make sure that we do not scan db for more than 30 blocks with
transfers. That's important for accounts with mostly incoming
transactions, because the range found on the first step might
contain any number of incoming transfers, but only 20-25 outgoing
transactions
4. when we found ~30 blocks in a given range, we update initial
range `from` block using the oldest found block
5. and now we scan db for erc20transfers using `eth_getLogs`
`oldest-found-eth-block`-`latest-block`, we make not more than 20 calls
6. when all blocks which contain incoming/outgoing transfers for a
given address are found, we save these blocks to db and mark that
transfers from these blocks are still to be fetched
7. Then we select latest ~30 (the number can be adjusted) blocks from
these which were found and fetch transfers, this requires 3-4
requests per transfer.
8. we persist scanned range so that we know were to start next time
9. we dispatch an event which tells client that transactions are found
10. client fetches latest 20 transfers
- when user presses "fetch more" button we check if app's db contains next
20 transfers, if not we scan chain again and return transfers after
small fixes
2019-12-18 11:01:46 +00:00
}
2023-11-02 17:24:23 +00:00
log . Debug ( "end transfersCommand" , "chain" , c . chainClient . NetworkID ( ) , "address" , c . address ,
2023-06-05 15:05:50 +00:00
"blocks.len" , len ( c . blockNums ) , "transfers.len" , len ( c . fetchedTransfers ) , "in" , time . Since ( startTs ) )
2023-06-01 13:09:50 +00:00
return nil
}
2023-09-06 19:15:07 +00:00
// saveAndConfirmPending ensures only the transaction that has owner (Address) as a sender is matched to the
// corresponding multi-transaction (by multi-transaction ID). This way we ensure that if receiver is in the list
// of accounts filter will discard the proper one
2023-08-01 18:50:30 +00:00
func ( c * transfersCommand ) saveAndConfirmPending ( allTransfers [ ] Transfer , blockNum * big . Int ) error {
2023-09-06 19:15:07 +00:00
tx , resErr := c . db . client . Begin ( )
if resErr != nil {
return resErr
2023-06-02 20:08:45 +00:00
}
2024-01-24 13:00:08 +00:00
notifyFunctions := c . confirmPendingTransactions ( tx , allTransfers )
2023-09-06 19:15:07 +00:00
defer func ( ) {
if resErr == nil {
commitErr := tx . Commit ( )
if commitErr != nil {
log . Error ( "failed to commit" , "error" , commitErr )
}
for _ , notify := range notifyFunctions {
notify ( )
}
} else {
rollbackErr := tx . Rollback ( )
if rollbackErr != nil {
log . Error ( "failed to rollback" , "error" , rollbackErr )
}
}
} ( )
2024-01-24 13:00:08 +00:00
resErr = saveTransfersMarkBlocksLoaded ( tx , c . chainClient . NetworkID ( ) , c . address , allTransfers , [ ] * big . Int { blockNum } )
if resErr != nil {
log . Error ( "SaveTransfers error" , "error" , resErr )
}
return resErr
}
2024-04-04 20:06:39 +00:00
func externalTransactionOrError ( err error , mTID int64 ) bool {
if err == sql . ErrNoRows {
// External transaction downloaded, ignore it
return true
} else if err != nil {
log . Warn ( "GetOwnedMultiTransactionID" , "error" , err )
return true
} else if mTID <= 0 {
// Existing external transaction, ignore it
return true
}
return false
}
2024-01-24 13:00:08 +00:00
func ( c * transfersCommand ) confirmPendingTransactions ( tx * sql . Tx , allTransfers [ ] Transfer ) ( notifyFunctions [ ] func ( ) ) {
notifyFunctions = make ( [ ] func ( ) , 0 )
2023-08-01 18:50:30 +00:00
// Confirm all pending transactions that are included in this block
2023-09-06 19:15:07 +00:00
for i , tr := range allTransfers {
chainID := w_common . ChainID ( tr . NetworkID )
txHash := tr . Receipt . TxHash
txType , mTID , err := transactions . GetOwnedPendingStatus ( tx , chainID , txHash , tr . Address )
if err == sql . ErrNoRows {
if tr . MultiTransactionID > 0 {
continue
} else {
// Outside transaction, already confirmed by another duplicate or not yet downloaded
2024-04-04 20:02:07 +00:00
existingMTID , err := GetOwnedMultiTransactionID ( tx , chainID , txHash , tr . Address )
2024-04-04 20:06:39 +00:00
if externalTransactionOrError ( err , existingMTID ) {
2023-09-06 19:15:07 +00:00
continue
}
mTID = w_common . NewAndSet ( existingMTID )
}
} else if err != nil {
log . Warn ( "GetOwnedPendingStatus" , "error" , err )
continue
2023-08-01 18:50:30 +00:00
}
2023-09-06 19:15:07 +00:00
if mTID != nil {
2024-03-12 09:15:30 +00:00
allTransfers [ i ] . MultiTransactionID = w_common . MultiTransactionIDType ( * mTID )
2023-08-01 18:50:30 +00:00
}
if txType != nil && * txType == transactions . WalletTransfer {
2023-09-06 19:15:07 +00:00
notify , err := c . pendingTxManager . DeleteBySQLTx ( tx , chainID , txHash )
2023-08-01 18:50:30 +00:00
if err != nil && err != transactions . ErrStillPending {
log . Error ( "DeleteBySqlTx error" , "error" , err )
}
notifyFunctions = append ( notifyFunctions , notify )
2023-06-02 20:08:45 +00:00
}
2023-06-13 14:06:36 +00:00
}
2024-01-24 13:00:08 +00:00
return notifyFunctions
2023-06-13 14:06:36 +00:00
}
2023-08-01 18:50:30 +00:00
// Mark all subTxs of a given Tx with the same multiTxID
2024-03-12 09:15:30 +00:00
func setMultiTxID ( tx Transaction , multiTxID w_common . MultiTransactionIDType ) {
2023-08-01 18:50:30 +00:00
for _ , subTx := range tx {
subTx . MultiTransactionID = multiTxID
}
}
2023-12-21 14:12:50 +00:00
func ( c * transfersCommand ) markMultiTxTokensAsPreviouslyOwned ( ctx context . Context , multiTransaction * MultiTransaction , ownerAddress common . Address ) {
if multiTransaction == nil {
return
}
if len ( multiTransaction . ToAsset ) > 0 && multiTransaction . ToNetworkID > 0 {
token := c . tokenManager . GetToken ( multiTransaction . ToNetworkID , multiTransaction . ToAsset )
2024-02-19 13:55:38 +00:00
_ , _ = c . tokenManager . MarkAsPreviouslyOwnedToken ( token , ownerAddress )
2023-12-21 14:12:50 +00:00
}
if len ( multiTransaction . FromAsset ) > 0 && multiTransaction . FromNetworkID > 0 {
token := c . tokenManager . GetToken ( multiTransaction . FromNetworkID , multiTransaction . FromAsset )
2024-02-19 13:55:38 +00:00
_ , _ = c . tokenManager . MarkAsPreviouslyOwnedToken ( token , ownerAddress )
2023-12-21 14:12:50 +00:00
}
}
2023-06-13 14:06:36 +00:00
func ( c * transfersCommand ) checkAndProcessSwapMultiTx ( ctx context . Context , tx Transaction ) ( bool , error ) {
for _ , subTx := range tx {
switch subTx . Type {
// If the Tx contains any uniswapV2Swap/uniswapV3Swap subTx, generate a Swap multiTx
case w_common . UniswapV2Swap , w_common . UniswapV3Swap :
multiTransaction , err := buildUniswapSwapMultitransaction ( ctx , c . chainClient , c . tokenManager , subTx )
2023-06-02 20:08:45 +00:00
if err != nil {
2023-06-13 14:06:36 +00:00
return false , err
}
if multiTransaction != nil {
id , err := c . transactionManager . InsertMultiTransaction ( multiTransaction )
if err != nil {
return false , err
}
setMultiTxID ( tx , id )
2023-12-21 14:12:50 +00:00
c . markMultiTxTokensAsPreviouslyOwned ( ctx , multiTransaction , subTx . Address )
2023-06-13 14:06:36 +00:00
return true , nil
2023-06-02 20:08:45 +00:00
}
}
}
2023-06-13 14:06:36 +00:00
return false , nil
2023-06-02 20:08:45 +00:00
}
2023-06-13 14:20:48 +00:00
func ( c * transfersCommand ) checkAndProcessBridgeMultiTx ( ctx context . Context , tx Transaction ) ( bool , error ) {
for _ , subTx := range tx {
switch subTx . Type {
// If the Tx contains any hopBridge subTx, create/update Bridge multiTx
case w_common . HopBridgeFrom , w_common . HopBridgeTo :
multiTransaction , err := buildHopBridgeMultitransaction ( ctx , c . chainClient , c . transactionManager , c . tokenManager , subTx )
if err != nil {
return false , err
}
if multiTransaction != nil {
2024-03-12 09:15:30 +00:00
setMultiTxID ( tx , multiTransaction . ID )
2023-12-21 14:12:50 +00:00
c . markMultiTxTokensAsPreviouslyOwned ( ctx , multiTransaction , subTx . Address )
2023-06-13 14:20:48 +00:00
return true , nil
}
}
}
return false , nil
}
2023-08-31 07:47:24 +00:00
func ( c * transfersCommand ) processUnknownErc20CommunityTransactions ( ctx context . Context , allTransfers [ ] Transfer ) {
for _ , tx := range allTransfers {
2023-12-06 10:26:53 +00:00
// To can be nil in case of erc20 contract creation
if tx . Type == w_common . Erc20Transfer && tx . Transaction . To ( ) != nil {
2023-08-31 07:47:24 +00:00
// Find token in db or if this is a community token, find its metadata
2023-12-21 14:12:50 +00:00
token := c . tokenManager . FindOrCreateTokenByAddress ( ctx , tx . NetworkID , * tx . Transaction . To ( ) )
2024-01-04 12:22:06 +00:00
if token != nil {
2024-02-19 13:55:38 +00:00
isFirst := false
2024-01-04 12:22:06 +00:00
if token . Verified || token . CommunityData != nil {
2024-02-19 13:55:38 +00:00
isFirst , _ = c . tokenManager . MarkAsPreviouslyOwnedToken ( token , tx . Address )
2024-01-04 12:22:06 +00:00
}
if token . CommunityData != nil {
2024-02-19 13:55:38 +00:00
go c . tokenManager . SignalCommunityTokenReceived ( tx . Address , tx . ID , tx . TokenValue , token , isFirst )
2024-01-04 12:22:06 +00:00
}
2023-12-21 14:12:50 +00:00
}
2023-08-31 07:47:24 +00:00
}
}
}
2023-06-02 20:08:45 +00:00
func ( c * transfersCommand ) processMultiTransactions ( ctx context . Context , allTransfers [ ] Transfer ) error {
2023-06-13 14:06:36 +00:00
txByTxHash := subTransactionListToTransactionsByTxHash ( allTransfers )
2023-06-02 20:08:45 +00:00
// Detect / Generate multitransactions
// Iterate over all detected transactions
2023-06-13 14:06:36 +00:00
for _ , tx := range txByTxHash {
// Then check for a Swap transaction
2023-06-13 14:20:48 +00:00
txProcessed , err := c . checkAndProcessSwapMultiTx ( ctx , tx )
if err != nil {
return err
}
if txProcessed {
continue
}
// Then check for a Bridge transaction
_ , err = c . checkAndProcessBridgeMultiTx ( ctx , tx )
2023-06-13 14:06:36 +00:00
if err != nil {
return err
2023-02-15 12:28:19 +00:00
}
}
status-im/status-react#9203 Faster tx fetching with less request
*** How it worked before this PR on multiaccount creation:
- On multiacc creation we scanned chain for eth and erc20 transfers. For
each address of a new empty multiaccount this scan required
1. two `eth_getBalance` requests to find out that there is no any
balance change between zero and the last block, for eth transfers
2. and `chain-size/100000` (currently ~100) `eth_getLogs` requests,
for erc20 transfers
- For some reason we scanned an address of the chat account as well, and
also accounts were not deduplicated. So even for an empty multiacc we
scanned chain twice for each chat and main wallet addresses, in result
app had to execute about 400 requests.
- As mentioned above, `eth_getBalance` requests were used to check if
there were any eth transfers, and that caused empty history in case
if user already used all available eth (so that both zero and latest
blocks show 0 eth for an address). There might have been transactions
but we wouldn't fetch/show them.
- There was no upper limit for the number of rpc requests during the
scan, so it could require indefinite number of requests; the scanning
algorithm was written so that we persisted the whole history of
transactions or tried to scan form the beginning again in case of
failure, giving up only after 10 minutes of failures. In result
addresses with sufficient number of transactions would never be fully
scanned and during these 10 minutes app could use gigabytes of
internet data.
- Failures were caused by `eth_getBlockByNumber`/`eth_getBlockByHash`
requests. These requests return significantly bigger responses than
`eth_getBalance`/`eth_transactionsCount` and it is likely that
execution of thousands of them in parallel caused failures for
accounts with hundreds of transactions. Even for an account with 12k
we could successfully determine blocks with transaction in a few
minutes using `eth_getBalance` requests, but `eth_getBlock...`
couldn't be processed for this acc.
- There was no caching for for `eth_getBalance` requests, and this
caused in average 3-4 times more such requests than is needed.
*** How it works now on multiaccount creation:
- On multiacc creation we scan chain for last ~30 eth transactions and
then check erc20 in the range where these eth transactions were found.
For an empty address in multiacc this means:
1. two `eth_getBalance` transactions to determine that there was no
balance change between zero and the last block; two
`eth_transactionsCount` requests to determine there are no outgoing
transactions for this address; total 4 requests for eth transfers
2. 20 `eth_getLogs` for erc20 transfers. This number can be lowered,
but that's not a big deal
- Deduplication of addresses is added and also we don't scan chat
account, so a new multiacc requires ~25 (we also request latest block
number and probably execute a few other calls) request to determine
that multiacc is empty (comparing to ~400 before)
- In case if address contains transactions we:
1. determine the range which contains 20-25 outgoing eth/erc20
transactions. This usually requires up to 10 `eth_transactionCount`
requests
2. then we scan chain for eth transfers using `eth_getBalance` and
`eth_transactionCount` (for double checking zero balances)
3. we make sure that we do not scan db for more than 30 blocks with
transfers. That's important for accounts with mostly incoming
transactions, because the range found on the first step might
contain any number of incoming transfers, but only 20-25 outgoing
transactions
4. when we found ~30 blocks in a given range, we update initial
range `from` block using the oldest found block
5. and now we scan db for erc20transfers using `eth_getLogs`
`oldest-found-eth-block`-`latest-block`, we make not more than 20 calls
6. when all blocks which contain incoming/outgoing transfers for a
given address are found, we save these blocks to db and mark that
transfers from these blocks are still to be fetched
7. Then we select latest ~30 (the number can be adjusted) blocks from
these which were found and fetch transfers, this requires 3-4
requests per transfer.
8. we persist scanned range so that we know were to start next time
9. we dispatch an event which tells client that transactions are found
10. client fetches latest 20 transfers
- when user presses "fetch more" button we check if app's db contains next
20 transfers, if not we scan chain again and return transfers after
small fixes
2019-12-18 11:01:46 +00:00
return nil
}
2023-10-04 12:00:12 +00:00
func ( c * transfersCommand ) notifyOfNewTransfers ( blockNum * big . Int , transfers [ ] Transfer ) {
2023-06-05 15:05:50 +00:00
if c . feed != nil {
if len ( transfers ) > 0 {
c . feed . Send ( walletevent . Event {
2023-10-04 12:00:12 +00:00
Type : EventNewTransfers ,
Accounts : [ ] common . Address { c . address } ,
ChainID : c . chainClient . NetworkID ( ) ,
BlockNumber : blockNum ,
2023-06-05 15:05:50 +00:00
} )
}
}
}
2023-11-27 21:28:16 +00:00
func transferTypeToEventType ( transferType w_common . Type ) walletevent . EventType {
switch transferType {
case w_common . EthTransfer :
return EventInternalETHTransferDetected
case w_common . Erc20Transfer :
return EventInternalERC20TransferDetected
case w_common . Erc721Transfer :
return EventInternalERC721TransferDetected
case w_common . Erc1155Transfer :
return EventInternalERC1155TransferDetected
default :
return ""
}
}
func ( c * transfersCommand ) notifyOfLatestTransfers ( transfers [ ] Transfer , transferType w_common . Type ) {
2023-10-09 12:43:53 +00:00
if c . feed != nil {
2024-01-08 05:21:50 +00:00
eventTransfers := make ( [ ] Transfer , 0 , len ( transfers ) )
2023-11-27 21:28:16 +00:00
latestTransferTimestamp := uint64 ( 0 )
2023-10-09 12:43:53 +00:00
for _ , transfer := range transfers {
2023-11-27 21:28:16 +00:00
if transfer . Type == transferType {
2024-01-08 05:21:50 +00:00
eventTransfers = append ( eventTransfers , transfer )
2023-11-27 21:28:16 +00:00
if transfer . Timestamp > latestTransferTimestamp {
latestTransferTimestamp = transfer . Timestamp
2023-10-09 12:43:53 +00:00
}
}
}
2024-01-08 05:21:50 +00:00
if len ( eventTransfers ) > 0 {
2023-10-09 12:43:53 +00:00
c . feed . Send ( walletevent . Event {
2024-01-08 05:21:50 +00:00
Type : transferTypeToEventType ( transferType ) ,
Accounts : [ ] common . Address { c . address } ,
ChainID : c . chainClient . NetworkID ( ) ,
At : int64 ( latestTransferTimestamp ) ,
EventParams : eventTransfers ,
2023-10-09 12:43:53 +00:00
} )
}
}
}
status-im/status-react#9203 Faster tx fetching with less request
*** How it worked before this PR on multiaccount creation:
- On multiacc creation we scanned chain for eth and erc20 transfers. For
each address of a new empty multiaccount this scan required
1. two `eth_getBalance` requests to find out that there is no any
balance change between zero and the last block, for eth transfers
2. and `chain-size/100000` (currently ~100) `eth_getLogs` requests,
for erc20 transfers
- For some reason we scanned an address of the chat account as well, and
also accounts were not deduplicated. So even for an empty multiacc we
scanned chain twice for each chat and main wallet addresses, in result
app had to execute about 400 requests.
- As mentioned above, `eth_getBalance` requests were used to check if
there were any eth transfers, and that caused empty history in case
if user already used all available eth (so that both zero and latest
blocks show 0 eth for an address). There might have been transactions
but we wouldn't fetch/show them.
- There was no upper limit for the number of rpc requests during the
scan, so it could require indefinite number of requests; the scanning
algorithm was written so that we persisted the whole history of
transactions or tried to scan form the beginning again in case of
failure, giving up only after 10 minutes of failures. In result
addresses with sufficient number of transactions would never be fully
scanned and during these 10 minutes app could use gigabytes of
internet data.
- Failures were caused by `eth_getBlockByNumber`/`eth_getBlockByHash`
requests. These requests return significantly bigger responses than
`eth_getBalance`/`eth_transactionsCount` and it is likely that
execution of thousands of them in parallel caused failures for
accounts with hundreds of transactions. Even for an account with 12k
we could successfully determine blocks with transaction in a few
minutes using `eth_getBalance` requests, but `eth_getBlock...`
couldn't be processed for this acc.
- There was no caching for for `eth_getBalance` requests, and this
caused in average 3-4 times more such requests than is needed.
*** How it works now on multiaccount creation:
- On multiacc creation we scan chain for last ~30 eth transactions and
then check erc20 in the range where these eth transactions were found.
For an empty address in multiacc this means:
1. two `eth_getBalance` transactions to determine that there was no
balance change between zero and the last block; two
`eth_transactionsCount` requests to determine there are no outgoing
transactions for this address; total 4 requests for eth transfers
2. 20 `eth_getLogs` for erc20 transfers. This number can be lowered,
but that's not a big deal
- Deduplication of addresses is added and also we don't scan chat
account, so a new multiacc requires ~25 (we also request latest block
number and probably execute a few other calls) request to determine
that multiacc is empty (comparing to ~400 before)
- In case if address contains transactions we:
1. determine the range which contains 20-25 outgoing eth/erc20
transactions. This usually requires up to 10 `eth_transactionCount`
requests
2. then we scan chain for eth transfers using `eth_getBalance` and
`eth_transactionCount` (for double checking zero balances)
3. we make sure that we do not scan db for more than 30 blocks with
transfers. That's important for accounts with mostly incoming
transactions, because the range found on the first step might
contain any number of incoming transfers, but only 20-25 outgoing
transactions
4. when we found ~30 blocks in a given range, we update initial
range `from` block using the oldest found block
5. and now we scan db for erc20transfers using `eth_getLogs`
`oldest-found-eth-block`-`latest-block`, we make not more than 20 calls
6. when all blocks which contain incoming/outgoing transfers for a
given address are found, we save these blocks to db and mark that
transfers from these blocks are still to be fetched
7. Then we select latest ~30 (the number can be adjusted) blocks from
these which were found and fetch transfers, this requires 3-4
requests per transfer.
8. we persist scanned range so that we know were to start next time
9. we dispatch an event which tells client that transactions are found
10. client fetches latest 20 transfers
- when user presses "fetch more" button we check if app's db contains next
20 transfers, if not we scan chain again and return transfers after
small fixes
2019-12-18 11:01:46 +00:00
type loadTransfersCommand struct {
2023-06-14 10:00:56 +00:00
accounts [ ] common . Address
db * Database
blockDAO * BlockDAO
2023-09-20 08:41:23 +00:00
chainClient chain . ClientInterface
2023-06-14 10:00:56 +00:00
blocksByAddress map [ common . Address ] [ ] * big . Int
transactionManager * TransactionManager
2023-08-01 18:50:30 +00:00
pendingTxManager * transactions . PendingTxTracker
2023-06-14 10:00:56 +00:00
blocksLimit int
tokenManager * token . Manager
feed * event . Feed
status-im/status-react#9203 Faster tx fetching with less request
*** How it worked before this PR on multiaccount creation:
- On multiacc creation we scanned chain for eth and erc20 transfers. For
each address of a new empty multiaccount this scan required
1. two `eth_getBalance` requests to find out that there is no any
balance change between zero and the last block, for eth transfers
2. and `chain-size/100000` (currently ~100) `eth_getLogs` requests,
for erc20 transfers
- For some reason we scanned an address of the chat account as well, and
also accounts were not deduplicated. So even for an empty multiacc we
scanned chain twice for each chat and main wallet addresses, in result
app had to execute about 400 requests.
- As mentioned above, `eth_getBalance` requests were used to check if
there were any eth transfers, and that caused empty history in case
if user already used all available eth (so that both zero and latest
blocks show 0 eth for an address). There might have been transactions
but we wouldn't fetch/show them.
- There was no upper limit for the number of rpc requests during the
scan, so it could require indefinite number of requests; the scanning
algorithm was written so that we persisted the whole history of
transactions or tried to scan form the beginning again in case of
failure, giving up only after 10 minutes of failures. In result
addresses with sufficient number of transactions would never be fully
scanned and during these 10 minutes app could use gigabytes of
internet data.
- Failures were caused by `eth_getBlockByNumber`/`eth_getBlockByHash`
requests. These requests return significantly bigger responses than
`eth_getBalance`/`eth_transactionsCount` and it is likely that
execution of thousands of them in parallel caused failures for
accounts with hundreds of transactions. Even for an account with 12k
we could successfully determine blocks with transaction in a few
minutes using `eth_getBalance` requests, but `eth_getBlock...`
couldn't be processed for this acc.
- There was no caching for for `eth_getBalance` requests, and this
caused in average 3-4 times more such requests than is needed.
*** How it works now on multiaccount creation:
- On multiacc creation we scan chain for last ~30 eth transactions and
then check erc20 in the range where these eth transactions were found.
For an empty address in multiacc this means:
1. two `eth_getBalance` transactions to determine that there was no
balance change between zero and the last block; two
`eth_transactionsCount` requests to determine there are no outgoing
transactions for this address; total 4 requests for eth transfers
2. 20 `eth_getLogs` for erc20 transfers. This number can be lowered,
but that's not a big deal
- Deduplication of addresses is added and also we don't scan chat
account, so a new multiacc requires ~25 (we also request latest block
number and probably execute a few other calls) request to determine
that multiacc is empty (comparing to ~400 before)
- In case if address contains transactions we:
1. determine the range which contains 20-25 outgoing eth/erc20
transactions. This usually requires up to 10 `eth_transactionCount`
requests
2. then we scan chain for eth transfers using `eth_getBalance` and
`eth_transactionCount` (for double checking zero balances)
3. we make sure that we do not scan db for more than 30 blocks with
transfers. That's important for accounts with mostly incoming
transactions, because the range found on the first step might
contain any number of incoming transfers, but only 20-25 outgoing
transactions
4. when we found ~30 blocks in a given range, we update initial
range `from` block using the oldest found block
5. and now we scan db for erc20transfers using `eth_getLogs`
`oldest-found-eth-block`-`latest-block`, we make not more than 20 calls
6. when all blocks which contain incoming/outgoing transfers for a
given address are found, we save these blocks to db and mark that
transfers from these blocks are still to be fetched
7. Then we select latest ~30 (the number can be adjusted) blocks from
these which were found and fetch transfers, this requires 3-4
requests per transfer.
8. we persist scanned range so that we know were to start next time
9. we dispatch an event which tells client that transactions are found
10. client fetches latest 20 transfers
- when user presses "fetch more" button we check if app's db contains next
20 transfers, if not we scan chain again and return transfers after
small fixes
2019-12-18 11:01:46 +00:00
}
2021-09-09 14:28:54 +00:00
func ( c * loadTransfersCommand ) Command ( ) async . Command {
return async . FiniteCommand {
status-im/status-react#9203 Faster tx fetching with less request
*** How it worked before this PR on multiaccount creation:
- On multiacc creation we scanned chain for eth and erc20 transfers. For
each address of a new empty multiaccount this scan required
1. two `eth_getBalance` requests to find out that there is no any
balance change between zero and the last block, for eth transfers
2. and `chain-size/100000` (currently ~100) `eth_getLogs` requests,
for erc20 transfers
- For some reason we scanned an address of the chat account as well, and
also accounts were not deduplicated. So even for an empty multiacc we
scanned chain twice for each chat and main wallet addresses, in result
app had to execute about 400 requests.
- As mentioned above, `eth_getBalance` requests were used to check if
there were any eth transfers, and that caused empty history in case
if user already used all available eth (so that both zero and latest
blocks show 0 eth for an address). There might have been transactions
but we wouldn't fetch/show them.
- There was no upper limit for the number of rpc requests during the
scan, so it could require indefinite number of requests; the scanning
algorithm was written so that we persisted the whole history of
transactions or tried to scan form the beginning again in case of
failure, giving up only after 10 minutes of failures. In result
addresses with sufficient number of transactions would never be fully
scanned and during these 10 minutes app could use gigabytes of
internet data.
- Failures were caused by `eth_getBlockByNumber`/`eth_getBlockByHash`
requests. These requests return significantly bigger responses than
`eth_getBalance`/`eth_transactionsCount` and it is likely that
execution of thousands of them in parallel caused failures for
accounts with hundreds of transactions. Even for an account with 12k
we could successfully determine blocks with transaction in a few
minutes using `eth_getBalance` requests, but `eth_getBlock...`
couldn't be processed for this acc.
- There was no caching for for `eth_getBalance` requests, and this
caused in average 3-4 times more such requests than is needed.
*** How it works now on multiaccount creation:
- On multiacc creation we scan chain for last ~30 eth transactions and
then check erc20 in the range where these eth transactions were found.
For an empty address in multiacc this means:
1. two `eth_getBalance` transactions to determine that there was no
balance change between zero and the last block; two
`eth_transactionsCount` requests to determine there are no outgoing
transactions for this address; total 4 requests for eth transfers
2. 20 `eth_getLogs` for erc20 transfers. This number can be lowered,
but that's not a big deal
- Deduplication of addresses is added and also we don't scan chat
account, so a new multiacc requires ~25 (we also request latest block
number and probably execute a few other calls) request to determine
that multiacc is empty (comparing to ~400 before)
- In case if address contains transactions we:
1. determine the range which contains 20-25 outgoing eth/erc20
transactions. This usually requires up to 10 `eth_transactionCount`
requests
2. then we scan chain for eth transfers using `eth_getBalance` and
`eth_transactionCount` (for double checking zero balances)
3. we make sure that we do not scan db for more than 30 blocks with
transfers. That's important for accounts with mostly incoming
transactions, because the range found on the first step might
contain any number of incoming transfers, but only 20-25 outgoing
transactions
4. when we found ~30 blocks in a given range, we update initial
range `from` block using the oldest found block
5. and now we scan db for erc20transfers using `eth_getLogs`
`oldest-found-eth-block`-`latest-block`, we make not more than 20 calls
6. when all blocks which contain incoming/outgoing transfers for a
given address are found, we save these blocks to db and mark that
transfers from these blocks are still to be fetched
7. Then we select latest ~30 (the number can be adjusted) blocks from
these which were found and fetch transfers, this requires 3-4
requests per transfer.
8. we persist scanned range so that we know were to start next time
9. we dispatch an event which tells client that transactions are found
10. client fetches latest 20 transfers
- when user presses "fetch more" button we check if app's db contains next
20 transfers, if not we scan chain again and return transfers after
small fixes
2019-12-18 11:01:46 +00:00
Interval : 5 * time . Second ,
Runable : c . Run ,
} . Run
}
2024-01-24 13:00:08 +00:00
// This command always returns nil, even if there is an error in one of the commands.
2024-01-19 12:43:25 +00:00
// `transferCommand`s retry until maxError, but this command doesn't retry.
// In case some transfer is not loaded after max retries, it will be retried only after restart of the app.
// Currently there is no implementation to keep retrying until success. I think this should be implemented
// in `transferCommand` with exponential backoff instead of `loadTransfersCommand` (issue #4608).
status-im/status-react#9203 Faster tx fetching with less request
*** How it worked before this PR on multiaccount creation:
- On multiacc creation we scanned chain for eth and erc20 transfers. For
each address of a new empty multiaccount this scan required
1. two `eth_getBalance` requests to find out that there is no any
balance change between zero and the last block, for eth transfers
2. and `chain-size/100000` (currently ~100) `eth_getLogs` requests,
for erc20 transfers
- For some reason we scanned an address of the chat account as well, and
also accounts were not deduplicated. So even for an empty multiacc we
scanned chain twice for each chat and main wallet addresses, in result
app had to execute about 400 requests.
- As mentioned above, `eth_getBalance` requests were used to check if
there were any eth transfers, and that caused empty history in case
if user already used all available eth (so that both zero and latest
blocks show 0 eth for an address). There might have been transactions
but we wouldn't fetch/show them.
- There was no upper limit for the number of rpc requests during the
scan, so it could require indefinite number of requests; the scanning
algorithm was written so that we persisted the whole history of
transactions or tried to scan form the beginning again in case of
failure, giving up only after 10 minutes of failures. In result
addresses with sufficient number of transactions would never be fully
scanned and during these 10 minutes app could use gigabytes of
internet data.
- Failures were caused by `eth_getBlockByNumber`/`eth_getBlockByHash`
requests. These requests return significantly bigger responses than
`eth_getBalance`/`eth_transactionsCount` and it is likely that
execution of thousands of them in parallel caused failures for
accounts with hundreds of transactions. Even for an account with 12k
we could successfully determine blocks with transaction in a few
minutes using `eth_getBalance` requests, but `eth_getBlock...`
couldn't be processed for this acc.
- There was no caching for for `eth_getBalance` requests, and this
caused in average 3-4 times more such requests than is needed.
*** How it works now on multiaccount creation:
- On multiacc creation we scan chain for last ~30 eth transactions and
then check erc20 in the range where these eth transactions were found.
For an empty address in multiacc this means:
1. two `eth_getBalance` transactions to determine that there was no
balance change between zero and the last block; two
`eth_transactionsCount` requests to determine there are no outgoing
transactions for this address; total 4 requests for eth transfers
2. 20 `eth_getLogs` for erc20 transfers. This number can be lowered,
but that's not a big deal
- Deduplication of addresses is added and also we don't scan chat
account, so a new multiacc requires ~25 (we also request latest block
number and probably execute a few other calls) request to determine
that multiacc is empty (comparing to ~400 before)
- In case if address contains transactions we:
1. determine the range which contains 20-25 outgoing eth/erc20
transactions. This usually requires up to 10 `eth_transactionCount`
requests
2. then we scan chain for eth transfers using `eth_getBalance` and
`eth_transactionCount` (for double checking zero balances)
3. we make sure that we do not scan db for more than 30 blocks with
transfers. That's important for accounts with mostly incoming
transactions, because the range found on the first step might
contain any number of incoming transfers, but only 20-25 outgoing
transactions
4. when we found ~30 blocks in a given range, we update initial
range `from` block using the oldest found block
5. and now we scan db for erc20transfers using `eth_getLogs`
`oldest-found-eth-block`-`latest-block`, we make not more than 20 calls
6. when all blocks which contain incoming/outgoing transfers for a
given address are found, we save these blocks to db and mark that
transfers from these blocks are still to be fetched
7. Then we select latest ~30 (the number can be adjusted) blocks from
these which were found and fetch transfers, this requires 3-4
requests per transfer.
8. we persist scanned range so that we know were to start next time
9. we dispatch an event which tells client that transactions are found
10. client fetches latest 20 transfers
- when user presses "fetch more" button we check if app's db contains next
20 transfers, if not we scan chain again and return transfers after
small fixes
2019-12-18 11:01:46 +00:00
func ( c * loadTransfersCommand ) Run ( parent context . Context ) ( err error ) {
2023-12-10 14:31:30 +00:00
return loadTransfers ( parent , c . blockDAO , c . db , c . chainClient , c . blocksLimit , c . blocksByAddress ,
c . transactionManager , c . pendingTxManager , c . tokenManager , c . feed )
status-im/status-react#9203 Faster tx fetching with less request
*** How it worked before this PR on multiaccount creation:
- On multiacc creation we scanned chain for eth and erc20 transfers. For
each address of a new empty multiaccount this scan required
1. two `eth_getBalance` requests to find out that there is no any
balance change between zero and the last block, for eth transfers
2. and `chain-size/100000` (currently ~100) `eth_getLogs` requests,
for erc20 transfers
- For some reason we scanned an address of the chat account as well, and
also accounts were not deduplicated. So even for an empty multiacc we
scanned chain twice for each chat and main wallet addresses, in result
app had to execute about 400 requests.
- As mentioned above, `eth_getBalance` requests were used to check if
there were any eth transfers, and that caused empty history in case
if user already used all available eth (so that both zero and latest
blocks show 0 eth for an address). There might have been transactions
but we wouldn't fetch/show them.
- There was no upper limit for the number of rpc requests during the
scan, so it could require indefinite number of requests; the scanning
algorithm was written so that we persisted the whole history of
transactions or tried to scan form the beginning again in case of
failure, giving up only after 10 minutes of failures. In result
addresses with sufficient number of transactions would never be fully
scanned and during these 10 minutes app could use gigabytes of
internet data.
- Failures were caused by `eth_getBlockByNumber`/`eth_getBlockByHash`
requests. These requests return significantly bigger responses than
`eth_getBalance`/`eth_transactionsCount` and it is likely that
execution of thousands of them in parallel caused failures for
accounts with hundreds of transactions. Even for an account with 12k
we could successfully determine blocks with transaction in a few
minutes using `eth_getBalance` requests, but `eth_getBlock...`
couldn't be processed for this acc.
- There was no caching for for `eth_getBalance` requests, and this
caused in average 3-4 times more such requests than is needed.
*** How it works now on multiaccount creation:
- On multiacc creation we scan chain for last ~30 eth transactions and
then check erc20 in the range where these eth transactions were found.
For an empty address in multiacc this means:
1. two `eth_getBalance` transactions to determine that there was no
balance change between zero and the last block; two
`eth_transactionsCount` requests to determine there are no outgoing
transactions for this address; total 4 requests for eth transfers
2. 20 `eth_getLogs` for erc20 transfers. This number can be lowered,
but that's not a big deal
- Deduplication of addresses is added and also we don't scan chat
account, so a new multiacc requires ~25 (we also request latest block
number and probably execute a few other calls) request to determine
that multiacc is empty (comparing to ~400 before)
- In case if address contains transactions we:
1. determine the range which contains 20-25 outgoing eth/erc20
transactions. This usually requires up to 10 `eth_transactionCount`
requests
2. then we scan chain for eth transfers using `eth_getBalance` and
`eth_transactionCount` (for double checking zero balances)
3. we make sure that we do not scan db for more than 30 blocks with
transfers. That's important for accounts with mostly incoming
transactions, because the range found on the first step might
contain any number of incoming transfers, but only 20-25 outgoing
transactions
4. when we found ~30 blocks in a given range, we update initial
range `from` block using the oldest found block
5. and now we scan db for erc20transfers using `eth_getLogs`
`oldest-found-eth-block`-`latest-block`, we make not more than 20 calls
6. when all blocks which contain incoming/outgoing transfers for a
given address are found, we save these blocks to db and mark that
transfers from these blocks are still to be fetched
7. Then we select latest ~30 (the number can be adjusted) blocks from
these which were found and fetch transfers, this requires 3-4
requests per transfer.
8. we persist scanned range so that we know were to start next time
9. we dispatch an event which tells client that transactions are found
10. client fetches latest 20 transfers
- when user presses "fetch more" button we check if app's db contains next
20 transfers, if not we scan chain again and return transfers after
small fixes
2019-12-18 11:01:46 +00:00
}
2023-11-27 10:08:17 +00:00
func loadTransfers ( ctx context . Context , blockDAO * BlockDAO , db * Database ,
2023-09-20 08:41:23 +00:00
chainClient chain . ClientInterface , blocksLimitPerAccount int , blocksByAddress map [ common . Address ] [ ] * big . Int ,
2023-08-01 18:50:30 +00:00
transactionManager * TransactionManager , pendingTxManager * transactions . PendingTxTracker ,
2023-06-21 14:09:55 +00:00
tokenManager * token . Manager , feed * event . Feed ) error {
2023-11-27 10:08:17 +00:00
log . Debug ( "loadTransfers start" , "chain" , chainClient . NetworkID ( ) , "limit" , blocksLimitPerAccount )
2023-05-08 06:02:00 +00:00
2021-09-09 14:28:54 +00:00
start := time . Now ( )
group := async . NewGroup ( ctx )
2023-11-27 10:08:17 +00:00
accounts := maps . Keys ( blocksByAddress )
2021-09-09 14:28:54 +00:00
for _ , address := range accounts {
2023-06-01 13:09:50 +00:00
transfers := & transfersCommand {
db : db ,
blockDAO : blockDAO ,
chainClient : chainClient ,
address : address ,
eth : & ETHDownloader {
2021-09-09 14:28:54 +00:00
chainClient : chainClient ,
2023-06-01 13:09:50 +00:00
accounts : [ ] common . Address { address } ,
2023-06-13 14:20:48 +00:00
signer : types . LatestSignerForChainID ( chainClient . ToBigInt ( ) ) ,
2023-06-01 13:09:50 +00:00
db : db ,
} ,
blockNums : blocksByAddress [ address ] ,
transactionManager : transactionManager ,
2023-06-21 14:09:55 +00:00
pendingTxManager : pendingTxManager ,
2023-06-02 20:08:45 +00:00
tokenManager : tokenManager ,
2023-06-14 10:00:56 +00:00
feed : feed ,
2021-09-09 14:28:54 +00:00
}
2023-06-01 13:09:50 +00:00
group . Add ( transfers . Command ( ) )
2021-09-09 14:28:54 +00:00
}
2023-06-01 13:09:50 +00:00
2021-09-09 14:28:54 +00:00
select {
case <- ctx . Done ( ) :
2023-12-10 14:31:30 +00:00
log . Debug ( "loadTransfers cancelled" , "chain" , chainClient . NetworkID ( ) , "error" , ctx . Err ( ) )
2021-09-09 14:28:54 +00:00
case <- group . WaitAsync ( ) :
2023-11-02 17:24:23 +00:00
log . Debug ( "loadTransfers finished for account" , "in" , time . Since ( start ) , "chain" , chainClient . NetworkID ( ) )
2021-09-09 14:28:54 +00:00
}
2023-12-10 14:31:30 +00:00
return nil
2021-09-09 14:28:54 +00:00
}
2021-11-24 12:59:45 +00:00
func isBinanceChain ( chainID uint64 ) bool {
return chainID == binancChainID || chainID == binanceTestChainID
}
2023-06-02 20:08:45 +00:00
// Ensure 1 DBHeader per Block Hash
func uniqueHeaderPerBlockHash ( allHeaders [ ] * DBHeader ) [ ] * DBHeader {
2023-06-01 13:09:50 +00:00
uniqHeadersByHash := map [ common . Hash ] * DBHeader { }
for _ , header := range allHeaders {
uniqHeader , ok := uniqHeadersByHash [ header . Hash ]
if ok {
2023-06-02 20:08:45 +00:00
if len ( header . PreloadedTransactions ) > 0 {
uniqHeader . PreloadedTransactions = append ( uniqHeader . PreloadedTransactions , header . PreloadedTransactions ... )
2023-06-01 13:09:50 +00:00
}
uniqHeadersByHash [ header . Hash ] = uniqHeader
} else {
uniqHeadersByHash [ header . Hash ] = header
}
}
uniqHeaders := [ ] * DBHeader { }
for _ , header := range uniqHeadersByHash {
uniqHeaders = append ( uniqHeaders , header )
}
return uniqHeaders
}
2023-06-02 20:08:45 +00:00
// Organize subTransactions by Transaction Hash
2023-06-13 14:06:36 +00:00
func subTransactionListToTransactionsByTxHash ( subTransactions [ ] Transfer ) map [ common . Hash ] Transaction {
rst := map [ common . Hash ] Transaction { }
2023-06-02 20:08:45 +00:00
for index := range subTransactions {
subTx := & subTransactions [ index ]
txHash := subTx . Transaction . Hash ( )
if _ , ok := rst [ txHash ] ; ! ok {
rst [ txHash ] = make ( [ ] * Transfer , 0 )
}
rst [ txHash ] = append ( rst [ txHash ] , subTx )
}
return rst
}
2024-02-02 10:42:56 +00:00
func IsTransferDetectionEvent ( ev walletevent . EventType ) bool {
if ev == EventInternalETHTransferDetected ||
ev == EventInternalERC20TransferDetected ||
ev == EventInternalERC721TransferDetected ||
ev == EventInternalERC1155TransferDetected {
return true
}
return false
}