2019-06-14 10:16:30 +00:00
|
|
|
package wallet
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"errors"
|
|
|
|
"math/big"
|
|
|
|
|
|
|
|
"github.com/ethereum/go-ethereum/common"
|
|
|
|
"github.com/ethereum/go-ethereum/common/hexutil"
|
|
|
|
"github.com/ethereum/go-ethereum/log"
|
|
|
|
)
|
|
|
|
|
2019-07-02 07:28:57 +00:00
|
|
|
var (
|
|
|
|
// ErrServiceNotInitialized returned when wallet is not initialized/started,.
|
|
|
|
ErrServiceNotInitialized = errors.New("wallet service is not initialized")
|
|
|
|
)
|
|
|
|
|
2019-06-14 10:16:30 +00:00
|
|
|
func NewAPI(s *Service) *API {
|
|
|
|
return &API{s}
|
|
|
|
}
|
|
|
|
|
|
|
|
// API is class with methods available over RPC.
|
|
|
|
type API struct {
|
|
|
|
s *Service
|
|
|
|
}
|
|
|
|
|
|
|
|
// GetTransfers returns transfers in range of blocks. If `end` is nil all transfers from `start` will be returned.
|
|
|
|
// TODO(dshulyak) benchmark loading many transfers from database. We can avoid json unmarshal/marshal if we will
|
|
|
|
// read header, tx and receipt as a raw json.
|
2019-07-15 11:16:07 +00:00
|
|
|
func (api *API) GetTransfers(ctx context.Context, start, end *hexutil.Big) ([]TransferView, error) {
|
2019-06-14 10:16:30 +00:00
|
|
|
log.Debug("call to get transfers", "start", start, "end", end)
|
|
|
|
if start == nil {
|
|
|
|
return nil, errors.New("start of the query must be provided. use 0 if you want to load all transfers")
|
|
|
|
}
|
|
|
|
if api.s.db == nil {
|
2019-07-02 07:28:57 +00:00
|
|
|
return nil, ErrServiceNotInitialized
|
2019-06-14 10:16:30 +00:00
|
|
|
}
|
|
|
|
rst, err := api.s.db.GetTransfers((*big.Int)(start), (*big.Int)(end))
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
log.Debug("result from database for transfers", "start", start, "end", end, "len", len(rst))
|
2019-07-15 11:16:07 +00:00
|
|
|
return castToTransferViews(rst), nil
|
2019-06-14 10:16:30 +00:00
|
|
|
}
|
|
|
|
|
status-im/status-react#9203 Faster tx fetching with less request
*** How it worked before this PR on multiaccount creation:
- On multiacc creation we scanned chain for eth and erc20 transfers. For
each address of a new empty multiaccount this scan required
1. two `eth_getBalance` requests to find out that there is no any
balance change between zero and the last block, for eth transfers
2. and `chain-size/100000` (currently ~100) `eth_getLogs` requests,
for erc20 transfers
- For some reason we scanned an address of the chat account as well, and
also accounts were not deduplicated. So even for an empty multiacc we
scanned chain twice for each chat and main wallet addresses, in result
app had to execute about 400 requests.
- As mentioned above, `eth_getBalance` requests were used to check if
there were any eth transfers, and that caused empty history in case
if user already used all available eth (so that both zero and latest
blocks show 0 eth for an address). There might have been transactions
but we wouldn't fetch/show them.
- There was no upper limit for the number of rpc requests during the
scan, so it could require indefinite number of requests; the scanning
algorithm was written so that we persisted the whole history of
transactions or tried to scan form the beginning again in case of
failure, giving up only after 10 minutes of failures. In result
addresses with sufficient number of transactions would never be fully
scanned and during these 10 minutes app could use gigabytes of
internet data.
- Failures were caused by `eth_getBlockByNumber`/`eth_getBlockByHash`
requests. These requests return significantly bigger responses than
`eth_getBalance`/`eth_transactionsCount` and it is likely that
execution of thousands of them in parallel caused failures for
accounts with hundreds of transactions. Even for an account with 12k
we could successfully determine blocks with transaction in a few
minutes using `eth_getBalance` requests, but `eth_getBlock...`
couldn't be processed for this acc.
- There was no caching for for `eth_getBalance` requests, and this
caused in average 3-4 times more such requests than is needed.
*** How it works now on multiaccount creation:
- On multiacc creation we scan chain for last ~30 eth transactions and
then check erc20 in the range where these eth transactions were found.
For an empty address in multiacc this means:
1. two `eth_getBalance` transactions to determine that there was no
balance change between zero and the last block; two
`eth_transactionsCount` requests to determine there are no outgoing
transactions for this address; total 4 requests for eth transfers
2. 20 `eth_getLogs` for erc20 transfers. This number can be lowered,
but that's not a big deal
- Deduplication of addresses is added and also we don't scan chat
account, so a new multiacc requires ~25 (we also request latest block
number and probably execute a few other calls) request to determine
that multiacc is empty (comparing to ~400 before)
- In case if address contains transactions we:
1. determine the range which contains 20-25 outgoing eth/erc20
transactions. This usually requires up to 10 `eth_transactionCount`
requests
2. then we scan chain for eth transfers using `eth_getBalance` and
`eth_transactionCount` (for double checking zero balances)
3. we make sure that we do not scan db for more than 30 blocks with
transfers. That's important for accounts with mostly incoming
transactions, because the range found on the first step might
contain any number of incoming transfers, but only 20-25 outgoing
transactions
4. when we found ~30 blocks in a given range, we update initial
range `from` block using the oldest found block
5. and now we scan db for erc20transfers using `eth_getLogs`
`oldest-found-eth-block`-`latest-block`, we make not more than 20 calls
6. when all blocks which contain incoming/outgoing transfers for a
given address are found, we save these blocks to db and mark that
transfers from these blocks are still to be fetched
7. Then we select latest ~30 (the number can be adjusted) blocks from
these which were found and fetch transfers, this requires 3-4
requests per transfer.
8. we persist scanned range so that we know were to start next time
9. we dispatch an event which tells client that transactions are found
10. client fetches latest 20 transfers
- when user presses "fetch more" button we check if app's db contains next
20 transfers, if not we scan chain again and return transfers after
small fixes
2019-12-18 11:01:46 +00:00
|
|
|
type StatsView struct {
|
|
|
|
BlocksStats map[int64]int64 `json:"blocksStats"`
|
|
|
|
TransfersCount int64 `json:"transfersCount"`
|
|
|
|
}
|
|
|
|
|
|
|
|
// GetTransfersFromBlock
|
|
|
|
func (api *API) GetTransfersFromBlock(ctx context.Context, address common.Address, block *hexutil.Big) ([]TransferView, error) {
|
|
|
|
log.Debug("[WalletAPI:: GetTransfersFromBlock] get transfers from block", "address", address, "block", block)
|
|
|
|
if api.s.db == nil {
|
|
|
|
return nil, ErrServiceNotInitialized
|
|
|
|
}
|
|
|
|
|
|
|
|
blocksByAddress := make(map[common.Address][]*big.Int)
|
|
|
|
blocksByAddress[address] = []*big.Int{block.ToInt()}
|
|
|
|
|
|
|
|
txCommand := &loadTransfersCommand{
|
|
|
|
accounts: []common.Address{address},
|
|
|
|
db: api.s.db,
|
|
|
|
chain: api.s.reactor.chain,
|
|
|
|
client: api.s.client,
|
|
|
|
blocksByAddress: blocksByAddress,
|
|
|
|
}
|
|
|
|
|
|
|
|
err := txCommand.Command()(ctx)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2019-06-14 10:16:30 +00:00
|
|
|
}
|
status-im/status-react#9203 Faster tx fetching with less request
*** How it worked before this PR on multiaccount creation:
- On multiacc creation we scanned chain for eth and erc20 transfers. For
each address of a new empty multiaccount this scan required
1. two `eth_getBalance` requests to find out that there is no any
balance change between zero and the last block, for eth transfers
2. and `chain-size/100000` (currently ~100) `eth_getLogs` requests,
for erc20 transfers
- For some reason we scanned an address of the chat account as well, and
also accounts were not deduplicated. So even for an empty multiacc we
scanned chain twice for each chat and main wallet addresses, in result
app had to execute about 400 requests.
- As mentioned above, `eth_getBalance` requests were used to check if
there were any eth transfers, and that caused empty history in case
if user already used all available eth (so that both zero and latest
blocks show 0 eth for an address). There might have been transactions
but we wouldn't fetch/show them.
- There was no upper limit for the number of rpc requests during the
scan, so it could require indefinite number of requests; the scanning
algorithm was written so that we persisted the whole history of
transactions or tried to scan form the beginning again in case of
failure, giving up only after 10 minutes of failures. In result
addresses with sufficient number of transactions would never be fully
scanned and during these 10 minutes app could use gigabytes of
internet data.
- Failures were caused by `eth_getBlockByNumber`/`eth_getBlockByHash`
requests. These requests return significantly bigger responses than
`eth_getBalance`/`eth_transactionsCount` and it is likely that
execution of thousands of them in parallel caused failures for
accounts with hundreds of transactions. Even for an account with 12k
we could successfully determine blocks with transaction in a few
minutes using `eth_getBalance` requests, but `eth_getBlock...`
couldn't be processed for this acc.
- There was no caching for for `eth_getBalance` requests, and this
caused in average 3-4 times more such requests than is needed.
*** How it works now on multiaccount creation:
- On multiacc creation we scan chain for last ~30 eth transactions and
then check erc20 in the range where these eth transactions were found.
For an empty address in multiacc this means:
1. two `eth_getBalance` transactions to determine that there was no
balance change between zero and the last block; two
`eth_transactionsCount` requests to determine there are no outgoing
transactions for this address; total 4 requests for eth transfers
2. 20 `eth_getLogs` for erc20 transfers. This number can be lowered,
but that's not a big deal
- Deduplication of addresses is added and also we don't scan chat
account, so a new multiacc requires ~25 (we also request latest block
number and probably execute a few other calls) request to determine
that multiacc is empty (comparing to ~400 before)
- In case if address contains transactions we:
1. determine the range which contains 20-25 outgoing eth/erc20
transactions. This usually requires up to 10 `eth_transactionCount`
requests
2. then we scan chain for eth transfers using `eth_getBalance` and
`eth_transactionCount` (for double checking zero balances)
3. we make sure that we do not scan db for more than 30 blocks with
transfers. That's important for accounts with mostly incoming
transactions, because the range found on the first step might
contain any number of incoming transfers, but only 20-25 outgoing
transactions
4. when we found ~30 blocks in a given range, we update initial
range `from` block using the oldest found block
5. and now we scan db for erc20transfers using `eth_getLogs`
`oldest-found-eth-block`-`latest-block`, we make not more than 20 calls
6. when all blocks which contain incoming/outgoing transfers for a
given address are found, we save these blocks to db and mark that
transfers from these blocks are still to be fetched
7. Then we select latest ~30 (the number can be adjusted) blocks from
these which were found and fetch transfers, this requires 3-4
requests per transfer.
8. we persist scanned range so that we know were to start next time
9. we dispatch an event which tells client that transactions are found
10. client fetches latest 20 transfers
- when user presses "fetch more" button we check if app's db contains next
20 transfers, if not we scan chain again and return transfers after
small fixes
2019-12-18 11:01:46 +00:00
|
|
|
|
|
|
|
rst, err := api.s.db.GetTransfersInRange(address, block.ToInt(), block.ToInt())
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return castToTransferViews(rst), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// GetTransfersByAddress returns transfers for a single address
|
|
|
|
func (api *API) GetTransfersByAddress(ctx context.Context, address common.Address, beforeBlock, limit *hexutil.Big) ([]TransferView, error) {
|
|
|
|
log.Info("call to get transfers for an address", "address", address, "block", beforeBlock, "limit", limit)
|
2019-06-14 10:16:30 +00:00
|
|
|
if api.s.db == nil {
|
2019-07-02 07:28:57 +00:00
|
|
|
return nil, ErrServiceNotInitialized
|
2019-06-14 10:16:30 +00:00
|
|
|
}
|
status-im/status-react#9203 Faster tx fetching with less request
*** How it worked before this PR on multiaccount creation:
- On multiacc creation we scanned chain for eth and erc20 transfers. For
each address of a new empty multiaccount this scan required
1. two `eth_getBalance` requests to find out that there is no any
balance change between zero and the last block, for eth transfers
2. and `chain-size/100000` (currently ~100) `eth_getLogs` requests,
for erc20 transfers
- For some reason we scanned an address of the chat account as well, and
also accounts were not deduplicated. So even for an empty multiacc we
scanned chain twice for each chat and main wallet addresses, in result
app had to execute about 400 requests.
- As mentioned above, `eth_getBalance` requests were used to check if
there were any eth transfers, and that caused empty history in case
if user already used all available eth (so that both zero and latest
blocks show 0 eth for an address). There might have been transactions
but we wouldn't fetch/show them.
- There was no upper limit for the number of rpc requests during the
scan, so it could require indefinite number of requests; the scanning
algorithm was written so that we persisted the whole history of
transactions or tried to scan form the beginning again in case of
failure, giving up only after 10 minutes of failures. In result
addresses with sufficient number of transactions would never be fully
scanned and during these 10 minutes app could use gigabytes of
internet data.
- Failures were caused by `eth_getBlockByNumber`/`eth_getBlockByHash`
requests. These requests return significantly bigger responses than
`eth_getBalance`/`eth_transactionsCount` and it is likely that
execution of thousands of them in parallel caused failures for
accounts with hundreds of transactions. Even for an account with 12k
we could successfully determine blocks with transaction in a few
minutes using `eth_getBalance` requests, but `eth_getBlock...`
couldn't be processed for this acc.
- There was no caching for for `eth_getBalance` requests, and this
caused in average 3-4 times more such requests than is needed.
*** How it works now on multiaccount creation:
- On multiacc creation we scan chain for last ~30 eth transactions and
then check erc20 in the range where these eth transactions were found.
For an empty address in multiacc this means:
1. two `eth_getBalance` transactions to determine that there was no
balance change between zero and the last block; two
`eth_transactionsCount` requests to determine there are no outgoing
transactions for this address; total 4 requests for eth transfers
2. 20 `eth_getLogs` for erc20 transfers. This number can be lowered,
but that's not a big deal
- Deduplication of addresses is added and also we don't scan chat
account, so a new multiacc requires ~25 (we also request latest block
number and probably execute a few other calls) request to determine
that multiacc is empty (comparing to ~400 before)
- In case if address contains transactions we:
1. determine the range which contains 20-25 outgoing eth/erc20
transactions. This usually requires up to 10 `eth_transactionCount`
requests
2. then we scan chain for eth transfers using `eth_getBalance` and
`eth_transactionCount` (for double checking zero balances)
3. we make sure that we do not scan db for more than 30 blocks with
transfers. That's important for accounts with mostly incoming
transactions, because the range found on the first step might
contain any number of incoming transfers, but only 20-25 outgoing
transactions
4. when we found ~30 blocks in a given range, we update initial
range `from` block using the oldest found block
5. and now we scan db for erc20transfers using `eth_getLogs`
`oldest-found-eth-block`-`latest-block`, we make not more than 20 calls
6. when all blocks which contain incoming/outgoing transfers for a
given address are found, we save these blocks to db and mark that
transfers from these blocks are still to be fetched
7. Then we select latest ~30 (the number can be adjusted) blocks from
these which were found and fetch transfers, this requires 3-4
requests per transfer.
8. we persist scanned range so that we know were to start next time
9. we dispatch an event which tells client that transactions are found
10. client fetches latest 20 transfers
- when user presses "fetch more" button we check if app's db contains next
20 transfers, if not we scan chain again and return transfers after
small fixes
2019-12-18 11:01:46 +00:00
|
|
|
rst, err := api.s.db.GetTransfersByAddress(address, beforeBlock.ToInt(), limit.ToInt().Int64())
|
2019-06-14 10:16:30 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
status-im/status-react#9203 Faster tx fetching with less request
*** How it worked before this PR on multiaccount creation:
- On multiacc creation we scanned chain for eth and erc20 transfers. For
each address of a new empty multiaccount this scan required
1. two `eth_getBalance` requests to find out that there is no any
balance change between zero and the last block, for eth transfers
2. and `chain-size/100000` (currently ~100) `eth_getLogs` requests,
for erc20 transfers
- For some reason we scanned an address of the chat account as well, and
also accounts were not deduplicated. So even for an empty multiacc we
scanned chain twice for each chat and main wallet addresses, in result
app had to execute about 400 requests.
- As mentioned above, `eth_getBalance` requests were used to check if
there were any eth transfers, and that caused empty history in case
if user already used all available eth (so that both zero and latest
blocks show 0 eth for an address). There might have been transactions
but we wouldn't fetch/show them.
- There was no upper limit for the number of rpc requests during the
scan, so it could require indefinite number of requests; the scanning
algorithm was written so that we persisted the whole history of
transactions or tried to scan form the beginning again in case of
failure, giving up only after 10 minutes of failures. In result
addresses with sufficient number of transactions would never be fully
scanned and during these 10 minutes app could use gigabytes of
internet data.
- Failures were caused by `eth_getBlockByNumber`/`eth_getBlockByHash`
requests. These requests return significantly bigger responses than
`eth_getBalance`/`eth_transactionsCount` and it is likely that
execution of thousands of them in parallel caused failures for
accounts with hundreds of transactions. Even for an account with 12k
we could successfully determine blocks with transaction in a few
minutes using `eth_getBalance` requests, but `eth_getBlock...`
couldn't be processed for this acc.
- There was no caching for for `eth_getBalance` requests, and this
caused in average 3-4 times more such requests than is needed.
*** How it works now on multiaccount creation:
- On multiacc creation we scan chain for last ~30 eth transactions and
then check erc20 in the range where these eth transactions were found.
For an empty address in multiacc this means:
1. two `eth_getBalance` transactions to determine that there was no
balance change between zero and the last block; two
`eth_transactionsCount` requests to determine there are no outgoing
transactions for this address; total 4 requests for eth transfers
2. 20 `eth_getLogs` for erc20 transfers. This number can be lowered,
but that's not a big deal
- Deduplication of addresses is added and also we don't scan chat
account, so a new multiacc requires ~25 (we also request latest block
number and probably execute a few other calls) request to determine
that multiacc is empty (comparing to ~400 before)
- In case if address contains transactions we:
1. determine the range which contains 20-25 outgoing eth/erc20
transactions. This usually requires up to 10 `eth_transactionCount`
requests
2. then we scan chain for eth transfers using `eth_getBalance` and
`eth_transactionCount` (for double checking zero balances)
3. we make sure that we do not scan db for more than 30 blocks with
transfers. That's important for accounts with mostly incoming
transactions, because the range found on the first step might
contain any number of incoming transfers, but only 20-25 outgoing
transactions
4. when we found ~30 blocks in a given range, we update initial
range `from` block using the oldest found block
5. and now we scan db for erc20transfers using `eth_getLogs`
`oldest-found-eth-block`-`latest-block`, we make not more than 20 calls
6. when all blocks which contain incoming/outgoing transfers for a
given address are found, we save these blocks to db and mark that
transfers from these blocks are still to be fetched
7. Then we select latest ~30 (the number can be adjusted) blocks from
these which were found and fetch transfers, this requires 3-4
requests per transfer.
8. we persist scanned range so that we know were to start next time
9. we dispatch an event which tells client that transactions are found
10. client fetches latest 20 transfers
- when user presses "fetch more" button we check if app's db contains next
20 transfers, if not we scan chain again and return transfers after
small fixes
2019-12-18 11:01:46 +00:00
|
|
|
|
|
|
|
transfersCount := big.NewInt(int64(len(rst)))
|
|
|
|
if limit.ToInt().Cmp(transfersCount) == 1 {
|
|
|
|
block, err := api.s.db.GetFirstKnownBlock(address)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
if block == nil {
|
|
|
|
return castToTransferViews(rst), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
from, err := findFirstRange(ctx, address, block, api.s.client)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
fromByAddress := map[common.Address]*big.Int{address: from}
|
|
|
|
toByAddress := map[common.Address]*big.Int{address: block}
|
|
|
|
|
|
|
|
balanceCache := newBalanceCache()
|
|
|
|
blocksCommand := &findAndCheckBlockRangeCommand{
|
|
|
|
accounts: []common.Address{address},
|
|
|
|
db: api.s.db,
|
|
|
|
chain: api.s.reactor.chain,
|
|
|
|
client: api.s.client,
|
|
|
|
balanceCache: balanceCache,
|
|
|
|
feed: api.s.feed,
|
|
|
|
fromByAddress: fromByAddress,
|
|
|
|
toByAddress: toByAddress,
|
|
|
|
}
|
|
|
|
|
|
|
|
if err = blocksCommand.Command()(ctx); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
blocks, err := api.s.db.GetBlocksByAddress(address, numberOfBlocksCheckedPerIteration)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
log.Info("checking blocks again", "blocks", len(blocks))
|
|
|
|
if len(blocks) > 0 {
|
|
|
|
txCommand := &loadTransfersCommand{
|
|
|
|
accounts: []common.Address{address},
|
|
|
|
db: api.s.db,
|
|
|
|
chain: api.s.reactor.chain,
|
|
|
|
client: api.s.client,
|
|
|
|
}
|
|
|
|
|
|
|
|
err = txCommand.Command()(ctx)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
rst, err = api.s.db.GetTransfersByAddress(address, beforeBlock.ToInt(), limit.ToInt().Int64())
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-07-15 11:16:07 +00:00
|
|
|
return castToTransferViews(rst), nil
|
2019-06-14 10:16:30 +00:00
|
|
|
}
|
2019-07-02 07:28:57 +00:00
|
|
|
|
|
|
|
// GetTokensBalances return mapping of token balances for every account.
|
|
|
|
func (api *API) GetTokensBalances(ctx context.Context, accounts, tokens []common.Address) (map[common.Address]map[common.Address]*big.Int, error) {
|
|
|
|
if api.s.client == nil {
|
|
|
|
return nil, ErrServiceNotInitialized
|
|
|
|
}
|
|
|
|
return GetTokensBalances(ctx, api.s.client, accounts, tokens)
|
|
|
|
}
|
2019-12-10 17:31:08 +00:00
|
|
|
|
|
|
|
func (api *API) GetCustomTokens(ctx context.Context) ([]*Token, error) {
|
|
|
|
log.Debug("call to get custom tokens")
|
|
|
|
rst, err := api.s.db.GetCustomTokens()
|
|
|
|
log.Debug("result from database for custom tokens", "len", len(rst))
|
|
|
|
return rst, err
|
|
|
|
}
|
|
|
|
|
|
|
|
func (api *API) AddCustomToken(ctx context.Context, token Token) error {
|
|
|
|
log.Debug("call to create or edit custom token")
|
|
|
|
err := api.s.db.AddCustomToken(token)
|
|
|
|
log.Debug("result from database for create or edit custom token", "err", err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
func (api *API) DeleteCustomToken(ctx context.Context, address common.Address) error {
|
|
|
|
log.Debug("call to remove custom token")
|
|
|
|
err := api.s.db.DeleteCustomToken(address)
|
|
|
|
log.Debug("result from database for remove custom token", "err", err)
|
|
|
|
return err
|
|
|
|
}
|