feat(Wallet): implement token list from Uniswap (#3273)

Fixes #8807

services/wallet/token/token.go

@@ -5,6 +5,7 @@ import (
 	"database/sql"
 	"errors"
 	"math/big"
+	"strconv"
 	"sync"
 	"time"
 
@@ -49,6 +50,9 @@ type Manager struct {
 	db             *sql.DB
 	RPCClient      *rpc.Client
 	networkManager *network.Manager
+	stores         []store
+	tokenList      []*Token
+	tokenMap       storeMap
 }
 
 func NewTokenManager(
@@ -56,49 +60,52 @@ func NewTokenManager(
 	RPCClient *rpc.Client,
 	networkManager *network.Manager,
 ) *Manager {
-	tokenManager := &Manager{db, RPCClient, networkManager}
+	defaultStore := newDefaultStore()
-
+	// Order of stores is important when merging token lists. The former prevale
-	overrideTokensInPlace(networkManager.GetConfiguredNetworks(), tokenStore)
+	tokenManager := &Manager{db, RPCClient, networkManager, []store{newUniswapStore(), defaultStore}, nil, nil}
 
 	return tokenManager
 }
 
 // overrideTokensInPlace overrides tokens in the store with the ones from the networks
 // BEWARE: overridden tokens will have their original address removed and replaced by the one in networks
-func overrideTokensInPlace(networks []params.Network, store map[uint64]map[common.Address]*Token) {
+func overrideTokensInPlace(networks []params.Network, tokens []*Token) {
 	for _, network := range networks {
 		if len(network.TokenOverrides) == 0 {
 			continue
 		}
 
-		// Map from original address to overridden address
-		overriddenMap := make(map[common.Address]common.Address, len(network.TokenOverrides))
-		tokensMap, ok := store[network.ChainID]
-		if !ok {
-			continue
-		}
 		for _, overrideToken := range network.TokenOverrides {
-			for _, token := range tokensMap {
+			for _, token := range tokens {
 				if token.Symbol == overrideToken.Symbol {
-					overriddenMap[token.Address] = overrideToken.Address
+					token.Address = overrideToken.Address
 				}
 			}
 		}
-		for originalAddress, newAddress := range overriddenMap {
+	}
-			newToken := *tokensMap[originalAddress]
+}
-			tokensMap[newAddress] = &newToken
-			newToken.Address = newAddress
 
-			delete(tokensMap, originalAddress)
+func mergeTokenLists(sliceLists [][]*Token) []*Token {
+	allKeys := make(map[string]bool)
+	res := []*Token{}
+	for _, list := range sliceLists {
+		for _, token := range list {
+			key := strconv.FormatUint(token.ChainID, 10) + token.Address.String()
+			if _, value := allKeys[key]; !value {
+				allKeys[key] = true
+				res = append(res, token)
+			}
 		}
 	}
+	return res
 }
 
 func (tm *Manager) inStore(address common.Address, chainID uint64) bool {
 	if address == nativeChainAddress {
 		return true
 	}
-	tokensMap, ok := tokenStore[chainID]
+
+	tokensMap, ok := tm.tokenMap[chainID]
 	if !ok {
 		return false
 	}
@@ -107,6 +114,31 @@ func (tm *Manager) inStore(address common.Address, chainID uint64) bool {
 	return ok
 }
 
+func (tm *Manager) areTokensFetched() bool {
+	for _, store := range tm.stores {
+		if !store.areTokensFetched() {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (tm *Manager) fetchTokens() []*Token {
+	result := make([]*Token, 0)
+	for _, store := range tm.stores {
+		tokens, err := store.GetTokens()
+		if err != nil {
+			log.Error("can't fetch tokens from store: %s", err)
+			continue
+		}
+
+		result = mergeTokenLists([][]*Token{result, tokens})
+	}
+
+	return result
+}
+
 func (tm *Manager) FindToken(network *params.Network, tokenSymbol string) *Token {
 	if tokenSymbol == network.NativeCurrencySymbol {
 		return tm.ToToken(network)
@@ -130,12 +162,12 @@ func (tm *Manager) FindToken(network *params.Network, tokenSymbol string) *Token
 }
 
 func (tm *Manager) FindSNT(chainID uint64) *Token {
-	tokensMap, ok := tokenStore[chainID]
+	tokens, err := tm.GetTokens(chainID)
-	if !ok {
+	if err != nil {
 		return nil
 	}
 
-	for _, token := range tokensMap {
+	for _, token := range tokens {
 		if token.Symbol == "SNT" || token.Symbol == "STT" {
 			return token
 		}
@@ -163,25 +195,27 @@ func (tm *Manager) GetAllTokensAndNativeCurrencies() ([]*Token, error) {
 }
 
 func (tm *Manager) GetAllTokens() ([]*Token, error) {
-	result := make([]*Token, 0)
+	if tm.areTokensFetched() {
-	for _, tokens := range tokenStore {
+		return tm.tokenList, nil
-		for _, token := range tokens {
-			result = append(result, token)
-		}
 	}
 
+	tm.tokenList = tm.fetchTokens()
+
 	tokens, err := tm.GetCustoms()
 	if err != nil {
-		return nil, err
+		log.Error("can't fetch custom tokens: %s", err)
 	}
 
-	result = append(result, tokens...)
+	tm.tokenList = append(tm.tokenList, tokens...)
+	tm.tokenMap = toTokenMap(tm.tokenList)
 
-	return result, nil
+	overrideTokensInPlace(tm.networkManager.GetConfiguredNetworks(), tm.tokenList)
+
+	return tm.tokenList, nil
 }
 
 func (tm *Manager) GetTokens(chainID uint64) ([]*Token, error) {
-	tokensMap, ok := tokenStore[chainID]
+	tokensMap, ok := tm.tokenMap[chainID]
 	if !ok {
 		return nil, errors.New("no tokens for this network")
 	}
@@ -348,7 +382,12 @@ func (tm *Manager) GetVisible(chainIDs []uint64) (map[uint64][]*Token, error) {
 		}
 
 		found := false
-		for _, token := range tokenStore[chainID] {
+		tokens, err := tm.GetTokens(chainID)
+		if err != nil {
+			continue
+		}
+
+		for _, token := range tokens {
 			if token.Address == address {
 				rst[chainID] = append(rst[chainID], token)
 				found = true

services/wallet/token/token_test.go

@@ -1,6 +1,7 @@
 package token
 
 import (
+	"reflect"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -14,7 +15,7 @@ import (
 func setupTestTokenDB(t *testing.T) (*Manager, func()) {
 	db, err := appdatabase.InitializeDB(":memory:", "wallet-token-tests", 1)
 	require.NoError(t, err)
-	return &Manager{db, nil, nil}, func() {
+	return &Manager{db, nil, nil, nil, nil, nil}, func() {
 		require.NoError(t, db.Close())
 	}
 }
@@ -74,35 +75,103 @@ func TestTokenOverride(t *testing.T) {
 			},
 		},
 	}
-	testTokenStore := map[uint64]map[common.Address]*Token{
+
-		1: {
+	tokenList := []*Token{
-			common.Address{1}: {
+		&Token{
-				Address: common.Address{1},
+			Address: common.Address{1},
-				Symbol:  "SNT",
+			Symbol:  "SNT",
-			},
+			ChainID: 1,
-			common.Address{2}: {
-				Address: common.Address{2},
-				Symbol:  "TNT",
-			},
 		},
-		2: {
+		&Token{
-			common.Address{3}: {
+			Address: common.Address{2},
-				Address: common.Address{3},
+			Symbol:  "TNT",
-				Symbol:  "STT",
+			ChainID: 1,
-			},
+		},
-			common.Address{4}: {
+		&Token{
-				Address: common.Address{4},
+			Address: common.Address{3},
-				Symbol:  "TTT",
+			Symbol:  "STT",
-			},
+			ChainID: 2,
+		},
+		&Token{
+			Address: common.Address{4},
+			Symbol:  "TTT",
+			ChainID: 2,
 		},
 	}
-	overrideTokensInPlace(networks, testTokenStore)
+	testStore := &DefaultStore{
-	_, found := testTokenStore[1][common.Address{1}]
+		tokenList,
+		false,
+	}
+
+	overrideTokensInPlace(networks, testStore.tokenList)
+	tokens, err := testStore.GetTokens()
+	require.NoError(t, err)
+	tokenMap := toTokenMap(tokens)
+	_, found := tokenMap[1][common.Address{1}]
 	require.False(t, found)
-	require.Equal(t, common.Address{11}, testTokenStore[1][common.Address{11}].Address)
+	require.Equal(t, common.Address{11}, tokenMap[1][common.Address{11}].Address)
-	require.Equal(t, common.Address{2}, testTokenStore[1][common.Address{2}].Address)
+	require.Equal(t, common.Address{2}, tokenMap[1][common.Address{2}].Address)
-	_, found = testTokenStore[2][common.Address{3}]
+	_, found = tokenMap[2][common.Address{3}]
 	require.False(t, found)
-	require.Equal(t, common.Address{33}, testTokenStore[2][common.Address{33}].Address)
+	require.Equal(t, common.Address{33}, tokenMap[2][common.Address{33}].Address)
-	require.Equal(t, common.Address{4}, testTokenStore[2][common.Address{4}].Address)
+	require.Equal(t, common.Address{4}, tokenMap[2][common.Address{4}].Address)
+}
+
+func TestMergeTokenLists(t *testing.T) {
+	tokenList1 := []*Token{
+		&Token{
+			Address: common.Address{1},
+			Symbol:  "SNT",
+			ChainID: 1,
+		},
+	}
+	tokenList1Copy := []*Token{
+		&Token{
+			Address: common.Address{1},
+			Symbol:  "SNT",
+			ChainID: 1,
+		},
+	}
+	tokenList2 := []*Token{
+		&Token{
+			Address: common.Address{3},
+			Symbol:  "STT",
+			ChainID: 2,
+		},
+		&Token{
+			Address: common.Address{4},
+			Symbol:  "TTT",
+			ChainID: 2,
+		},
+	}
+	tokenList1Plus2 := []*Token{
+		&Token{
+			Address: common.Address{1},
+			Symbol:  "SNT",
+			ChainID: 1,
+		},
+		&Token{
+			Address: common.Address{3},
+			Symbol:  "STT",
+			ChainID: 2,
+		},
+		&Token{
+			Address: common.Address{4},
+			Symbol:  "TTT",
+			ChainID: 2,
+		},
+	}
+	tokenListEmpty := []*Token{}
+
+	mergedList := mergeTokenLists([][]*Token{tokenListEmpty, tokenListEmpty})
+	require.Equal(t, 0, len(mergedList))
+
+	mergedList = mergeTokenLists([][]*Token{tokenListEmpty, tokenList1})
+	require.True(t, reflect.DeepEqual(mergedList, tokenList1))
+
+	mergedList = mergeTokenLists([][]*Token{tokenList1, tokenList1Copy})
+	require.True(t, reflect.DeepEqual(mergedList, tokenList1))
+
+	mergedList = mergeTokenLists([][]*Token{tokenList1, tokenList2})
+	require.True(t, reflect.DeepEqual(mergedList, tokenList1Plus2))
 }

services/wallet/token/uniswap_tokenstore.go

@@ -0,0 +1,54 @@
+package token
+
+import (
+	"io/ioutil"
+	"net/http"
+	"time"
+)
+
+type uniswapStore struct {
+	client        *http.Client
+	tokensFetched bool
+}
+
+const uniswapTokensURL = "https://gateway.ipfs.io/ipns/tokens.uniswap.org" // nolint:gosec
+const tokenListSchemaURL = "https://uniswap.org/tokenlist.schema.json"     // nolint:gosec
+
+func newUniswapStore() *uniswapStore {
+	return &uniswapStore{client: &http.Client{Timeout: time.Minute}, tokensFetched: false}
+}
+
+func (ts *uniswapStore) doQuery(url string) (*http.Response, error) {
+	return ts.client.Get(url)
+}
+
+func (ts *uniswapStore) areTokensFetched() bool {
+	return ts.tokensFetched
+}
+
+func (ts *uniswapStore) GetTokens() ([]*Token, error) {
+	resp, err := ts.doQuery(uniswapTokensURL)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	// In an unlikely case when token list is fetched fine,
+	// but fails to validate against the schema, we don't want
+	// to refetch the tokens on every GetTokens call as it will
+	// still fail but will be wasting CPU cycles until restart,
+	// so lets keep tokensFetched before validate() call
+	ts.tokensFetched = true
+
+	_, err = validateDocument(string(body), tokenListSchemaURL)
+	if err != nil {
+		return nil, err
+	}
+
+	return bytesToTokens(body)
+}