status-im
/
status-go
mirror of https://github.com/status-im/status-go.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix_: re-trying http requests up to the predefined max retries 

This commit should fix the issue with Cryptocompare and CoinGecko providers down.
Often happens that the first request fails and because of that:
- the Router cannot complete calculation successfully
- some tests are failing
- some users very often see a red banner line saying the providers are down
This commit is contained in:
Sale Djenic 2024-05-22 13:56:30 +02:00 committed by saledjenic
parent dcef87af3f
commit 534c756bf5
4 changed files with 107 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
services/wallet/thirdparty/coingecko/client.go vendored
View File

@ -1,13 +1,12 @@
package coingecko
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
"sync"
"time"
"github.com/status-im/status-go/services/wallet/thirdparty"
"github.com/status-im/status-go/services/wallet/thirdparty/utils"
@ -83,14 +82,18 @@ type GeckoToken struct {
}
type Client struct {
client *http.Client
httpClient *thirdparty.HTTPClient
tokens map[string][]GeckoToken
tokensURL string
fetchTokensMutex sync.Mutex
}
func NewClient() *Client {
return &Client{client: &http.Client{Timeout: time.Minute}, tokens: make(map[string][]GeckoToken), tokensURL: fmt.Sprintf("%scoins/list?include_platform=true", baseURL)}
return &Client{
httpClient: thirdparty.NewHTTPClient(),
tokens: make(map[string][]GeckoToken),
tokensURL: fmt.Sprintf("%scoins/list", baseURL),
}
}
func (gt *GeckoToken) UnmarshalJSON(data []byte) error {
@ -124,15 +127,6 @@ func (gt *GeckoToken) UnmarshalJSON(data []byte) error {
return nil
}
func (c *Client) DoQuery(url string) (*http.Response, error) {
resp, err := c.client.Get(url)
if err != nil {
return nil, err
}
return resp, nil
}
func mapTokensToSymbols(tokens []GeckoToken, tokenMap map[string][]GeckoToken) {
for _, token := range tokens {
symbol := strings.ToUpper(token.Symbol)
@ -174,19 +168,17 @@ func (c *Client) getTokens() (map[string][]GeckoToken, error) {
return c.tokens, nil
}
resp, err := c.DoQuery(c.tokensURL)
if err != nil {
return nil, err
}
defer resp.Body.Close()
params := url.Values{}
params.Add("include_platform", "true")
body, err := ioutil.ReadAll(resp.Body)
url := c.tokensURL
response, err := c.httpClient.DoGetRequest(context.Background(), url, params)
if err != nil {
return nil, err
}
var tokens []GeckoToken
err = json.Unmarshal(body, &tokens)
err = json.Unmarshal(response, &tokens)
if err != nil {
return nil, err
}
@ -218,22 +210,21 @@ func (c *Client) FetchPrices(symbols []string, currencies []string) (map[string]
if err != nil {
return nil, err
}
url := fmt.Sprintf("%ssimple/price?ids=%s&vs_currencies=%s", baseURL, strings.Join(ids, ","), strings.Join(currencies, ","))
resp, err := c.DoQuery(url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
params := url.Values{}
params.Add("ids", strings.Join(ids, ","))
params.Add("vs_currencies", strings.Join(currencies, ","))
url := fmt.Sprintf("%ssimple/price", baseURL)
response, err := c.httpClient.DoGetRequest(context.Background(), url, params)
if err != nil {
return nil, err
}
prices := make(map[string]map[string]float64)
err = json.Unmarshal(body, &prices)
err = json.Unmarshal(response, &prices)
if err != nil {
return nil, fmt.Errorf("%s - %s", err, string(body))
return nil, fmt.Errorf("%s - %s", err, string(response))
}
tokens, err := c.getTokens()
@ -279,23 +270,26 @@ func (c *Client) FetchTokenMarketValues(symbols []string, currency string) (map[
if err != nil {
return nil, err
}
url := fmt.Sprintf("%scoins/markets?ids=%s&vs_currency=%s&order=market_cap_desc&per_page=250&page=1&sparkline=false&price_change_percentage=%s", baseURL, strings.Join(ids, ","), currency, "1h%2C24h")
resp, err := c.DoQuery(url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
params := url.Values{}
params.Add("ids", strings.Join(ids, ","))
params.Add("vs_currency", currency)
params.Add("order", "market_cap_desc")
params.Add("per_page", "250")
params.Add("page", "1")
params.Add("sparkline", "false")
params.Add("price_change_percentage", "1h,24h")
body, err := ioutil.ReadAll(resp.Body)
url := fmt.Sprintf("%scoins/markets", baseURL)
response, err := c.httpClient.DoGetRequest(context.Background(), url, params)
if err != nil {
return nil, err
}
var marketValues []GeckoMarketValues
err = json.Unmarshal(body, &marketValues)
err = json.Unmarshal(response, &marketValues)
if err != nil {
return nil, fmt.Errorf("%s - %s", err, string(body))
return nil, fmt.Errorf("%s - %s", err, string(response))
}
tokens, err := c.getTokens()
@ -344,19 +338,18 @@ func (c *Client) FetchHistoricalDailyPrices(symbol string, currency string, limi
return nil, err
}
url := fmt.Sprintf("%scoins/%s/market_chart?vs_currency=%s&days=30", baseURL, id, currency)
resp, err := c.DoQuery(url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
params := url.Values{}
params.Add("vs_currency", currency)
params.Add("days", "30")
body, err := ioutil.ReadAll(resp.Body)
url := fmt.Sprintf("%scoins/%s/market_chart", baseURL, id)
response, err := c.httpClient.DoGetRequest(context.Background(), url, params)
if err != nil {
return nil, err
}
var container HistoricalPriceContainer
err = json.Unmarshal(body, &container)
err = json.Unmarshal(response, &container)
if err != nil {
return nil, err
}

8
services/wallet/thirdparty/coingecko/client_test.go vendored
View File

@ -7,6 +7,8 @@ import (
"reflect"
"testing"
"github.com/status-im/status-go/services/wallet/thirdparty"
"github.com/stretchr/testify/require"
)
@ -68,7 +70,7 @@ func TestGetTokensSuccess(t *testing.T) {
defer stop()
geckoClient := &Client{
client: srv.Client(),
httpClient: thirdparty.NewHTTPClient(),
tokens: make(map[string][]GeckoToken),
tokensURL: srv.URL,
}
@ -150,7 +152,7 @@ func TestGetTokensEthPlatform(t *testing.T) {
defer stop()
geckoClient := &Client{
client: srv.Client(),
httpClient: thirdparty.NewHTTPClient(),
tokens: make(map[string][]GeckoToken),
tokensURL: srv.URL,
}
@ -166,7 +168,7 @@ func TestGetTokensFailure(t *testing.T) {
defer stop()
geckoClient := &Client{
client: srv.Client(),
httpClient: thirdparty.NewHTTPClient(),
tokens: make(map[string][]GeckoToken),
tokensURL: srv.URL,
}

104
services/wallet/thirdparty/cryptocompare/client.go vendored
View File

@ -1,18 +1,18 @@
package cryptocompare
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
"time"
"github.com/status-im/status-go/services/wallet/thirdparty"
"github.com/status-im/status-go/services/wallet/thirdparty/utils"
)
const baseURL = "https://min-api.cryptocompare.com"
const extraParamStatus = "Status.im"
type HistoricalPricesContainer struct {
Aggregated bool `json:"Aggregated"`
@ -34,20 +34,13 @@ type MarketValuesContainer struct {
}
type Client struct {
client *http.Client
httpClient *thirdparty.HTTPClient
}
func NewClient() *Client {
return &Client{client: &http.Client{Timeout: time.Minute}}
}
func (c *Client) DoQuery(url string) (*http.Response, error) {
resp, err := c.client.Get(url)
if err != nil {
return nil, err
return &Client{
httpClient: thirdparty.NewHTTPClient(),
}
return resp, nil
}
func (c *Client) FetchPrices(symbols []string, currencies []string) (map[string]map[string]float64, error) {
@ -56,22 +49,22 @@ func (c *Client) FetchPrices(symbols []string, currencies []string) (map[string]
realCurrencies := utils.RenameSymbols(currencies)
for _, smbls := range chunks {
realSymbols := utils.RenameSymbols(smbls)
url := fmt.Sprintf("%s/data/pricemulti?fsyms=%s&tsyms=%s&extraParams=Status.im", baseURL, strings.Join(realSymbols, ","), strings.Join(realCurrencies, ","))
resp, err := c.DoQuery(url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
params := url.Values{}
params.Add("fsyms", strings.Join(realSymbols, ","))
params.Add("tsyms", strings.Join(realCurrencies, ","))
params.Add("extraParams", extraParamStatus)
url := fmt.Sprintf("%s/data/pricemulti", baseURL)
response, err := c.httpClient.DoGetRequest(context.Background(), url, params)
if err != nil {
return nil, err
}
prices := make(map[string]map[string]float64)
err = json.Unmarshal(body, &prices)
err = json.Unmarshal(response, &prices)
if err != nil {
return nil, fmt.Errorf("%s - %s", err, string(body))
return nil, fmt.Errorf("%s - %s", err, string(response))
}
for _, symbol := range smbls {
@ -86,19 +79,13 @@ func (c *Client) FetchPrices(symbols []string, currencies []string) (map[string]
func (c *Client) FetchTokenDetails(symbols []string) (map[string]thirdparty.TokenDetails, error) {
url := fmt.Sprintf("%s/data/all/coinlist", baseURL)
resp, err := c.DoQuery(url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
response, err := c.httpClient.DoGetRequest(context.Background(), url, nil)
if err != nil {
return nil, err
}
container := TokenDetailsContainer{}
err = json.Unmarshal(body, &container)
err = json.Unmarshal(response, &container)
if err != nil {
return nil, err
}
@ -118,26 +105,26 @@ func (c *Client) FetchTokenMarketValues(symbols []string, currency string) (map[
item := map[string]thirdparty.TokenMarketValues{}
for _, smbls := range chunks {
realSymbols := utils.RenameSymbols(smbls)
url := fmt.Sprintf("%s/data/pricemultifull?fsyms=%s&tsyms=%s&extraParams=Status.im", baseURL, strings.Join(realSymbols, ","), realCurrency)
resp, err := c.DoQuery(url)
if err != nil {
return item, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
params := url.Values{}
params.Add("fsyms", strings.Join(realSymbols, ","))
params.Add("tsyms", realCurrency)
params.Add("extraParams", extraParamStatus)
url := fmt.Sprintf("%s/data/pricemultifull", baseURL)
response, err := c.httpClient.DoGetRequest(context.Background(), url, params)
if err != nil {
return item, err
return nil, err
}
container := MarketValuesContainer{}
err = json.Unmarshal(body, &container)
err = json.Unmarshal(response, &container)
if len(container.Raw) == 0 {
return nil, fmt.Errorf("no data found - %s", string(body))
return nil, fmt.Errorf("no data found - %s", string(response))
}
if err != nil {
return nil, fmt.Errorf("%s - %s", err, string(body))
return nil, fmt.Errorf("%s - %s", err, string(response))
}
for _, symbol := range smbls {
@ -150,20 +137,21 @@ func (c *Client) FetchTokenMarketValues(symbols []string, currency string) (map[
func (c *Client) FetchHistoricalHourlyPrices(symbol string, currency string, limit int, aggregate int) ([]thirdparty.HistoricalPrice, error) {
item := []thirdparty.HistoricalPrice{}
url := fmt.Sprintf("%s/data/v2/histohour?fsym=%s&tsym=%s&aggregate=%d&limit=%d&extraParams=Status.im", baseURL, utils.GetRealSymbol(symbol), currency, aggregate, limit)
resp, err := c.DoQuery(url)
if err != nil {
return item, err
}
defer resp.Body.Close()
params := url.Values{}
params.Add("fsym", utils.GetRealSymbol(symbol))
params.Add("tsym", currency)
params.Add("aggregate", fmt.Sprintf("%d", aggregate))
params.Add("limit", fmt.Sprintf("%d", limit))
params.Add("extraParams", extraParamStatus)
body, err := ioutil.ReadAll(resp.Body)
url := fmt.Sprintf("%s/data/v2/histohour", baseURL)
response, err := c.httpClient.DoGetRequest(context.Background(), url, params)
if err != nil {
return item, err
}
container := HistoricalPricesData{}
err = json.Unmarshal(body, &container)
err = json.Unmarshal(response, &container)
if err != nil {
return item, err
}
@ -176,20 +164,22 @@ func (c *Client) FetchHistoricalHourlyPrices(symbol string, currency string, lim
func (c *Client) FetchHistoricalDailyPrices(symbol string, currency string, limit int, allData bool, aggregate int) ([]thirdparty.HistoricalPrice, error) {
item := []thirdparty.HistoricalPrice{}
url := fmt.Sprintf("%s/data/v2/histoday?fsym=%s&tsym=%s&aggregate=%d&limit=%d&allData=%v&extraParams=Status.im", baseURL, utils.GetRealSymbol(symbol), currency, aggregate, limit, allData)
resp, err := c.DoQuery(url)
if err != nil {
return item, err
}
defer resp.Body.Close()
params := url.Values{}
params.Add("fsym", utils.GetRealSymbol(symbol))
params.Add("tsym", currency)
params.Add("aggregate", fmt.Sprintf("%d", aggregate))
params.Add("limit", fmt.Sprintf("%d", limit))
params.Add("allData", fmt.Sprintf("%v", allData))
params.Add("extraParams", extraParamStatus)
body, err := ioutil.ReadAll(resp.Body)
url := fmt.Sprintf("%s/data/v2/histoday", baseURL)
response, err := c.httpClient.DoGetRequest(context.Background(), url, params)
if err != nil {
return item, err
}
container := HistoricalPricesData{}
err = json.Unmarshal(body, &container)
err = json.Unmarshal(response, &container)
if err != nil {
return item, err
}

10
services/wallet/thirdparty/http_client.go vendored
View File

@ -11,6 +11,7 @@ import (
)
const requestTimeout = 5 * time.Second
const maxNumOfRequestRetries = 5
type HTTPClient struct {
client *http.Client
@ -34,7 +35,14 @@ func (c *HTTPClient) DoGetRequest(ctx context.Context, url string, params netUrl
return nil, err
}
resp, err := c.client.Do(req)
var resp *http.Response
for i := 0; i < maxNumOfRequestRetries; i++ {
resp, err = c.client.Do(req)
if err == nil || i == maxNumOfRequestRetries-1 {
break
}
time.Sleep(200 * time.Millisecond)
}
if err != nil {
return nil, err
}