diff --git a/services/wallet/collectibles/collectibles.go b/services/wallet/collectibles/collectibles.go index 4ba3f166f..bd6d7b1d3 100644 --- a/services/wallet/collectibles/collectibles.go +++ b/services/wallet/collectibles/collectibles.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math/big" + "net/http" "strings" "sync" "time" @@ -12,6 +13,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" "github.com/status-im/status-go/contracts/collectibles" "github.com/status-im/status-go/rpc" "github.com/status-im/status-go/services/wallet/bigint" @@ -20,9 +22,6 @@ import ( "github.com/status-im/status-go/services/wallet/thirdparty/opensea" ) -const FetchNoLimit = 0 -const FetchFromStartCursor = "" - const requestTimeout = 5 * time.Second const hystrixContractOwnershipClientName = "contractOwnershipClient" @@ -40,6 +39,7 @@ type Manager struct { fallbackContractOwnershipProvider thirdparty.CollectibleContractOwnershipProvider metadataProvider thirdparty.CollectibleMetadataProvider opensea *opensea.Client + httpClient *http.Client collectiblesDataCache map[string]thirdparty.CollectibleData collectiblesDataCacheLock sync.RWMutex collectionsDataCache map[string]thirdparty.CollectionData @@ -59,8 +59,11 @@ func NewManager(rpcClient *rpc.Client, mainContractOwnershipProvider thirdparty. mainContractOwnershipProvider: mainContractOwnershipProvider, fallbackContractOwnershipProvider: fallbackContractOwnershipProvider, opensea: opensea, - collectiblesDataCache: make(map[string]thirdparty.CollectibleData), - collectionsDataCache: make(map[string]thirdparty.CollectionData), + httpClient: &http.Client{ + Timeout: requestTimeout, + }, + collectiblesDataCache: make(map[string]thirdparty.CollectibleData), + collectionsDataCache: make(map[string]thirdparty.CollectionData), } } @@ -93,6 +96,25 @@ func makeContractOwnershipCall(main func() (any, error), fallback func() (any, e } } +func (o *Manager) doContentTypeRequest(url string) (string, error) { + req, err := http.NewRequest(http.MethodHead, url, nil) + if err != nil { + return "", err + } + + resp, err := o.httpClient.Do(req) + if err != nil { + return "", err + } + defer func() { + if err := resp.Body.Close(); err != nil { + log.Error("failed to close head request body", "err", err) + } + }() + + return resp.Header.Get("Content-Type"), nil +} + // Used to break circular dependency, call once as soon as possible after initialization func (o *Manager) SetMetadataProvider(metadataProvider thirdparty.CollectibleMetadataProvider) { o.metadataProvider = metadataProvider @@ -129,7 +151,7 @@ func (o *Manager) FetchBalancesByOwnerAndContractAddress(chainID walletCommon.Ch } // Try with more direct endpoint first (OpenSea) - assetsContainer, err := o.FetchAllAssetsByOwnerAndContractAddress(chainID, ownerAddress, contractAddresses, FetchFromStartCursor, FetchNoLimit) + assetsContainer, err := o.FetchAllAssetsByOwnerAndContractAddress(chainID, ownerAddress, contractAddresses, thirdparty.FetchFromStartCursor, thirdparty.FetchNoLimit) if err == thirdparty.ErrChainIDNotSupported { // Use contract ownership providers for _, contractAddress := range contractAddresses { @@ -279,6 +301,7 @@ func (o *Manager) processFullCollectibleData(assets []thirdparty.FullCollectible for idx, asset := range assets { id := asset.CollectibleData.ID + // Get Metadata from alternate source if empty if isMetadataEmpty(asset.CollectibleData) { if o.metadataProvider == nil { return fmt.Errorf("CollectibleMetadataProvider not available") @@ -309,6 +332,15 @@ func (o *Manager) processFullCollectibleData(assets []thirdparty.FullCollectible } } + // Get Animation MediaType + if len(assets[idx].CollectibleData.AnimationURL) > 0 { + contentType, err := o.doContentTypeRequest(assets[idx].CollectibleData.AnimationURL) + if err != nil { + assets[idx].CollectibleData.AnimationURL = "" + } + assets[idx].CollectibleData.AnimationMediaType = contentType + } + o.setCacheCollectibleData(assets[idx].CollectibleData) if assets[idx].CollectionData != nil { o.setCacheCollectionData(*assets[idx].CollectionData) diff --git a/services/wallet/collectibles/commands.go b/services/wallet/collectibles/commands.go index ede337b65..085b4d6b4 100644 --- a/services/wallet/collectibles/commands.go +++ b/services/wallet/collectibles/commands.go @@ -136,7 +136,7 @@ func (c *loadOwnedCollectiblesCommand) Run(parent context.Context) (err error) { log.Debug("start loadOwnedCollectiblesCommand", "chain", c.chainID, "account", c.account) pageNr := 0 - cursor := FetchFromStartCursor + cursor := thirdparty.FetchFromStartCursor c.triggerEvent(EventCollectiblesOwnershipUpdateStarted, c.chainID, c.account, "") // Fetch collectibles in chunks @@ -161,7 +161,7 @@ func (c *loadOwnedCollectiblesCommand) Run(parent context.Context) (err error) { pageNr++ cursor = partialOwnership.NextCursor - if cursor == FetchFromStartCursor { + if cursor == thirdparty.FetchFromStartCursor { err = c.ownershipDB.Update(c.chainID, c.account, c.partialOwnership) if err != nil { log.Error("failed updating ownershipDB in loadOwnedCollectiblesCommand", "chain", c.chainID, "account", c.account, "error", err) diff --git a/services/wallet/thirdparty/alchemy/client.go b/services/wallet/thirdparty/alchemy/client.go index 24a06c9b2..57f6a5588 100644 --- a/services/wallet/thirdparty/alchemy/client.go +++ b/services/wallet/thirdparty/alchemy/client.go @@ -10,11 +10,12 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/status-im/status-go/services/wallet/bigint" walletCommon "github.com/status-im/status-go/services/wallet/common" "github.com/status-im/status-go/services/wallet/thirdparty" ) +const AlchemyID = "alchemy" + func getBaseURL(chainID walletCommon.ChainID) (string, error) { switch uint64(chainID) { case walletCommon.EthereumMainnet: @@ -37,7 +38,7 @@ func getBaseURL(chainID walletCommon.ChainID) (string, error) { } func (o *Client) ID() string { - return "alchemy" + return AlchemyID } func (o *Client) IsChainSupported(chainID walletCommon.ChainID) bool { @@ -62,21 +63,6 @@ func getNFTBaseURL(chainID walletCommon.ChainID, apiKey string) (string, error) return fmt.Sprintf("%s/nft/v2/%s", baseURL, getAPIKeySubpath(apiKey)), nil } -type TokenBalance struct { - TokenID *bigint.HexBigInt `json:"tokenId"` - Balance *bigint.BigInt `json:"balance"` -} - -type CollectibleOwner struct { - OwnerAddress common.Address `json:"ownerAddress"` - TokenBalances []TokenBalance `json:"tokenBalances"` -} - -type CollectibleContractOwnership struct { - Owners []CollectibleOwner `json:"ownerAddresses"` - PageKey string `json:"pageKey"` -} - type Client struct { thirdparty.CollectibleContractOwnershipProvider client *http.Client @@ -102,33 +88,6 @@ func (o *Client) doQuery(url string) (*http.Response, error) { return resp, nil } -func alchemyOwnershipToCommon(contractAddress common.Address, alchemyOwnership CollectibleContractOwnership) (*thirdparty.CollectibleContractOwnership, error) { - owners := make([]thirdparty.CollectibleOwner, 0, len(alchemyOwnership.Owners)) - for _, alchemyOwner := range alchemyOwnership.Owners { - balances := make([]thirdparty.TokenBalance, 0, len(alchemyOwner.TokenBalances)) - - for _, alchemyBalance := range alchemyOwner.TokenBalances { - balances = append(balances, thirdparty.TokenBalance{ - TokenID: &bigint.BigInt{Int: alchemyBalance.TokenID.Int}, - Balance: alchemyBalance.Balance, - }) - } - owner := thirdparty.CollectibleOwner{ - OwnerAddress: alchemyOwner.OwnerAddress, - TokenBalances: balances, - } - - owners = append(owners, owner) - } - - ownership := thirdparty.CollectibleContractOwnership{ - ContractAddress: contractAddress, - Owners: owners, - } - - return &ownership, nil -} - func (o *Client) FetchCollectibleOwnersByContractAddress(chainID walletCommon.ChainID, contractAddress common.Address) (*thirdparty.CollectibleContractOwnership, error) { queryParams := url.Values{ "contractAddress": {contractAddress.String()}, diff --git a/services/wallet/thirdparty/alchemy/types.go b/services/wallet/thirdparty/alchemy/types.go new file mode 100644 index 000000000..bf1fd3726 --- /dev/null +++ b/services/wallet/thirdparty/alchemy/types.go @@ -0,0 +1,49 @@ +package alchemy + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/status-im/status-go/services/wallet/bigint" + "github.com/status-im/status-go/services/wallet/thirdparty" +) + +type TokenBalance struct { + TokenID *bigint.HexBigInt `json:"tokenId"` + Balance *bigint.BigInt `json:"balance"` +} + +type CollectibleOwner struct { + OwnerAddress common.Address `json:"ownerAddress"` + TokenBalances []TokenBalance `json:"tokenBalances"` +} + +type CollectibleContractOwnership struct { + Owners []CollectibleOwner `json:"ownerAddresses"` + PageKey string `json:"pageKey"` +} + +func alchemyOwnershipToCommon(contractAddress common.Address, alchemyOwnership CollectibleContractOwnership) (*thirdparty.CollectibleContractOwnership, error) { + owners := make([]thirdparty.CollectibleOwner, 0, len(alchemyOwnership.Owners)) + for _, alchemyOwner := range alchemyOwnership.Owners { + balances := make([]thirdparty.TokenBalance, 0, len(alchemyOwner.TokenBalances)) + + for _, alchemyBalance := range alchemyOwner.TokenBalances { + balances = append(balances, thirdparty.TokenBalance{ + TokenID: &bigint.BigInt{Int: alchemyBalance.TokenID.Int}, + Balance: alchemyBalance.Balance, + }) + } + owner := thirdparty.CollectibleOwner{ + OwnerAddress: alchemyOwner.OwnerAddress, + TokenBalances: balances, + } + + owners = append(owners, owner) + } + + ownership := thirdparty.CollectibleContractOwnership{ + ContractAddress: contractAddress, + Owners: owners, + } + + return &ownership, nil +} diff --git a/services/wallet/thirdparty/collectible_types.go b/services/wallet/thirdparty/collectible_types.go index 54ac472da..9ee46e453 100644 --- a/services/wallet/thirdparty/collectible_types.go +++ b/services/wallet/thirdparty/collectible_types.go @@ -13,6 +13,9 @@ var ( ErrChainIDNotSupported = errors.New("chainID not supported") ) +const FetchNoLimit = 0 +const FetchFromStartCursor = "" + type CollectibleProvider interface { ID() string IsChainSupported(chainID w_common.ChainID) bool diff --git a/services/wallet/thirdparty/infura/client.go b/services/wallet/thirdparty/infura/client.go index 688e0852d..4935db31f 100644 --- a/services/wallet/thirdparty/infura/client.go +++ b/services/wallet/thirdparty/infura/client.go @@ -9,25 +9,12 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/status-im/status-go/services/wallet/bigint" walletCommon "github.com/status-im/status-go/services/wallet/common" "github.com/status-im/status-go/services/wallet/thirdparty" ) const baseURL = "https://nft.api.infura.io" - -type CollectibleOwner struct { - ContractAddress common.Address `json:"tokenAddress"` - TokenID *bigint.BigInt `json:"tokenId"` - Amount *bigint.BigInt `json:"amount"` - OwnerAddress common.Address `json:"ownerOf"` -} - -type CollectibleContractOwnership struct { - Owners []CollectibleOwner `json:"owners"` - Network string `json:"network"` - Cursor string `json:"cursor"` -} +const InfuraID = "infura" type Client struct { thirdparty.CollectibleContractOwnershipProvider @@ -64,44 +51,18 @@ func (o *Client) doQuery(url string) (*http.Response, error) { } func (o *Client) ID() string { - return "infura" + return InfuraID } func (o *Client) IsChainSupported(chainID walletCommon.ChainID) bool { switch uint64(chainID) { - case walletCommon.EthereumMainnet, walletCommon.EthereumGoerli, walletCommon.EthereumSepolia, walletCommon.ArbitrumMainnet: + case walletCommon.EthereumMainnet, walletCommon.ArbitrumMainnet: + case walletCommon.EthereumGoerli, walletCommon.EthereumSepolia: return true } return false } -func infuraOwnershipToCommon(contractAddress common.Address, ownersMap map[common.Address][]CollectibleOwner) (*thirdparty.CollectibleContractOwnership, error) { - owners := make([]thirdparty.CollectibleOwner, 0, len(ownersMap)) - - for ownerAddress, ownerTokens := range ownersMap { - tokenBalances := make([]thirdparty.TokenBalance, 0, len(ownerTokens)) - - for _, token := range ownerTokens { - tokenBalances = append(tokenBalances, thirdparty.TokenBalance{ - TokenID: token.TokenID, - Balance: token.Amount, - }) - } - - owners = append(owners, thirdparty.CollectibleOwner{ - OwnerAddress: ownerAddress, - TokenBalances: tokenBalances, - }) - } - - ownership := thirdparty.CollectibleContractOwnership{ - ContractAddress: contractAddress, - Owners: owners, - } - - return &ownership, nil -} - func (o *Client) FetchCollectibleOwnersByContractAddress(chainID walletCommon.ChainID, contractAddress common.Address) (*thirdparty.CollectibleContractOwnership, error) { cursor := "" ownersMap := make(map[common.Address][]CollectibleOwner) diff --git a/services/wallet/thirdparty/infura/types.go b/services/wallet/thirdparty/infura/types.go new file mode 100644 index 000000000..7b94dc232 --- /dev/null +++ b/services/wallet/thirdparty/infura/types.go @@ -0,0 +1,47 @@ +package infura + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/status-im/status-go/services/wallet/bigint" + "github.com/status-im/status-go/services/wallet/thirdparty" +) + +type CollectibleOwner struct { + ContractAddress common.Address `json:"tokenAddress"` + TokenID *bigint.BigInt `json:"tokenId"` + Amount *bigint.BigInt `json:"amount"` + OwnerAddress common.Address `json:"ownerOf"` +} + +type CollectibleContractOwnership struct { + Owners []CollectibleOwner `json:"owners"` + Network string `json:"network"` + Cursor string `json:"cursor"` +} + +func infuraOwnershipToCommon(contractAddress common.Address, ownersMap map[common.Address][]CollectibleOwner) (*thirdparty.CollectibleContractOwnership, error) { + owners := make([]thirdparty.CollectibleOwner, 0, len(ownersMap)) + + for ownerAddress, ownerTokens := range ownersMap { + tokenBalances := make([]thirdparty.TokenBalance, 0, len(ownerTokens)) + + for _, token := range ownerTokens { + tokenBalances = append(tokenBalances, thirdparty.TokenBalance{ + TokenID: token.TokenID, + Balance: token.Amount, + }) + } + + owners = append(owners, thirdparty.CollectibleOwner{ + OwnerAddress: ownerAddress, + TokenBalances: tokenBalances, + }) + } + + ownership := thirdparty.CollectibleContractOwnership{ + ContractAddress: contractAddress, + Owners: owners, + } + + return &ownership, nil +} diff --git a/services/wallet/thirdparty/opensea/client.go b/services/wallet/thirdparty/opensea/client.go index 08e36b858..edae618ba 100644 --- a/services/wallet/thirdparty/opensea/client.go +++ b/services/wallet/thirdparty/opensea/client.go @@ -3,22 +3,14 @@ package opensea import ( "encoding/json" "fmt" - "io/ioutil" - "net/http" "net/url" "strconv" "strings" - "sync" "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/log" - "golang.org/x/text/cases" - "golang.org/x/text/language" - - "github.com/status-im/status-go/services/wallet/bigint" walletCommon "github.com/status-im/status-go/services/wallet/common" "github.com/status-im/status-go/services/wallet/connection" "github.com/status-im/status-go/services/wallet/thirdparty" @@ -26,9 +18,11 @@ import ( ) const ( - EventCollectibleStatusChanged walletevent.EventType = "wallet-collectible-status-changed" + EventCollectibleStatusChanged walletevent.EventType = "wallet-collectible-opensea-v1-status-changed" ) +const OpenseaV1ID = "openseaV1" + const AssetLimit = 200 const CollectionLimit = 300 @@ -38,8 +32,6 @@ const GetRequestWaitTime = 300 * time.Millisecond const ChainIDRequiringAPIKey = walletCommon.EthereumMainnet -const FetchNoLimit = 0 - type urlGetter func(walletCommon.ChainID, string) (string, error) func getBaseURL(chainID walletCommon.ChainID) (string, error) { @@ -55,7 +47,7 @@ func getBaseURL(chainID walletCommon.ChainID) (string, error) { } func (o *Client) ID() string { - return "opensea" + return OpenseaV1ID } func (o *Client) IsChainSupported(chainID walletCommon.ChainID) bool { @@ -72,279 +64,6 @@ func getURL(chainID walletCommon.ChainID, path string) (string, error) { return fmt.Sprintf("%s/%s", baseURL, path), nil } -func chainStringToChainID(chainString string) walletCommon.ChainID { - chainID := walletCommon.UnknownChainID - switch chainString { - case "ethereum": - chainID = walletCommon.EthereumMainnet - case "arbitrum": - chainID = walletCommon.ArbitrumMainnet - case "optimism": - chainID = walletCommon.OptimismMainnet - case "goerli": - chainID = walletCommon.EthereumGoerli - case "arbitrum_goerli": - chainID = walletCommon.ArbitrumGoerli - case "optimism_goerli": - chainID = walletCommon.OptimismGoerli - } - return walletCommon.ChainID(chainID) -} - -type TraitValue string - -func (st *TraitValue) UnmarshalJSON(b []byte) error { - var item interface{} - if err := json.Unmarshal(b, &item); err != nil { - return err - } - - switch v := item.(type) { - case float64: - *st = TraitValue(strconv.FormatFloat(v, 'f', 2, 64)) - case int: - *st = TraitValue(strconv.Itoa(v)) - case string: - *st = TraitValue(v) - - } - return nil -} - -type AssetContainer struct { - Assets []Asset `json:"assets"` - NextCursor string `json:"next"` - PreviousCursor string `json:"previous"` -} - -type Contract struct { - Address string `json:"address"` - ChainIdentifier string `json:"chain_identifier"` -} - -type Trait struct { - TraitType string `json:"trait_type"` - Value TraitValue `json:"value"` - DisplayType string `json:"display_type"` - MaxValue string `json:"max_value"` -} - -type PaymentToken struct { - ID int `json:"id"` - Symbol string `json:"symbol"` - Address string `json:"address"` - ImageURL string `json:"image_url"` - Name string `json:"name"` - Decimals int `json:"decimals"` - EthPrice string `json:"eth_price"` - UsdPrice string `json:"usd_price"` -} - -type LastSale struct { - PaymentToken PaymentToken `json:"payment_token"` -} - -type SellOrder struct { - CurrentPrice string `json:"current_price"` -} - -type Asset struct { - ID int `json:"id"` - TokenID *bigint.BigInt `json:"token_id"` - Name string `json:"name"` - Description string `json:"description"` - Permalink string `json:"permalink"` - ImageThumbnailURL string `json:"image_thumbnail_url"` - ImageURL string `json:"image_url"` - AnimationURL string `json:"animation_url"` - AnimationMediaType string `json:"animation_media_type"` - Contract Contract `json:"asset_contract"` - Collection Collection `json:"collection"` - Traits []Trait `json:"traits"` - LastSale LastSale `json:"last_sale"` - SellOrders []SellOrder `json:"sell_orders"` - BackgroundColor string `json:"background_color"` - TokenURI string `json:"token_metadata"` -} - -type CollectionTrait struct { - Min float64 `json:"min"` - Max float64 `json:"max"` -} - -type Collection struct { - Name string `json:"name"` - Slug string `json:"slug"` - ImageURL string `json:"image_url"` - Traits map[string]CollectionTrait `json:"traits"` -} - -type OwnedCollection struct { - Collection - OwnedAssetCount *bigint.BigInt `json:"owned_asset_count"` -} - -func (c *Asset) id() thirdparty.CollectibleUniqueID { - return thirdparty.CollectibleUniqueID{ - ContractID: thirdparty.ContractID{ - ChainID: chainStringToChainID(c.Contract.ChainIdentifier), - Address: common.HexToAddress(c.Contract.Address), - }, - TokenID: c.TokenID, - } -} - -func openseaToCollectibleTraits(traits []Trait) []thirdparty.CollectibleTrait { - ret := make([]thirdparty.CollectibleTrait, 0, len(traits)) - caser := cases.Title(language.Und, cases.NoLower) - for _, orig := range traits { - dest := thirdparty.CollectibleTrait{ - TraitType: strings.Replace(orig.TraitType, "_", " ", 1), - Value: caser.String(string(orig.Value)), - DisplayType: orig.DisplayType, - MaxValue: orig.MaxValue, - } - - ret = append(ret, dest) - } - return ret -} - -func (c *Asset) toCollectionData() thirdparty.CollectionData { - ret := thirdparty.CollectionData{ - ID: c.id().ContractID, - Name: c.Collection.Name, - Slug: c.Collection.Slug, - ImageURL: c.Collection.ImageURL, - Traits: make(map[string]thirdparty.CollectionTrait), - } - for traitType, trait := range c.Collection.Traits { - ret.Traits[traitType] = thirdparty.CollectionTrait{ - Min: trait.Min, - Max: trait.Max, - } - } - return ret -} - -func (c *Asset) toCollectiblesData() thirdparty.CollectibleData { - return thirdparty.CollectibleData{ - ID: c.id(), - Name: c.Name, - Description: c.Description, - Permalink: c.Permalink, - ImageURL: c.ImageURL, - AnimationURL: c.AnimationURL, - AnimationMediaType: c.AnimationMediaType, - Traits: openseaToCollectibleTraits(c.Traits), - BackgroundColor: c.BackgroundColor, - TokenURI: c.TokenURI, - } -} - -func (c *Asset) toCommon() thirdparty.FullCollectibleData { - collection := c.toCollectionData() - return thirdparty.FullCollectibleData{ - CollectibleData: c.toCollectiblesData(), - CollectionData: &collection, - } -} - -type HTTPClient struct { - client *http.Client - getRequestLock sync.RWMutex -} - -func newHTTPClient() *HTTPClient { - return &HTTPClient{ - client: &http.Client{ - Timeout: RequestTimeout, - }, - } -} - -func (o *HTTPClient) doGetRequest(url string, apiKey string) ([]byte, error) { - // Ensure only one thread makes a request at a time - o.getRequestLock.Lock() - defer o.getRequestLock.Unlock() - - retryCount := 0 - statusCode := http.StatusOK - - // Try to do the request without an apiKey first - tmpAPIKey := "" - - for { - req, err := http.NewRequest(http.MethodGet, url, nil) - if err != nil { - return nil, err - } - - req.Header.Set("Content-Type", "application/json") - req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:96.0) Gecko/20100101 Firefox/96.0") - if len(tmpAPIKey) > 0 { - req.Header.Set("X-API-KEY", tmpAPIKey) - } - - resp, err := o.client.Do(req) - if err != nil { - return nil, err - } - defer func() { - if err := resp.Body.Close(); err != nil { - log.Error("failed to close opensea request body", "err", err) - } - }() - - statusCode = resp.StatusCode - switch resp.StatusCode { - case http.StatusOK: - body, err := ioutil.ReadAll(resp.Body) - return body, err - case http.StatusTooManyRequests: - if retryCount < GetRequestRetryMaxCount { - // sleep and retry - time.Sleep(GetRequestWaitTime) - retryCount++ - continue - } - // break and error - case http.StatusForbidden: - // Request requires an apiKey, set it and retry - if tmpAPIKey == "" && apiKey != "" { - tmpAPIKey = apiKey - // sleep and retry - time.Sleep(GetRequestWaitTime) - continue - } - // break and error - default: - // break and error - } - break - } - return nil, fmt.Errorf("unsuccessful request: %d %s", statusCode, http.StatusText(statusCode)) -} - -func (o *HTTPClient) doContentTypeRequest(url string) (string, error) { - req, err := http.NewRequest(http.MethodHead, url, nil) - if err != nil { - return "", err - } - - resp, err := o.client.Do(req) - if err != nil { - return "", err - } - defer func() { - if err := resp.Body.Close(); err != nil { - log.Error("failed to close head request body", "err", err) - } - }() - - return resp.Header.Get("Content-Type"), nil -} - type Client struct { client *HTTPClient apiKey string @@ -453,7 +172,7 @@ func (o *Client) FetchAssetsByCollectibleUniqueID(uniqueIDs []thirdparty.Collect queryParams.Add("asset_contract_addresses", id.ContractID.Address.String()) } - data, err := o.fetchAssets(chainID, queryParams, FetchNoLimit) + data, err := o.fetchAssets(chainID, queryParams, thirdparty.FetchNoLimit) if err != nil { return nil, err } @@ -472,7 +191,7 @@ func (o *Client) fetchAssets(chainID walletCommon.ChainID, queryParams url.Value } tmpLimit := AssetLimit - if limit > FetchNoLimit && limit < tmpLimit { + if limit > thirdparty.FetchNoLimit && limit < tmpLimit { tmpLimit = limit } @@ -503,12 +222,6 @@ func (o *Client) fetchAssets(chainID walletCommon.ChainID, queryParams url.Value } for _, asset := range container.Assets { - if len(asset.AnimationURL) > 0 { - asset.AnimationMediaType, err = o.client.doContentTypeRequest(asset.AnimationURL) - if err != nil { - asset.AnimationURL = "" - } - } assets.Items = append(assets.Items, asset.toCommon()) } assets.NextCursor = container.NextCursor @@ -519,7 +232,7 @@ func (o *Client) fetchAssets(chainID walletCommon.ChainID, queryParams url.Value queryParams["cursor"] = []string{assets.NextCursor} - if limit > FetchNoLimit && len(assets.Items) >= limit { + if limit > thirdparty.FetchNoLimit && len(assets.Items) >= limit { break } } @@ -586,12 +299,6 @@ func (o *Client) fetchOpenseaAssets(chainID walletCommon.ChainID, queryParams ur asset.Traits[i].TraitType = strings.Replace(asset.Traits[i].TraitType, "_", " ", 1) asset.Traits[i].Value = TraitValue(strings.Title(string(asset.Traits[i].Value))) } - if len(asset.AnimationURL) > 0 { - asset.AnimationMediaType, err = o.client.doContentTypeRequest(asset.AnimationURL) - if err != nil { - asset.AnimationURL = "" - } - } assets.Assets = append(assets.Assets, asset) } assets.NextCursor = container.NextCursor diff --git a/services/wallet/thirdparty/opensea/http_client.go b/services/wallet/thirdparty/opensea/http_client.go new file mode 100644 index 000000000..5e7182782 --- /dev/null +++ b/services/wallet/thirdparty/opensea/http_client.go @@ -0,0 +1,87 @@ +package opensea + +import ( + "fmt" + "io/ioutil" + "net/http" + "sync" + "time" + + "github.com/ethereum/go-ethereum/log" +) + +type HTTPClient struct { + client *http.Client + getRequestLock sync.RWMutex +} + +func newHTTPClient() *HTTPClient { + return &HTTPClient{ + client: &http.Client{ + Timeout: RequestTimeout, + }, + } +} + +func (o *HTTPClient) doGetRequest(url string, apiKey string) ([]byte, error) { + // Ensure only one thread makes a request at a time + o.getRequestLock.Lock() + defer o.getRequestLock.Unlock() + + retryCount := 0 + statusCode := http.StatusOK + + // Try to do the request without an apiKey first + tmpAPIKey := "" + + for { + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:96.0) Gecko/20100101 Firefox/96.0") + if len(tmpAPIKey) > 0 { + req.Header.Set("X-API-KEY", tmpAPIKey) + } + + resp, err := o.client.Do(req) + if err != nil { + return nil, err + } + defer func() { + if err := resp.Body.Close(); err != nil { + log.Error("failed to close opensea request body", "err", err) + } + }() + + statusCode = resp.StatusCode + switch resp.StatusCode { + case http.StatusOK: + body, err := ioutil.ReadAll(resp.Body) + return body, err + case http.StatusTooManyRequests: + if retryCount < GetRequestRetryMaxCount { + // sleep and retry + time.Sleep(GetRequestWaitTime) + retryCount++ + continue + } + // break and error + case http.StatusForbidden: + // Request requires an apiKey, set it and retry + if tmpAPIKey == "" && apiKey != "" { + tmpAPIKey = apiKey + // sleep and retry + time.Sleep(GetRequestWaitTime) + continue + } + // break and error + default: + // break and error + } + break + } + return nil, fmt.Errorf("unsuccessful request: %d %s", statusCode, http.StatusText(statusCode)) +} diff --git a/services/wallet/thirdparty/opensea/types.go b/services/wallet/thirdparty/opensea/types.go new file mode 100644 index 000000000..24b6616a6 --- /dev/null +++ b/services/wallet/thirdparty/opensea/types.go @@ -0,0 +1,190 @@ +package opensea + +import ( + "encoding/json" + "strconv" + "strings" + + "github.com/ethereum/go-ethereum/common" + "github.com/status-im/status-go/services/wallet/bigint" + walletCommon "github.com/status-im/status-go/services/wallet/common" + "github.com/status-im/status-go/services/wallet/thirdparty" + "golang.org/x/text/cases" + "golang.org/x/text/language" +) + +func chainStringToChainID(chainString string) walletCommon.ChainID { + chainID := walletCommon.UnknownChainID + switch chainString { + case "ethereum": + chainID = walletCommon.EthereumMainnet + case "arbitrum": + chainID = walletCommon.ArbitrumMainnet + case "optimism": + chainID = walletCommon.OptimismMainnet + case "goerli": + chainID = walletCommon.EthereumGoerli + case "arbitrum_goerli": + chainID = walletCommon.ArbitrumGoerli + case "optimism_goerli": + chainID = walletCommon.OptimismGoerli + } + return walletCommon.ChainID(chainID) +} + +type TraitValue string + +func (st *TraitValue) UnmarshalJSON(b []byte) error { + var item interface{} + if err := json.Unmarshal(b, &item); err != nil { + return err + } + + switch v := item.(type) { + case float64: + *st = TraitValue(strconv.FormatFloat(v, 'f', 2, 64)) + case int: + *st = TraitValue(strconv.Itoa(v)) + case string: + *st = TraitValue(v) + + } + return nil +} + +type AssetContainer struct { + Assets []Asset `json:"assets"` + NextCursor string `json:"next"` + PreviousCursor string `json:"previous"` +} + +type Contract struct { + Address string `json:"address"` + ChainIdentifier string `json:"chain_identifier"` +} + +type Trait struct { + TraitType string `json:"trait_type"` + Value TraitValue `json:"value"` + DisplayType string `json:"display_type"` + MaxValue string `json:"max_value"` +} + +type PaymentToken struct { + ID int `json:"id"` + Symbol string `json:"symbol"` + Address string `json:"address"` + ImageURL string `json:"image_url"` + Name string `json:"name"` + Decimals int `json:"decimals"` + EthPrice string `json:"eth_price"` + UsdPrice string `json:"usd_price"` +} + +type LastSale struct { + PaymentToken PaymentToken `json:"payment_token"` +} + +type SellOrder struct { + CurrentPrice string `json:"current_price"` +} + +type Asset struct { + ID int `json:"id"` + TokenID *bigint.BigInt `json:"token_id"` + Name string `json:"name"` + Description string `json:"description"` + Permalink string `json:"permalink"` + ImageThumbnailURL string `json:"image_thumbnail_url"` + ImageURL string `json:"image_url"` + AnimationURL string `json:"animation_url"` + Contract Contract `json:"asset_contract"` + Collection Collection `json:"collection"` + Traits []Trait `json:"traits"` + LastSale LastSale `json:"last_sale"` + SellOrders []SellOrder `json:"sell_orders"` + BackgroundColor string `json:"background_color"` + TokenURI string `json:"token_metadata"` +} + +type CollectionTrait struct { + Min float64 `json:"min"` + Max float64 `json:"max"` +} + +type Collection struct { + Name string `json:"name"` + Slug string `json:"slug"` + ImageURL string `json:"image_url"` + Traits map[string]CollectionTrait `json:"traits"` +} + +type OwnedCollection struct { + Collection + OwnedAssetCount *bigint.BigInt `json:"owned_asset_count"` +} + +func (c *Asset) id() thirdparty.CollectibleUniqueID { + return thirdparty.CollectibleUniqueID{ + ContractID: thirdparty.ContractID{ + ChainID: chainStringToChainID(c.Contract.ChainIdentifier), + Address: common.HexToAddress(c.Contract.Address), + }, + TokenID: c.TokenID, + } +} + +func openseaToCollectibleTraits(traits []Trait) []thirdparty.CollectibleTrait { + ret := make([]thirdparty.CollectibleTrait, 0, len(traits)) + caser := cases.Title(language.Und, cases.NoLower) + for _, orig := range traits { + dest := thirdparty.CollectibleTrait{ + TraitType: strings.Replace(orig.TraitType, "_", " ", 1), + Value: caser.String(string(orig.Value)), + DisplayType: orig.DisplayType, + MaxValue: orig.MaxValue, + } + + ret = append(ret, dest) + } + return ret +} + +func (c *Asset) toCollectionData() thirdparty.CollectionData { + ret := thirdparty.CollectionData{ + ID: c.id().ContractID, + Name: c.Collection.Name, + Slug: c.Collection.Slug, + ImageURL: c.Collection.ImageURL, + Traits: make(map[string]thirdparty.CollectionTrait), + } + for traitType, trait := range c.Collection.Traits { + ret.Traits[traitType] = thirdparty.CollectionTrait{ + Min: trait.Min, + Max: trait.Max, + } + } + return ret +} + +func (c *Asset) toCollectiblesData() thirdparty.CollectibleData { + return thirdparty.CollectibleData{ + ID: c.id(), + Name: c.Name, + Description: c.Description, + Permalink: c.Permalink, + ImageURL: c.ImageURL, + AnimationURL: c.AnimationURL, + Traits: openseaToCollectibleTraits(c.Traits), + BackgroundColor: c.BackgroundColor, + TokenURI: c.TokenURI, + } +} + +func (c *Asset) toCommon() thirdparty.FullCollectibleData { + collection := c.toCollectionData() + return thirdparty.FullCollectibleData{ + CollectibleData: c.toCollectiblesData(), + CollectionData: &collection, + } +}