status-im
/
status-go
mirror of https://github.com/status-im/status-go.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat: fetch collection metadata when missing

This commit is contained in:
Dario Gabriel Lipicar 2023-08-03 09:24:23 -03:00 committed by dlipicar
parent 1f510eae70
commit d5974dd52e
8 changed files with 402 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
services/wallet/collectibles/collectibles.go
View File

@ -42,6 +42,7 @@ type Manager struct {
rpcClient *rpc.Client
contractOwnershipProviders []thirdparty.CollectibleContractOwnershipProvider
accountOwnershipProviders []thirdparty.CollectibleAccountOwnershipProvider
collectibleDataProviders []thirdparty.CollectibleDataProvider
metadataProvider thirdparty.CollectibleMetadataProvider
opensea *opensea.Client
httpClient *http.Client
@ -51,7 +52,7 @@ type Manager struct {
collectionsDataCacheLock sync.RWMutex
}
func NewManager(rpcClient *rpc.Client, contractOwnershipProviders []thirdparty.CollectibleContractOwnershipProvider, accountOwnershipProviders []thirdparty.CollectibleAccountOwnershipProvider, opensea *opensea.Client) *Manager {
func NewManager(rpcClient *rpc.Client, contractOwnershipProviders []thirdparty.CollectibleContractOwnershipProvider, accountOwnershipProviders []thirdparty.CollectibleAccountOwnershipProvider, collectibleDataProviders []thirdparty.CollectibleDataProvider, opensea *opensea.Client) *Manager {
hystrix.ConfigureCommand(hystrixContractOwnershipClientName, hystrix.CommandConfig{
Timeout: 10000,
MaxConcurrentRequests: 100,
@ -63,6 +64,7 @@ func NewManager(rpcClient *rpc.Client, contractOwnershipProviders []thirdparty.C
rpcClient: rpcClient,
contractOwnershipProviders: contractOwnershipProviders,
accountOwnershipProviders: accountOwnershipProviders,
collectibleDataProviders: collectibleDataProviders,
opensea: opensea,
httpClient: &http.Client{
Timeout: requestTimeout,
@ -72,6 +74,14 @@ func NewManager(rpcClient *rpc.Client, contractOwnershipProviders []thirdparty.C
}
}
func refMapToList[K comparable, T any](m map[K]*T) []T {
list := make([]T, 0, len(m))
for _, v := range m {
list = append(list, *v)
}
return list
}
func makeContractOwnershipCall(main func() (any, error), fallback func() (any, error)) (any, error) {
resultChan := make(chan any, 1)
errChan := hystrix.Go(hystrixContractOwnershipClientName, func() error {
@ -246,22 +256,57 @@ func (o *Manager) FetchCollectibleOwnershipByOwner(chainID walletCommon.ChainID,
}
func (o *Manager) FetchAssetsByCollectibleUniqueID(uniqueIDs []thirdparty.CollectibleUniqueID) ([]thirdparty.FullCollectibleData, error) {
idsToFetch := o.getIDsNotInCollectiblesDataCache(uniqueIDs)
if len(idsToFetch) > 0 {
fetchedAssets, err := o.opensea.FetchAssetsByCollectibleUniqueID(idsToFetch)
if err != nil {
return nil, err
}
idsPerChainID := thirdparty.GroupCollectibleUIDsByChainID(o.getIDsNotInCollectiblesDataCache(uniqueIDs))
err = o.processFullCollectibleData(fetchedAssets)
if err != nil {
return nil, err
for chainID, idsToFetch := range idsPerChainID {
for _, provider := range o.collectibleDataProviders {
if !provider.IsChainSupported(chainID) {
continue
}
fetchedAssets, err := o.opensea.FetchAssetsByCollectibleUniqueID(idsToFetch)
if err != nil {
return nil, err
}
err = o.processFullCollectibleData(fetchedAssets)
if err != nil {
return nil, err
}
break
}
}
return o.getCacheFullCollectibleData(uniqueIDs), nil
}
func (o *Manager) FetchCollectionsDataByContractID(ids []thirdparty.ContractID) ([]thirdparty.CollectionData, error) {
idsPerChainID := thirdparty.GroupContractIDsByChainID(o.getIDsNotInCollectionDataCache(ids))
for chainID, idsToFetch := range idsPerChainID {
for _, provider := range o.collectibleDataProviders {
if !provider.IsChainSupported(chainID) {
continue
}
fetchedCollections, err := provider.FetchCollectionsDataByContractID(idsToFetch)
if err != nil {
return nil, err
}
err = o.processCollectionData(fetchedCollections)
if err != nil {
return nil, err
}
break
}
}
return refMapToList(o.getCacheCollectionData(ids)), nil
}
func (o *Manager) getContractOwnershipProviders(chainID walletCommon.ChainID) (mainProvider thirdparty.CollectibleContractOwnershipProvider, fallbackProvider thirdparty.CollectibleContractOwnershipProvider) {
mainProvider = nil
fallbackProvider = nil
@ -347,6 +392,8 @@ func (o *Manager) fetchTokenURI(id thirdparty.CollectibleUniqueID) (string, erro
}
func (o *Manager) processFullCollectibleData(assets []thirdparty.FullCollectibleData) error {
missingCollectionIDs := make([]thirdparty.ContractID, 0)
for idx, asset := range assets {
id := asset.CollectibleData.ID
@ -393,10 +440,26 @@ func (o *Manager) processFullCollectibleData(assets []thirdparty.FullCollectible
o.setCacheCollectibleData(assets[idx].CollectibleData)
if assets[idx].CollectionData != nil {
o.setCacheCollectionData(*assets[idx].CollectionData)
} else {
missingCollectionIDs = append(missingCollectionIDs, id.ContractID)
}
// TODO: Fetch collection metadata separately
}
if len(missingCollectionIDs) > 0 {
// Calling this ensures collection data is fetched and cached (if not already available)
_, err := o.FetchCollectionsDataByContractID(missingCollectionIDs)
if err != nil {
return err
}
}
return nil
}
func (o *Manager) processCollectionData(collections []thirdparty.CollectionData) error {
for _, collection := range collections {
o.setCacheCollectionData(collection)
}
return nil
}
@ -441,6 +504,26 @@ func (o *Manager) setCacheCollectibleData(data thirdparty.CollectibleData) {
o.collectiblesDataCache[data.ID.HashKey()] = data
}
func (o *Manager) isIDInCollectionDataCache(id thirdparty.ContractID) bool {
o.collectionsDataCacheLock.RLock()
defer o.collectionsDataCacheLock.RUnlock()
if _, ok := o.collectionsDataCache[id.HashKey()]; ok {
return true
}
return false
}
func (o *Manager) getIDsNotInCollectionDataCache(ids []thirdparty.ContractID) []thirdparty.ContractID {
idsToFetch := make([]thirdparty.ContractID, 0, len(ids))
for _, id := range ids {
if o.isIDInCollectionDataCache(id) {
continue
}
idsToFetch = append(idsToFetch, id)
}
return idsToFetch
}
func (o *Manager) getCacheCollectionData(ids []thirdparty.ContractID) map[string]*thirdparty.CollectionData {
o.collectionsDataCacheLock.RLock()
defer o.collectionsDataCacheLock.RUnlock()

8
services/wallet/service.go
View File

@ -121,7 +121,13 @@ func NewService(
alchemyClient,
}
collectiblesManager := collectibles.NewManager(rpcClient, contractOwnershipProviders, accountOwnershipProviders, openseaClient)
collectibleDataProviders := []thirdparty.CollectibleDataProvider{
openseaClient,
infuraClient,
alchemyClient,
}
collectiblesManager := collectibles.NewManager(rpcClient, contractOwnershipProviders, accountOwnershipProviders, collectibleDataProviders, openseaClient)
collectibles := collectibles.NewService(db, walletFeed, accountsDB, accountFeed, rpcClient.NetworkManager, collectiblesManager)
return &Service{
db: db,

196
services/wallet/thirdparty/alchemy/client.go vendored
View File

@ -6,6 +6,7 @@ import (
"io/ioutil"
"net/http"
"net/url"
"strings"
"sync"
"time"
@ -15,6 +16,8 @@ import (
)
const AlchemyID = "alchemy"
const nftMetadataBatchLimit = 100
const contractMetadataBatchLimit = 100
func getBaseURL(chainID walletCommon.ChainID) (string, error) {
switch uint64(chainID) {
@ -88,6 +91,31 @@ func (o *Client) doQuery(url string) (*http.Response, error) {
return resp, nil
}
func (o *Client) doPostWithJSON(url string, payload any) (*http.Response, error) {
payloadJSON, err := json.Marshal(payload)
if err != nil {
return nil, err
}
payloadString := string(payloadJSON)
payloadReader := strings.NewReader(payloadString)
req, err := http.NewRequest("POST", url, payloadReader)
if err != nil {
return nil, err
}
req.Header.Add("accept", "application/json")
req.Header.Add("content-type", "application/json")
resp, err := o.client.Do(req)
if err != nil {
return nil, err
}
return resp, nil
}
func (o *Client) FetchCollectibleOwnersByContractAddress(chainID walletCommon.ChainID, contractAddress common.Address) (*thirdparty.CollectibleContractOwnership, error) {
ownership := thirdparty.CollectibleContractOwnership{
ContractAddress: contractAddress,
@ -192,13 +220,13 @@ func (o *Client) fetchOwnedAssets(chainID walletCommon.ChainID, owner common.Add
return nil, fmt.Errorf("invalid json: %s", string(body))
}
container := NFTList{}
container := OwnedNFTList{}
err = json.Unmarshal(body, &container)
if err != nil {
return nil, err
}
assets.Items = append(assets.Items, container.toCommon(chainID)...)
assets.Items = append(assets.Items, alchemyToCollectiblesData(chainID, container.OwnedNFTs)...)
assets.NextCursor = container.PageKey
if len(assets.NextCursor) == 0 {
@ -214,3 +242,167 @@ func (o *Client) fetchOwnedAssets(chainID walletCommon.ChainID, owner common.Add
return assets, nil
}
func getCollectibleUniqueIDBatches(ids []thirdparty.CollectibleUniqueID) []BatchTokenIDs {
batches := make([]BatchTokenIDs, 0)
for startIdx := 0; startIdx < len(ids); startIdx += nftMetadataBatchLimit {
endIdx := startIdx + nftMetadataBatchLimit
if endIdx > len(ids) {
endIdx = len(ids)
}
pageIDs := ids[startIdx:endIdx]
batchIDs := BatchTokenIDs{
IDs: make([]TokenID, 0, len(pageIDs)),
}
for _, id := range pageIDs {
batchID := TokenID{
ContractAddress: id.ContractID.Address,
TokenID: id.TokenID,
}
batchIDs.IDs = append(batchIDs.IDs, batchID)
}
batches = append(batches, batchIDs)
}
return batches
}
func (o *Client) fetchAssetsByBatchTokenIDs(chainID walletCommon.ChainID, batchIDs BatchTokenIDs) ([]thirdparty.FullCollectibleData, error) {
baseURL, err := getNFTBaseURL(chainID, o.apiKeys[uint64(chainID)])
if err != nil {
return nil, err
}
url := fmt.Sprintf("%s/getNFTMetadataBatch", baseURL)
resp, err := o.doPostWithJSON(url, batchIDs)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
// if Json is not returned there must be an error
if !json.Valid(body) {
return nil, fmt.Errorf("invalid json: %s", string(body))
}
assets := NFTList{}
err = json.Unmarshal(body, &assets)
if err != nil {
return nil, err
}
ret := alchemyToCollectiblesData(chainID, assets.NFTs)
return ret, nil
}
func (o *Client) FetchAssetsByCollectibleUniqueID(uniqueIDs []thirdparty.CollectibleUniqueID) ([]thirdparty.FullCollectibleData, error) {
ret := make([]thirdparty.FullCollectibleData, 0, len(uniqueIDs))
idsPerChainID := thirdparty.GroupCollectibleUIDsByChainID(uniqueIDs)
for chainID, ids := range idsPerChainID {
batches := getCollectibleUniqueIDBatches(ids)
for _, batch := range batches {
assets, err := o.fetchAssetsByBatchTokenIDs(chainID, batch)
if err != nil {
return nil, err
}
ret = append(ret, assets...)
}
}
return ret, nil
}
func getContractAddressBatches(ids []thirdparty.ContractID) []BatchContractAddresses {
batches := make([]BatchContractAddresses, 0)
for startIdx := 0; startIdx < len(ids); startIdx += contractMetadataBatchLimit {
endIdx := startIdx + contractMetadataBatchLimit
if endIdx > len(ids) {
endIdx = len(ids)
}
pageIDs := ids[startIdx:endIdx]
batchIDs := BatchContractAddresses{
Addresses: make([]common.Address, 0, len(pageIDs)),
}
for _, id := range pageIDs {
batchIDs.Addresses = append(batchIDs.Addresses, id.Address)
}
batches = append(batches, batchIDs)
}
return batches
}
func (o *Client) fetchCollectionsDataByBatchContractAddresses(chainID walletCommon.ChainID, batchAddresses BatchContractAddresses) ([]thirdparty.CollectionData, error) {
baseURL, err := getNFTBaseURL(chainID, o.apiKeys[uint64(chainID)])
if err != nil {
return nil, err
}
url := fmt.Sprintf("%s/getContractMetadataBatch", baseURL)
resp, err := o.doPostWithJSON(url, batchAddresses)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
// if Json is not returned there must be an error
if !json.Valid(body) {
return nil, fmt.Errorf("invalid json: %s", string(body))
}
collections := ContractList{}
err = json.Unmarshal(body, &collections)
if err != nil {
return nil, err
}
ret := alchemyToCollectionsData(chainID, collections.Contracts)
return ret, nil
}
func (o *Client) FetchCollectionsDataByContractID(contractIDs []thirdparty.ContractID) ([]thirdparty.CollectionData, error) {
ret := make([]thirdparty.CollectionData, 0, len(contractIDs))
idsPerChainID := thirdparty.GroupContractIDsByChainID(contractIDs)
for chainID, ids := range idsPerChainID {
batches := getContractAddressBatches(ids)
for _, batch := range batches {
contractsData, err := o.fetchCollectionsDataByBatchContractAddresses(chainID, batch)
if err != nil {
return nil, err
}
ret = append(ret, contractsData...)
}
}
return ret, nil
}

50
services/wallet/thirdparty/alchemy/types.go vendored
View File

@ -95,6 +95,10 @@ type Contract struct {
OpenSeaMetadata OpenSeaMetadata `json:"openSeaMetadata"`
}
type ContractList struct {
Contracts []Contract `json:"contracts"`
}
type Image struct {
ImageURL string `json:"pngUrl"`
CachedAnimationURL string `json:"cachedUrl"`
@ -111,12 +115,29 @@ type Asset struct {
TokenURI string `json:"tokenUri"`
}
type NFTList struct {
type OwnedNFTList struct {
OwnedNFTs []Asset `json:"ownedNfts"`
TotalCount *bigint.BigInt `json:"totalCount"`
PageKey string `json:"pageKey"`
}
type NFTList struct {
NFTs []Asset `json:"nfts"`
}
type BatchContractAddresses struct {
Addresses []common.Address `json:"contractAddresses"`
}
type BatchTokenIDs struct {
IDs []TokenID `json:"tokens"`
}
type TokenID struct {
ContractAddress common.Address `json:"contractAddress"`
TokenID *bigint.BigInt `json:"tokenId"`
}
func alchemyToCollectibleTraits(attributes []Attribute) []thirdparty.CollectibleTrait {
ret := make([]thirdparty.CollectibleTrait, 0, len(attributes))
caser := cases.Title(language.Und, cases.NoLower)
@ -131,11 +152,11 @@ func alchemyToCollectibleTraits(attributes []Attribute) []thirdparty.Collectible
return ret
}
func (c *Asset) toCollectionData(id thirdparty.ContractID) thirdparty.CollectionData {
func (c *Contract) toCollectionData(id thirdparty.ContractID) thirdparty.CollectionData {
ret := thirdparty.CollectionData{
ID: id,
Name: c.Contract.Name,
ImageURL: c.Contract.OpenSeaMetadata.ImageURL,
Name: c.Name,
ImageURL: c.OpenSeaMetadata.ImageURL,
}
return ret
}
@ -152,16 +173,16 @@ func (c *Asset) toCollectiblesData(id thirdparty.CollectibleUniqueID) thirdparty
}
func (c *Asset) toCommon(id thirdparty.CollectibleUniqueID) thirdparty.FullCollectibleData {
contractData := c.toCollectionData(id.ContractID)
contractData := c.Contract.toCollectionData(id.ContractID)
return thirdparty.FullCollectibleData{
CollectibleData: c.toCollectiblesData(id),
CollectionData: &contractData,
}
}
func (l *NFTList) toCommon(chainID walletCommon.ChainID) []thirdparty.FullCollectibleData {
ret := make([]thirdparty.FullCollectibleData, 0, len(l.OwnedNFTs))
for _, asset := range l.OwnedNFTs {
func alchemyToCollectiblesData(chainID walletCommon.ChainID, l []Asset) []thirdparty.FullCollectibleData {
ret := make([]thirdparty.FullCollectibleData, 0, len(l))
for _, asset := range l {
id := thirdparty.CollectibleUniqueID{
ContractID: thirdparty.ContractID{
ChainID: chainID,
@ -174,3 +195,16 @@ func (l *NFTList) toCommon(chainID walletCommon.ChainID) []thirdparty.FullCollec
}
return ret
}
func alchemyToCollectionsData(chainID walletCommon.ChainID, l []Contract) []thirdparty.CollectionData {
ret := make([]thirdparty.CollectionData, 0, len(l))
for _, contract := range l {
id := thirdparty.ContractID{
ChainID: chainID,
Address: contract.Address,
}
item := contract.toCollectionData(id)
ret = append(ret, item)
}
return ret
}

19
services/wallet/thirdparty/collectible_types.go vendored
View File

@ -52,6 +52,19 @@ func GroupCollectibleUIDsByChainID(uids []CollectibleUniqueID) map[w_common.Chai
return ret
}
func GroupContractIDsByChainID(ids []ContractID) map[w_common.ChainID][]ContractID {
ret := make(map[w_common.ChainID][]ContractID)
for _, id := range ids {
if _, ok := ret[id.ChainID]; !ok {
ret[id.ChainID] = make([]ContractID, 0, len(ids))
}
ret[id.ChainID] = append(ret[id.ChainID], id)
}
return ret
}
type CollectionTrait struct {
Min float64 `json:"min"`
Max float64 `json:"max"`
@ -154,3 +167,9 @@ type CollectibleAccountOwnershipProvider interface {
FetchAllAssetsByOwner(chainID w_common.ChainID, owner common.Address, cursor string, limit int) (*FullCollectibleDataContainer, error)
FetchAllAssetsByOwnerAndContractAddress(chainID w_common.ChainID, owner common.Address, contractAddresses []common.Address, cursor string, limit int) (*FullCollectibleDataContainer, error)
}
type CollectibleDataProvider interface {
CollectibleProvider
FetchAssetsByCollectibleUniqueID(uniqueIDs []CollectibleUniqueID) ([]FullCollectibleData, error)
FetchCollectionsDataByContractID(ids []ContractID) ([]CollectionData, error)
}

2
services/wallet/thirdparty/infura/client.go vendored
View File

@ -217,7 +217,7 @@ func (o *Client) FetchAssetsByCollectibleUniqueID(uniqueIDs []thirdparty.Collect
return ret, nil
}
func (o *Client) FetchCollectionDataByContractID(contractIDs []thirdparty.ContractID) ([]thirdparty.CollectionData, error) {
func (o *Client) FetchCollectionsDataByContractID(contractIDs []thirdparty.ContractID) ([]thirdparty.CollectionData, error) {
ret := make([]thirdparty.CollectionData, 0, len(contractIDs))
for _, id := range contractIDs {

34
services/wallet/thirdparty/opensea/client.go vendored
View File

@ -119,6 +119,40 @@ func (o *Client) FetchAllCollectionsByOwner(chainID walletCommon.ChainID, owner
return collections, nil
}
func (o *Client) FetchCollectionsDataByContractID(ids []thirdparty.ContractID) ([]thirdparty.CollectionData, error) {
ret := make([]thirdparty.CollectionData, 0, len(ids))
for _, id := range ids {
path := fmt.Sprintf("asset_contract/%s", id.Address.String())
url, err := o.urlGetter(id.ChainID, path)
if err != nil {
return nil, err
}
body, err := o.client.doGetRequest(url, o.apiKey)
if err != nil {
o.connectionStatus.SetIsConnected(false)
return nil, err
}
o.connectionStatus.SetIsConnected(true)
// if Json is not returned there must be an error
if !json.Valid(body) {
return nil, fmt.Errorf("invalid json: %s", string(body))
}
var tmp AssetContract
err = json.Unmarshal(body, &tmp)
if err != nil {
return nil, err
}
ret = append(ret, tmp.Collection.toCollectionData(id))
}
return ret, nil
}
func (o *Client) FetchAllAssetsByOwnerAndCollection(chainID walletCommon.ChainID, owner common.Address, collectionSlug string, cursor string, limit int) (*thirdparty.FullCollectibleDataContainer, error) {
queryParams := url.Values{
"owner": {owner.String()},

18
services/wallet/thirdparty/opensea/types.go vendored
View File

@ -126,6 +126,10 @@ type OwnedCollection struct {
OwnedAssetCount *bigint.BigInt `json:"owned_asset_count"`
}
type AssetContract struct {
Collection Collection `json:"collection"`
}
func (c *Asset) id() thirdparty.CollectibleUniqueID {
return thirdparty.CollectibleUniqueID{
ContractID: thirdparty.ContractID{
@ -152,15 +156,15 @@ func openseaToCollectibleTraits(traits []Trait) []thirdparty.CollectibleTrait {
return ret
}
func (c *Asset) toCollectionData() thirdparty.CollectionData {
func (c *Collection) toCollectionData(id thirdparty.ContractID) thirdparty.CollectionData {
ret := thirdparty.CollectionData{
ID: c.id().ContractID,
Name: c.Collection.Name,
Slug: c.Collection.Slug,
ImageURL: c.Collection.ImageURL,
ID: id,
Name: c.Name,
Slug: c.Slug,
ImageURL: c.ImageURL,
Traits: make(map[string]thirdparty.CollectionTrait),
}
for traitType, trait := range c.Collection.Traits {
for traitType, trait := range c.Traits {
ret.Traits[traitType] = thirdparty.CollectionTrait{
Min: trait.Min,
Max: trait.Max,
@ -184,7 +188,7 @@ func (c *Asset) toCollectiblesData() thirdparty.CollectibleData {
}
func (c *Asset) toCommon() thirdparty.FullCollectibleData {
collection := c.toCollectionData()
collection := c.Collection.toCollectionData(c.id().ContractID)
return thirdparty.FullCollectibleData{
CollectibleData: c.toCollectiblesData(),
CollectionData: &collection,