feat: lazy load collectibles metadata

This commit is contained in:
Dario Gabriel Lipicar 2023-12-13 09:19:25 -03:00 committed by dlipicar
parent e5ce11f067
commit 959dcbdea5
18 changed files with 554 additions and 357 deletions

View File

@ -59,6 +59,7 @@ import (
) )
const infinityString = "∞" const infinityString = "∞"
const providerID = "community"
// EnvelopeEventsHandler used for two different event types. // EnvelopeEventsHandler used for two different event types.
type EnvelopeEventsHandler interface { type EnvelopeEventsHandler interface {
@ -548,7 +549,7 @@ func (s *Service) FillCollectibleMetadata(collectible *thirdparty.FullCollectibl
return fmt.Errorf("invalid communityID") return fmt.Errorf("invalid communityID")
} }
community, err := s.fetchCommunity(communityID) community, err := s.fetchCommunity(communityID, true)
if err != nil { if err != nil {
return err return err
@ -582,6 +583,7 @@ func (s *Service) FillCollectibleMetadata(collectible *thirdparty.FullCollectibl
imagePayload, _ := images.GetPayloadFromURI(tokenMetadata.GetImage()) imagePayload, _ := images.GetPayloadFromURI(tokenMetadata.GetImage())
collectible.CollectibleData.Provider = providerID
collectible.CollectibleData.Name = tokenMetadata.GetName() collectible.CollectibleData.Name = tokenMetadata.GetName()
collectible.CollectibleData.Description = tokenMetadata.GetDescription() collectible.CollectibleData.Description = tokenMetadata.GetDescription()
collectible.CollectibleData.ImagePayload = imagePayload collectible.CollectibleData.ImagePayload = imagePayload
@ -593,10 +595,13 @@ func (s *Service) FillCollectibleMetadata(collectible *thirdparty.FullCollectibl
CommunityID: communityID, CommunityID: communityID,
} }
} }
collectible.CollectionData.Provider = providerID
collectible.CollectionData.Name = tokenMetadata.GetName() collectible.CollectionData.Name = tokenMetadata.GetName()
collectible.CollectionData.ImagePayload = imagePayload collectible.CollectionData.ImagePayload = imagePayload
collectible.CommunityInfo = &thirdparty.CollectibleCommunityInfo{ collectible.CommunityInfo = communityToInfo(community)
collectible.CollectibleCommunityInfo = &thirdparty.CollectibleCommunityInfo{
PrivilegesLevel: privilegesLevel, PrivilegesLevel: privilegesLevel,
} }
@ -614,25 +619,28 @@ func permissionTypeToPrivilegesLevel(permissionType protobuf.CommunityTokenPermi
} }
} }
func (s *Service) FetchCommunityInfo(communityID string) (*thirdparty.CommunityInfo, error) { func communityToInfo(community *communities.Community) *thirdparty.CommunityInfo {
community, err := s.fetchCommunity(communityID)
if err != nil {
return nil, err
}
if community == nil { if community == nil {
return nil, nil return nil
} }
communityInfo := &thirdparty.CommunityInfo{ return &thirdparty.CommunityInfo{
CommunityName: community.Name(), CommunityName: community.Name(),
CommunityColor: community.Color(), CommunityColor: community.Color(),
CommunityImagePayload: fetchCommunityImage(community), CommunityImagePayload: fetchCommunityImage(community),
} }
return communityInfo, nil
} }
func (s *Service) fetchCommunity(communityID string) (*communities.Community, error) { func (s *Service) FetchCommunityInfo(communityID string) (*thirdparty.CommunityInfo, error) {
community, err := s.fetchCommunity(communityID, false)
if err != nil {
return nil, err
}
return communityToInfo(community), nil
}
func (s *Service) fetchCommunity(communityID string, tryDatabase bool) (*communities.Community, error) {
if s.messenger == nil { if s.messenger == nil {
return nil, fmt.Errorf("messenger not ready") return nil, fmt.Errorf("messenger not ready")
} }
@ -646,7 +654,7 @@ func (s *Service) fetchCommunity(communityID string) (*communities.Community, er
community, err := s.messenger.FetchCommunity(&protocol.FetchCommunityRequest{ community, err := s.messenger.FetchCommunity(&protocol.FetchCommunityRequest{
CommunityKey: communityID, CommunityKey: communityID,
Shard: shard, Shard: shard,
TryDatabase: true, TryDatabase: tryDatabase,
WaitForResponse: true, WaitForResponse: true,
}) })

View File

@ -153,7 +153,7 @@ func (s *Service) GetActivityCollectiblesAsync(requestID int32, chainIDs []w_com
return nil, err return nil, err
} }
data, err := s.collectibles.FetchAssetsByCollectibleUniqueID(ctx, collectibles) data, err := s.collectibles.FetchAssetsByCollectibleUniqueID(ctx, collectibles, true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -222,7 +222,7 @@ func (s *Service) getActivityDetails(ctx context.Context, entries []Entry) ([]*E
log.Debug("wallet.activity.Service lazyLoadDetails", "entries.len", len(entries), "ids.len", len(ids)) log.Debug("wallet.activity.Service lazyLoadDetails", "entries.len", len(entries), "ids.len", len(ids))
colData, err := s.collectibles.FetchAssetsByCollectibleUniqueID(ctx, ids) colData, err := s.collectibles.FetchAssetsByCollectibleUniqueID(ctx, ids, true)
if err != nil { if err != nil {
log.Error("Error fetching collectible details", "error", err) log.Error("Error fetching collectible details", "error", err)
return nil, err return nil, err

View File

@ -29,7 +29,7 @@ type mockCollectiblesManager struct {
mock.Mock mock.Mock
} }
func (m *mockCollectiblesManager) FetchAssetsByCollectibleUniqueID(ctx context.Context, uniqueIDs []thirdparty.CollectibleUniqueID) ([]thirdparty.FullCollectibleData, error) { func (m *mockCollectiblesManager) FetchAssetsByCollectibleUniqueID(ctx context.Context, uniqueIDs []thirdparty.CollectibleUniqueID, asyncFetch bool) ([]thirdparty.FullCollectibleData, error) {
args := m.Called(uniqueIDs) args := m.Called(uniqueIDs)
res := args.Get(0) res := args.Get(0)
if res == nil { if res == nil {

View File

@ -336,24 +336,6 @@ func (api *API) GetCollectiblesByUniqueIDAsync(requestID int32, uniqueIDs []thir
return nil return nil
} }
// @deprecated
func (api *API) GetCollectiblesByOwnerWithCursor(ctx context.Context, chainID wcommon.ChainID, owner common.Address, cursor string, limit int) (*thirdparty.FullCollectibleDataContainer, error) {
log.Debug("call to GetCollectiblesByOwnerWithCursor")
return api.s.collectiblesManager.FetchAllAssetsByOwner(ctx, chainID, owner, cursor, limit, thirdparty.FetchFromAnyProvider)
}
// @deprecated
func (api *API) GetCollectiblesByOwnerAndContractAddressWithCursor(ctx context.Context, chainID wcommon.ChainID, owner common.Address, contractAddresses []common.Address, cursor string, limit int) (*thirdparty.FullCollectibleDataContainer, error) {
log.Debug("call to GetCollectiblesByOwnerAndContractAddressWithCursor")
return api.s.collectiblesManager.FetchAllAssetsByOwnerAndContractAddress(ctx, chainID, owner, contractAddresses, cursor, limit, thirdparty.FetchFromAnyProvider)
}
// @deprecated
func (api *API) GetCollectiblesByUniqueID(ctx context.Context, uniqueIDs []thirdparty.CollectibleUniqueID) ([]thirdparty.FullCollectibleData, error) {
log.Debug("call to GetCollectiblesByUniqueID")
return api.s.collectiblesManager.FetchAssetsByCollectibleUniqueID(ctx, uniqueIDs)
}
func (api *API) GetCollectibleOwnersByContractAddress(ctx context.Context, chainID wcommon.ChainID, contractAddress common.Address) (*thirdparty.CollectibleContractOwnership, error) { func (api *API) GetCollectibleOwnersByContractAddress(ctx context.Context, chainID wcommon.ChainID, contractAddress common.Address) (*thirdparty.CollectibleContractOwnership, error) {
log.Debug("call to GetCollectibleOwnersByContractAddress") log.Debug("call to GetCollectibleOwnersByContractAddress")
return api.s.collectiblesManager.FetchCollectibleOwnersByContractAddress(ctx, chainID, contractAddress) return api.s.collectiblesManager.FetchCollectibleOwnersByContractAddress(ctx, chainID, contractAddress)

View File

@ -1,13 +1,11 @@
package collectibles package collectibles
import ( import (
"fmt"
"math/big" "math/big"
"testing" "testing"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/status-im/status-go/protocol/communities/token"
"github.com/status-im/status-go/services/wallet/bigint" "github.com/status-im/status-go/services/wallet/bigint"
w_common "github.com/status-im/status-go/services/wallet/common" w_common "github.com/status-im/status-go/services/wallet/common"
"github.com/status-im/status-go/services/wallet/thirdparty" "github.com/status-im/status-go/services/wallet/thirdparty"
@ -25,74 +23,11 @@ func setupCollectibleDataDBTest(t *testing.T) (*CollectibleDataDB, func()) {
} }
} }
func generateTestCollectiblesData(count int) (result []thirdparty.CollectibleData) {
result = make([]thirdparty.CollectibleData, 0, count)
for i := 0; i < count; i++ {
bigI := big.NewInt(int64(i))
newCollectible := thirdparty.CollectibleData{
ID: thirdparty.CollectibleUniqueID{
ContractID: thirdparty.ContractID{
ChainID: w_common.ChainID(i % 4),
Address: common.BigToAddress(bigI),
},
TokenID: &bigint.BigInt{Int: bigI},
},
Provider: fmt.Sprintf("provider-%d", i),
Name: fmt.Sprintf("name-%d", i),
Description: fmt.Sprintf("description-%d", i),
Permalink: fmt.Sprintf("permalink-%d", i),
ImageURL: fmt.Sprintf("imageurl-%d", i),
ImagePayload: []byte(fmt.Sprintf("imagepayload-%d", i)),
AnimationURL: fmt.Sprintf("animationurl-%d", i),
AnimationMediaType: fmt.Sprintf("animationmediatype-%d", i),
Traits: []thirdparty.CollectibleTrait{
{
TraitType: fmt.Sprintf("traittype-%d", i),
Value: fmt.Sprintf("traitvalue-%d", i),
DisplayType: fmt.Sprintf("displaytype-%d", i),
MaxValue: fmt.Sprintf("maxvalue-%d", i),
},
{
TraitType: fmt.Sprintf("traittype-%d", i),
Value: fmt.Sprintf("traitvalue-%d", i),
DisplayType: fmt.Sprintf("displaytype-%d", i),
MaxValue: fmt.Sprintf("maxvalue-%d", i),
},
{
TraitType: fmt.Sprintf("traittype-%d", i),
Value: fmt.Sprintf("traitvalue-%d", i),
DisplayType: fmt.Sprintf("displaytype-%d", i),
MaxValue: fmt.Sprintf("maxvalue-%d", i),
},
},
BackgroundColor: fmt.Sprintf("backgroundcolor-%d", i),
TokenURI: fmt.Sprintf("tokenuri-%d", i),
CommunityID: fmt.Sprintf("communityid-%d", i%5),
}
if i%5 == 0 {
newCollectible.CommunityID = ""
}
result = append(result, newCollectible)
}
return result
}
func generateTestCommunityData(count int) []thirdparty.CollectibleCommunityInfo {
result := make([]thirdparty.CollectibleCommunityInfo, 0, count)
for i := 0; i < count; i++ {
newCommunityInfo := thirdparty.CollectibleCommunityInfo{
PrivilegesLevel: token.PrivilegesLevel(i) % (token.CommunityLevel + 1),
}
result = append(result, newCommunityInfo)
}
return result
}
func TestUpdateCollectiblesData(t *testing.T) { func TestUpdateCollectiblesData(t *testing.T) {
db, cleanDB := setupCollectibleDataDBTest(t) db, cleanDB := setupCollectibleDataDBTest(t)
defer cleanDB() defer cleanDB()
data := generateTestCollectiblesData(50) data := thirdparty.GenerateTestCollectiblesData(50)
var err error var err error
@ -168,8 +103,8 @@ func TestUpdateCommunityData(t *testing.T) {
defer cleanDB() defer cleanDB()
const nData = 50 const nData = 50
data := generateTestCollectiblesData(nData) data := thirdparty.GenerateTestCollectiblesData(nData)
communityData := generateTestCommunityData(nData) communityData := thirdparty.GenerateTestCollectiblesCommunityData(nData)
var err error var err error

View File

@ -1,7 +1,6 @@
package collectibles package collectibles
import ( import (
"fmt"
"math/big" "math/big"
"testing" "testing"
@ -23,41 +22,11 @@ func setupCollectionDataDBTest(t *testing.T) (*CollectionDataDB, func()) {
} }
} }
func generateTestCollectionsData(count int) (result []thirdparty.CollectionData) {
result = make([]thirdparty.CollectionData, 0, count)
for i := 0; i < count; i++ {
bigI := big.NewInt(int64(count))
traits := make(map[string]thirdparty.CollectionTrait)
for j := 0; j < 3; j++ {
traits[fmt.Sprintf("traittype-%d", j)] = thirdparty.CollectionTrait{
Min: float64(i+j) / 2,
Max: float64(i+j) * 2,
}
}
newCollection := thirdparty.CollectionData{
ID: thirdparty.ContractID{
ChainID: w_common.ChainID(i),
Address: common.BigToAddress(bigI),
},
Provider: fmt.Sprintf("provider-%d", i),
Name: fmt.Sprintf("name-%d", i),
Slug: fmt.Sprintf("slug-%d", i),
ImageURL: fmt.Sprintf("imageurl-%d", i),
ImagePayload: []byte(fmt.Sprintf("imagepayload-%d", i)),
Traits: traits,
CommunityID: fmt.Sprintf("community-%d", i),
}
result = append(result, newCollection)
}
return result
}
func TestUpdateCollectionsData(t *testing.T) { func TestUpdateCollectionsData(t *testing.T) {
db, cleanDB := setupCollectionDataDBTest(t) db, cleanDB := setupCollectionDataDBTest(t)
defer cleanDB() defer cleanDB()
data := generateTestCollectionsData(50) data := thirdparty.GenerateTestCollectionsData(50)
var err error var err error

View File

@ -35,8 +35,8 @@ func TestFilterOwnedCollectibles(t *testing.T) {
cDB := NewCollectibleDataDB(db) cDB := NewCollectibleDataDB(db)
const nData = 50 const nData = 50
data := generateTestCollectiblesData(nData) data := thirdparty.GenerateTestCollectiblesData(nData)
communityData := generateTestCommunityData(nData) communityData := thirdparty.GenerateTestCollectiblesCommunityData(nData)
ownerAddresses := []common.Address{ ownerAddresses := []common.Address{
common.HexToAddress("0x1234"), common.HexToAddress("0x1234"),

View File

@ -3,6 +3,7 @@ package collectibles
import ( import (
"context" "context"
"database/sql" "database/sql"
"encoding/json"
"errors" "errors"
"math/big" "math/big"
"net/http" "net/http"
@ -18,6 +19,7 @@ import (
"github.com/status-im/status-go/contracts/community-tokens/collectibles" "github.com/status-im/status-go/contracts/community-tokens/collectibles"
"github.com/status-im/status-go/rpc" "github.com/status-im/status-go/rpc"
"github.com/status-im/status-go/server" "github.com/status-im/status-go/server"
"github.com/status-im/status-go/services/wallet/async"
"github.com/status-im/status-go/services/wallet/bigint" "github.com/status-im/status-go/services/wallet/bigint"
walletCommon "github.com/status-im/status-go/services/wallet/common" walletCommon "github.com/status-im/status-go/services/wallet/common"
"github.com/status-im/status-go/services/wallet/community" "github.com/status-im/status-go/services/wallet/community"
@ -27,6 +29,7 @@ import (
) )
const requestTimeout = 5 * time.Second const requestTimeout = 5 * time.Second
const signalUpdatedCollectiblesDataPageSize = 10
const hystrixContractOwnershipClientName = "contractOwnershipClient" const hystrixContractOwnershipClientName = "contractOwnershipClient"
@ -45,7 +48,7 @@ var (
) )
type ManagerInterface interface { type ManagerInterface interface {
FetchAssetsByCollectibleUniqueID(ctx context.Context, uniqueIDs []thirdparty.CollectibleUniqueID) ([]thirdparty.FullCollectibleData, error) FetchAssetsByCollectibleUniqueID(ctx context.Context, uniqueIDs []thirdparty.CollectibleUniqueID, asyncFetch bool) ([]thirdparty.FullCollectibleData, error)
} }
type Manager struct { type Manager struct {
@ -61,11 +64,13 @@ type Manager struct {
collectiblesDataDB *CollectibleDataDB collectiblesDataDB *CollectibleDataDB
collectionsDataDB *CollectionDataDB collectionsDataDB *CollectionDataDB
communityManager *community.Manager communityManager *community.Manager
ownershipDB *OwnershipDB
mediaServer *server.MediaServer mediaServer *server.MediaServer
statuses map[string]*connection.Status statuses map[string]*connection.Status
statusNotifier *connection.StatusNotifier statusNotifier *connection.StatusNotifier
feed *event.Feed
} }
func NewManager( func NewManager(
@ -139,9 +144,11 @@ func NewManager(
collectiblesDataDB: NewCollectibleDataDB(db), collectiblesDataDB: NewCollectibleDataDB(db),
collectionsDataDB: NewCollectionDataDB(db), collectionsDataDB: NewCollectionDataDB(db),
communityManager: communityManager, communityManager: communityManager,
ownershipDB: ownershipDB,
mediaServer: mediaServer, mediaServer: mediaServer,
statuses: statuses, statuses: statuses,
statusNotifier: statusNotifier, statusNotifier: statusNotifier,
feed: feed,
} }
} }
@ -262,7 +269,7 @@ func (o *Manager) FetchAllAssetsByOwnerAndContractAddress(ctx context.Context, c
continue continue
} }
err = o.processFullCollectibleData(ctx, assetContainer.Items) _, err = o.processFullCollectibleData(ctx, assetContainer.Items, true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -295,7 +302,7 @@ func (o *Manager) FetchAllAssetsByOwner(ctx context.Context, chainID walletCommo
continue continue
} }
err = o.processFullCollectibleData(ctx, assetContainer.Items) _, err = o.processFullCollectibleData(ctx, assetContainer.Items, true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -321,7 +328,10 @@ func (o *Manager) FetchCollectibleOwnershipByOwner(ctx context.Context, chainID
return &ret, nil return &ret, nil
} }
func (o *Manager) FetchAssetsByCollectibleUniqueID(ctx context.Context, uniqueIDs []thirdparty.CollectibleUniqueID) ([]thirdparty.FullCollectibleData, error) { // Returns collectible metadata for the given unique IDs.
// If asyncFetch is true, empty metadata will be returned for any missing collectibles and an EventCollectiblesDataUpdated will be sent when the data is ready.
// If asyncFetch is false, it will wait for all collectibles' metadata to be retrieved before returning.
func (o *Manager) FetchAssetsByCollectibleUniqueID(ctx context.Context, uniqueIDs []thirdparty.CollectibleUniqueID, asyncFetch bool) ([]thirdparty.FullCollectibleData, error) {
missingIDs, err := o.collectiblesDataDB.GetIDsNotInDB(uniqueIDs) missingIDs, err := o.collectiblesDataDB.GetIDsNotInDB(uniqueIDs)
if err != nil { if err != nil {
return nil, err return nil, err
@ -329,27 +339,40 @@ func (o *Manager) FetchAssetsByCollectibleUniqueID(ctx context.Context, uniqueID
missingIDsPerChainID := thirdparty.GroupCollectibleUIDsByChainID(missingIDs) missingIDsPerChainID := thirdparty.GroupCollectibleUIDsByChainID(missingIDs)
for chainID, idsToFetch := range missingIDsPerChainID { group := async.NewGroup(ctx)
defer o.checkConnectionStatus(chainID) group.Add(func(ctx context.Context) error {
for chainID, idsToFetch := range missingIDsPerChainID {
defer o.checkConnectionStatus(chainID)
for _, provider := range o.collectibleDataProviders { for _, provider := range o.collectibleDataProviders {
if !provider.IsChainSupported(chainID) { if !provider.IsChainSupported(chainID) {
continue continue
}
fetchedAssets, err := provider.FetchAssetsByCollectibleUniqueID(ctx, idsToFetch)
if err != nil {
log.Error("FetchAssetsByCollectibleUniqueID failed for", "provider", provider.ID(), "chainID", chainID, "err", err)
continue
}
updatedCollectibles, err := o.processFullCollectibleData(ctx, fetchedAssets, asyncFetch)
if err != nil {
log.Error("processFullCollectibleData failed for", "provider", provider.ID(), "chainID", chainID, "len(fetchedAssets)", len(fetchedAssets), "err", err)
return err
}
if asyncFetch {
o.signalUpdatedCollectiblesData(updatedCollectibles)
}
break
} }
fetchedAssets, err := provider.FetchAssetsByCollectibleUniqueID(ctx, idsToFetch)
if err != nil {
log.Error("FetchAssetsByCollectibleUniqueID failed for", "provider", provider.ID(), "chainID", chainID, "err", err)
continue
}
err = o.processFullCollectibleData(ctx, fetchedAssets)
if err != nil {
return nil, err
}
break
} }
return nil
})
if !asyncFetch {
group.Wait()
} }
return o.getCacheFullCollectibleData(uniqueIDs) return o.getCacheFullCollectibleData(uniqueIDs)
@ -445,12 +468,6 @@ func (o *Manager) FetchCollectibleOwnersByContractAddress(ctx context.Context, c
return owners.(*thirdparty.CollectibleContractOwnership), nil return owners.(*thirdparty.CollectibleContractOwnership), nil
} }
func isMetadataEmpty(asset thirdparty.CollectibleData) bool {
return asset.Name == "" &&
asset.Description == "" &&
asset.ImageURL == ""
}
func (o *Manager) fetchTokenURI(ctx context.Context, id thirdparty.CollectibleUniqueID) (string, error) { func (o *Manager) fetchTokenURI(ctx context.Context, id thirdparty.CollectibleUniqueID) (string, error) {
if id.TokenID == nil { if id.TokenID == nil {
return "", errors.New("empty token ID") return "", errors.New("empty token ID")
@ -482,9 +499,18 @@ func (o *Manager) fetchTokenURI(ctx context.Context, id thirdparty.CollectibleUn
return tokenURI, err return tokenURI, err
} }
func (o *Manager) processFullCollectibleData(ctx context.Context, assets []thirdparty.FullCollectibleData) error { func isMetadataEmpty(asset thirdparty.CollectibleData) bool {
return asset.Description == "" &&
asset.ImageURL == ""
}
// Processes collectible metadata obtained from a provider and ensures any missing data is fetched.
// If asyncFetch is true, community collectibles metadata will be fetched async and an EventCollectiblesDataUpdated will be sent when the data is ready.
// If asyncFetch is false, it will wait for all community collectibles' metadata to be retrieved before returning.
func (o *Manager) processFullCollectibleData(ctx context.Context, assets []thirdparty.FullCollectibleData, asyncFetch bool) ([]thirdparty.CollectibleUniqueID, error) {
fullyFetchedAssets := make(map[string]*thirdparty.FullCollectibleData) fullyFetchedAssets := make(map[string]*thirdparty.FullCollectibleData)
communityCollectibles := make(map[string][]*thirdparty.FullCollectibleData) communityCollectibles := make(map[string][]*thirdparty.FullCollectibleData)
processedIDs := make([]thirdparty.CollectibleUniqueID, 0, len(assets))
// Start with all assets, remove if any of the fetch steps fail // Start with all assets, remove if any of the fetch steps fail
for idx := range assets { for idx := range assets {
@ -493,15 +519,19 @@ func (o *Manager) processFullCollectibleData(ctx context.Context, assets []third
fullyFetchedAssets[id.HashKey()] = asset fullyFetchedAssets[id.HashKey()] = asset
} }
// Detect community collectibles
for _, asset := range fullyFetchedAssets { for _, asset := range fullyFetchedAssets {
// Only check community ownership if metadata is empty // Only check community ownership if metadata is empty
if isMetadataEmpty(asset.CollectibleData) { if isMetadataEmpty(asset.CollectibleData) {
// Get TokenURI if not given by provider
err := o.fillTokenURI(ctx, asset) err := o.fillTokenURI(ctx, asset)
if err != nil { if err != nil {
log.Error("fillTokenURI failed", "err", err) log.Error("fillTokenURI failed", "err", err)
delete(fullyFetchedAssets, asset.CollectibleData.ID.HashKey()) delete(fullyFetchedAssets, asset.CollectibleData.ID.HashKey())
continue continue
} }
// Get CommunityID if obtainable from TokenURI
err = o.fillCommunityID(asset) err = o.fillCommunityID(asset)
if err != nil { if err != nil {
log.Error("fillCommunityID failed", "err", err) log.Error("fillCommunityID failed", "err", err)
@ -509,25 +539,32 @@ func (o *Manager) processFullCollectibleData(ctx context.Context, assets []third
continue continue
} }
// Get metadata from community if community collectible
communityID := asset.CollectibleData.CommunityID communityID := asset.CollectibleData.CommunityID
if communityID != "" { if communityID != "" {
if _, ok := communityCollectibles[communityID]; !ok { if _, ok := communityCollectibles[communityID]; !ok {
communityCollectibles[communityID] = make([]*thirdparty.FullCollectibleData, 0) communityCollectibles[communityID] = make([]*thirdparty.FullCollectibleData, 0)
} }
communityCollectibles[communityID] = append(communityCollectibles[communityID], asset) communityCollectibles[communityID] = append(communityCollectibles[communityID], asset)
// Community collectibles are handled separately, remove from list
delete(fullyFetchedAssets, asset.CollectibleData.ID.HashKey())
} }
} }
} }
// Community collectibles are grouped by community ID // Community collectibles are grouped by community ID
// If fetching data for one community fails (for example, owner node is down),
// skip and continue with the other communities.
for communityID, communityAssets := range communityCollectibles { for communityID, communityAssets := range communityCollectibles {
err := o.fillCommunityInfo(communityID, communityAssets) if asyncFetch {
if err != nil { o.fetchCommunityAssetsAsync(ctx, communityID, communityAssets)
log.Error("fillCommunityInfo failed", "communityID", communityID, "err", err) } else {
for _, communityAsset := range communityAssets { err := o.fetchCommunityAssets(communityID, communityAssets)
delete(fullyFetchedAssets, communityAsset.CollectibleData.ID.HashKey()) if err != nil {
log.Error("fetchCommunityAssets failed", "communityID", communityID, "err", err)
continue
}
for _, asset := range communityAssets {
processedIDs = append(processedIDs, asset.CollectibleData.ID)
} }
} }
} }
@ -548,6 +585,7 @@ func (o *Manager) processFullCollectibleData(ctx context.Context, assets []third
for _, asset := range fullyFetchedAssets { for _, asset := range fullyFetchedAssets {
id := asset.CollectibleData.ID id := asset.CollectibleData.ID
processedIDs = append(processedIDs, id)
collectiblesData = append(collectiblesData, asset.CollectibleData) collectiblesData = append(collectiblesData, asset.CollectibleData)
if asset.CollectionData != nil { if asset.CollectionData != nil {
@ -559,32 +597,23 @@ func (o *Manager) processFullCollectibleData(ctx context.Context, assets []third
err := o.collectiblesDataDB.SetData(collectiblesData) err := o.collectiblesDataDB.SetData(collectiblesData)
if err != nil { if err != nil {
return err return nil, err
}
for _, asset := range assets {
if asset.CommunityInfo != nil {
err = o.collectiblesDataDB.SetCommunityInfo(asset.CollectibleData.ID, *asset.CommunityInfo)
if err != nil {
return err
}
}
} }
err = o.collectionsDataDB.SetData(collectionsData) err = o.collectionsDataDB.SetData(collectionsData)
if err != nil { if err != nil {
return err return nil, err
} }
if len(missingCollectionIDs) > 0 { if len(missingCollectionIDs) > 0 {
// Calling this ensures collection data is fetched and cached (if not already available) // Calling this ensures collection data is fetched and cached (if not already available)
_, err := o.FetchCollectionsDataByContractID(ctx, missingCollectionIDs) _, err := o.FetchCollectionsDataByContractID(ctx, missingCollectionIDs)
if err != nil { if err != nil {
return err return nil, err
} }
} }
return nil return processedIDs, nil
} }
func (o *Manager) fillTokenURI(ctx context.Context, asset *thirdparty.FullCollectibleData) error { func (o *Manager) fillTokenURI(ctx context.Context, asset *thirdparty.FullCollectibleData) error {
@ -616,9 +645,10 @@ func (o *Manager) fillCommunityID(asset *thirdparty.FullCollectibleData) error {
return nil return nil
} }
func (o *Manager) fillCommunityInfo(communityID string, communityAssets []*thirdparty.FullCollectibleData) error { func (o *Manager) fetchCommunityAssets(communityID string, communityAssets []*thirdparty.FullCollectibleData) error {
communityInfo, err := o.communityManager.FetchCommunityInfo(communityID) communityInfo, err := o.communityManager.FetchCommunityInfo(communityID)
if err != nil { if err != nil {
log.Error("fetchCommunityInfo failed", "communityID", communityID, "err", err)
return err return err
} }
@ -626,6 +656,41 @@ func (o *Manager) fillCommunityInfo(communityID string, communityAssets []*third
for _, communityAsset := range communityAssets { for _, communityAsset := range communityAssets {
err := o.communityManager.FillCollectibleMetadata(communityAsset) err := o.communityManager.FillCollectibleMetadata(communityAsset)
if err != nil { if err != nil {
log.Error("FillCollectibleMetadata failed", "communityID", communityID, "err", err)
return err
}
}
} else {
log.Warn("fetchCommunityAssets community not found", "communityID", communityID)
}
collectiblesData := make([]thirdparty.CollectibleData, 0, len(communityAssets))
collectionsData := make([]thirdparty.CollectionData, 0, len(communityAssets))
for _, asset := range communityAssets {
collectiblesData = append(collectiblesData, asset.CollectibleData)
if asset.CollectionData != nil {
collectionsData = append(collectionsData, *asset.CollectionData)
}
}
err = o.collectiblesDataDB.SetData(collectiblesData)
if err != nil {
log.Error("collectiblesDataDB SetData failed", "communityID", communityID, "err", err)
return err
}
err = o.collectionsDataDB.SetData(collectionsData)
if err != nil {
log.Error("collectionsDataDB SetData failed", "communityID", communityID, "err", err)
return err
}
for _, asset := range communityAssets {
if asset.CollectibleCommunityInfo != nil {
err = o.collectiblesDataDB.SetCommunityInfo(asset.CollectibleData.ID, *asset.CollectibleCommunityInfo)
if err != nil {
log.Error("collectiblesDataDB SetCommunityInfo failed", "communityID", communityID, "err", err)
return err return err
} }
} }
@ -634,6 +699,27 @@ func (o *Manager) fillCommunityInfo(communityID string, communityAssets []*third
return nil return nil
} }
func (o *Manager) fetchCommunityAssetsAsync(ctx context.Context, communityID string, communityAssets []*thirdparty.FullCollectibleData) {
if len(communityAssets) == 0 {
return
}
go func() {
err := o.fetchCommunityAssets(communityID, communityAssets)
if err != nil {
log.Error("fetchCommunityAssets failed", "communityID", communityID, "err", err)
return
}
// Metadata is up to date in db at this point, fetch and send Event.
ids := make([]thirdparty.CollectibleUniqueID, 0, len(communityAssets))
for _, asset := range communityAssets {
ids = append(ids, asset.CollectibleData.ID)
}
o.signalUpdatedCollectiblesData(ids)
}()
}
func (o *Manager) fillAnimationMediatype(ctx context.Context, asset *thirdparty.FullCollectibleData) error { func (o *Manager) fillAnimationMediatype(ctx context.Context, asset *thirdparty.FullCollectibleData) error {
if len(asset.CollectibleData.AnimationURL) > 0 { if len(asset.CollectibleData.AnimationURL) > 0 {
contentType, err := o.doContentTypeRequest(ctx, asset.CollectibleData.AnimationURL) contentType, err := o.doContentTypeRequest(ctx, asset.CollectibleData.AnimationURL)
@ -690,15 +776,27 @@ func (o *Manager) getCacheFullCollectibleData(uniqueIDs []thirdparty.Collectible
collectionData.ImageURL = o.mediaServer.MakeWalletCollectionImagesURL(collectionData.ID) collectionData.ImageURL = o.mediaServer.MakeWalletCollectionImagesURL(collectionData.ID)
} }
communityInfo, err := o.collectiblesDataDB.GetCommunityInfo(id) communityInfo, _, err := o.communityManager.GetCommunityInfo(collectibleData.CommunityID)
if err != nil {
return nil, err
}
collectibleCommunityInfo, err := o.collectiblesDataDB.GetCommunityInfo(id)
if err != nil {
return nil, err
}
ownership, err := o.ownershipDB.GetOwnership(id)
if err != nil { if err != nil {
return nil, err return nil, err
} }
fullData := thirdparty.FullCollectibleData{ fullData := thirdparty.FullCollectibleData{
CollectibleData: collectibleData, CollectibleData: collectibleData,
CollectionData: &collectionData, CollectionData: &collectionData,
CommunityInfo: communityInfo, CommunityInfo: communityInfo,
CollectibleCommunityInfo: collectibleCommunityInfo,
Ownership: ownership,
} }
ret = append(ret, fullData) ret = append(ret, fullData)
} }
@ -723,3 +821,36 @@ func (o *Manager) checkConnectionStatus(chainID walletCommon.ChainID) {
} }
o.statuses[chainID.String()].SetIsConnected(false) o.statuses[chainID.String()].SetIsConnected(false)
} }
func (o *Manager) signalUpdatedCollectiblesData(ids []thirdparty.CollectibleUniqueID) {
// We limit how much collectibles data we send in each event to avoid problems on the client side
for startIdx := 0; startIdx < len(ids); startIdx += signalUpdatedCollectiblesDataPageSize {
endIdx := startIdx + signalUpdatedCollectiblesDataPageSize
if endIdx > len(ids) {
endIdx = len(ids)
}
pageIDs := ids[startIdx:endIdx]
collectibles, err := o.getCacheFullCollectibleData(pageIDs)
if err != nil {
log.Error("Error getting FullCollectibleData from cache: %v", err)
return
}
// Send update event with most complete data type available
details := fullCollectiblesDataToDetails(collectibles)
payload, err := json.Marshal(details)
if err != nil {
log.Error("Error marshaling response: %v", err)
return
}
event := walletevent.Event{
Type: EventCollectiblesDataUpdated,
Message: string(payload),
}
o.feed.Send(event)
}
}

View File

@ -29,6 +29,7 @@ const (
EventCollectiblesOwnershipUpdateFinished walletevent.EventType = "wallet-collectibles-ownership-update-finished" EventCollectiblesOwnershipUpdateFinished walletevent.EventType = "wallet-collectibles-ownership-update-finished"
EventCollectiblesOwnershipUpdateFinishedWithError walletevent.EventType = "wallet-collectibles-ownership-update-finished-with-error" EventCollectiblesOwnershipUpdateFinishedWithError walletevent.EventType = "wallet-collectibles-ownership-update-finished-with-error"
EventCommunityCollectiblesReceived walletevent.EventType = "wallet-collectibles-community-collectibles-received" EventCommunityCollectiblesReceived walletevent.EventType = "wallet-collectibles-community-collectibles-received"
EventCollectiblesDataUpdated walletevent.EventType = "wallet-collectibles-data-updated"
EventOwnedCollectiblesFilteringDone walletevent.EventType = "wallet-owned-collectibles-filtering-done" EventOwnedCollectiblesFilteringDone walletevent.EventType = "wallet-owned-collectibles-filtering-done"
EventGetCollectiblesDetailsDone walletevent.EventType = "wallet-get-collectibles-details-done" EventGetCollectiblesDetailsDone walletevent.EventType = "wallet-get-collectibles-details-done"
@ -76,6 +77,7 @@ type Service struct {
communityManager *community.Manager communityManager *community.Manager
walletFeed *event.Feed walletFeed *event.Feed
scheduler *async.MultiClientScheduler scheduler *async.MultiClientScheduler
group *async.Group
} }
func NewService( func NewService(
@ -95,6 +97,7 @@ func NewService(
communityManager: communityManager, communityManager: communityManager,
walletFeed: walletFeed, walletFeed: walletFeed,
scheduler: async.NewMultiClientScheduler(), scheduler: async.NewMultiClientScheduler(),
group: async.NewGroup(context.Background()),
} }
s.controller.SetReceivedCollectiblesCb(s.notifyCommunityCollectiblesReceived) s.controller.SetReceivedCollectiblesCb(s.notifyCommunityCollectiblesReceived)
return s return s
@ -380,136 +383,32 @@ func (s *Service) collectibleIDsToDataType(ctx context.Context, ids []thirdparty
case CollectibleDataTypeUniqueID: case CollectibleDataTypeUniqueID:
return idsToCollectibles(ids), nil return idsToCollectibles(ids), nil
case CollectibleDataTypeHeader, CollectibleDataTypeDetails, CollectibleDataTypeCommunityHeader: case CollectibleDataTypeHeader, CollectibleDataTypeDetails, CollectibleDataTypeCommunityHeader:
collectibles, err := s.manager.FetchAssetsByCollectibleUniqueID(ctx, ids) collectibles, err := s.manager.FetchAssetsByCollectibleUniqueID(ctx, ids, true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
switch dataType { switch dataType {
case CollectibleDataTypeHeader: case CollectibleDataTypeHeader:
return s.fullCollectiblesDataToHeaders(collectibles) return fullCollectiblesDataToHeaders(collectibles), nil
case CollectibleDataTypeDetails: case CollectibleDataTypeDetails:
return s.fullCollectiblesDataToDetails(collectibles) return fullCollectiblesDataToDetails(collectibles), nil
case CollectibleDataTypeCommunityHeader: case CollectibleDataTypeCommunityHeader:
return s.fullCollectiblesDataToCommunityHeader(collectibles) return fullCollectiblesDataToCommunityHeader(collectibles), nil
} }
} }
return nil, errors.New("unknown data type") return nil, errors.New("unknown data type")
} }
func idsToCollectibles(ids []thirdparty.CollectibleUniqueID) []Collectible {
res := make([]Collectible, 0, len(ids))
for _, id := range ids {
c := idToCollectible(id)
res = append(res, c)
}
return res
}
func (s *Service) fullCollectiblesDataToHeaders(data []thirdparty.FullCollectibleData) ([]Collectible, error) {
res := make([]Collectible, 0, len(data))
for _, c := range data {
header := fullCollectibleDataToHeader(c)
ownership, err := s.ownershipDB.GetOwnership(c.CollectibleData.ID)
if err != nil {
return nil, err
}
header.Ownership = ownership
if c.CollectibleData.CommunityID != "" {
communityInfo, _, err := s.communityManager.GetCommunityInfo(c.CollectibleData.CommunityID)
if err != nil {
return nil, err
}
communityData := communityInfoToData(c.CollectibleData.CommunityID, communityInfo, c.CommunityInfo)
header.CommunityData = &communityData
}
res = append(res, header)
}
return res, nil
}
func (s *Service) fullCollectiblesDataToDetails(data []thirdparty.FullCollectibleData) ([]Collectible, error) {
res := make([]Collectible, 0, len(data))
for _, c := range data {
details := fullCollectibleDataToDetails(c)
ownership, err := s.ownershipDB.GetOwnership(c.CollectibleData.ID)
if err != nil {
return nil, err
}
details.Ownership = ownership
if c.CollectibleData.CommunityID != "" {
communityInfo, _, err := s.communityManager.GetCommunityInfo(c.CollectibleData.CommunityID)
if err != nil {
return nil, err
}
communityData := communityInfoToData(c.CollectibleData.CommunityID, communityInfo, c.CommunityInfo)
details.CommunityData = &communityData
}
res = append(res, details)
}
return res, nil
}
func (s *Service) fullCollectiblesDataToCommunityHeader(data []thirdparty.FullCollectibleData) ([]Collectible, error) {
res := make([]Collectible, 0, len(data))
for _, c := range data {
collectibleID := c.CollectibleData.ID
communityID := c.CollectibleData.CommunityID
if communityID == "" {
continue
}
communityInfo, _, err := s.communityManager.GetCommunityInfo(communityID)
if err != nil {
log.Error("Error fetching community info", "error", err)
continue
}
communityData := communityInfoToData(communityID, communityInfo, c.CommunityInfo)
header := Collectible{
ID: collectibleID,
CollectibleData: &CollectibleData{
Name: c.CollectibleData.Name,
},
CommunityData: &communityData,
}
res = append(res, header)
}
return res, nil
}
func (s *Service) notifyCommunityCollectiblesReceived(ownedCollectibles OwnedCollectibles) { func (s *Service) notifyCommunityCollectiblesReceived(ownedCollectibles OwnedCollectibles) {
ctx := context.Background() ctx := context.Background()
collectiblesData, err := s.manager.FetchAssetsByCollectibleUniqueID(ctx, ownedCollectibles.ids) collectiblesData, err := s.manager.FetchAssetsByCollectibleUniqueID(ctx, ownedCollectibles.ids, false)
if err != nil { if err != nil {
log.Error("Error fetching collectibles data", "error", err) log.Error("Error fetching collectibles data", "error", err)
return return
} }
communityCollectibles, err := s.fullCollectiblesDataToCommunityHeader(collectiblesData) communityCollectibles := fullCollectiblesDataToCommunityHeader(collectiblesData)
if err != nil {
log.Error("Error converting received collectibles data", "error", err)
return
}
if len(communityCollectibles) == 0 { if len(communityCollectibles) == 0 {
return return

View File

@ -47,6 +47,17 @@ func idToCollectible(id thirdparty.CollectibleUniqueID) Collectible {
return ret return ret
} }
func idsToCollectibles(ids []thirdparty.CollectibleUniqueID) []Collectible {
res := make([]Collectible, 0, len(ids))
for _, id := range ids {
c := idToCollectible(id)
res = append(res, c)
}
return res
}
func fullCollectibleDataToHeader(c thirdparty.FullCollectibleData) Collectible { func fullCollectibleDataToHeader(c thirdparty.FullCollectibleData) Collectible {
ret := Collectible{ ret := Collectible{
DataType: CollectibleDataTypeHeader, DataType: CollectibleDataTypeHeader,
@ -66,13 +77,28 @@ func fullCollectibleDataToHeader(c thirdparty.FullCollectibleData) Collectible {
ImageURL: c.CollectionData.ImageURL, ImageURL: c.CollectionData.ImageURL,
} }
} }
if c.CollectibleData.CommunityID != "" {
communityData := communityInfoToData(c.CollectibleData.CommunityID, c.CommunityInfo, c.CollectibleCommunityInfo)
ret.CommunityData = &communityData
}
ret.Ownership = c.Ownership
return ret return ret
} }
func fullCollectiblesDataToHeaders(data []thirdparty.FullCollectibleData) []Collectible {
res := make([]Collectible, 0, len(data))
for _, c := range data {
header := fullCollectibleDataToHeader(c)
res = append(res, header)
}
return res
}
func fullCollectibleDataToDetails(c thirdparty.FullCollectibleData) Collectible { func fullCollectibleDataToDetails(c thirdparty.FullCollectibleData) Collectible {
ret := Collectible{ ret := Collectible{
DataType: CollectibleDataTypeHeader, DataType: CollectibleDataTypeDetails,
ID: c.CollectibleData.ID, ID: c.CollectibleData.ID,
CollectibleData: &CollectibleData{ CollectibleData: &CollectibleData{
Name: c.CollectibleData.Name, Name: c.CollectibleData.Name,
@ -91,9 +117,53 @@ func fullCollectibleDataToDetails(c thirdparty.FullCollectibleData) Collectible
ImageURL: c.CollectionData.ImageURL, ImageURL: c.CollectionData.ImageURL,
} }
} }
if c.CollectibleData.CommunityID != "" {
communityData := communityInfoToData(c.CollectibleData.CommunityID, c.CommunityInfo, c.CollectibleCommunityInfo)
ret.CommunityData = &communityData
}
ret.Ownership = c.Ownership
return ret return ret
} }
func fullCollectiblesDataToDetails(data []thirdparty.FullCollectibleData) []Collectible {
res := make([]Collectible, 0, len(data))
for _, c := range data {
details := fullCollectibleDataToDetails(c)
res = append(res, details)
}
return res
}
func fullCollectiblesDataToCommunityHeader(data []thirdparty.FullCollectibleData) []Collectible {
res := make([]Collectible, 0, len(data))
for _, c := range data {
collectibleID := c.CollectibleData.ID
communityID := c.CollectibleData.CommunityID
if communityID == "" {
continue
}
communityData := communityInfoToData(communityID, c.CommunityInfo, c.CollectibleCommunityInfo)
header := Collectible{
ID: collectibleID,
CollectibleData: &CollectibleData{
Name: c.CollectibleData.Name,
},
CommunityData: &communityData,
Ownership: c.Ownership,
}
res = append(res, header)
}
return res
}
func communityInfoToData(communityID string, community *thirdparty.CommunityInfo, communityCollectible *thirdparty.CollectibleCommunityInfo) CommunityData { func communityInfoToData(communityID string, community *thirdparty.CommunityInfo, communityCollectible *thirdparty.CollectibleCommunityInfo) CommunityData {
ret := CommunityData{ ret := CommunityData{
ID: communityID, ID: communityID,

View File

@ -0,0 +1,84 @@
package collectibles
import (
"testing"
"github.com/status-im/status-go/services/wallet/thirdparty"
"github.com/stretchr/testify/require"
)
func getCommunityCollectible() thirdparty.FullCollectibleData {
return thirdparty.GenerateTestFullCollectiblesData(1)[0]
}
func getNonCommunityCollectible() thirdparty.FullCollectibleData {
c := thirdparty.GenerateTestFullCollectiblesData(1)[0]
c.CollectibleData.CommunityID = ""
c.CollectionData.CommunityID = ""
c.CommunityInfo = nil
c.CollectibleCommunityInfo = nil
return c
}
func TestFullCollectibleToHeader(t *testing.T) {
communityCollectible := getCommunityCollectible()
communityHeader := fullCollectibleDataToHeader(communityCollectible)
require.Equal(t, CollectibleDataTypeHeader, communityHeader.DataType)
require.Equal(t, communityCollectible.CollectibleData.ID, communityHeader.ID)
require.NotEmpty(t, communityHeader.CollectibleData)
require.NotEmpty(t, communityHeader.CollectionData)
require.NotEmpty(t, communityHeader.CommunityData)
require.NotEmpty(t, communityHeader.Ownership)
nonCommunityCollectible := getNonCommunityCollectible()
nonCommunityHeader := fullCollectibleDataToHeader(nonCommunityCollectible)
require.Equal(t, CollectibleDataTypeHeader, nonCommunityHeader.DataType)
require.Equal(t, nonCommunityCollectible.CollectibleData.ID, nonCommunityHeader.ID)
require.NotEmpty(t, nonCommunityHeader.CollectibleData)
require.NotEmpty(t, nonCommunityHeader.CollectionData)
require.Empty(t, nonCommunityHeader.CommunityData)
require.NotEmpty(t, nonCommunityHeader.Ownership)
}
func TestFullCollectibleToDetails(t *testing.T) {
communityCollectible := getCommunityCollectible()
communityDetails := fullCollectibleDataToDetails(communityCollectible)
require.Equal(t, CollectibleDataTypeDetails, communityDetails.DataType)
require.Equal(t, communityCollectible.CollectibleData.ID, communityDetails.ID)
require.NotEmpty(t, communityDetails.CollectibleData)
require.NotEmpty(t, communityDetails.CollectionData)
require.NotEmpty(t, communityDetails.CommunityData)
require.NotEmpty(t, communityDetails.Ownership)
nonCommunityCollectible := getNonCommunityCollectible()
nonCommunityDetails := fullCollectibleDataToDetails(nonCommunityCollectible)
require.Equal(t, CollectibleDataTypeDetails, nonCommunityDetails.DataType)
require.Equal(t, nonCommunityCollectible.CollectibleData.ID, nonCommunityDetails.ID)
require.NotEmpty(t, nonCommunityDetails.CollectibleData)
require.NotEmpty(t, nonCommunityDetails.CollectionData)
require.Empty(t, nonCommunityDetails.CommunityData)
require.NotEmpty(t, nonCommunityDetails.Ownership)
}
func TestFullCollectiblesToCommunityHeader(t *testing.T) {
collectibles := make([]thirdparty.FullCollectibleData, 0, 10)
for i := 0; i < 10; i++ {
if i%2 == 0 {
collectibles = append(collectibles, getCommunityCollectible())
} else {
collectibles = append(collectibles, getNonCommunityCollectible())
}
}
communityHeaders := fullCollectiblesDataToCommunityHeader(collectibles)
require.Equal(t, 5, len(communityHeaders))
}

View File

@ -81,6 +81,10 @@ func (o *DataDB) SetCommunityInfo(id string, c *thirdparty.CommunityInfo) (err e
} }
func (o *DataDB) GetCommunityInfo(id string) (*thirdparty.CommunityInfo, *InfoState, error) { func (o *DataDB) GetCommunityInfo(id string) (*thirdparty.CommunityInfo, *InfoState, error) {
if id == "" {
return nil, nil, nil
}
var info thirdparty.CommunityInfo var info thirdparty.CommunityInfo
var state InfoState var state InfoState
var row *sql.Row var row *sql.Row

View File

@ -1,7 +1,6 @@
package community package community
import ( import (
"fmt"
"testing" "testing"
"github.com/status-im/status-go/services/wallet/thirdparty" "github.com/status-im/status-go/services/wallet/thirdparty"
@ -19,27 +18,11 @@ func setupCommunityDataDBTest(t *testing.T) (*DataDB, func()) {
} }
} }
func generateTestCommunityInfo(count int) map[string]thirdparty.CommunityInfo {
result := make(map[string]thirdparty.CommunityInfo)
for i := 0; i < count; i++ {
communityID := fmt.Sprintf("communityid-%d", i)
newCommunity := thirdparty.CommunityInfo{
CommunityName: fmt.Sprintf("communityname-%d", i),
CommunityColor: fmt.Sprintf("communitycolor-%d", i),
CommunityImage: fmt.Sprintf("communityimage-%d", i),
CommunityImagePayload: []byte(fmt.Sprintf("communityimagepayload-%d", i)),
}
result[communityID] = newCommunity
}
return result
}
func TestUpdateCommunityInfo(t *testing.T) { func TestUpdateCommunityInfo(t *testing.T) {
db, cleanup := setupCommunityDataDBTest(t) db, cleanup := setupCommunityDataDBTest(t)
defer cleanup() defer cleanup()
communityData := generateTestCommunityInfo(10) communityData := thirdparty.GenerateTestCommunityInfo(10)
extraCommunityID := "extra-community-id" extraCommunityID := "extra-community-id"
for communityID, communityInfo := range communityData { for communityID, communityInfo := range communityData {

View File

@ -2,16 +2,12 @@ package community
import ( import (
"database/sql" "database/sql"
"fmt"
"time"
"github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log"
"github.com/status-im/status-go/server" "github.com/status-im/status-go/server"
"github.com/status-im/status-go/services/wallet/thirdparty" "github.com/status-im/status-go/services/wallet/thirdparty"
) )
const failedCommunityFetchRetryDelay = 1 * time.Hour
type Manager struct { type Manager struct {
db *DataDB db *DataDB
communityInfoProvider thirdparty.CommunityInfoProvider communityInfoProvider thirdparty.CommunityInfoProvider
@ -53,36 +49,7 @@ func (cm *Manager) setCommunityInfo(id string, c *thirdparty.CommunityInfo) (err
return cm.db.SetCommunityInfo(id, c) return cm.db.SetCommunityInfo(id, c)
} }
func (cm *Manager) mustFetchCommunityInfo(communityID string) bool {
// See if we have cached data
_, state, err := cm.GetCommunityInfo(communityID)
if err != nil {
return true
}
// If we don't have a state, this community has never been fetched before
if state == nil {
return true
}
// If the last fetch was successful, we can safely refresh our cache
if state.LastUpdateSuccesful {
return true
}
// If the last fetch was not successful, we should only retry after a delay
if time.Unix(int64(state.LastUpdateTimestamp), 0).Add(failedCommunityFetchRetryDelay).Before(time.Now()) {
return true
}
return false
}
func (cm *Manager) FetchCommunityInfo(communityID string) (*thirdparty.CommunityInfo, error) { func (cm *Manager) FetchCommunityInfo(communityID string) (*thirdparty.CommunityInfo, error) {
if !cm.mustFetchCommunityInfo(communityID) {
return nil, fmt.Errorf("backing off fetchCommunityInfo for id: %s", communityID)
}
communityInfo, err := cm.communityInfoProvider.FetchCommunityInfo(communityID) communityInfo, err := cm.communityInfoProvider.FetchCommunityInfo(communityID)
if err != nil { if err != nil {
dbErr := cm.setCommunityInfo(communityID, nil) dbErr := cm.setCommunityInfo(communityID, nil)

View File

@ -194,6 +194,7 @@ func (c *Asset) toCollectiblesData(id thirdparty.CollectibleUniqueID) thirdparty
ImageURL: c.Image.ImageURL, ImageURL: c.Image.ImageURL,
AnimationURL: c.Image.CachedAnimationURL, AnimationURL: c.Image.CachedAnimationURL,
Traits: alchemyToCollectibleTraits(rawMetadata.Attributes), Traits: alchemyToCollectibleTraits(rawMetadata.Attributes),
TokenURI: c.TokenURI,
} }
} }

View File

@ -146,9 +146,11 @@ type CollectibleCommunityInfo struct {
// Combined Collection+Collectible info returned by the CollectibleProvider // Combined Collection+Collectible info returned by the CollectibleProvider
// Some providers may not return the CollectionData in the same API call, so it's optional // Some providers may not return the CollectionData in the same API call, so it's optional
type FullCollectibleData struct { type FullCollectibleData struct {
CollectibleData CollectibleData CollectibleData CollectibleData
CollectionData *CollectionData CollectionData *CollectionData
CommunityInfo *CollectibleCommunityInfo CommunityInfo *CommunityInfo
CollectibleCommunityInfo *CollectibleCommunityInfo
Ownership []AccountBalance
} }
type CollectiblesContainer[T any] struct { type CollectiblesContainer[T any] struct {

View File

@ -9,9 +9,9 @@ type CommunityInfo struct {
} }
type CommunityInfoProvider interface { type CommunityInfoProvider interface {
GetCommunityID(tokenURI string) string
FetchCommunityInfo(communityID string) (*CommunityInfo, error) FetchCommunityInfo(communityID string) (*CommunityInfo, error)
// Collectible-related methods // Collectible-related methods
GetCommunityID(tokenURI string) string
FillCollectibleMetadata(collectible *FullCollectibleData) error FillCollectibleMetadata(collectible *FullCollectibleData) error
} }

162
services/wallet/thirdparty/test_utils.go vendored Normal file
View File

@ -0,0 +1,162 @@
package thirdparty
import (
"fmt"
"math/big"
"math/rand"
"github.com/ethereum/go-ethereum/common"
"github.com/status-im/status-go/protocol/communities/token"
"github.com/status-im/status-go/services/wallet/bigint"
w_common "github.com/status-im/status-go/services/wallet/common"
)
func GenerateTestCollectiblesData(count int) (result []CollectibleData) {
base := rand.Intn(100) // nolint: gosec
result = make([]CollectibleData, 0, count)
for i := base; i < count+base; i++ {
bigI := big.NewInt(int64(i))
newCollectible := CollectibleData{
ID: CollectibleUniqueID{
ContractID: ContractID{
ChainID: w_common.ChainID(i % 4),
Address: common.BigToAddress(bigI),
},
TokenID: &bigint.BigInt{Int: bigI},
},
Provider: fmt.Sprintf("provider-%d", i),
Name: fmt.Sprintf("name-%d", i),
Description: fmt.Sprintf("description-%d", i),
Permalink: fmt.Sprintf("permalink-%d", i),
ImageURL: fmt.Sprintf("imageurl-%d", i),
ImagePayload: []byte(fmt.Sprintf("imagepayload-%d", i)),
AnimationURL: fmt.Sprintf("animationurl-%d", i),
AnimationMediaType: fmt.Sprintf("animationmediatype-%d", i),
Traits: []CollectibleTrait{
{
TraitType: fmt.Sprintf("traittype-%d", i),
Value: fmt.Sprintf("traitvalue-%d", i),
DisplayType: fmt.Sprintf("displaytype-%d", i),
MaxValue: fmt.Sprintf("maxvalue-%d", i),
},
{
TraitType: fmt.Sprintf("traittype-%d", i),
Value: fmt.Sprintf("traitvalue-%d", i),
DisplayType: fmt.Sprintf("displaytype-%d", i),
MaxValue: fmt.Sprintf("maxvalue-%d", i),
},
{
TraitType: fmt.Sprintf("traittype-%d", i),
Value: fmt.Sprintf("traitvalue-%d", i),
DisplayType: fmt.Sprintf("displaytype-%d", i),
MaxValue: fmt.Sprintf("maxvalue-%d", i),
},
},
BackgroundColor: fmt.Sprintf("backgroundcolor-%d", i),
TokenURI: fmt.Sprintf("tokenuri-%d", i),
CommunityID: fmt.Sprintf("communityid-%d", i%5),
}
result = append(result, newCollectible)
}
return result
}
func GenerateTestCollectiblesCommunityData(count int) []CollectibleCommunityInfo {
base := rand.Intn(100) // nolint: gosec
result := make([]CollectibleCommunityInfo, 0, count)
for i := base; i < count+base; i++ {
newCommunityInfo := CollectibleCommunityInfo{
PrivilegesLevel: token.PrivilegesLevel(i) % (token.CommunityLevel + 1),
}
result = append(result, newCommunityInfo)
}
return result
}
func GenerateTestCollectiblesOwnership(count int) []AccountBalance {
base := rand.Intn(100) // nolint: gosec
ret := make([]AccountBalance, 0, count)
for i := base; i < count+base; i++ {
ret = append(ret, AccountBalance{
Address: common.HexToAddress(fmt.Sprintf("0x%x", i)),
Balance: &bigint.BigInt{Int: big.NewInt(int64(i))},
})
}
return ret
}
func GenerateTestCollectionsData(count int) (result []CollectionData) {
base := rand.Intn(100) // nolint: gosec
result = make([]CollectionData, 0, count)
for i := base; i < count+base; i++ {
bigI := big.NewInt(int64(count))
traits := make(map[string]CollectionTrait)
for j := 0; j < 3; j++ {
traits[fmt.Sprintf("traittype-%d", j)] = CollectionTrait{
Min: float64(i+j) / 2,
Max: float64(i+j) * 2,
}
}
newCollection := CollectionData{
ID: ContractID{
ChainID: w_common.ChainID(i),
Address: common.BigToAddress(bigI),
},
Provider: fmt.Sprintf("provider-%d", i),
Name: fmt.Sprintf("name-%d", i),
Slug: fmt.Sprintf("slug-%d", i),
ImageURL: fmt.Sprintf("imageurl-%d", i),
ImagePayload: []byte(fmt.Sprintf("imagepayload-%d", i)),
Traits: traits,
CommunityID: fmt.Sprintf("community-%d", i),
}
result = append(result, newCollection)
}
return result
}
func GenerateTestCommunityInfo(count int) map[string]CommunityInfo {
base := rand.Intn(100) // nolint: gosec
result := make(map[string]CommunityInfo)
for i := base; i < count+base; i++ {
communityID := fmt.Sprintf("communityid-%d", i)
newCommunity := CommunityInfo{
CommunityName: fmt.Sprintf("communityname-%d", i),
CommunityColor: fmt.Sprintf("communitycolor-%d", i),
CommunityImage: fmt.Sprintf("communityimage-%d", i),
CommunityImagePayload: []byte(fmt.Sprintf("communityimagepayload-%d", i)),
}
result[communityID] = newCommunity
}
return result
}
func GenerateTestFullCollectiblesData(count int) []FullCollectibleData {
collectiblesData := GenerateTestCollectiblesData(count)
collectionsData := GenerateTestCollectionsData(count)
communityInfoMap := GenerateTestCommunityInfo(count)
communityInfo := make([]CommunityInfo, 0, count)
for _, info := range communityInfoMap {
communityInfo = append(communityInfo, info)
}
communityData := GenerateTestCollectiblesCommunityData(count)
ret := make([]FullCollectibleData, 0, count)
for i := 0; i < count; i++ {
ret = append(ret, FullCollectibleData{
CollectibleData: collectiblesData[i],
CollectionData: &collectionsData[i],
CommunityInfo: &communityInfo[i],
CollectibleCommunityInfo: &communityData[i],
Ownership: GenerateTestCollectiblesOwnership(rand.Intn(5) + 1), // nolint: gosec
})
}
return ret
}