status-im
/
status-go
mirror of https://github.com/status-im/status-go.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat: implement search api 

Issue #13921
This commit is contained in:
Dario Gabriel Lipicar 2024-03-13 11:59:18 -03:00 committed by dlipicar
parent 6e5c91f2d7
commit 58b57b12a3
7 changed files with 485 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
services/wallet/api.go
View File

@ -346,6 +346,16 @@ func (api *API) GetCollectibleOwnersByContractAddress(ctx context.Context, chain
return api.s.collectiblesManager.FetchCollectibleOwnersByContractAddress(ctx, chainID, contractAddress)
}
func (api *API) SearchCollectibles(ctx context.Context, chainID wcommon.ChainID, text string, cursor string, limit int, providerID string) (*thirdparty.FullCollectibleDataContainer, error) {
log.Debug("call to SearchCollectibles")
return api.s.collectiblesManager.SearchCollectibles(ctx, chainID, text, cursor, limit, providerID)
}
func (api *API) SearchCollections(ctx context.Context, chainID wcommon.ChainID, text string, cursor string, limit int, providerID string) (*thirdparty.CollectionDataContainer, error) {
log.Debug("call to SearchCollections")
return api.s.collectiblesManager.SearchCollections(ctx, chainID, text, cursor, limit, providerID)
}
/*
Collectibles API End
*/

122
services/wallet/collectibles/manager.go
View File

@ -51,12 +51,8 @@ type ManagerInterface interface {
}
type Manager struct {
rpcClient *rpc.Client
contractOwnershipProviders []thirdparty.CollectibleContractOwnershipProvider
accountOwnershipProviders []thirdparty.CollectibleAccountOwnershipProvider
collectibleDataProviders []thirdparty.CollectibleDataProvider
collectionDataProviders []thirdparty.CollectionDataProvider
collectibleProviders []thirdparty.CollectibleProvider
rpcClient *rpc.Client
providers thirdparty.CollectibleProviders
httpClient *http.Client
@ -77,10 +73,7 @@ func NewManager(
db *sql.DB,
rpcClient *rpc.Client,
communityManager *community.Manager,
contractOwnershipProviders []thirdparty.CollectibleContractOwnershipProvider,
accountOwnershipProviders []thirdparty.CollectibleAccountOwnershipProvider,
collectibleDataProviders []thirdparty.CollectibleDataProvider,
collectionDataProviders []thirdparty.CollectionDataProvider,
providers thirdparty.CollectibleProviders,
mediaServer *server.MediaServer,
feed *event.Feed) *Manager {
@ -106,32 +99,9 @@ func NewManager(
feed,
)
// Get list of all providers
collectibleProvidersMap := make(map[string]thirdparty.CollectibleProvider)
collectibleProviders := make([]thirdparty.CollectibleProvider, 0)
for _, provider := range contractOwnershipProviders {
collectibleProvidersMap[provider.ID()] = provider
}
for _, provider := range accountOwnershipProviders {
collectibleProvidersMap[provider.ID()] = provider
}
for _, provider := range collectibleDataProviders {
collectibleProvidersMap[provider.ID()] = provider
}
for _, provider := range collectionDataProviders {
collectibleProvidersMap[provider.ID()] = provider
}
for _, provider := range collectibleProvidersMap {
collectibleProviders = append(collectibleProviders, provider)
}
return &Manager{
rpcClient: rpcClient,
contractOwnershipProviders: contractOwnershipProviders,
accountOwnershipProviders: accountOwnershipProviders,
collectibleDataProviders: collectibleDataProviders,
collectionDataProviders: collectionDataProviders,
collectibleProviders: collectibleProviders,
rpcClient: rpcClient,
providers: providers,
httpClient: &http.Client{
Timeout: requestTimeout,
},
@ -219,7 +189,7 @@ func (o *Manager) FetchAllAssetsByOwnerAndContractAddress(ctx context.Context, c
defer o.checkConnectionStatus(chainID)
cmd := circuitbreaker.Command{}
for _, provider := range o.accountOwnershipProviders {
for _, provider := range o.providers.AccountOwnershipProviders {
if !provider.IsChainSupported(chainID) {
continue
}
@ -263,7 +233,7 @@ func (o *Manager) FetchAllAssetsByOwner(ctx context.Context, chainID walletCommo
defer o.checkConnectionStatus(chainID)
cmd := circuitbreaker.Command{}
for _, provider := range o.accountOwnershipProviders {
for _, provider := range o.providers.AccountOwnershipProviders {
if !provider.IsChainSupported(chainID) {
continue
}
@ -456,7 +426,7 @@ func (o *Manager) FetchMissingAssetsByCollectibleUniqueID(ctx context.Context, u
func (o *Manager) fetchMissingAssetsForChainByCollectibleUniqueID(ctx context.Context, chainID walletCommon.ChainID, idsToFetch []thirdparty.CollectibleUniqueID) ([]thirdparty.FullCollectibleData, error) {
cmd := circuitbreaker.Command{}
for _, provider := range o.collectibleDataProviders {
for _, provider := range o.providers.CollectibleDataProviders {
if !provider.IsChainSupported(chainID) {
continue
}
@ -499,7 +469,7 @@ func (o *Manager) FetchCollectionsDataByContractID(ctx context.Context, ids []th
defer o.checkConnectionStatus(chainID)
cmd := circuitbreaker.Command{}
for _, provider := range o.collectionDataProviders {
for _, provider := range o.providers.CollectionDataProviders {
if !provider.IsChainSupported(chainID) {
continue
}
@ -553,7 +523,7 @@ func (o *Manager) FetchCollectibleOwnersByContractAddress(ctx context.Context, c
defer o.checkConnectionStatus(chainID)
cmd := circuitbreaker.Command{}
for _, provider := range o.contractOwnershipProviders {
for _, provider := range o.providers.ContractOwnershipProviders {
if !provider.IsChainSupported(chainID) {
continue
}
@ -939,7 +909,7 @@ func (o *Manager) ResetConnectionStatus() {
}
func (o *Manager) checkConnectionStatus(chainID walletCommon.ChainID) {
for _, provider := range o.collectibleProviders {
for _, provider := range o.providers.GetProviderList() {
if provider.IsChainSupported(chainID) && provider.IsConnected() {
o.statuses[chainID.String()].SetIsConnected(true)
return
@ -996,3 +966,73 @@ func (o *Manager) getCircuitBreaker(chainID walletCommon.ChainID) *circuitbreake
}
return cb.(*circuitbreaker.CircuitBreaker)
}
func (o *Manager) SearchCollectibles(ctx context.Context, chainID walletCommon.ChainID, text string, cursor string, limit int, providerID string) (*thirdparty.FullCollectibleDataContainer, error) {
defer o.checkConnectionStatus(chainID)
anyProviderAvailable := false
for _, provider := range o.providers.SearchProviders {
if !provider.IsChainSupported(chainID) {
continue
}
anyProviderAvailable = true
if providerID != thirdparty.FetchFromAnyProvider && providerID != provider.ID() {
continue
}
// TODO (#13951): Be smarter about how we handle the user-entered string
collections := []common.Address{}
container, err := provider.SearchCollectibles(ctx, chainID, collections, text, cursor, limit)
if err != nil {
log.Error("FetchAllAssetsByOwner failed for", "provider", provider.ID(), "chainID", chainID, "err", err)
continue
}
_, err = o.processFullCollectibleData(ctx, container.Items, true)
if err != nil {
return nil, err
}
return container, nil
}
if anyProviderAvailable {
return nil, ErrAllProvidersFailedForChainID
}
return nil, ErrNoProvidersAvailableForChainID
}
func (o *Manager) SearchCollections(ctx context.Context, chainID walletCommon.ChainID, query string, cursor string, limit int, providerID string) (*thirdparty.CollectionDataContainer, error) {
defer o.checkConnectionStatus(chainID)
anyProviderAvailable := false
for _, provider := range o.providers.SearchProviders {
if !provider.IsChainSupported(chainID) {
continue
}
anyProviderAvailable = true
if providerID != thirdparty.FetchFromAnyProvider && providerID != provider.ID() {
continue
}
// TODO (#13951): Be smarter about how we handle the user-entered string
container, err := provider.SearchCollections(ctx, chainID, query, cursor, limit)
if err != nil {
log.Error("FetchAllAssetsByOwner failed for", "provider", provider.ID(), "chainID", chainID, "err", err)
continue
}
err = o.processCollectionData(ctx, container.Items)
if err != nil {
return nil, err
}
return container, nil
}
if anyProviderAvailable {
return nil, ErrAllProvidersFailedForChainID
}
return nil, ErrNoProvidersAvailableForChainID
}

23
services/wallet/service.go
View File

@ -121,7 +121,7 @@ func NewService(
raribleClient := rarible.NewClient(config.WalletConfig.RaribleMainnetAPIKey, config.WalletConfig.RaribleTestnetAPIKey)
alchemyClient := alchemy.NewClient(config.WalletConfig.AlchemyAPIKeys)
// Try OpenSea, Infura, Alchemy in that order
// Collectible providers in priority order (i.e. provider N+1 will be tried only if provider N fails)
contractOwnershipProviders := []thirdparty.CollectibleContractOwnershipProvider{
raribleClient,
alchemyClient,
@ -145,7 +145,26 @@ func NewService(
openseaV2Client,
}
collectiblesManager := collectibles.NewManager(db, rpcClient, communityManager, contractOwnershipProviders, accountOwnershipProviders, collectibleDataProviders, collectionDataProviders, mediaServer, feed)
collectibleSearchProviders := []thirdparty.CollectibleSearchProvider{
raribleClient,
}
collectibleProviders := thirdparty.CollectibleProviders{
ContractOwnershipProviders: contractOwnershipProviders,
AccountOwnershipProviders: accountOwnershipProviders,
CollectibleDataProviders: collectibleDataProviders,
CollectionDataProviders: collectionDataProviders,
SearchProviders: collectibleSearchProviders,
}
collectiblesManager := collectibles.NewManager(
db,
rpcClient,
communityManager,
collectibleProviders,
mediaServer,
feed,
)
collectibles := collectibles.NewService(db, feed, accountsDB, accountFeed, settingsFeed, communityManager, rpcClient.NetworkManager, collectiblesManager)
activity := activity.NewService(db, tokenManager, collectiblesManager, feed, pendingTxManager)

41
services/wallet/thirdparty/collectible_types.go vendored
View File

@ -278,3 +278,44 @@ type CollectionDataProvider interface {
CollectibleProvider
FetchCollectionsDataByContractID(ctx context.Context, ids []ContractID) ([]CollectionData, error)
}
type CollectibleSearchProvider interface {
CollectibleProvider
SearchCollections(ctx context.Context, chainID w_common.ChainID, text string, cursor string, limit int) (*CollectionDataContainer, error)
SearchCollectibles(ctx context.Context, chainID w_common.ChainID, collections []common.Address, text string, cursor string, limit int) (*FullCollectibleDataContainer, error)
}
type CollectibleProviders struct {
ContractOwnershipProviders []CollectibleContractOwnershipProvider
AccountOwnershipProviders []CollectibleAccountOwnershipProvider
CollectibleDataProviders []CollectibleDataProvider
CollectionDataProviders []CollectionDataProvider
SearchProviders []CollectibleSearchProvider
}
func (p *CollectibleProviders) GetProviderList() []CollectibleProvider {
ret := make([]CollectibleProvider, 0)
uniqueProviders := make(map[string]CollectibleProvider)
for _, provider := range p.ContractOwnershipProviders {
uniqueProviders[provider.ID()] = provider
}
for _, provider := range p.AccountOwnershipProviders {
uniqueProviders[provider.ID()] = provider
}
for _, provider := range p.CollectibleDataProviders {
uniqueProviders[provider.ID()] = provider
}
for _, provider := range p.CollectionDataProviders {
uniqueProviders[provider.ID()] = provider
}
for _, provider := range p.SearchProviders {
uniqueProviders[provider.ID()] = provider
}
for _, provider := range uniqueProviders {
ret = append(ret, provider)
}
return ret
}

183
services/wallet/thirdparty/rarible/client.go vendored
View File

@ -24,6 +24,8 @@ import (
const ownedNFTLimit = 100
const collectionOwnershipLimit = 50
const nftMetadataBatchLimit = 50
const searchCollectiblesLimit = 1000
const searchCollectionsLimit = 1000
func (o *Client) ID() string {
return RaribleID
@ -432,3 +434,184 @@ func (o *Client) FetchCollectionsDataByContractID(ctx context.Context, contractI
return ret, nil
}
func (o *Client) searchCollectibles(ctx context.Context, chainID walletCommon.ChainID, collections []common.Address, fullText CollectibleFilterFullText, sort CollectibleFilterContainerSort, cursor string, limit int) (*thirdparty.FullCollectibleDataContainer, error) {
baseURL, err := getItemBaseURL(chainID)
if err != nil {
return nil, err
}
url := fmt.Sprintf("%s/search", baseURL)
ret := &thirdparty.FullCollectibleDataContainer{
Provider: o.ID(),
Items: make([]thirdparty.FullCollectibleData, 0),
PreviousCursor: cursor,
NextCursor: "",
}
if fullText.Text == "" {
return ret, nil
}
tmpLimit := searchCollectiblesLimit
if limit > thirdparty.FetchNoLimit && limit < tmpLimit {
tmpLimit = limit
}
blockchainString := chainIDToChainString(chainID)
filterContainer := CollectibleFilterContainer{
Cursor: cursor,
Limit: tmpLimit,
Filter: CollectibleFilter{
Blockchains: []string{blockchainString},
Deleted: false,
FullText: fullText,
},
Sort: sort,
}
for _, collection := range collections {
filterContainer.Filter.Collections = append(filterContainer.Filter.Collections, fmt.Sprintf("%s:%s", blockchainString, collection.String()))
}
for {
resp, err := o.doPostWithJSON(ctx, url, filterContainer, o.getAPIKey(chainID))
if err != nil {
if ctx.Err() == nil {
o.connectionStatus.SetIsConnected(false)
}
return nil, err
}
o.connectionStatus.SetIsConnected(true)
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
// if Json is not returned there must be an error
if !json.Valid(body) {
return nil, fmt.Errorf("invalid json: %s", string(body))
}
var collectibles CollectiblesContainer
err = json.Unmarshal(body, &collectibles)
if err != nil {
return nil, err
}
ret.Items = append(ret.Items, raribleToCollectiblesData(collectibles.Collectibles, chainID.IsMainnet())...)
ret.NextCursor = collectibles.Continuation
if len(ret.NextCursor) == 0 {
break
}
filterContainer.Cursor = ret.NextCursor
if limit != thirdparty.FetchNoLimit && len(ret.Items) >= limit {
break
}
}
return ret, nil
}
func (o *Client) searchCollections(ctx context.Context, chainID walletCommon.ChainID, text string, cursor string, limit int) (*thirdparty.CollectionDataContainer, error) {
baseURL, err := getCollectionBaseURL(chainID)
if err != nil {
return nil, err
}
url := fmt.Sprintf("%s/search", baseURL)
ret := &thirdparty.CollectionDataContainer{
Provider: o.ID(),
Items: make([]thirdparty.CollectionData, 0),
PreviousCursor: cursor,
NextCursor: "",
}
if text == "" {
return ret, nil
}
tmpLimit := searchCollectionsLimit
if limit > thirdparty.FetchNoLimit && limit < tmpLimit {
tmpLimit = limit
}
filterContainer := CollectionFilterContainer{
Cursor: cursor,
Limit: tmpLimit,
Filter: CollectionFilter{
Blockchains: []string{chainIDToChainString(chainID)},
Text: text,
},
}
for {
resp, err := o.doPostWithJSON(ctx, url, filterContainer, o.getAPIKey(chainID))
if err != nil {
if ctx.Err() == nil {
o.connectionStatus.SetIsConnected(false)
}
return nil, err
}
o.connectionStatus.SetIsConnected(true)
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
// if Json is not returned there must be an error
if !json.Valid(body) {
return nil, fmt.Errorf("invalid json: %s", string(body))
}
var collections CollectionsContainer
err = json.Unmarshal(body, &collections)
if err != nil {
return nil, err
}
ret.Items = append(ret.Items, raribleToCollectionsData(collections.Collections, chainID.IsMainnet())...)
ret.NextCursor = collections.Continuation
if len(ret.NextCursor) == 0 {
break
}
filterContainer.Cursor = ret.NextCursor
if limit != thirdparty.FetchNoLimit && len(ret.Items) >= limit {
break
}
}
return ret, nil
}
func (o *Client) SearchCollections(ctx context.Context, chainID walletCommon.ChainID, text string, cursor string, limit int) (*thirdparty.CollectionDataContainer, error) {
return o.searchCollections(ctx, chainID, text, cursor, limit)
}
func (o *Client) SearchCollectibles(ctx context.Context, chainID walletCommon.ChainID, collections []common.Address, text string, cursor string, limit int) (*thirdparty.FullCollectibleDataContainer, error) {
fullText := CollectibleFilterFullText{
Text: text,
Fields: []string{
CollectibleFilterFullTextFieldName,
},
}
sort := CollectibleFilterContainerSortRelevance
return o.searchCollectibles(ctx, chainID, collections, fullText, sort, cursor, limit)
}

86
services/wallet/thirdparty/rarible/client_int_test.go vendored Normal file
View File

@ -0,0 +1,86 @@
package rarible
import (
"context"
"os"
"testing"
"time"
"github.com/ethereum/go-ethereum/common"
walletCommon "github.com/status-im/status-go/services/wallet/common"
"github.com/status-im/status-go/services/wallet/thirdparty"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
)
func TestRaribleClientIntegrationSuite(t *testing.T) {
suite.Run(t, new(RaribleClientIntegrationSuite))
}
type RaribleClientIntegrationSuite struct {
suite.Suite
client *Client
}
func (s *RaribleClientIntegrationSuite) SetupTest() {
mainnetKey := os.Getenv("STATUS_BUILD_RARIBLE_MAINNET_API_KEY")
if mainnetKey == "" {
mainnetKey = os.Getenv("STATUS_RUNTIME_RARIBLE_MAINNET_API_KEY")
}
testnetKey := os.Getenv("STATUS_BUILD_RARIBLE_TESTNET_API_KEY")
if testnetKey == "" {
testnetKey = os.Getenv("STATUS_RUNTIME_RARIBLE_TESTNET_API_KEY")
}
s.client = NewClient(mainnetKey, testnetKey)
}
func (s *RaribleClientIntegrationSuite) TestAPIKeysAvailable() {
// TODO #13953: Enable for nightly runs
s.T().Skip("integration test")
assert.NotEmpty(s.T(), s.client.mainnetAPIKey)
assert.NotEmpty(s.T(), s.client.testnetAPIKey)
}
func (s *RaribleClientIntegrationSuite) TestSearchCollections() {
// TODO #13953: Enable for nightly runs
s.T().Skip("integration test")
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
collections, err := s.client.SearchCollections(
ctx,
walletCommon.ChainID(walletCommon.EthereumMainnet),
"CryptoKitties",
thirdparty.FetchFromStartCursor,
10,
)
s.Require().NoError(err)
s.Require().NotNil(collections)
s.Require().NotEmpty(collections.Items)
}
func (s *RaribleClientIntegrationSuite) TestSearchCollectibles() {
// TODO #13953: Enable for nightly runs
s.T().Skip("integration test")
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
collectibles, err := s.client.SearchCollectibles(
ctx,
walletCommon.ChainID(walletCommon.EthereumMainnet),
[]common.Address{common.HexToAddress("0x06012c8cf97BEaD5deAe237070F9587f8E7A266d")},
"Furbeard",
thirdparty.FetchFromStartCursor,
10,
)
s.Require().NoError(err)
s.Require().NotNil(collectibles)
s.Require().NotEmpty(collectibles.Items)
}

63
services/wallet/thirdparty/rarible/types.go vendored
View File

@ -110,6 +110,51 @@ type BatchTokenIDs struct {
IDs []string `json:"ids"`
}
type CollectibleFilterFullTextField = string
const (
CollectibleFilterFullTextFieldName = "NAME"
CollectibleFilterFullTextFieldDescription = "DESCRIPTION"
)
type CollectibleFilterFullText struct {
Text string `json:"text"`
Fields []CollectibleFilterFullTextField `json:"fields"`
}
type CollectibleFilter struct {
Blockchains []string `json:"blockchains"`
Collections []string `json:"collections,omitempty"`
Deleted bool `json:"deleted"`
FullText CollectibleFilterFullText `json:"fullText"`
}
type CollectibleFilterContainerSort = string
const (
CollectibleFilterContainerSortRelevance = "RELEVANCE"
CollectibleFilterContainerSortLatest = "LATEST"
CollectibleFilterContainerSortEarliest = "EARLIEST"
)
type CollectibleFilterContainer struct {
Limit int `json:"size"`
Cursor string `json:"continuation"`
Filter CollectibleFilter `json:"filter"`
Sort CollectibleFilterContainerSort `json:"sort"`
}
type CollectionFilter struct {
Blockchains []string `json:"blockchains"`
Text string `json:"text"`
}
type CollectionFilterContainer struct {
Limit int `json:"size"`
Cursor string `json:"continuation"`
Filter CollectionFilter `json:"filter"`
}
type CollectiblesContainer struct {
Continuation string `json:"continuation"`
Collectibles []Collectible `json:"items"`
@ -157,6 +202,11 @@ func (st *AttributeValue) UnmarshalJSON(b []byte) error {
return nil
}
type CollectionsContainer struct {
Continuation string `json:"continuation"`
Collections []Collection `json:"collections"`
}
type Collection struct {
ID string `json:"id"`
Blockchain string `json:"blockchain"`
@ -247,6 +297,19 @@ func raribleToCollectiblesData(l []Collectible, isMainnet bool) []thirdparty.Ful
return ret
}
func raribleToCollectionsData(l []Collection, isMainnet bool) []thirdparty.CollectionData {
ret := make([]thirdparty.CollectionData, 0, len(l))
for _, c := range l {
id, err := raribleContractIDToUniqueID(c.ID, isMainnet)
if err != nil {
continue
}
item := c.toCommon(id)
ret = append(ret, item)
}
return ret
}
func (c *Collection) toCommon(id thirdparty.ContractID) thirdparty.CollectionData {
ret := thirdparty.CollectionData{
ID: id,