From ecc8b4cb5513ed29deb4fdc85eaeed5e96d1e562 Mon Sep 17 00:00:00 2001 From: Cuteivist Date: Tue, 3 Oct 2023 12:49:04 +0200 Subject: [PATCH] feat: Wallet activity collectibles model (#4074) --- services/wallet/activity/filter.go | 28 +++++++ services/wallet/activity/filter_test.go | 79 ++++++++++++++++++- services/wallet/activity/get_collectibles.sql | 33 ++++++++ services/wallet/activity/service.go | 63 +++++++++++++++ services/wallet/api.go | 8 ++ services/wallet/collectibles/ownership_db.go | 25 +----- .../wallet/thirdparty/collectible_types.go | 23 ++++++ 7 files changed, 233 insertions(+), 26 deletions(-) create mode 100644 services/wallet/activity/get_collectibles.sql diff --git a/services/wallet/activity/filter.go b/services/wallet/activity/filter.go index 400bdf701..3ef992036 100644 --- a/services/wallet/activity/filter.go +++ b/services/wallet/activity/filter.go @@ -11,11 +11,15 @@ import ( eth "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/status-im/status-go/services/wallet/common" + "github.com/status-im/status-go/services/wallet/thirdparty" "github.com/status-im/status-go/transactions" ) const NoLimitTimestampForPeriod = 0 +//go:embed get_collectibles.sql +var getCollectiblesQueryFormatString string + //go:embed oldest_timestamp.sql var oldestTimestampQueryFormatString string @@ -158,3 +162,27 @@ func GetOldestTimestamp(ctx context.Context, db *sql.DB, addresses []eth.Address return timestamp, nil } + +func GetActivityCollectibles(ctx context.Context, db *sql.DB, chainIDs []common.ChainID, owners []eth.Address, offset int, limit int) ([]thirdparty.CollectibleUniqueID, error) { + filterAllAddresses := len(owners) == 0 + involvedAddresses := noEntriesInTmpTableSQLValues + if !filterAllAddresses { + involvedAddresses = joinAddresses(owners) + } + + includeAllNetworks := len(chainIDs) == 0 + networks := noEntriesInTmpTableSQLValues + if !includeAllNetworks { + networks = joinItems(chainIDs, nil) + } + + queryString := fmt.Sprintf(getCollectiblesQueryFormatString, involvedAddresses, networks) + + rows, err := db.QueryContext(ctx, queryString, filterAllAddresses, includeAllNetworks, limit, offset) + if err != nil { + return nil, err + } + defer rows.Close() + + return thirdparty.RowsToCollectibles(rows) +} diff --git a/services/wallet/activity/filter_test.go b/services/wallet/activity/filter_test.go index e9f765b89..ceb39b9a7 100644 --- a/services/wallet/activity/filter_test.go +++ b/services/wallet/activity/filter_test.go @@ -8,8 +8,10 @@ import ( eth "github.com/ethereum/go-ethereum/common" + "github.com/status-im/status-go/services/wallet/bigint" "github.com/status-im/status-go/services/wallet/common" "github.com/status-im/status-go/services/wallet/testutils" + "github.com/status-im/status-go/services/wallet/thirdparty" "github.com/status-im/status-go/services/wallet/transfer" "github.com/status-im/status-go/t/helpers" "github.com/status-im/status-go/walletdatabase" @@ -29,7 +31,7 @@ func setupTestFilterDB(t *testing.T) (db *sql.DB, close func()) { // insertTestData inserts 6 extractable activity entries: 2 transfers, 2 pending transactions and 2 multi transactions func insertTestData(t *testing.T, db *sql.DB, nullifyToForIndexes []int) (trs []transfer.TestTransfer, toTrs []eth.Address, multiTxs []transfer.TestMultiTransaction) { // Add 6 extractable transactions - trs, _, toTrs = transfer.GenerateTestTransfers(t, db, 0, 7) + trs, _, toTrs = transfer.GenerateTestTransfers(t, db, 0, 10) multiTxs = []transfer.TestMultiTransaction{ transfer.GenerateTestBridgeMultiTransaction(trs[0], trs[1]), transfer.GenerateTestSwapMultiTransaction(trs[2], testutils.SntSymbol, 100), @@ -58,6 +60,13 @@ func insertTestData(t *testing.T, db *sql.DB, nullifyToForIndexes []int) (trs [] transfer.InsertTestTransferWithOptions(t, db, trs[i].To, &trs[i], &transfer.TestTransferOptions{ NullifyAddresses: nullifyAddresses, }) + } else if i >= 7 && i < 10 { + ci := i - 7 + trs[i].ChainID = transfer.TestCollectibles[ci].ChainID + transfer.InsertTestTransferWithOptions(t, db, trs[i].To, &trs[i], &transfer.TestTransferOptions{ + TokenID: transfer.TestCollectibles[ci].TokenID, + TokenAddress: transfer.TestCollectibles[ci].TokenAddress, + }) } else { for j := range nullifyToForIndexes { if i == nullifyToForIndexes[j] { @@ -97,7 +106,7 @@ func TestGetRecipients(t *testing.T) { entries, hasMore, err := GetRecipients(context.Background(), db, []common.ChainID{}, []eth.Address{}, 0, 15) require.NoError(t, err) require.False(t, hasMore) - require.Equal(t, 6, len(entries)) + require.Equal(t, 9, len(entries)) for i := range entries { found := false for j := range toTrs { @@ -141,7 +150,7 @@ func TestGetRecipients_NullAddresses(t *testing.T) { entries, hasMore, err := GetRecipients(context.Background(), db, []common.ChainID{}, []eth.Address{}, 0, 15) require.NoError(t, err) require.False(t, hasMore) - require.Equal(t, 3, len(entries)) + require.Equal(t, 6, len(entries)) } func TestGetOldestTimestampEmptyDB(t *testing.T) { @@ -226,3 +235,67 @@ func TestGetOldestTimestamp_NullAddresses(t *testing.T) { require.NoError(t, err) require.Equal(t, int64(0), timestamp) } + +func TestGetActivityCollectiblesEmptyDB(t *testing.T) { + db, close := setupTestFilterDB(t) + defer close() + + collectibles, err := GetActivityCollectibles(context.Background(), db, []common.ChainID{}, []eth.Address{}, 0, 10) + require.NoError(t, err) + require.Equal(t, 0, len(collectibles)) +} + +func TestGetActivityCollectibles(t *testing.T) { + db, close := setupTestFilterDB(t) + defer close() + + trs, _, _ := insertTestData(t, db, nil) + + // Extract all collectibles + collectibles, err := GetActivityCollectibles(context.Background(), db, []common.ChainID{}, []eth.Address{}, 0, 10) + require.NoError(t, err) + require.Equal(t, 3, len(collectibles)) + + // Extract collectibles for a specific chain + collectibles, err = GetActivityCollectibles(context.Background(), db, []common.ChainID{1}, []eth.Address{}, 0, 10) + require.NoError(t, err) + require.Equal(t, 2, len(collectibles)) + require.Equal(t, thirdparty.CollectibleUniqueID{ + TokenID: &bigint.BigInt{Int: transfer.TestCollectibles[0].TokenID}, + ContractID: thirdparty.ContractID{ + ChainID: transfer.TestCollectibles[1].ChainID, + Address: transfer.TestCollectibles[1].TokenAddress, + }, + }, collectibles[0]) + require.Equal(t, thirdparty.CollectibleUniqueID{ + TokenID: &bigint.BigInt{Int: transfer.TestCollectibles[1].TokenID}, + ContractID: thirdparty.ContractID{ + ChainID: transfer.TestCollectibles[0].ChainID, + Address: transfer.TestCollectibles[0].TokenAddress, + }, + }, collectibles[1]) + + // Extract collectibles for a specific sender addresses + collectibles, err = GetActivityCollectibles(context.Background(), db, []common.ChainID{}, []eth.Address{trs[8].From}, 0, 10) + require.NoError(t, err) + require.Equal(t, 1, len(collectibles)) + require.Equal(t, thirdparty.CollectibleUniqueID{ + TokenID: &bigint.BigInt{Int: transfer.TestCollectibles[1].TokenID}, + ContractID: thirdparty.ContractID{ + ChainID: transfer.TestCollectibles[1].ChainID, + Address: transfer.TestCollectibles[1].TokenAddress, + }, + }, collectibles[0]) + + // Extract collectibles for a specific recipient addresses + collectibles, err = GetActivityCollectibles(context.Background(), db, []common.ChainID{}, []eth.Address{trs[7].To}, 0, 10) + require.NoError(t, err) + require.Equal(t, 1, len(collectibles)) + require.Equal(t, thirdparty.CollectibleUniqueID{ + TokenID: &bigint.BigInt{Int: transfer.TestCollectibles[0].TokenID}, + ContractID: thirdparty.ContractID{ + ChainID: transfer.TestCollectibles[0].ChainID, + Address: transfer.TestCollectibles[0].TokenAddress, + }, + }, collectibles[0]) +} diff --git a/services/wallet/activity/get_collectibles.sql b/services/wallet/activity/get_collectibles.sql new file mode 100644 index 000000000..873edb2a5 --- /dev/null +++ b/services/wallet/activity/get_collectibles.sql @@ -0,0 +1,33 @@ +-- Query for getting collectibles from transfers +-- It can be filtered by owner addresses and networks +WITH filter_conditions AS ( + SELECT + ? AS filterAllAddresses, + ? AS includeAllNetworks + ), + owner_addresses(address) AS ( + VALUES + %s + ), + filter_networks(network_id) AS ( + VALUES + %s + ) +SELECT network_id, token_address, token_id +FROM + transfers, filter_conditions +WHERE +token_id IS NOT NULL +AND token_address IS NOT NULL +AND ( + filterAllAddresses + OR tx_from_address IN owner_addresses + OR tx_to_address IN owner_addresses +) +AND ( + includeAllNetworks + OR network_id IN filter_networks +) +GROUP BY + token_id, token_address +LIMIT ? OFFSET ? \ No newline at end of file diff --git a/services/wallet/activity/service.go b/services/wallet/activity/service.go index df8db05eb..cd265ee3c 100644 --- a/services/wallet/activity/service.go +++ b/services/wallet/activity/service.go @@ -26,6 +26,7 @@ const ( EventActivityFilteringUpdate walletevent.EventType = "wallet-activity-filtering-entries-updated" EventActivityGetRecipientsDone walletevent.EventType = "wallet-activity-get-recipients-result" EventActivityGetOldestTimestampDone walletevent.EventType = "wallet-activity-get-oldest-timestamp-result" + EventActivityGetCollectibles walletevent.EventType = "wallet-activity-get-collectibles" ) var ( @@ -41,6 +42,10 @@ var ( ID: 3, Policy: async.ReplacementPolicyCancelOld, } + getCollectiblesTask = async.TaskType{ + ID: 4, + Policy: async.ReplacementPolicyCancelOld, + } ) // Service provides an async interface, ensuring only one filter request, of each type, is running at a time. It also provides lazy load of NFT info and token mapping @@ -113,6 +118,64 @@ func (s *Service) FilterActivityAsync(requestID int32, addresses []common.Addres }) } +type CollectibleHeader struct { + ID thirdparty.CollectibleUniqueID `json:"id"` + Name string `json:"name"` + ImageURL string `json:"image_url"` +} + +type GetollectiblesResponse struct { + Collectibles []CollectibleHeader `json:"collectibles"` + Offset int `json:"offset"` + // Used to indicate that there might be more collectibles that were not returned + // based on a simple heuristic + HasMore bool `json:"hasMore"` + ErrorCode ErrorCode `json:"errorCode"` +} + +func (s *Service) GetActivityCollectiblesAsync(requestID int32, chainIDs []w_common.ChainID, addresses []common.Address, offset int, limit int) { + s.scheduler.Enqueue(requestID, getCollectiblesTask, func(ctx context.Context) (interface{}, error) { + collectibles, err := GetActivityCollectibles(ctx, s.db, chainIDs, addresses, offset, limit) + + if err != nil { + return nil, err + } + + data, err := s.collectibles.FetchAssetsByCollectibleUniqueID(collectibles) + if err != nil { + return nil, err + } + + res := make([]CollectibleHeader, 0, len(data)) + + for _, c := range data { + res = append(res, CollectibleHeader{ + ID: c.CollectibleData.ID, + Name: c.CollectibleData.Name, + ImageURL: c.CollectibleData.ImageURL, + }) + } + + return res, err + }, func(result interface{}, taskType async.TaskType, err error) { + res := GetollectiblesResponse{ + ErrorCode: ErrorCodeFailed, + } + + if errors.Is(err, context.Canceled) || errors.Is(err, async.ErrTaskOverwritten) { + res.ErrorCode = ErrorCodeTaskCanceled + } else if err == nil { + collectibles := result.([]CollectibleHeader) + res.Collectibles = collectibles + res.Offset = offset + res.HasMore = len(collectibles) == limit + res.ErrorCode = ErrorCodeSuccess + } + + s.sendResponseEvent(&requestID, EventActivityGetCollectibles, res, err) + }) +} + func (s *Service) GetMultiTxDetails(ctx context.Context, multiTxID int) (*EntryDetails, error) { return getMultiTxDetails(ctx, s.db, multiTxID) } diff --git a/services/wallet/api.go b/services/wallet/api.go index 2a8e9a35a..441c1e217 100644 --- a/services/wallet/api.go +++ b/services/wallet/api.go @@ -617,6 +617,14 @@ func (api *API) GetOldestActivityTimestampAsync(requestID int32, addresses []com return nil } +func (api *API) GetActivityCollectiblesAsync(requestID int32, chainIDs []wcommon.ChainID, addresses []common.Address, offset int, limit int) error { + log.Debug("wallet.api.GetActivityCollectiblesAsync", "addresses.len", len(addresses), "chainIDs.len", len(chainIDs), "offset", offset, "limit", limit) + + api.s.activity.GetActivityCollectiblesAsync(requestID, chainIDs, addresses, offset, limit) + + return nil +} + func (api *API) FetchChainIDForURL(ctx context.Context, rpcURL string) (*big.Int, error) { log.Debug("wallet.api.VerifyURL", rpcURL) diff --git a/services/wallet/collectibles/ownership_db.go b/services/wallet/collectibles/ownership_db.go index cd02eb752..7fe94b03f 100644 --- a/services/wallet/collectibles/ownership_db.go +++ b/services/wallet/collectibles/ownership_db.go @@ -111,27 +111,6 @@ func (o *OwnershipDB) Update(chainID w_common.ChainID, ownerAddress common.Addre return } -func rowsToCollectibles(rows *sql.Rows) ([]thirdparty.CollectibleUniqueID, error) { - var ids []thirdparty.CollectibleUniqueID - for rows.Next() { - id := thirdparty.CollectibleUniqueID{ - TokenID: &bigint.BigInt{Int: big.NewInt(0)}, - } - err := rows.Scan( - &id.ContractID.ChainID, - &id.ContractID.Address, - (*bigint.SQLBigIntBytes)(id.TokenID.Int), - ) - if err != nil { - return nil, err - } - - ids = append(ids, id) - } - - return ids, nil -} - func (o *OwnershipDB) GetOwnedCollectibles(chainIDs []w_common.ChainID, ownerAddresses []common.Address, offset int, limit int) ([]thirdparty.CollectibleUniqueID, error) { query, args, err := sqlx.In(fmt.Sprintf(`SELECT %s FROM collectibles_ownership_cache @@ -153,7 +132,7 @@ func (o *OwnershipDB) GetOwnedCollectibles(chainIDs []w_common.ChainID, ownerAdd } defer rows.Close() - return rowsToCollectibles(rows) + return thirdparty.RowsToCollectibles(rows) } func (o *OwnershipDB) GetOwnedCollectible(chainID w_common.ChainID, ownerAddresses common.Address, contractAddress common.Address, tokenID *big.Int) (*thirdparty.CollectibleUniqueID, error) { @@ -173,7 +152,7 @@ func (o *OwnershipDB) GetOwnedCollectible(chainID w_common.ChainID, ownerAddress } defer rows.Close() - ids, err := rowsToCollectibles(rows) + ids, err := thirdparty.RowsToCollectibles(rows) if err != nil { return nil, err } diff --git a/services/wallet/thirdparty/collectible_types.go b/services/wallet/thirdparty/collectible_types.go index de0259f5a..0e438db72 100644 --- a/services/wallet/thirdparty/collectible_types.go +++ b/services/wallet/thirdparty/collectible_types.go @@ -1,8 +1,10 @@ package thirdparty import ( + "database/sql" "errors" "fmt" + "math/big" "github.com/ethereum/go-ethereum/common" "github.com/status-im/status-go/protocol/communities/token" @@ -45,6 +47,27 @@ func (k *CollectibleUniqueID) Same(other *CollectibleUniqueID) bool { return k.ContractID.ChainID == other.ContractID.ChainID && k.ContractID.Address == other.ContractID.Address && k.TokenID.Cmp(other.TokenID.Int) == 0 } +func RowsToCollectibles(rows *sql.Rows) ([]CollectibleUniqueID, error) { + var ids []CollectibleUniqueID + for rows.Next() { + id := CollectibleUniqueID{ + TokenID: &bigint.BigInt{Int: big.NewInt(0)}, + } + err := rows.Scan( + &id.ContractID.ChainID, + &id.ContractID.Address, + (*bigint.SQLBigIntBytes)(id.TokenID.Int), + ) + if err != nil { + return nil, err + } + + ids = append(ids, id) + } + + return ids, nil +} + func GroupCollectibleUIDsByChainID(uids []CollectibleUniqueID) map[w_common.ChainID][]CollectibleUniqueID { ret := make(map[w_common.ChainID][]CollectibleUniqueID)