From d07f9b5b162c078f71bf19a189e2e26c118a0db4 Mon Sep 17 00:00:00 2001 From: Ivan Belyakov Date: Tue, 16 Jul 2024 16:02:52 +0200 Subject: [PATCH] fix(wallet)_: no balance chart for tokens, due to wrong symbol (ETH) used instead. Fixed padding points being removed from final result, regression. Edge points not added per address as it does not make sense. Fixed padding points number with respect to edge points number. Padding points now duplicate previous entry. Fixed timestamp boundaries to ignore addresses, as we want the whole range for all passed addresses. Fixed missing indices in balance_history table and clean up of duplicate rows. Removed ERC1155 from balance history sql query --- services/wallet/history/balance.go | 112 +++++----- services/wallet/history/balance_db.go | 2 +- services/wallet/history/balance_test.go | 41 ++++ services/wallet/history/service.go | 27 +-- services/wallet/history/service_test.go | 58 ++++-- services/wallet/token/token_test.go | 4 +- walletdatabase/migrations/bindata.go | 194 +++++++++++------- ...indices_balance_history_remove_dups.up.sql | 19 ++ 8 files changed, 292 insertions(+), 165 deletions(-) create mode 100644 walletdatabase/migrations/sql/1721136888_recreate_indices_balance_history_remove_dups.up.sql diff --git a/services/wallet/history/balance.go b/services/wallet/history/balance.go index e573d0a5e..0d0c010ae 100644 --- a/services/wallet/history/balance.go +++ b/services/wallet/history/balance.go @@ -72,82 +72,73 @@ func (b *Balance) get(ctx context.Context, chainID uint64, currency string, addr func (b *Balance) addEdgePoints(chainID uint64, currency string, addresses []common.Address, fromTimestamp, toTimestamp uint64, data []*entry) (res []*entry, err error) { log.Debug("Adding edge points", "chainID", chainID, "currency", currency, "address", addresses, "fromTimestamp", fromTimestamp) + if len(addresses) == 0 { + return nil, errors.New("addresses must not be empty") + } + res = data - for _, address := range addresses { - var firstEntry *entry + var firstEntry *entry - if len(data) > 0 { - for _, entry := range data { - if entry.address == address { - firstEntry = entry - break - } - } + if len(data) > 0 { + firstEntry = data[0] + } else { + firstEntry = &entry{ + chainID: chainID, + address: addresses[0], + tokenSymbol: currency, + timestamp: int64(fromTimestamp), } - if firstEntry == nil { - firstEntry = &entry{ + } + + previous, err := b.db.getEntryPreviousTo(firstEntry) + if err != nil { + return nil, err + } + + firstTimestamp, lastTimestamp := timestampBoundaries(fromTimestamp, toTimestamp, data) + + if previous != nil { + previous.timestamp = int64(firstTimestamp) // We might need to use another minimal offset respecting the time interval + previous.block = nil + res = append([]*entry{previous}, res...) + } else { + // Add a zero point at the beginning to draw a line from + res = append([]*entry{ + { chainID: chainID, - address: address, + address: addresses[0], tokenSymbol: currency, - timestamp: int64(fromTimestamp), - } - } + timestamp: int64(firstTimestamp), + balance: big.NewInt(0), + }, + }, res...) + } - previous, err := b.db.getEntryPreviousTo(firstEntry) - if err != nil { - return nil, err - } - - firstTimestamp, lastTimestamp := timestampBoundaries(fromTimestamp, toTimestamp, address, data) - - if previous != nil { - previous.timestamp = int64(firstTimestamp) // We might need to use another minimal offset respecting the time interval - previous.block = nil - res = append([]*entry{previous}, res...) - } else { - // Add a zero point at the beginning to draw a line from - res = append([]*entry{ - { - chainID: chainID, - address: address, - tokenSymbol: currency, - timestamp: int64(firstTimestamp), - balance: big.NewInt(0), - }, - }, res...) - } - - if res[len(res)-1].timestamp < int64(lastTimestamp) { - // Add a last point to draw a line to - res = append(res, &entry{ - chainID: chainID, - address: address, - tokenSymbol: currency, - timestamp: int64(lastTimestamp), - balance: res[len(res)-1].balance, - }) - } + lastPoint := res[len(res)-1] + if lastPoint.timestamp < int64(lastTimestamp) { + // Add a last point to draw a line to + res = append(res, &entry{ + chainID: chainID, + address: lastPoint.address, + tokenSymbol: currency, + timestamp: int64(lastTimestamp), + balance: lastPoint.balance, + }) } return res, nil } -func timestampBoundaries(fromTimestamp, toTimestamp uint64, address common.Address, data []*entry) (firstTimestamp, lastTimestamp uint64) { +func timestampBoundaries(fromTimestamp, toTimestamp uint64, data []*entry) (firstTimestamp, lastTimestamp uint64) { firstTimestamp = fromTimestamp if fromTimestamp == 0 { if len(data) > 0 { - for _, entry := range data { - if entry.address == address { - if entry.timestamp == 0 { - panic("data[0].timestamp must never be 0") - } - firstTimestamp = uint64(entry.timestamp) - 1 - break - } + if data[0].timestamp == 0 { + panic("data[0].timestamp must never be 0") } - } - if firstTimestamp == fromTimestamp { + firstTimestamp = uint64(data[0].timestamp) - 1 + } else { firstTimestamp = genesisTimestamp } } @@ -197,6 +188,7 @@ func addPaddingPoints(currency string, addresses []common.Address, toTimestamp u tokenSymbol: currency, timestamp: paddingTimestamp, balance: data[j-1].balance, // take the previous balance + chainID: data[j-1].chainID, } res[index] = entry diff --git a/services/wallet/history/balance_db.go b/services/wallet/history/balance_db.go index 504ae83c0..5be594084 100644 --- a/services/wallet/history/balance_db.go +++ b/services/wallet/history/balance_db.go @@ -64,7 +64,7 @@ func (b *BalanceDB) add(entry *entry) error { } func (b *BalanceDB) getEntriesWithoutBalances(chainID uint64, address common.Address) (entries []*entry, err error) { - rows, err := b.db.Query("SELECT blk_number, tr.timestamp, token_address from transfers tr LEFT JOIN balance_history bh ON bh.block = tr.blk_number WHERE tr.network_id = ? AND tr.address = ? AND tr.type != 'erc721' AND bh.block IS NULL", + rows, err := b.db.Query("SELECT blk_number, tr.timestamp, token_address from transfers tr LEFT JOIN balance_history bh ON bh.block = tr.blk_number WHERE tr.network_id = ? AND tr.address = ? AND tr.type != 'erc721' AND tr.type !='erc1155' AND bh.block IS NULL", chainID, address) if err == sql.ErrNoRows { return nil, nil diff --git a/services/wallet/history/balance_test.go b/services/wallet/history/balance_test.go index 039cb8922..b550470c2 100644 --- a/services/wallet/history/balance_test.go +++ b/services/wallet/history/balance_test.go @@ -471,6 +471,47 @@ func TestBalance_addEdgePoints(t *testing.T) { }, wantErr: false, }, + { + name: "addToNonEmptyData", + fields: fields{ + db: walletDB, + }, + args: args{ + chainID: 111, + currency: "SNT", + addresses: []common.Address{common.Address{1}}, + fromTimestamp: 1, + toTimestamp: 2, + data: []*entry{ + { + chainID: 111, + balance: big.NewInt(2), + timestamp: 2, + block: big.NewInt(2), + tokenSymbol: "SNT", + address: common.Address{1}, + }, + }, + }, + wantRes: []*entry{ + { + chainID: 111, + balance: big.NewInt(0), + timestamp: 1, + tokenSymbol: "SNT", + address: common.Address{1}, + }, + { + chainID: 111, + balance: big.NewInt(2), + timestamp: 2, + block: big.NewInt(2), + tokenSymbol: "SNT", + address: common.Address{1}, + }, + }, + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/services/wallet/history/service.go b/services/wallet/history/service.go index 859363506..3e7f51d36 100644 --- a/services/wallet/history/service.go +++ b/services/wallet/history/service.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "errors" + "fmt" "math" "math/big" "reflect" @@ -44,6 +45,10 @@ type ValuePoint struct { Timestamp uint64 `json:"time"` } +func (vp *ValuePoint) String() string { + return fmt.Sprintf("%d: %f", vp.Timestamp, vp.Value) +} + type Service struct { balance *Balance db *sql.DB @@ -136,14 +141,10 @@ func (s *Service) mergeChainsBalances(chainIDs []uint64, addresses []common.Addr return allData[i].timestamp < allData[j].timestamp }) - log.Debug("Sorted balances", "len", len(allData)) - for _, entry := range allData { - log.Debug("Sorted balances", "entry", entry) - } - // Add padding points to make chart look nice + numEdgePoints := 2 * len(chainIDs) // 2 edge points per chain if len(allData) < minPointsForGraph { - allData, _ = addPaddingPoints(tokenSymbol, addresses, toTimestamp, allData, minPointsForGraph) + allData, _ = addPaddingPoints(tokenSymbol, addresses, toTimestamp, allData, minPointsForGraph+numEdgePoints) } return entriesToDataPoints(allData) @@ -171,9 +172,6 @@ func entriesToDataPoints(data []*entry) ([]*DataPoint, error) { updateBalanceMap := func(balanceMap map[AddressKey]*big.Int, entries []*entry) map[AddressKey]*big.Int { // Update balance map for this timestamp for _, entry := range entries { - if entry.chainID == 0 { - continue - } key := AddressKey{ Address: entry.address, ChainID: entry.chainID, @@ -433,7 +431,7 @@ func (s *Service) addEntriesToDB(ctx context.Context, client chain.ClientInterfa if balance == nil { balance, err = client.BalanceAt(ctx, common.Address(address), entry.block) - if balance == nil { + if err != nil { log.Error("Error getting balance", "chainID", network.ChainID, "address", address.String(), "err", err, "unwrapped", errors.Unwrap(err)) return err } @@ -546,17 +544,10 @@ func transfersToEntries(address common.Address, block *big.Int, transfers []tran entries := make([]*entry, 0) for _, transfer := range transfers { - if transfer.Address != address { - panic("Address mismatch") // coding error - } - - if transfer.BlockNumber.Cmp(block) != 0 { - panic("Block number mismatch") // coding error - } entry := &entry{ chainID: transfer.NetworkID, address: transfer.Address, - tokenAddress: transfer.Receipt.ContractAddress, + tokenAddress: transfer.Log.Address, block: transfer.BlockNumber, timestamp: (int64)(transfer.Timestamp), } diff --git a/services/wallet/history/service_test.go b/services/wallet/history/service_test.go index 36ea2b1b8..e2f2c182e 100644 --- a/services/wallet/history/service_test.go +++ b/services/wallet/history/service_test.go @@ -226,18 +226,18 @@ func Test_entriesToDataPoints(t *testing.T) { }, // Padding { - chainID: 0, - balance: big.NewInt(6), + chainID: 3, + balance: big.NewInt(3), timestamp: 2, }, { - chainID: 0, - balance: big.NewInt(6), + chainID: 3, + balance: big.NewInt(3), timestamp: 3, }, { - chainID: 0, - balance: big.NewInt(6), + chainID: 3, + balance: big.NewInt(3), timestamp: 4, }, // Right edge - same timestamp @@ -304,16 +304,36 @@ func Test_entriesToDataPoints(t *testing.T) { timestamp: 2, address: common.Address{1}, }, + // padding - duplicate last point, just update timestamp + { + chainID: 1, + balance: big.NewInt(1), + timestamp: 3, + address: common.Address{1}, + }, + { + chainID: 1, + balance: big.NewInt(1), + timestamp: 4, + address: common.Address{1}, + }, + { + chainID: 1, + balance: big.NewInt(1), + timestamp: 5, + address: common.Address{1}, + }, + // real points { chainID: 1, balance: big.NewInt(2), - timestamp: 3, + timestamp: 6, address: common.Address{2}, }, { chainID: 1, balance: big.NewInt(4), - timestamp: 4, + timestamp: 7, address: common.Address{2}, }, }, @@ -327,14 +347,28 @@ func Test_entriesToDataPoints(t *testing.T) { Balance: (*hexutil.Big)(big.NewInt(12)), Timestamp: 2, }, + // padding { - Balance: (*hexutil.Big)(big.NewInt(8)), + Balance: (*hexutil.Big)(big.NewInt(12)), Timestamp: 3, }, { - Balance: (*hexutil.Big)(big.NewInt(10)), + Balance: (*hexutil.Big)(big.NewInt(12)), Timestamp: 4, }, + { + Balance: (*hexutil.Big)(big.NewInt(12)), + Timestamp: 5, + }, + // real points + { + Balance: (*hexutil.Big)(big.NewInt(8)), + Timestamp: 6, + }, + { + Balance: (*hexutil.Big)(big.NewInt(10)), + Timestamp: 7, + }, }, wantErr: false, }, @@ -344,11 +378,11 @@ func Test_entriesToDataPoints(t *testing.T) { t.Run(tt.name, func(t *testing.T) { got, err := entriesToDataPoints(tt.args.data) if (err != nil) != tt.wantErr { - t.Errorf("entriesToDataPoints() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("entriesToDataPoints() name: %s, error = %v, wantErr = %v", tt.name, err, tt.wantErr) return } if !reflect.DeepEqual(got, tt.want) { - t.Errorf("entriesToDataPoints() = %v, want %v", got, tt.want) + t.Errorf("entriesToDataPoints() name: %s, got: %v, want: %v", tt.name, got, tt.want) } }) } diff --git a/services/wallet/token/token_test.go b/services/wallet/token/token_test.go index bc72ef59c..1b9b1ba97 100644 --- a/services/wallet/token/token_test.go +++ b/services/wallet/token/token_test.go @@ -296,7 +296,7 @@ func TestGetTokenHistoricalBalance(t *testing.T) { // Test multiple values. Must return the most recent one historyBalance.SetInt64(int64(100)) - _, err = manager.db.Exec("INSERT INTO balance_history (currency, chain_id, address, timestamp, balance, block) VALUES (?, ?, ?, ?, ?, ?)", testSymbol, chainID, account, timestamp-200, (*bigint.SQLBigIntBytes)(historyBalance), block) + _, err = manager.db.Exec("INSERT INTO balance_history (currency, chain_id, address, timestamp, balance, block) VALUES (?, ?, ?, ?, ?, ?)", testSymbol, chainID, account, timestamp-200, (*bigint.SQLBigIntBytes)(historyBalance), block+1) require.NoError(t, err) historyBalance.SetInt64(int64(50)) @@ -306,7 +306,7 @@ func TestGetTokenHistoricalBalance(t *testing.T) { historyBalance.SetInt64(int64(50)) chainID = uint64(2) - _, err = manager.db.Exec("INSERT INTO balance_history (currency, chain_id, address, timestamp, balance, block) VALUES (?, ?, ?, ?, ?, ?)", testSymbol, chainID, account, timestamp-1, (*bigint.SQLBigIntBytes)(historyBalance), block) + _, err = manager.db.Exec("INSERT INTO balance_history (currency, chain_id, address, timestamp, balance, block) VALUES (?, ?, ?, ?, ?, ?)", testSymbol, chainID, account, timestamp-1, (*bigint.SQLBigIntBytes)(historyBalance), block+2) require.NoError(t, err) chainID = uint64(1) diff --git a/walletdatabase/migrations/bindata.go b/walletdatabase/migrations/bindata.go index 6ebf27dec..188848509 100644 --- a/walletdatabase/migrations/bindata.go +++ b/walletdatabase/migrations/bindata.go @@ -30,6 +30,7 @@ // 1715839555_rename_chain_prefixes.up.sql (259B) // 1716313614_add_rpc_limits_table.up.sql (203B) // 1716912885_add_wallet_connect_dapps.up.sql (750B) +// 1721136888_recreate_indices_balance_history_remove_dups.up.sql (861B) // doc.go (94B) package migrations @@ -40,6 +41,7 @@ import ( "crypto/sha256" "fmt" "io" + "io/ioutil" "os" "path/filepath" "strings" @@ -49,7 +51,7 @@ import ( func bindataRead(data []byte, name string) ([]byte, error) { gz, err := gzip.NewReader(bytes.NewBuffer(data)) if err != nil { - return nil, fmt.Errorf("read %q: %w", name, err) + return nil, fmt.Errorf("read %q: %v", name, err) } var buf bytes.Buffer @@ -57,7 +59,7 @@ func bindataRead(data []byte, name string) ([]byte, error) { clErr := gz.Close() if err != nil { - return nil, fmt.Errorf("read %q: %w", name, err) + return nil, fmt.Errorf("read %q: %v", name, err) } if clErr != nil { return nil, err @@ -698,6 +700,26 @@ func _1716912885_add_wallet_connect_dappsUpSql() (*asset, error) { return a, nil } +var __1721136888_recreate_indices_balance_history_remove_dupsUpSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x92\xcd\xae\x9b\x30\x14\x84\xf7\x7e\x8a\x59\x36\x12\x59\xf4\x6f\x43\x56\x29\x38\x92\xa5\x14\x5a\x70\xa4\xec\x90\x31\x27\x8d\x15\x62\x53\x63\x16\xbc\x7d\x45\x52\xaa\x08\x55\xca\x55\x74\x1f\xe0\x7c\xa3\xef\xcc\xac\xd7\x28\x03\x75\xf8\x18\x23\xf1\xa4\x02\x41\x21\xd0\xb5\x73\x5e\xf9\x11\x41\xd5\x2d\x21\x38\xf4\xc1\x79\xc2\x60\xcd\xef\x81\xe0\x49\x3b\xdf\xf4\x2c\x29\xf8\x56\x72\xc8\xed\xb7\x3d\x87\xd8\x21\xcb\x25\xf8\x51\x94\xb2\x44\xad\x5a\x65\x35\x55\x67\x33\x5d\x8e\xd5\x84\xc4\xb6\x64\x25\xdf\xf3\x44\x22\x15\xa5\x14\x59\x22\xa1\xcf\xca\xd8\xca\x34\x11\x54\xd3\x78\xea\xfb\x08\x7a\xf0\x9e\xac\x1e\x23\xd4\xad\xd3\x97\x08\xc1\x5c\xa9\x0f\xea\xda\x45\x33\x96\xed\x8a\xfc\xfb\x32\x63\xc3\xd8\x2c\xf3\x29\x86\xf4\x83\xd5\x93\x4e\x38\x13\x9c\x37\xbf\x8c\x55\xed\xdd\x87\xa5\x7c\xcf\x25\xc7\x13\xc8\xe7\x18\xc2\xf6\xe4\xc3\x42\x1b\xb5\xd2\x17\x18\x1b\xdc\xff\xd8\x22\x2b\x79\x21\x21\x32\x99\x2f\xd9\xf8\xf0\x9a\xed\x6a\xfe\xda\x3b\x3e\xeb\x56\xc8\x83\xec\x97\x18\xa9\x77\xdd\xcd\x68\x51\x3f\x4b\x8b\xfc\xc7\xdf\x92\x9f\x50\xbe\xc6\x28\x48\xdf\x67\x34\x91\x8c\x6d\x8c\xa6\x7f\x43\x39\x64\xe2\xe7\x81\x43\x64\x29\x3f\x3e\xd9\x8b\x69\xc8\x06\x73\x1a\x2b\xb2\xc1\x8f\xc8\xb3\x57\x9e\xb9\xda\xcc\xc9\x6f\x89\x3c\x99\x36\x90\xbf\x05\x1a\xea\x5f\x8c\x7c\x28\x60\xb5\x61\x7f\x02\x00\x00\xff\xff\x4e\xfa\x92\xa6\x5d\x03\x00\x00") + +func _1721136888_recreate_indices_balance_history_remove_dupsUpSqlBytes() ([]byte, error) { + return bindataRead( + __1721136888_recreate_indices_balance_history_remove_dupsUpSql, + "1721136888_recreate_indices_balance_history_remove_dups.up.sql", + ) +} + +func _1721136888_recreate_indices_balance_history_remove_dupsUpSql() (*asset, error) { + bytes, err := _1721136888_recreate_indices_balance_history_remove_dupsUpSqlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "1721136888_recreate_indices_balance_history_remove_dups.up.sql", size: 861, mode: os.FileMode(0644), modTime: time.Unix(1700000000, 0)} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x10, 0x61, 0x2d, 0xbc, 0x57, 0xf8, 0x2, 0x5d, 0xa7, 0xab, 0xdd, 0xf1, 0x5a, 0x9f, 0xb6, 0x3c, 0xe, 0x7, 0x39, 0x45, 0xd4, 0x85, 0x79, 0x23, 0xa7, 0xb3, 0x61, 0xe1, 0x96, 0x8b, 0x2d, 0x14}} + return a, nil +} + var _docGo = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x2c\xcb\x41\x0e\x02\x31\x08\x05\xd0\x7d\x4f\xf1\x2f\x00\xe8\xca\xc4\xc4\xc3\xa0\x43\x08\x19\x5b\xc6\x96\xfb\xc7\x4d\xdf\xfe\x5d\xfa\x39\xd5\x0d\xeb\xf7\x6d\x4d\xc4\xf3\xe9\x36\x6c\x6a\x19\x3c\xe9\x1d\xe3\xd0\x52\x50\xcf\xa3\xa2\xdb\xeb\xfe\xb8\x6d\xa0\xeb\x74\xf4\xf0\xa9\x15\x39\x16\x28\xc1\x2c\x7b\xb0\x27\x58\xda\x3f\x00\x00\xff\xff\x57\xd4\xd5\x90\x5e\x00\x00\x00") func docGoBytes() ([]byte, error) { @@ -809,53 +831,80 @@ func AssetNames() []string { // _bindata is a table, holding each asset generator, mapped to its name. var _bindata = map[string]func() (*asset, error){ - "1691753758_initial.up.sql": _1691753758_initialUpSql, - "1692701329_add_collectibles_and_collections_data_cache.up.sql": _1692701329_add_collectibles_and_collections_data_cacheUpSql, - "1692701339_add_scope_to_pending.up.sql": _1692701339_add_scope_to_pendingUpSql, - "1694540071_add_collectibles_ownership_update_timestamp.up.sql": _1694540071_add_collectibles_ownership_update_timestampUpSql, - "1694692748_add_raw_balance_to_token_balances.up.sql": _1694692748_add_raw_balance_to_token_balancesUpSql, + "1691753758_initial.up.sql": _1691753758_initialUpSql, + + "1692701329_add_collectibles_and_collections_data_cache.up.sql": _1692701329_add_collectibles_and_collections_data_cacheUpSql, + + "1692701339_add_scope_to_pending.up.sql": _1692701339_add_scope_to_pendingUpSql, + + "1694540071_add_collectibles_ownership_update_timestamp.up.sql": _1694540071_add_collectibles_ownership_update_timestampUpSql, + + "1694692748_add_raw_balance_to_token_balances.up.sql": _1694692748_add_raw_balance_to_token_balancesUpSql, + "1695133989_add_community_id_to_collectibles_and_collections_data_cache.up.sql": _1695133989_add_community_id_to_collectibles_and_collections_data_cacheUpSql, - "1695932536_balance_history_v2.up.sql": _1695932536_balance_history_v2UpSql, - "1696853635_input_data.up.sql": _1696853635_input_dataUpSql, - "1698117918_add_community_id_to_tokens.up.sql": _1698117918_add_community_id_to_tokensUpSql, - "1698257443_add_community_metadata_to_wallet_db.up.sql": _1698257443_add_community_metadata_to_wallet_dbUpSql, - "1699987075_add_timestamp_and_state_to_community_data_cache.up.sql": _1699987075_add_timestamp_and_state_to_community_data_cacheUpSql, - "1700414564_add_wallet_connect_pairings_table.up.sql": _1700414564_add_wallet_connect_pairings_tableUpSql, - "1701101493_add_token_blocks_range.up.sql": _1701101493_add_token_blocks_rangeUpSql, - "1702467441_wallet_connect_sessions_instead_of_pairings.up.sql": _1702467441_wallet_connect_sessions_instead_of_pairingsUpSql, - "1702577524_add_community_collections_and_collectibles_images_cache.up.sql": _1702577524_add_community_collections_and_collectibles_images_cacheUpSql, - "1702867707_add_balance_to_collectibles_ownership_cache.up.sql": _1702867707_add_balance_to_collectibles_ownership_cacheUpSql, - "1703686612_add_color_to_saved_addresses.up.sql": _1703686612_add_color_to_saved_addressesUpSql, + + "1695932536_balance_history_v2.up.sql": _1695932536_balance_history_v2UpSql, + + "1696853635_input_data.up.sql": _1696853635_input_dataUpSql, + + "1698117918_add_community_id_to_tokens.up.sql": _1698117918_add_community_id_to_tokensUpSql, + + "1698257443_add_community_metadata_to_wallet_db.up.sql": _1698257443_add_community_metadata_to_wallet_dbUpSql, + + "1699987075_add_timestamp_and_state_to_community_data_cache.up.sql": _1699987075_add_timestamp_and_state_to_community_data_cacheUpSql, + + "1700414564_add_wallet_connect_pairings_table.up.sql": _1700414564_add_wallet_connect_pairings_tableUpSql, + + "1701101493_add_token_blocks_range.up.sql": _1701101493_add_token_blocks_rangeUpSql, + + "1702467441_wallet_connect_sessions_instead_of_pairings.up.sql": _1702467441_wallet_connect_sessions_instead_of_pairingsUpSql, + + "1702577524_add_community_collections_and_collectibles_images_cache.up.sql": _1702577524_add_community_collections_and_collectibles_images_cacheUpSql, + + "1702867707_add_balance_to_collectibles_ownership_cache.up.sql": _1702867707_add_balance_to_collectibles_ownership_cacheUpSql, + + "1703686612_add_color_to_saved_addresses.up.sql": _1703686612_add_color_to_saved_addressesUpSql, + "1704701942_remove_favourite_and_change_primary_key_for_saved_addresses.up.sql": _1704701942_remove_favourite_and_change_primary_key_for_saved_addressesUpSql, - "1704913491_add_type_and_tx_timestamp_to_collectibles_ownership_cache.up.sql": _1704913491_add_type_and_tx_timestamp_to_collectibles_ownership_cacheUpSql, - "1705664490_add_balance_check_fields_blocks_ranges_sequential.up.sql": _1705664490_add_balance_check_fields_blocks_ranges_sequentialUpSql, - "1706531789_remove_gasfee-only-eth-transfers.up.sql": _1706531789_remove_gasfeeOnlyEthTransfersUpSql, - "1707160323_add_contract_type_table.up.sql": _1707160323_add_contract_type_tableUpSql, - "1708089811_add_nullable_fiesl_blocks_ranges.up.sql": _1708089811_add_nullable_fiesl_blocks_rangesUpSql, - "1710189541_add_nonce_to_pending_transactions.up.sql": _1710189541_add_nonce_to_pending_transactionsUpSql, - "1712567001_add_soulbound_collectible_cache.up.sql": _1712567001_add_soulbound_collectible_cacheUpSql, - "1714670633_add_id_to_multi_transaction_table.up.sql": _1714670633_add_id_to_multi_transaction_tableUpSql, - "1715637927_add_collection_socials.up.sql": _1715637927_add_collection_socialsUpSql, - "1715839555_rename_chain_prefixes.up.sql": _1715839555_rename_chain_prefixesUpSql, - "1716313614_add_rpc_limits_table.up.sql": _1716313614_add_rpc_limits_tableUpSql, - "1716912885_add_wallet_connect_dapps.up.sql": _1716912885_add_wallet_connect_dappsUpSql, + + "1704913491_add_type_and_tx_timestamp_to_collectibles_ownership_cache.up.sql": _1704913491_add_type_and_tx_timestamp_to_collectibles_ownership_cacheUpSql, + + "1705664490_add_balance_check_fields_blocks_ranges_sequential.up.sql": _1705664490_add_balance_check_fields_blocks_ranges_sequentialUpSql, + + "1706531789_remove_gasfee-only-eth-transfers.up.sql": _1706531789_remove_gasfeeOnlyEthTransfersUpSql, + + "1707160323_add_contract_type_table.up.sql": _1707160323_add_contract_type_tableUpSql, + + "1708089811_add_nullable_fiesl_blocks_ranges.up.sql": _1708089811_add_nullable_fiesl_blocks_rangesUpSql, + + "1710189541_add_nonce_to_pending_transactions.up.sql": _1710189541_add_nonce_to_pending_transactionsUpSql, + + "1712567001_add_soulbound_collectible_cache.up.sql": _1712567001_add_soulbound_collectible_cacheUpSql, + + "1714670633_add_id_to_multi_transaction_table.up.sql": _1714670633_add_id_to_multi_transaction_tableUpSql, + + "1715637927_add_collection_socials.up.sql": _1715637927_add_collection_socialsUpSql, + + "1715839555_rename_chain_prefixes.up.sql": _1715839555_rename_chain_prefixesUpSql, + + "1716313614_add_rpc_limits_table.up.sql": _1716313614_add_rpc_limits_tableUpSql, + + "1716912885_add_wallet_connect_dapps.up.sql": _1716912885_add_wallet_connect_dappsUpSql, + + "1721136888_recreate_indices_balance_history_remove_dups.up.sql": _1721136888_recreate_indices_balance_history_remove_dupsUpSql, + "doc.go": docGo, } -// AssetDebug is true if the assets were built with the debug flag enabled. -const AssetDebug = false - // AssetDir returns the file names below a certain // directory embedded in the file by go-bindata. // For example if you run go-bindata on data/... and data contains the // following hierarchy: -// -// data/ -// foo.txt -// img/ -// a.png -// b.png -// +// data/ +// foo.txt +// img/ +// a.png +// b.png // then AssetDir("data") would return []string{"foo.txt", "img"}, // AssetDir("data/img") would return []string{"a.png", "b.png"}, // AssetDir("foo.txt") and AssetDir("notexist") would return an error, and @@ -888,37 +937,38 @@ type bintree struct { } var _bintree = &bintree{nil, map[string]*bintree{ - "1691753758_initial.up.sql": {_1691753758_initialUpSql, map[string]*bintree{}}, - "1692701329_add_collectibles_and_collections_data_cache.up.sql": {_1692701329_add_collectibles_and_collections_data_cacheUpSql, map[string]*bintree{}}, - "1692701339_add_scope_to_pending.up.sql": {_1692701339_add_scope_to_pendingUpSql, map[string]*bintree{}}, - "1694540071_add_collectibles_ownership_update_timestamp.up.sql": {_1694540071_add_collectibles_ownership_update_timestampUpSql, map[string]*bintree{}}, - "1694692748_add_raw_balance_to_token_balances.up.sql": {_1694692748_add_raw_balance_to_token_balancesUpSql, map[string]*bintree{}}, - "1695133989_add_community_id_to_collectibles_and_collections_data_cache.up.sql": {_1695133989_add_community_id_to_collectibles_and_collections_data_cacheUpSql, map[string]*bintree{}}, - "1695932536_balance_history_v2.up.sql": {_1695932536_balance_history_v2UpSql, map[string]*bintree{}}, - "1696853635_input_data.up.sql": {_1696853635_input_dataUpSql, map[string]*bintree{}}, - "1698117918_add_community_id_to_tokens.up.sql": {_1698117918_add_community_id_to_tokensUpSql, map[string]*bintree{}}, - "1698257443_add_community_metadata_to_wallet_db.up.sql": {_1698257443_add_community_metadata_to_wallet_dbUpSql, map[string]*bintree{}}, - "1699987075_add_timestamp_and_state_to_community_data_cache.up.sql": {_1699987075_add_timestamp_and_state_to_community_data_cacheUpSql, map[string]*bintree{}}, - "1700414564_add_wallet_connect_pairings_table.up.sql": {_1700414564_add_wallet_connect_pairings_tableUpSql, map[string]*bintree{}}, - "1701101493_add_token_blocks_range.up.sql": {_1701101493_add_token_blocks_rangeUpSql, map[string]*bintree{}}, - "1702467441_wallet_connect_sessions_instead_of_pairings.up.sql": {_1702467441_wallet_connect_sessions_instead_of_pairingsUpSql, map[string]*bintree{}}, - "1702577524_add_community_collections_and_collectibles_images_cache.up.sql": {_1702577524_add_community_collections_and_collectibles_images_cacheUpSql, map[string]*bintree{}}, - "1702867707_add_balance_to_collectibles_ownership_cache.up.sql": {_1702867707_add_balance_to_collectibles_ownership_cacheUpSql, map[string]*bintree{}}, - "1703686612_add_color_to_saved_addresses.up.sql": {_1703686612_add_color_to_saved_addressesUpSql, map[string]*bintree{}}, - "1704701942_remove_favourite_and_change_primary_key_for_saved_addresses.up.sql": {_1704701942_remove_favourite_and_change_primary_key_for_saved_addressesUpSql, map[string]*bintree{}}, - "1704913491_add_type_and_tx_timestamp_to_collectibles_ownership_cache.up.sql": {_1704913491_add_type_and_tx_timestamp_to_collectibles_ownership_cacheUpSql, map[string]*bintree{}}, - "1705664490_add_balance_check_fields_blocks_ranges_sequential.up.sql": {_1705664490_add_balance_check_fields_blocks_ranges_sequentialUpSql, map[string]*bintree{}}, - "1706531789_remove_gasfee-only-eth-transfers.up.sql": {_1706531789_remove_gasfeeOnlyEthTransfersUpSql, map[string]*bintree{}}, - "1707160323_add_contract_type_table.up.sql": {_1707160323_add_contract_type_tableUpSql, map[string]*bintree{}}, - "1708089811_add_nullable_fiesl_blocks_ranges.up.sql": {_1708089811_add_nullable_fiesl_blocks_rangesUpSql, map[string]*bintree{}}, - "1710189541_add_nonce_to_pending_transactions.up.sql": {_1710189541_add_nonce_to_pending_transactionsUpSql, map[string]*bintree{}}, - "1712567001_add_soulbound_collectible_cache.up.sql": {_1712567001_add_soulbound_collectible_cacheUpSql, map[string]*bintree{}}, - "1714670633_add_id_to_multi_transaction_table.up.sql": {_1714670633_add_id_to_multi_transaction_tableUpSql, map[string]*bintree{}}, - "1715637927_add_collection_socials.up.sql": {_1715637927_add_collection_socialsUpSql, map[string]*bintree{}}, - "1715839555_rename_chain_prefixes.up.sql": {_1715839555_rename_chain_prefixesUpSql, map[string]*bintree{}}, - "1716313614_add_rpc_limits_table.up.sql": {_1716313614_add_rpc_limits_tableUpSql, map[string]*bintree{}}, - "1716912885_add_wallet_connect_dapps.up.sql": {_1716912885_add_wallet_connect_dappsUpSql, map[string]*bintree{}}, - "doc.go": {docGo, map[string]*bintree{}}, + "1691753758_initial.up.sql": &bintree{_1691753758_initialUpSql, map[string]*bintree{}}, + "1692701329_add_collectibles_and_collections_data_cache.up.sql": &bintree{_1692701329_add_collectibles_and_collections_data_cacheUpSql, map[string]*bintree{}}, + "1692701339_add_scope_to_pending.up.sql": &bintree{_1692701339_add_scope_to_pendingUpSql, map[string]*bintree{}}, + "1694540071_add_collectibles_ownership_update_timestamp.up.sql": &bintree{_1694540071_add_collectibles_ownership_update_timestampUpSql, map[string]*bintree{}}, + "1694692748_add_raw_balance_to_token_balances.up.sql": &bintree{_1694692748_add_raw_balance_to_token_balancesUpSql, map[string]*bintree{}}, + "1695133989_add_community_id_to_collectibles_and_collections_data_cache.up.sql": &bintree{_1695133989_add_community_id_to_collectibles_and_collections_data_cacheUpSql, map[string]*bintree{}}, + "1695932536_balance_history_v2.up.sql": &bintree{_1695932536_balance_history_v2UpSql, map[string]*bintree{}}, + "1696853635_input_data.up.sql": &bintree{_1696853635_input_dataUpSql, map[string]*bintree{}}, + "1698117918_add_community_id_to_tokens.up.sql": &bintree{_1698117918_add_community_id_to_tokensUpSql, map[string]*bintree{}}, + "1698257443_add_community_metadata_to_wallet_db.up.sql": &bintree{_1698257443_add_community_metadata_to_wallet_dbUpSql, map[string]*bintree{}}, + "1699987075_add_timestamp_and_state_to_community_data_cache.up.sql": &bintree{_1699987075_add_timestamp_and_state_to_community_data_cacheUpSql, map[string]*bintree{}}, + "1700414564_add_wallet_connect_pairings_table.up.sql": &bintree{_1700414564_add_wallet_connect_pairings_tableUpSql, map[string]*bintree{}}, + "1701101493_add_token_blocks_range.up.sql": &bintree{_1701101493_add_token_blocks_rangeUpSql, map[string]*bintree{}}, + "1702467441_wallet_connect_sessions_instead_of_pairings.up.sql": &bintree{_1702467441_wallet_connect_sessions_instead_of_pairingsUpSql, map[string]*bintree{}}, + "1702577524_add_community_collections_and_collectibles_images_cache.up.sql": &bintree{_1702577524_add_community_collections_and_collectibles_images_cacheUpSql, map[string]*bintree{}}, + "1702867707_add_balance_to_collectibles_ownership_cache.up.sql": &bintree{_1702867707_add_balance_to_collectibles_ownership_cacheUpSql, map[string]*bintree{}}, + "1703686612_add_color_to_saved_addresses.up.sql": &bintree{_1703686612_add_color_to_saved_addressesUpSql, map[string]*bintree{}}, + "1704701942_remove_favourite_and_change_primary_key_for_saved_addresses.up.sql": &bintree{_1704701942_remove_favourite_and_change_primary_key_for_saved_addressesUpSql, map[string]*bintree{}}, + "1704913491_add_type_and_tx_timestamp_to_collectibles_ownership_cache.up.sql": &bintree{_1704913491_add_type_and_tx_timestamp_to_collectibles_ownership_cacheUpSql, map[string]*bintree{}}, + "1705664490_add_balance_check_fields_blocks_ranges_sequential.up.sql": &bintree{_1705664490_add_balance_check_fields_blocks_ranges_sequentialUpSql, map[string]*bintree{}}, + "1706531789_remove_gasfee-only-eth-transfers.up.sql": &bintree{_1706531789_remove_gasfeeOnlyEthTransfersUpSql, map[string]*bintree{}}, + "1707160323_add_contract_type_table.up.sql": &bintree{_1707160323_add_contract_type_tableUpSql, map[string]*bintree{}}, + "1708089811_add_nullable_fiesl_blocks_ranges.up.sql": &bintree{_1708089811_add_nullable_fiesl_blocks_rangesUpSql, map[string]*bintree{}}, + "1710189541_add_nonce_to_pending_transactions.up.sql": &bintree{_1710189541_add_nonce_to_pending_transactionsUpSql, map[string]*bintree{}}, + "1712567001_add_soulbound_collectible_cache.up.sql": &bintree{_1712567001_add_soulbound_collectible_cacheUpSql, map[string]*bintree{}}, + "1714670633_add_id_to_multi_transaction_table.up.sql": &bintree{_1714670633_add_id_to_multi_transaction_tableUpSql, map[string]*bintree{}}, + "1715637927_add_collection_socials.up.sql": &bintree{_1715637927_add_collection_socialsUpSql, map[string]*bintree{}}, + "1715839555_rename_chain_prefixes.up.sql": &bintree{_1715839555_rename_chain_prefixesUpSql, map[string]*bintree{}}, + "1716313614_add_rpc_limits_table.up.sql": &bintree{_1716313614_add_rpc_limits_tableUpSql, map[string]*bintree{}}, + "1716912885_add_wallet_connect_dapps.up.sql": &bintree{_1716912885_add_wallet_connect_dappsUpSql, map[string]*bintree{}}, + "1721136888_recreate_indices_balance_history_remove_dups.up.sql": &bintree{_1721136888_recreate_indices_balance_history_remove_dupsUpSql, map[string]*bintree{}}, + "doc.go": &bintree{docGo, map[string]*bintree{}}, }} // RestoreAsset restores an asset under the given directory. @@ -935,7 +985,7 @@ func RestoreAsset(dir, name string) error { if err != nil { return err } - err = os.WriteFile(_filePath(dir, name), data, info.Mode()) + err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode()) if err != nil { return err } diff --git a/walletdatabase/migrations/sql/1721136888_recreate_indices_balance_history_remove_dups.up.sql b/walletdatabase/migrations/sql/1721136888_recreate_indices_balance_history_remove_dups.up.sql new file mode 100644 index 000000000..87a3dc922 --- /dev/null +++ b/walletdatabase/migrations/sql/1721136888_recreate_indices_balance_history_remove_dups.up.sql @@ -0,0 +1,19 @@ +-- Step 1: Create a temporary table to store unique records +CREATE TABLE IF NOT EXISTS balance_history_temp AS +SELECT DISTINCT chain_id, address, currency, block, timestamp, balance +FROM balance_history; + +-- Step 2: Truncate the original table +DELETE FROM balance_history; + +-- Step 3: Insert unique records back into the original table +INSERT INTO balance_history (chain_id, address, currency, block, timestamp, balance) +SELECT chain_id, address, currency, block, timestamp, balance +FROM balance_history_temp; + +-- Step 4: Drop the temporary table +DROP TABLE balance_history_temp; + +-- Step 5: Recreate the indices +CREATE UNIQUE INDEX IF NOT EXISTS balance_history_identify_entry ON balance_history (chain_id, address, currency, block); +CREATE INDEX IF NOT EXISTS balance_history_filter_entries ON balance_history (chain_id, address, currency, block, timestamp);