diff --git a/services/wallet/history/balance.go b/services/wallet/history/balance.go index e573d0a5e..0d0c010ae 100644 --- a/services/wallet/history/balance.go +++ b/services/wallet/history/balance.go @@ -72,82 +72,73 @@ func (b *Balance) get(ctx context.Context, chainID uint64, currency string, addr func (b *Balance) addEdgePoints(chainID uint64, currency string, addresses []common.Address, fromTimestamp, toTimestamp uint64, data []*entry) (res []*entry, err error) { log.Debug("Adding edge points", "chainID", chainID, "currency", currency, "address", addresses, "fromTimestamp", fromTimestamp) + if len(addresses) == 0 { + return nil, errors.New("addresses must not be empty") + } + res = data - for _, address := range addresses { - var firstEntry *entry + var firstEntry *entry - if len(data) > 0 { - for _, entry := range data { - if entry.address == address { - firstEntry = entry - break - } - } + if len(data) > 0 { + firstEntry = data[0] + } else { + firstEntry = &entry{ + chainID: chainID, + address: addresses[0], + tokenSymbol: currency, + timestamp: int64(fromTimestamp), } - if firstEntry == nil { - firstEntry = &entry{ + } + + previous, err := b.db.getEntryPreviousTo(firstEntry) + if err != nil { + return nil, err + } + + firstTimestamp, lastTimestamp := timestampBoundaries(fromTimestamp, toTimestamp, data) + + if previous != nil { + previous.timestamp = int64(firstTimestamp) // We might need to use another minimal offset respecting the time interval + previous.block = nil + res = append([]*entry{previous}, res...) + } else { + // Add a zero point at the beginning to draw a line from + res = append([]*entry{ + { chainID: chainID, - address: address, + address: addresses[0], tokenSymbol: currency, - timestamp: int64(fromTimestamp), - } - } + timestamp: int64(firstTimestamp), + balance: big.NewInt(0), + }, + }, res...) + } - previous, err := b.db.getEntryPreviousTo(firstEntry) - if err != nil { - return nil, err - } - - firstTimestamp, lastTimestamp := timestampBoundaries(fromTimestamp, toTimestamp, address, data) - - if previous != nil { - previous.timestamp = int64(firstTimestamp) // We might need to use another minimal offset respecting the time interval - previous.block = nil - res = append([]*entry{previous}, res...) - } else { - // Add a zero point at the beginning to draw a line from - res = append([]*entry{ - { - chainID: chainID, - address: address, - tokenSymbol: currency, - timestamp: int64(firstTimestamp), - balance: big.NewInt(0), - }, - }, res...) - } - - if res[len(res)-1].timestamp < int64(lastTimestamp) { - // Add a last point to draw a line to - res = append(res, &entry{ - chainID: chainID, - address: address, - tokenSymbol: currency, - timestamp: int64(lastTimestamp), - balance: res[len(res)-1].balance, - }) - } + lastPoint := res[len(res)-1] + if lastPoint.timestamp < int64(lastTimestamp) { + // Add a last point to draw a line to + res = append(res, &entry{ + chainID: chainID, + address: lastPoint.address, + tokenSymbol: currency, + timestamp: int64(lastTimestamp), + balance: lastPoint.balance, + }) } return res, nil } -func timestampBoundaries(fromTimestamp, toTimestamp uint64, address common.Address, data []*entry) (firstTimestamp, lastTimestamp uint64) { +func timestampBoundaries(fromTimestamp, toTimestamp uint64, data []*entry) (firstTimestamp, lastTimestamp uint64) { firstTimestamp = fromTimestamp if fromTimestamp == 0 { if len(data) > 0 { - for _, entry := range data { - if entry.address == address { - if entry.timestamp == 0 { - panic("data[0].timestamp must never be 0") - } - firstTimestamp = uint64(entry.timestamp) - 1 - break - } + if data[0].timestamp == 0 { + panic("data[0].timestamp must never be 0") } - } - if firstTimestamp == fromTimestamp { + firstTimestamp = uint64(data[0].timestamp) - 1 + } else { firstTimestamp = genesisTimestamp } } @@ -197,6 +188,7 @@ func addPaddingPoints(currency string, addresses []common.Address, toTimestamp u tokenSymbol: currency, timestamp: paddingTimestamp, balance: data[j-1].balance, // take the previous balance + chainID: data[j-1].chainID, } res[index] = entry diff --git a/services/wallet/history/balance_db.go b/services/wallet/history/balance_db.go index 504ae83c0..5be594084 100644 --- a/services/wallet/history/balance_db.go +++ b/services/wallet/history/balance_db.go @@ -64,7 +64,7 @@ func (b *BalanceDB) add(entry *entry) error { } func (b *BalanceDB) getEntriesWithoutBalances(chainID uint64, address common.Address) (entries []*entry, err error) { - rows, err := b.db.Query("SELECT blk_number, tr.timestamp, token_address from transfers tr LEFT JOIN balance_history bh ON bh.block = tr.blk_number WHERE tr.network_id = ? AND tr.address = ? AND tr.type != 'erc721' AND bh.block IS NULL", + rows, err := b.db.Query("SELECT blk_number, tr.timestamp, token_address from transfers tr LEFT JOIN balance_history bh ON bh.block = tr.blk_number WHERE tr.network_id = ? AND tr.address = ? AND tr.type != 'erc721' AND tr.type !='erc1155' AND bh.block IS NULL", chainID, address) if err == sql.ErrNoRows { return nil, nil diff --git a/services/wallet/history/balance_test.go b/services/wallet/history/balance_test.go index 039cb8922..b550470c2 100644 --- a/services/wallet/history/balance_test.go +++ b/services/wallet/history/balance_test.go @@ -471,6 +471,47 @@ func TestBalance_addEdgePoints(t *testing.T) { }, wantErr: false, }, + { + name: "addToNonEmptyData", + fields: fields{ + db: walletDB, + }, + args: args{ + chainID: 111, + currency: "SNT", + addresses: []common.Address{common.Address{1}}, + fromTimestamp: 1, + toTimestamp: 2, + data: []*entry{ + { + chainID: 111, + balance: big.NewInt(2), + timestamp: 2, + block: big.NewInt(2), + tokenSymbol: "SNT", + address: common.Address{1}, + }, + }, + }, + wantRes: []*entry{ + { + chainID: 111, + balance: big.NewInt(0), + timestamp: 1, + tokenSymbol: "SNT", + address: common.Address{1}, + }, + { + chainID: 111, + balance: big.NewInt(2), + timestamp: 2, + block: big.NewInt(2), + tokenSymbol: "SNT", + address: common.Address{1}, + }, + }, + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/services/wallet/history/service.go b/services/wallet/history/service.go index 859363506..3e7f51d36 100644 --- a/services/wallet/history/service.go +++ b/services/wallet/history/service.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "errors" + "fmt" "math" "math/big" "reflect" @@ -44,6 +45,10 @@ type ValuePoint struct { Timestamp uint64 `json:"time"` } +func (vp *ValuePoint) String() string { + return fmt.Sprintf("%d: %f", vp.Timestamp, vp.Value) +} + type Service struct { balance *Balance db *sql.DB @@ -136,14 +141,10 @@ func (s *Service) mergeChainsBalances(chainIDs []uint64, addresses []common.Addr return allData[i].timestamp < allData[j].timestamp }) - log.Debug("Sorted balances", "len", len(allData)) - for _, entry := range allData { - log.Debug("Sorted balances", "entry", entry) - } - // Add padding points to make chart look nice + numEdgePoints := 2 * len(chainIDs) // 2 edge points per chain if len(allData) < minPointsForGraph { - allData, _ = addPaddingPoints(tokenSymbol, addresses, toTimestamp, allData, minPointsForGraph) + allData, _ = addPaddingPoints(tokenSymbol, addresses, toTimestamp, allData, minPointsForGraph+numEdgePoints) } return entriesToDataPoints(allData) @@ -171,9 +172,6 @@ func entriesToDataPoints(data []*entry) ([]*DataPoint, error) { updateBalanceMap := func(balanceMap map[AddressKey]*big.Int, entries []*entry) map[AddressKey]*big.Int { // Update balance map for this timestamp for _, entry := range entries { - if entry.chainID == 0 { - continue - } key := AddressKey{ Address: entry.address, ChainID: entry.chainID, @@ -433,7 +431,7 @@ func (s *Service) addEntriesToDB(ctx context.Context, client chain.ClientInterfa if balance == nil { balance, err = client.BalanceAt(ctx, common.Address(address), entry.block) - if balance == nil { + if err != nil { log.Error("Error getting balance", "chainID", network.ChainID, "address", address.String(), "err", err, "unwrapped", errors.Unwrap(err)) return err } @@ -546,17 +544,10 @@ func transfersToEntries(address common.Address, block *big.Int, transfers []tran entries := make([]*entry, 0) for _, transfer := range transfers { - if transfer.Address != address { - panic("Address mismatch") // coding error - } - - if transfer.BlockNumber.Cmp(block) != 0 { - panic("Block number mismatch") // coding error - } entry := &entry{ chainID: transfer.NetworkID, address: transfer.Address, - tokenAddress: transfer.Receipt.ContractAddress, + tokenAddress: transfer.Log.Address, block: transfer.BlockNumber, timestamp: (int64)(transfer.Timestamp), } diff --git a/services/wallet/history/service_test.go b/services/wallet/history/service_test.go index 36ea2b1b8..e2f2c182e 100644 --- a/services/wallet/history/service_test.go +++ b/services/wallet/history/service_test.go @@ -226,18 +226,18 @@ func Test_entriesToDataPoints(t *testing.T) { }, // Padding { - chainID: 0, - balance: big.NewInt(6), + chainID: 3, + balance: big.NewInt(3), timestamp: 2, }, { - chainID: 0, - balance: big.NewInt(6), + chainID: 3, + balance: big.NewInt(3), timestamp: 3, }, { - chainID: 0, - balance: big.NewInt(6), + chainID: 3, + balance: big.NewInt(3), timestamp: 4, }, // Right edge - same timestamp @@ -304,16 +304,36 @@ func Test_entriesToDataPoints(t *testing.T) { timestamp: 2, address: common.Address{1}, }, + // padding - duplicate last point, just update timestamp + { + chainID: 1, + balance: big.NewInt(1), + timestamp: 3, + address: common.Address{1}, + }, + { + chainID: 1, + balance: big.NewInt(1), + timestamp: 4, + address: common.Address{1}, + }, + { + chainID: 1, + balance: big.NewInt(1), + timestamp: 5, + address: common.Address{1}, + }, + // real points { chainID: 1, balance: big.NewInt(2), - timestamp: 3, + timestamp: 6, address: common.Address{2}, }, { chainID: 1, balance: big.NewInt(4), - timestamp: 4, + timestamp: 7, address: common.Address{2}, }, }, @@ -327,14 +347,28 @@ func Test_entriesToDataPoints(t *testing.T) { Balance: (*hexutil.Big)(big.NewInt(12)), Timestamp: 2, }, + // padding { - Balance: (*hexutil.Big)(big.NewInt(8)), + Balance: (*hexutil.Big)(big.NewInt(12)), Timestamp: 3, }, { - Balance: (*hexutil.Big)(big.NewInt(10)), + Balance: (*hexutil.Big)(big.NewInt(12)), Timestamp: 4, }, + { + Balance: (*hexutil.Big)(big.NewInt(12)), + Timestamp: 5, + }, + // real points + { + Balance: (*hexutil.Big)(big.NewInt(8)), + Timestamp: 6, + }, + { + Balance: (*hexutil.Big)(big.NewInt(10)), + Timestamp: 7, + }, }, wantErr: false, }, @@ -344,11 +378,11 @@ func Test_entriesToDataPoints(t *testing.T) { t.Run(tt.name, func(t *testing.T) { got, err := entriesToDataPoints(tt.args.data) if (err != nil) != tt.wantErr { - t.Errorf("entriesToDataPoints() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("entriesToDataPoints() name: %s, error = %v, wantErr = %v", tt.name, err, tt.wantErr) return } if !reflect.DeepEqual(got, tt.want) { - t.Errorf("entriesToDataPoints() = %v, want %v", got, tt.want) + t.Errorf("entriesToDataPoints() name: %s, got: %v, want: %v", tt.name, got, tt.want) } }) } diff --git a/services/wallet/token/token_test.go b/services/wallet/token/token_test.go index bc72ef59c..1b9b1ba97 100644 --- a/services/wallet/token/token_test.go +++ b/services/wallet/token/token_test.go @@ -296,7 +296,7 @@ func TestGetTokenHistoricalBalance(t *testing.T) { // Test multiple values. Must return the most recent one historyBalance.SetInt64(int64(100)) - _, err = manager.db.Exec("INSERT INTO balance_history (currency, chain_id, address, timestamp, balance, block) VALUES (?, ?, ?, ?, ?, ?)", testSymbol, chainID, account, timestamp-200, (*bigint.SQLBigIntBytes)(historyBalance), block) + _, err = manager.db.Exec("INSERT INTO balance_history (currency, chain_id, address, timestamp, balance, block) VALUES (?, ?, ?, ?, ?, ?)", testSymbol, chainID, account, timestamp-200, (*bigint.SQLBigIntBytes)(historyBalance), block+1) require.NoError(t, err) historyBalance.SetInt64(int64(50)) @@ -306,7 +306,7 @@ func TestGetTokenHistoricalBalance(t *testing.T) { historyBalance.SetInt64(int64(50)) chainID = uint64(2) - _, err = manager.db.Exec("INSERT INTO balance_history (currency, chain_id, address, timestamp, balance, block) VALUES (?, ?, ?, ?, ?, ?)", testSymbol, chainID, account, timestamp-1, (*bigint.SQLBigIntBytes)(historyBalance), block) + _, err = manager.db.Exec("INSERT INTO balance_history (currency, chain_id, address, timestamp, balance, block) VALUES (?, ?, ?, ?, ?, ?)", testSymbol, chainID, account, timestamp-1, (*bigint.SQLBigIntBytes)(historyBalance), block+2) require.NoError(t, err) chainID = uint64(1) diff --git a/walletdatabase/migrations/bindata.go b/walletdatabase/migrations/bindata.go index 6ebf27dec..188848509 100644 --- a/walletdatabase/migrations/bindata.go +++ b/walletdatabase/migrations/bindata.go @@ -30,6 +30,7 @@ // 1715839555_rename_chain_prefixes.up.sql (259B) // 1716313614_add_rpc_limits_table.up.sql (203B) // 1716912885_add_wallet_connect_dapps.up.sql (750B) +// 1721136888_recreate_indices_balance_history_remove_dups.up.sql (861B) // doc.go (94B) package migrations @@ -40,6 +41,7 @@ import ( "crypto/sha256" "fmt" "io" + "io/ioutil" "os" "path/filepath" "strings" @@ -49,7 +51,7 @@ import ( func bindataRead(data []byte, name string) ([]byte, error) { gz, err := gzip.NewReader(bytes.NewBuffer(data)) if err != nil { - return nil, fmt.Errorf("read %q: %w", name, err) + return nil, fmt.Errorf("read %q: %v", name, err) } var buf bytes.Buffer @@ -57,7 +59,7 @@ func bindataRead(data []byte, name string) ([]byte, error) { clErr := gz.Close() if err != nil { - return nil, fmt.Errorf("read %q: %w", name, err) + return nil, fmt.Errorf("read %q: %v", name, err) } if clErr != nil { return nil, err @@ -698,6 +700,26 @@ func _1716912885_add_wallet_connect_dappsUpSql() (*asset, error) { return a, nil } +var __1721136888_recreate_indices_balance_history_remove_dupsUpSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x92\xcd\xae\x9b\x30\x14\x84\xf7\x7e\x8a\x59\x36\x12\x59\xf4\x6f\x43\x56\x29\x38\x92\xa5\x14\x5a\x70\xa4\xec\x90\x31\x27\x8d\x15\x62\x53\x63\x16\xbc\x7d\x45\x52\xaa\x08\x55\xca\x55\x74\x1f\xe0\x7c\xa3\xef\xcc\xac\xd7\x28\x03\x75\xf8\x18\x23\xf1\xa4\x02\x41\x21\xd0\xb5\x73\x5e\xf9\x11\x41\xd5\x2d\x21\x38\xf4\xc1\x79\xc2\x60\xcd\xef\x81\xe0\x49\x3b\xdf\xf4\x2c\x29\xf8\x56\x72\xc8\xed\xb7\x3d\x87\xd8\x21\xcb\x25\xf8\x51\x94\xb2\x44\xad\x5a\x65\x35\x55\x67\x33\x5d\x8e\xd5\x84\xc4\xb6\x64\x25\xdf\xf3\x44\x22\x15\xa5\x14\x59\x22\xa1\xcf\xca\xd8\xca\x34\x11\x54\xd3\x78\xea\xfb\x08\x7a\xf0\x9e\xac\x1e\x23\xd4\xad\xd3\x97\x08\xc1\x5c\xa9\x0f\xea\xda\x45\x33\x96\xed\x8a\xfc\xfb\x32\x63\xc3\xd8\x2c\xf3\x29\x86\xf4\x83\xd5\x93\x4e\x38\x13\x9c\x37\xbf\x8c\x55\xed\xdd\x87\xa5\x7c\xcf\x25\xc7\x13\xc8\xe7\x18\xc2\xf6\xe4\xc3\x42\x1b\xb5\xd2\x17\x18\x1b\xdc\xff\xd8\x22\x2b\x79\x21\x21\x32\x99\x2f\xd9\xf8\xf0\x9a\xed\x6a\xfe\xda\x3b\x3e\xeb\x56\xc8\x83\xec\x97\x18\xa9\x77\xdd\xcd\x68\x51\x3f\x4b\x8b\xfc\xc7\xdf\x92\x9f\x50\xbe\xc6\x28\x48\xdf\x67\x34\x91\x8c\x6d\x8c\xa6\x7f\x43\x39\x64\xe2\xe7\x81\x43\x64\x29\x3f\x3e\xd9\x8b\x69\xc8\x06\x73\x1a\x2b\xb2\xc1\x8f\xc8\xb3\x57\x9e\xb9\xda\xcc\xc9\x6f\x89\x3c\x99\x36\x90\xbf\x05\x1a\xea\x5f\x8c\x7c\x28\x60\xb5\x61\x7f\x02\x00\x00\xff\xff\x4e\xfa\x92\xa6\x5d\x03\x00\x00") + +func _1721136888_recreate_indices_balance_history_remove_dupsUpSqlBytes() ([]byte, error) { + return bindataRead( + __1721136888_recreate_indices_balance_history_remove_dupsUpSql, + "1721136888_recreate_indices_balance_history_remove_dups.up.sql", + ) +} + +func _1721136888_recreate_indices_balance_history_remove_dupsUpSql() (*asset, error) { + bytes, err := _1721136888_recreate_indices_balance_history_remove_dupsUpSqlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "1721136888_recreate_indices_balance_history_remove_dups.up.sql", size: 861, mode: os.FileMode(0644), modTime: time.Unix(1700000000, 0)} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x10, 0x61, 0x2d, 0xbc, 0x57, 0xf8, 0x2, 0x5d, 0xa7, 0xab, 0xdd, 0xf1, 0x5a, 0x9f, 0xb6, 0x3c, 0xe, 0x7, 0x39, 0x45, 0xd4, 0x85, 0x79, 0x23, 0xa7, 0xb3, 0x61, 0xe1, 0x96, 0x8b, 0x2d, 0x14}} + return a, nil +} + var _docGo = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x2c\xcb\x41\x0e\x02\x31\x08\x05\xd0\x7d\x4f\xf1\x2f\x00\xe8\xca\xc4\xc4\xc3\xa0\x43\x08\x19\x5b\xc6\x96\xfb\xc7\x4d\xdf\xfe\x5d\xfa\x39\xd5\x0d\xeb\xf7\x6d\x4d\xc4\xf3\xe9\x36\x6c\x6a\x19\x3c\xe9\x1d\xe3\xd0\x52\x50\xcf\xa3\xa2\xdb\xeb\xfe\xb8\x6d\xa0\xeb\x74\xf4\xf0\xa9\x15\x39\x16\x28\xc1\x2c\x7b\xb0\x27\x58\xda\x3f\x00\x00\xff\xff\x57\xd4\xd5\x90\x5e\x00\x00\x00") func docGoBytes() ([]byte, error) { @@ -809,53 +831,80 @@ func AssetNames() []string { // _bindata is a table, holding each asset generator, mapped to its name. var _bindata = map[string]func() (*asset, error){ - "1691753758_initial.up.sql": _1691753758_initialUpSql, - "1692701329_add_collectibles_and_collections_data_cache.up.sql": _1692701329_add_collectibles_and_collections_data_cacheUpSql, - "1692701339_add_scope_to_pending.up.sql": _1692701339_add_scope_to_pendingUpSql, - "1694540071_add_collectibles_ownership_update_timestamp.up.sql": _1694540071_add_collectibles_ownership_update_timestampUpSql, - "1694692748_add_raw_balance_to_token_balances.up.sql": _1694692748_add_raw_balance_to_token_balancesUpSql, + "1691753758_initial.up.sql": _1691753758_initialUpSql, + + "1692701329_add_collectibles_and_collections_data_cache.up.sql": _1692701329_add_collectibles_and_collections_data_cacheUpSql, + + "1692701339_add_scope_to_pending.up.sql": _1692701339_add_scope_to_pendingUpSql, + + "1694540071_add_collectibles_ownership_update_timestamp.up.sql": _1694540071_add_collectibles_ownership_update_timestampUpSql, + + "1694692748_add_raw_balance_to_token_balances.up.sql": _1694692748_add_raw_balance_to_token_balancesUpSql, + "1695133989_add_community_id_to_collectibles_and_collections_data_cache.up.sql": _1695133989_add_community_id_to_collectibles_and_collections_data_cacheUpSql, - "1695932536_balance_history_v2.up.sql": _1695932536_balance_history_v2UpSql, - "1696853635_input_data.up.sql": _1696853635_input_dataUpSql, - "1698117918_add_community_id_to_tokens.up.sql": _1698117918_add_community_id_to_tokensUpSql, - "1698257443_add_community_metadata_to_wallet_db.up.sql": _1698257443_add_community_metadata_to_wallet_dbUpSql, - "1699987075_add_timestamp_and_state_to_community_data_cache.up.sql": _1699987075_add_timestamp_and_state_to_community_data_cacheUpSql, - "1700414564_add_wallet_connect_pairings_table.up.sql": _1700414564_add_wallet_connect_pairings_tableUpSql, - "1701101493_add_token_blocks_range.up.sql": _1701101493_add_token_blocks_rangeUpSql, - "1702467441_wallet_connect_sessions_instead_of_pairings.up.sql": _1702467441_wallet_connect_sessions_instead_of_pairingsUpSql, - "1702577524_add_community_collections_and_collectibles_images_cache.up.sql": _1702577524_add_community_collections_and_collectibles_images_cacheUpSql, - "1702867707_add_balance_to_collectibles_ownership_cache.up.sql": _1702867707_add_balance_to_collectibles_ownership_cacheUpSql, - "1703686612_add_color_to_saved_addresses.up.sql": _1703686612_add_color_to_saved_addressesUpSql, + + "1695932536_balance_history_v2.up.sql": _1695932536_balance_history_v2UpSql, + + "1696853635_input_data.up.sql": _1696853635_input_dataUpSql, + + "1698117918_add_community_id_to_tokens.up.sql": _1698117918_add_community_id_to_tokensUpSql, + + "1698257443_add_community_metadata_to_wallet_db.up.sql": _1698257443_add_community_metadata_to_wallet_dbUpSql, + + "1699987075_add_timestamp_and_state_to_community_data_cache.up.sql": _1699987075_add_timestamp_and_state_to_community_data_cacheUpSql, + + "1700414564_add_wallet_connect_pairings_table.up.sql": _1700414564_add_wallet_connect_pairings_tableUpSql, + + "1701101493_add_token_blocks_range.up.sql": _1701101493_add_token_blocks_rangeUpSql, + + "1702467441_wallet_connect_sessions_instead_of_pairings.up.sql": _1702467441_wallet_connect_sessions_instead_of_pairingsUpSql, + + "1702577524_add_community_collections_and_collectibles_images_cache.up.sql": _1702577524_add_community_collections_and_collectibles_images_cacheUpSql, + + "1702867707_add_balance_to_collectibles_ownership_cache.up.sql": _1702867707_add_balance_to_collectibles_ownership_cacheUpSql, + + "1703686612_add_color_to_saved_addresses.up.sql": _1703686612_add_color_to_saved_addressesUpSql, + "1704701942_remove_favourite_and_change_primary_key_for_saved_addresses.up.sql": _1704701942_remove_favourite_and_change_primary_key_for_saved_addressesUpSql, - "1704913491_add_type_and_tx_timestamp_to_collectibles_ownership_cache.up.sql": _1704913491_add_type_and_tx_timestamp_to_collectibles_ownership_cacheUpSql, - "1705664490_add_balance_check_fields_blocks_ranges_sequential.up.sql": _1705664490_add_balance_check_fields_blocks_ranges_sequentialUpSql, - "1706531789_remove_gasfee-only-eth-transfers.up.sql": _1706531789_remove_gasfeeOnlyEthTransfersUpSql, - "1707160323_add_contract_type_table.up.sql": _1707160323_add_contract_type_tableUpSql, - "1708089811_add_nullable_fiesl_blocks_ranges.up.sql": _1708089811_add_nullable_fiesl_blocks_rangesUpSql, - "1710189541_add_nonce_to_pending_transactions.up.sql": _1710189541_add_nonce_to_pending_transactionsUpSql, - "1712567001_add_soulbound_collectible_cache.up.sql": _1712567001_add_soulbound_collectible_cacheUpSql, - "1714670633_add_id_to_multi_transaction_table.up.sql": _1714670633_add_id_to_multi_transaction_tableUpSql, - "1715637927_add_collection_socials.up.sql": _1715637927_add_collection_socialsUpSql, - "1715839555_rename_chain_prefixes.up.sql": _1715839555_rename_chain_prefixesUpSql, - "1716313614_add_rpc_limits_table.up.sql": _1716313614_add_rpc_limits_tableUpSql, - "1716912885_add_wallet_connect_dapps.up.sql": _1716912885_add_wallet_connect_dappsUpSql, + + "1704913491_add_type_and_tx_timestamp_to_collectibles_ownership_cache.up.sql": _1704913491_add_type_and_tx_timestamp_to_collectibles_ownership_cacheUpSql, + + "1705664490_add_balance_check_fields_blocks_ranges_sequential.up.sql": _1705664490_add_balance_check_fields_blocks_ranges_sequentialUpSql, + + "1706531789_remove_gasfee-only-eth-transfers.up.sql": _1706531789_remove_gasfeeOnlyEthTransfersUpSql, + + "1707160323_add_contract_type_table.up.sql": _1707160323_add_contract_type_tableUpSql, + + "1708089811_add_nullable_fiesl_blocks_ranges.up.sql": _1708089811_add_nullable_fiesl_blocks_rangesUpSql, + + "1710189541_add_nonce_to_pending_transactions.up.sql": _1710189541_add_nonce_to_pending_transactionsUpSql, + + "1712567001_add_soulbound_collectible_cache.up.sql": _1712567001_add_soulbound_collectible_cacheUpSql, + + "1714670633_add_id_to_multi_transaction_table.up.sql": _1714670633_add_id_to_multi_transaction_tableUpSql, + + "1715637927_add_collection_socials.up.sql": _1715637927_add_collection_socialsUpSql, + + "1715839555_rename_chain_prefixes.up.sql": _1715839555_rename_chain_prefixesUpSql, + + "1716313614_add_rpc_limits_table.up.sql": _1716313614_add_rpc_limits_tableUpSql, + + "1716912885_add_wallet_connect_dapps.up.sql": _1716912885_add_wallet_connect_dappsUpSql, + + "1721136888_recreate_indices_balance_history_remove_dups.up.sql": _1721136888_recreate_indices_balance_history_remove_dupsUpSql, + "doc.go": docGo, } -// AssetDebug is true if the assets were built with the debug flag enabled. -const AssetDebug = false - // AssetDir returns the file names below a certain // directory embedded in the file by go-bindata. // For example if you run go-bindata on data/... and data contains the // following hierarchy: -// -// data/ -// foo.txt -// img/ -// a.png -// b.png -// +// data/ +// foo.txt +// img/ +// a.png +// b.png // then AssetDir("data") would return []string{"foo.txt", "img"}, // AssetDir("data/img") would return []string{"a.png", "b.png"}, // AssetDir("foo.txt") and AssetDir("notexist") would return an error, and @@ -888,37 +937,38 @@ type bintree struct { } var _bintree = &bintree{nil, map[string]*bintree{ - "1691753758_initial.up.sql": {_1691753758_initialUpSql, map[string]*bintree{}}, - "1692701329_add_collectibles_and_collections_data_cache.up.sql": {_1692701329_add_collectibles_and_collections_data_cacheUpSql, map[string]*bintree{}}, - "1692701339_add_scope_to_pending.up.sql": {_1692701339_add_scope_to_pendingUpSql, map[string]*bintree{}}, - "1694540071_add_collectibles_ownership_update_timestamp.up.sql": {_1694540071_add_collectibles_ownership_update_timestampUpSql, map[string]*bintree{}}, - "1694692748_add_raw_balance_to_token_balances.up.sql": {_1694692748_add_raw_balance_to_token_balancesUpSql, map[string]*bintree{}}, - "1695133989_add_community_id_to_collectibles_and_collections_data_cache.up.sql": {_1695133989_add_community_id_to_collectibles_and_collections_data_cacheUpSql, map[string]*bintree{}}, - "1695932536_balance_history_v2.up.sql": {_1695932536_balance_history_v2UpSql, map[string]*bintree{}}, - "1696853635_input_data.up.sql": {_1696853635_input_dataUpSql, map[string]*bintree{}}, - "1698117918_add_community_id_to_tokens.up.sql": {_1698117918_add_community_id_to_tokensUpSql, map[string]*bintree{}}, - "1698257443_add_community_metadata_to_wallet_db.up.sql": {_1698257443_add_community_metadata_to_wallet_dbUpSql, map[string]*bintree{}}, - "1699987075_add_timestamp_and_state_to_community_data_cache.up.sql": {_1699987075_add_timestamp_and_state_to_community_data_cacheUpSql, map[string]*bintree{}}, - "1700414564_add_wallet_connect_pairings_table.up.sql": {_1700414564_add_wallet_connect_pairings_tableUpSql, map[string]*bintree{}}, - "1701101493_add_token_blocks_range.up.sql": {_1701101493_add_token_blocks_rangeUpSql, map[string]*bintree{}}, - "1702467441_wallet_connect_sessions_instead_of_pairings.up.sql": {_1702467441_wallet_connect_sessions_instead_of_pairingsUpSql, map[string]*bintree{}}, - "1702577524_add_community_collections_and_collectibles_images_cache.up.sql": {_1702577524_add_community_collections_and_collectibles_images_cacheUpSql, map[string]*bintree{}}, - "1702867707_add_balance_to_collectibles_ownership_cache.up.sql": {_1702867707_add_balance_to_collectibles_ownership_cacheUpSql, map[string]*bintree{}}, - "1703686612_add_color_to_saved_addresses.up.sql": {_1703686612_add_color_to_saved_addressesUpSql, map[string]*bintree{}}, - "1704701942_remove_favourite_and_change_primary_key_for_saved_addresses.up.sql": {_1704701942_remove_favourite_and_change_primary_key_for_saved_addressesUpSql, map[string]*bintree{}}, - "1704913491_add_type_and_tx_timestamp_to_collectibles_ownership_cache.up.sql": {_1704913491_add_type_and_tx_timestamp_to_collectibles_ownership_cacheUpSql, map[string]*bintree{}}, - "1705664490_add_balance_check_fields_blocks_ranges_sequential.up.sql": {_1705664490_add_balance_check_fields_blocks_ranges_sequentialUpSql, map[string]*bintree{}}, - "1706531789_remove_gasfee-only-eth-transfers.up.sql": {_1706531789_remove_gasfeeOnlyEthTransfersUpSql, map[string]*bintree{}}, - "1707160323_add_contract_type_table.up.sql": {_1707160323_add_contract_type_tableUpSql, map[string]*bintree{}}, - "1708089811_add_nullable_fiesl_blocks_ranges.up.sql": {_1708089811_add_nullable_fiesl_blocks_rangesUpSql, map[string]*bintree{}}, - "1710189541_add_nonce_to_pending_transactions.up.sql": {_1710189541_add_nonce_to_pending_transactionsUpSql, map[string]*bintree{}}, - "1712567001_add_soulbound_collectible_cache.up.sql": {_1712567001_add_soulbound_collectible_cacheUpSql, map[string]*bintree{}}, - "1714670633_add_id_to_multi_transaction_table.up.sql": {_1714670633_add_id_to_multi_transaction_tableUpSql, map[string]*bintree{}}, - "1715637927_add_collection_socials.up.sql": {_1715637927_add_collection_socialsUpSql, map[string]*bintree{}}, - "1715839555_rename_chain_prefixes.up.sql": {_1715839555_rename_chain_prefixesUpSql, map[string]*bintree{}}, - "1716313614_add_rpc_limits_table.up.sql": {_1716313614_add_rpc_limits_tableUpSql, map[string]*bintree{}}, - "1716912885_add_wallet_connect_dapps.up.sql": {_1716912885_add_wallet_connect_dappsUpSql, map[string]*bintree{}}, - "doc.go": {docGo, map[string]*bintree{}}, + "1691753758_initial.up.sql": &bintree{_1691753758_initialUpSql, map[string]*bintree{}}, + "1692701329_add_collectibles_and_collections_data_cache.up.sql": &bintree{_1692701329_add_collectibles_and_collections_data_cacheUpSql, map[string]*bintree{}}, + "1692701339_add_scope_to_pending.up.sql": &bintree{_1692701339_add_scope_to_pendingUpSql, map[string]*bintree{}}, + "1694540071_add_collectibles_ownership_update_timestamp.up.sql": &bintree{_1694540071_add_collectibles_ownership_update_timestampUpSql, map[string]*bintree{}}, + "1694692748_add_raw_balance_to_token_balances.up.sql": &bintree{_1694692748_add_raw_balance_to_token_balancesUpSql, map[string]*bintree{}}, + "1695133989_add_community_id_to_collectibles_and_collections_data_cache.up.sql": &bintree{_1695133989_add_community_id_to_collectibles_and_collections_data_cacheUpSql, map[string]*bintree{}}, + "1695932536_balance_history_v2.up.sql": &bintree{_1695932536_balance_history_v2UpSql, map[string]*bintree{}}, + "1696853635_input_data.up.sql": &bintree{_1696853635_input_dataUpSql, map[string]*bintree{}}, + "1698117918_add_community_id_to_tokens.up.sql": &bintree{_1698117918_add_community_id_to_tokensUpSql, map[string]*bintree{}}, + "1698257443_add_community_metadata_to_wallet_db.up.sql": &bintree{_1698257443_add_community_metadata_to_wallet_dbUpSql, map[string]*bintree{}}, + "1699987075_add_timestamp_and_state_to_community_data_cache.up.sql": &bintree{_1699987075_add_timestamp_and_state_to_community_data_cacheUpSql, map[string]*bintree{}}, + "1700414564_add_wallet_connect_pairings_table.up.sql": &bintree{_1700414564_add_wallet_connect_pairings_tableUpSql, map[string]*bintree{}}, + "1701101493_add_token_blocks_range.up.sql": &bintree{_1701101493_add_token_blocks_rangeUpSql, map[string]*bintree{}}, + "1702467441_wallet_connect_sessions_instead_of_pairings.up.sql": &bintree{_1702467441_wallet_connect_sessions_instead_of_pairingsUpSql, map[string]*bintree{}}, + "1702577524_add_community_collections_and_collectibles_images_cache.up.sql": &bintree{_1702577524_add_community_collections_and_collectibles_images_cacheUpSql, map[string]*bintree{}}, + "1702867707_add_balance_to_collectibles_ownership_cache.up.sql": &bintree{_1702867707_add_balance_to_collectibles_ownership_cacheUpSql, map[string]*bintree{}}, + "1703686612_add_color_to_saved_addresses.up.sql": &bintree{_1703686612_add_color_to_saved_addressesUpSql, map[string]*bintree{}}, + "1704701942_remove_favourite_and_change_primary_key_for_saved_addresses.up.sql": &bintree{_1704701942_remove_favourite_and_change_primary_key_for_saved_addressesUpSql, map[string]*bintree{}}, + "1704913491_add_type_and_tx_timestamp_to_collectibles_ownership_cache.up.sql": &bintree{_1704913491_add_type_and_tx_timestamp_to_collectibles_ownership_cacheUpSql, map[string]*bintree{}}, + "1705664490_add_balance_check_fields_blocks_ranges_sequential.up.sql": &bintree{_1705664490_add_balance_check_fields_blocks_ranges_sequentialUpSql, map[string]*bintree{}}, + "1706531789_remove_gasfee-only-eth-transfers.up.sql": &bintree{_1706531789_remove_gasfeeOnlyEthTransfersUpSql, map[string]*bintree{}}, + "1707160323_add_contract_type_table.up.sql": &bintree{_1707160323_add_contract_type_tableUpSql, map[string]*bintree{}}, + "1708089811_add_nullable_fiesl_blocks_ranges.up.sql": &bintree{_1708089811_add_nullable_fiesl_blocks_rangesUpSql, map[string]*bintree{}}, + "1710189541_add_nonce_to_pending_transactions.up.sql": &bintree{_1710189541_add_nonce_to_pending_transactionsUpSql, map[string]*bintree{}}, + "1712567001_add_soulbound_collectible_cache.up.sql": &bintree{_1712567001_add_soulbound_collectible_cacheUpSql, map[string]*bintree{}}, + "1714670633_add_id_to_multi_transaction_table.up.sql": &bintree{_1714670633_add_id_to_multi_transaction_tableUpSql, map[string]*bintree{}}, + "1715637927_add_collection_socials.up.sql": &bintree{_1715637927_add_collection_socialsUpSql, map[string]*bintree{}}, + "1715839555_rename_chain_prefixes.up.sql": &bintree{_1715839555_rename_chain_prefixesUpSql, map[string]*bintree{}}, + "1716313614_add_rpc_limits_table.up.sql": &bintree{_1716313614_add_rpc_limits_tableUpSql, map[string]*bintree{}}, + "1716912885_add_wallet_connect_dapps.up.sql": &bintree{_1716912885_add_wallet_connect_dappsUpSql, map[string]*bintree{}}, + "1721136888_recreate_indices_balance_history_remove_dups.up.sql": &bintree{_1721136888_recreate_indices_balance_history_remove_dupsUpSql, map[string]*bintree{}}, + "doc.go": &bintree{docGo, map[string]*bintree{}}, }} // RestoreAsset restores an asset under the given directory. @@ -935,7 +985,7 @@ func RestoreAsset(dir, name string) error { if err != nil { return err } - err = os.WriteFile(_filePath(dir, name), data, info.Mode()) + err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode()) if err != nil { return err } diff --git a/walletdatabase/migrations/sql/1721136888_recreate_indices_balance_history_remove_dups.up.sql b/walletdatabase/migrations/sql/1721136888_recreate_indices_balance_history_remove_dups.up.sql new file mode 100644 index 000000000..87a3dc922 --- /dev/null +++ b/walletdatabase/migrations/sql/1721136888_recreate_indices_balance_history_remove_dups.up.sql @@ -0,0 +1,19 @@ +-- Step 1: Create a temporary table to store unique records +CREATE TABLE IF NOT EXISTS balance_history_temp AS +SELECT DISTINCT chain_id, address, currency, block, timestamp, balance +FROM balance_history; + +-- Step 2: Truncate the original table +DELETE FROM balance_history; + +-- Step 3: Insert unique records back into the original table +INSERT INTO balance_history (chain_id, address, currency, block, timestamp, balance) +SELECT chain_id, address, currency, block, timestamp, balance +FROM balance_history_temp; + +-- Step 4: Drop the temporary table +DROP TABLE balance_history_temp; + +-- Step 5: Recreate the indices +CREATE UNIQUE INDEX IF NOT EXISTS balance_history_identify_entry ON balance_history (chain_id, address, currency, block); +CREATE INDEX IF NOT EXISTS balance_history_filter_entries ON balance_history (chain_id, address, currency, block, timestamp);