From 579f7e4a523b2734a589bc452d4f391334627a0c Mon Sep 17 00:00:00 2001 From: Stefan Date: Thu, 14 Sep 2023 23:50:51 +0200 Subject: [PATCH] chore(wallet) optimize the filer query Main changes: - Use tr_type instead of IN clause - Use binary (X'...' syntax) directly into the query instead of converting DB values to HEX - Found to be slightly faster than query parameters in the dedicated benchmark - Didn't see much improvement in filter benchmarks - Tried various combinations of optimizations but without impressive performance results Benchmark results: | Name | Original | tr_type | join | hex | no-db | db_only | last | net_j | |:-----------------------|:-----------|:----------|:---------|:---------|:----------|:----------|---------:|---------:| | RAM_NoFilter-10 | 49580229 | 51253242 | 51112462 | 50915133 | 121217817 | 141691008 | 50908642 | 50239712 | | SSD_NoFilter-10 | 49963604 | 51393588 | 51213038 | 50881483 | 120785679 | 141063467 | 50462767 | 49676867 | | SSD_MovingWindow-10 | 53695712 | 54155292 | 54161733 | 54061325 | 126966633 | 146866017 | 53479929 | 53350475 | | SSD_AllAddr_AllTos-10 | 41382804 | 41195225 | 51684175 | 52107262 | 64348100 | 97608833 | 50523529 | 49968321 | | SSD_OneAddress-10 | 34945275 | 35103850 | 31066429 | 31328762 | 50927300 | 54322971 | 30098529 | 30252546 | | FilterSend_AllAddr-10 | 39546808 | 37566604 | 38389725 | 38260738 | 114820458 | 125588408 | 37127625 | 36864575 | | FilterSend_6Addr-10 | 41221458 | 41111225 | 40848288 | 40135492 | 118629700 | 128200467 | 38942521 | 39012100 | | FilterThreeNetworks-10 | - | - | - | - | - | - | 50058929 | 49854450 | Update status-desktop: #11036 --- services/wallet/activity/activity.go | 18 +- services/wallet/activity/activity_test.go | 157 +--------- services/wallet/activity/benchmarks_test.go | 303 ++++++++++++++++++++ services/wallet/activity/filter.go | 6 +- services/wallet/activity/filter.sql | 143 ++++----- services/wallet/transfer/testutils.go | 17 +- 6 files changed, 387 insertions(+), 257 deletions(-) create mode 100644 services/wallet/activity/benchmarks_test.go diff --git a/services/wallet/activity/activity.go b/services/wallet/activity/activity.go index f421c8c77..0878ef075 100644 --- a/services/wallet/activity/activity.go +++ b/services/wallet/activity/activity.go @@ -285,7 +285,7 @@ func joinItems[T interface{}](items []T, itemConversion func(T) string) string { func joinAddresses(addresses []eth.Address) string { return joinItems(addresses, func(a eth.Address) string { - return fmt.Sprintf("'%s'", strings.ToUpper(hex.EncodeToString(a[:]))) + return fmt.Sprintf("X'%s'", hex.EncodeToString(a[:])) }) } @@ -337,8 +337,6 @@ type FilterDependencies struct { // allAddresses optimization indicates if the passed addresses include all the owners in the wallet DB // // Adding a no-limit option was never considered or required. -// -// TODO: optimization: consider implementing nullable []byte instead of using strings for addresses or insert binary (X'...' syntax) directly into the query func getActivityEntries(ctx context.Context, deps FilterDependencies, addresses []eth.Address, allAddresses bool, chainIDs []common.ChainID, filter Filter, offset int, limit int) ([]Entry, error) { if len(addresses) == 0 { return nil, errors.New("no addresses provided") @@ -367,8 +365,7 @@ func getActivityEntries(ctx context.Context, deps FilterDependencies, addresses if sliceChecksCondition(filter.Assets, func(item *Token) bool { return item.TokenType == Erc20 }) { assetsERC20 = joinItems(filter.Assets, func(item Token) string { if item.TokenType == Erc20 { - // SQL HEX() (Blob->Hex) conversion returns uppercase digits with no 0x prefix - return fmt.Sprintf("%d, '%s'", item.ChainID, strings.ToUpper(item.Address.Hex()[2:])) + return fmt.Sprintf("%d, X'%s'", item.ChainID, item.Address.Hex()[2:]) } return "" }) @@ -379,14 +376,13 @@ func getActivityEntries(ctx context.Context, deps FilterDependencies, addresses assetsERC721 := noEntriesInThreeColumnsTmpTableSQLValues if !includeAllCollectibles && !filter.FilterOutCollectibles { assetsERC721 = joinItems(filter.Collectibles, func(item Token) string { - // SQL HEX() (Blob->Hex) conversion returns uppercase digits with no 0x prefix - tokenID := strings.ToUpper(item.TokenID.String()[2:]) - address := strings.ToUpper(item.Address.Hex()[2:]) + tokenID := item.TokenID.String()[2:] + address := item.Address.Hex()[2:] + // SQLite mandates that byte length is an even number which hexutil.EncodeBig doesn't guarantee if len(tokenID)%2 == 1 { - // Hex length must be divisable by 2, otherwise append '0' at the beginning tokenID = "0" + tokenID } - return fmt.Sprintf("%d, '%s', '%s'", item.ChainID, tokenID, address) + return fmt.Sprintf("%d, X'%s', X'%s'", item.ChainID, tokenID, address) }) } @@ -436,7 +432,7 @@ func getActivityEntries(ctx context.Context, deps FilterDependencies, addresses // The duplicated temporary table UNION with CTE acts as an optimization // As soon as we use filter_addresses CTE or filter_addresses_table temp table // or switch them alternatively for JOIN or IN clauses the performance drops significantly - _, err := deps.db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS filter_addresses_table; CREATE TEMP TABLE filter_addresses_table (address VARCHAR PRIMARY KEY); INSERT OR IGNORE INTO filter_addresses_table (address) VALUES %s;\n", involvedAddresses)) + _, err := deps.db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS filter_addresses_table; CREATE TEMP TABLE filter_addresses_table (address VARCHAR PRIMARY KEY); INSERT INTO filter_addresses_table (address) VALUES %s;\n", involvedAddresses)) if err != nil { return nil, err } diff --git a/services/wallet/activity/activity_test.go b/services/wallet/activity/activity_test.go index 0f57c8865..93ab777b6 100644 --- a/services/wallet/activity/activity_test.go +++ b/services/wallet/activity/activity_test.go @@ -638,8 +638,6 @@ func TestGetActivityEntriesFilterByType(t *testing.T) { multiTxs[3] = transfer.GenerateTestBridgeMultiTransaction(trs[6], trs[7]) multiTxs[4] = transfer.GenerateTestSendMultiTransaction(trs[8]) // trs[9] - allAddresses := append(append(append(tdFromAdds, tdToAddrs...), fromAddrs...), toAddrs...) - var lastMT transfer.MultiTransactionIDType for i := range trs { if i%2 == 0 { @@ -649,7 +647,11 @@ func TestGetActivityEntriesFilterByType(t *testing.T) { transfer.InsertTestTransfer(t, deps.db, trs[i].To, &trs[i]) } - trsSpecial, _, _ := transfer.GenerateTestTransfers(t, deps.db, 100, 2) + trsSpecial, fromSpecial, toSpecial := transfer.GenerateTestTransfers(t, deps.db, 100, 2) + + // Here not to include the modified To and From addresses + allAddresses := append(append(append(append(append(tdFromAdds, tdToAddrs...), fromAddrs...), toAddrs...), fromSpecial...), toSpecial...) + // Insert MintAT trsSpecial[0].From = eth.HexToAddress("0x0") transfer.InsertTestTransferWithOptions(t, deps.db, trsSpecial[0].To, &trsSpecial[0], &transfer.TestTransferOptions{ @@ -731,6 +733,7 @@ func TestGetActivityEntriesFilterByType(t *testing.T) { filter.Types = []Type{SendAT} entries, err = getActivityEntries(context.Background(), deps, allAddresses, true, []common.ChainID{}, filter, 0, 15) require.NoError(t, err) + // We have 6 but one is not matched because is a receive, having owner the to address require.Equal(t, 5, len(entries)) } @@ -1294,151 +1297,3 @@ func TestGetMultiTxDetails(t *testing.T) { require.Equal(t, td.multiTx1Tr2.BlkNumber, details.BlockNumber) require.Equal(t, td.multiTx1Tr1.Contract, *details.Contract) } - -func setupBenchmark(b *testing.B, inMemory bool, resultCount int) (deps FilterDependencies, close func(), accounts []eth.Address) { - deps, close = setupTestActivityDBStorageChoice(b, inMemory) - - const transactionCount = 100000 - const mtSendRatio = 0.2 // 20% - const mtSwapRatio = 0.1 // 10% - const mtBridgeRatio = 0.1 // 10% - const pendingCount = 10 - const mtSendCount = int(float64(transactionCount) * mtSendRatio) - const mtSwapCount = int(float64(transactionCount) * mtSwapRatio) - // Bridge requires two transactions - const mtBridgeCount = int(float64(transactionCount) * (mtBridgeRatio / 2)) - - trs, _, _ := transfer.GenerateTestTransfers(b, deps.db, 0, transactionCount) - - accounts = []eth.Address{trs[0].From, trs[1].From, trs[2].From, trs[3].To, trs[4].To, trs[5].To} - - i := 0 - multiTxs := make([]transfer.TestMultiTransaction, mtSendCount+mtSwapCount+mtBridgeCount) - for ; i < mtSendCount; i++ { - multiTxs[i] = transfer.GenerateTestSendMultiTransaction(trs[i]) - trs[i].From = accounts[i%len(accounts)] - multiTxs[i].FromAddress = trs[i].From - - multiTxs[i].MultiTransactionID = transfer.InsertTestMultiTransaction(b, deps.db, &multiTxs[i]) - trs[i].MultiTransactionID = multiTxs[i].MultiTransactionID - } - - for j := 0; j < mtSwapCount; i, j = i+1, j+1 { - multiTxs[i] = transfer.GenerateTestSwapMultiTransaction(trs[i], testutils.SntSymbol, int64(i)) - trs[i].From = accounts[i%len(accounts)] - multiTxs[i].FromAddress = trs[i].From - - multiTxs[i].MultiTransactionID = transfer.InsertTestMultiTransaction(b, deps.db, &multiTxs[i]) - trs[i].MultiTransactionID = multiTxs[i].MultiTransactionID - } - - for mtIdx := 0; mtIdx < mtBridgeCount; i, mtIdx = i+2, mtIdx+1 { - firstTrIdx := i - secondTrIdx := i + 1 - multiTxs[mtIdx] = transfer.GenerateTestBridgeMultiTransaction(trs[firstTrIdx], trs[secondTrIdx]) - trs[firstTrIdx].From = accounts[i%len(accounts)] - trs[secondTrIdx].To = accounts[(i+3)%len(accounts)] - multiTxs[mtIdx].FromAddress = trs[firstTrIdx].From - multiTxs[mtIdx].ToAddress = trs[secondTrIdx].To - multiTxs[mtIdx].FromAddress = trs[i].From - - multiTxs[mtIdx].MultiTransactionID = transfer.InsertTestMultiTransaction(b, deps.db, &multiTxs[mtIdx]) - trs[firstTrIdx].MultiTransactionID = multiTxs[mtIdx].MultiTransactionID - trs[secondTrIdx].MultiTransactionID = multiTxs[mtIdx].MultiTransactionID - } - - for i = 0; i < transactionCount-pendingCount; i++ { - trs[i].From = accounts[i%len(accounts)] - transfer.InsertTestTransfer(b, deps.db, trs[i].From, &trs[i]) - } - - for ; i < transactionCount; i++ { - trs[i].From = accounts[i%len(accounts)] - transfer.InsertTestPendingTransaction(b, deps.db, &trs[i]) - } - - return -} - -func BenchmarkGetActivityEntries(bArg *testing.B) { - type params struct { - inMemory bool - resultCount int - generateTestParameters func([]eth.Address) (addresses []eth.Address, allAddresses bool, filter *Filter, startIndex int) - } - testCases := []struct { - name string - params params - }{ - { - "RAM_NoFilter", - params{ - true, - 10, - func(addresses []eth.Address) ([]eth.Address, bool, *Filter, int) { - return addresses, true, &Filter{}, 0 - }, - }, - }, - { - "SSD_NoFilter", - params{ - false, - 10, - func(addresses []eth.Address) ([]eth.Address, bool, *Filter, int) { - return addresses, true, &Filter{}, 0 - }, - }, - }, - { - "SSD_MovingWindow", - params{ - false, - 10, - func(addresses []eth.Address) ([]eth.Address, bool, *Filter, int) { - return addresses, true, &Filter{}, 200 - }, - }, - }, - { - "SSD_AllAddresses_AllTos", - params{ - false, - 10, - func(addresses []eth.Address) ([]eth.Address, bool, *Filter, int) { - return addresses, true, &Filter{CounterpartyAddresses: addresses[3:]}, 0 - }, - }, - }, - { - "SSD_OneAddress", - params{ - false, - 10, - func(addresses []eth.Address) ([]eth.Address, bool, *Filter, int) { - return addresses[0:1], false, &Filter{}, 0 - }, - }, - }, - } - - deps, closeFn, accounts := setupBenchmark(bArg, true, 10) - defer closeFn() - - const resultCount = 10 - for _, tc := range testCases { - addresses, allAddresses, filter, startIndex := tc.params.generateTestParameters(accounts) - bArg.Run(tc.name, func(b *testing.B) { - // Reset timer after setup - b.ResetTimer() - - // Run benchmark - for i := 0; i < b.N; i++ { - res, err := getActivityEntries(context.Background(), deps, addresses, allAddresses, allNetworksFilter(), *filter, startIndex, resultCount) - if err != nil || len(res) != resultCount { - b.Error(err) - } - } - }) - } -} diff --git a/services/wallet/activity/benchmarks_test.go b/services/wallet/activity/benchmarks_test.go new file mode 100644 index 000000000..4014f60e9 --- /dev/null +++ b/services/wallet/activity/benchmarks_test.go @@ -0,0 +1,303 @@ +package activity + +import ( + "context" + "fmt" + "testing" + + eth "github.com/ethereum/go-ethereum/common" + "github.com/status-im/status-go/services/wallet/common" + "github.com/status-im/status-go/services/wallet/testutils" + "github.com/status-im/status-go/services/wallet/transfer" +) + +func setupBenchmark(b *testing.B, accountsCount int, inMemory bool) (deps FilterDependencies, close func(), accounts []eth.Address) { + deps, close = setupTestActivityDBStorageChoice(b, inMemory) + + const transactionCount = 100000 + const mtSendRatio = 0.2 // 20% + const mtSwapRatio = 0.1 // 10% + const mtBridgeRatio = 0.1 // 10% + const pendingCount = 10 + const mtSendCount = int(float64(transactionCount) * mtSendRatio) + const mtSwapCount = int(float64(transactionCount) * mtSwapRatio) + // Bridge requires two transactions + const mtBridgeCount = int(float64(transactionCount) * (mtBridgeRatio / 2)) + + trs, _, _ := transfer.GenerateTestTransfers(b, deps.db, 0, transactionCount) + + accounts = []eth.Address{} + for i := 0; i < accountsCount; i++ { + if i%2 == 0 { + accounts = append(accounts, trs[i].From) + } else { + accounts = append(accounts, trs[i].To) + } + } + + i := 0 + multiTxs := make([]transfer.TestMultiTransaction, mtSendCount+mtSwapCount+mtBridgeCount) + for ; i < mtSendCount; i++ { + multiTxs[i] = transfer.GenerateTestSendMultiTransaction(trs[i]) + trs[i].From = accounts[i%len(accounts)] + multiTxs[i].FromAddress = trs[i].From + // Currently the network ID is not filled in for send transactions + multiTxs[i].FromNetworkID = nil + multiTxs[i].ToNetworkID = nil + + multiTxs[i].MultiTransactionID = transfer.InsertTestMultiTransaction(b, deps.db, &multiTxs[i]) + trs[i].MultiTransactionID = multiTxs[i].MultiTransactionID + } + + for j := 0; j < mtSwapCount; i, j = i+1, j+1 { + multiTxs[i] = transfer.GenerateTestSwapMultiTransaction(trs[i], testutils.SntSymbol, int64(i)) + trs[i].From = accounts[i%len(accounts)] + multiTxs[i].FromAddress = trs[i].From + + multiTxs[i].MultiTransactionID = transfer.InsertTestMultiTransaction(b, deps.db, &multiTxs[i]) + trs[i].MultiTransactionID = multiTxs[i].MultiTransactionID + } + + for mtIdx := 0; mtIdx < mtBridgeCount; i, mtIdx = i+2, mtIdx+1 { + firstTrIdx := i + secondTrIdx := i + 1 + multiTxs[mtIdx] = transfer.GenerateTestBridgeMultiTransaction(trs[firstTrIdx], trs[secondTrIdx]) + trs[firstTrIdx].From = accounts[i%len(accounts)] + trs[secondTrIdx].To = accounts[(i+3)%len(accounts)] + multiTxs[mtIdx].FromAddress = trs[firstTrIdx].From + multiTxs[mtIdx].ToAddress = trs[secondTrIdx].To + multiTxs[mtIdx].FromAddress = trs[i].From + + multiTxs[mtIdx].MultiTransactionID = transfer.InsertTestMultiTransaction(b, deps.db, &multiTxs[mtIdx]) + trs[firstTrIdx].MultiTransactionID = multiTxs[mtIdx].MultiTransactionID + trs[secondTrIdx].MultiTransactionID = multiTxs[mtIdx].MultiTransactionID + } + + for i = 0; i < transactionCount-pendingCount; i++ { + trs[i].From = accounts[i%len(accounts)] + transfer.InsertTestTransfer(b, deps.db, trs[i].From, &trs[i]) + } + + for ; i < transactionCount; i++ { + trs[i].From = accounts[i%len(accounts)] + transfer.InsertTestPendingTransaction(b, deps.db, &trs[i]) + } + + return +} + +var allNetEnabled = []common.ChainID(nil) + +func BenchmarkGetActivityEntries(bArg *testing.B) { + deps, closeFn, accounts := setupBenchmark(bArg, 6, true) + defer closeFn() + + type params struct { + inMemory bool + // resultCount must be nil to expect as many requested + resultCount *int + generateTestParameters func() (addresses []eth.Address, allAddresses bool, networks []common.ChainID, filter *Filter, startIndex int) + } + testCases := []struct { + name string + params params + }{ + { + "RAM_NoFilter", + params{ + true, + nil, + func() ([]eth.Address, bool, []common.ChainID, *Filter, int) { + return accounts, true, allNetEnabled, &Filter{}, 0 + }, + }, + }, + { + "SSD_NoFilter", + params{ + false, + nil, + func() ([]eth.Address, bool, []common.ChainID, *Filter, int) { + return accounts, true, allNetEnabled, &Filter{}, 0 + }, + }, + }, + { + "SSD_MovingWindow", + params{ + false, + nil, + func() ([]eth.Address, bool, []common.ChainID, *Filter, int) { + return accounts, true, allNetEnabled, &Filter{}, 200 + }, + }, + }, + { + "SSD_AllAddr_AllTos", + params{ + false, + nil, + func() ([]eth.Address, bool, []common.ChainID, *Filter, int) { + return accounts, true, allNetEnabled, &Filter{CounterpartyAddresses: accounts[3:]}, 0 + }, + }, + }, + { + "SSD_OneAddress", + params{ + false, + nil, + func() ([]eth.Address, bool, []common.ChainID, *Filter, int) { + return accounts[0:1], false, allNetEnabled, &Filter{}, 0 + }, + }, + }, + // All memory from here + { + "FilterSend_AllAddr", + params{ + true, + nil, + func() ([]eth.Address, bool, []common.ChainID, *Filter, int) { + return accounts, true, allNetEnabled, &Filter{ + Types: []Type{SendAT}, + }, 0 + }, + }, + }, + { + "FilterSend_6Addr", + params{ + true, + nil, + func() ([]eth.Address, bool, []common.ChainID, *Filter, int) { + return accounts[len(accounts)-6:], false, allNetEnabled, &Filter{ + Types: []Type{SendAT}, + }, 0 + }, + }, + }, + { + "FilterThreeNetworks", + params{ + true, + nil, + func() ([]eth.Address, bool, []common.ChainID, *Filter, int) { + return accounts, true, []common.ChainID{}, &Filter{}, 0 + }, + }, + }, + } + + const resultCount = 100 + for _, tc := range testCases { + addresses, allAddresses, nets, filter, startIndex := tc.params.generateTestParameters() + networks := allNetworksFilter() + if len(nets) > 0 { + networks = nets + } + + bArg.Run(tc.name, func(b *testing.B) { + // Reset timer after setup + b.ResetTimer() + + // Run benchmark + for i := 0; i < b.N; i++ { + res, err := getActivityEntries(context.Background(), deps, addresses, allAddresses, networks, *filter, startIndex, resultCount) + if err != nil { + b.Error(err) + } else if tc.params.resultCount != nil { + if len(res) != *tc.params.resultCount { + b.Error("Got less then expected") + } + } else if len(res) != resultCount { + b.Error("Got less than requested") + } + } + }) + } +} + +func BenchmarkSQLQuery(b *testing.B) { + type params struct { + query string + args []interface{} + expectedResCount int + } + + deps, closeFn, accounts := setupBenchmark(b, 10000, true) + defer closeFn() + + var addrValuesStr, insertAddrValuesStr, addrPlaceholdersStr string + var refAccounts []interface{} + for _, acc := range accounts { + addrValuesStr += fmt.Sprintf("X'%s',", acc.Hex()[2:]) + insertAddrValuesStr += fmt.Sprintf("(X'%s'),", acc.Hex()[2:]) + addrPlaceholdersStr += "?," + refAccounts = append(refAccounts, acc) + } + addrValuesStr = addrValuesStr[:len(addrValuesStr)-1] + insertAddrValuesStr = insertAddrValuesStr[:len(insertAddrValuesStr)-1] + addrPlaceholdersStr = addrPlaceholdersStr[:len(addrPlaceholdersStr)-1] + + if _, err := deps.db.Exec(fmt.Sprintf("CREATE TEMP TABLE filter_addresses_table (address VARCHAR PRIMARY KEY); INSERT INTO filter_addresses_table (address) VALUES %s;", insertAddrValuesStr)); err != nil { + b.Fatal("failed to create temporary table", err) + } + + testCases := []struct { + name string + args params + res testing.BenchmarkResult + }{ + { + name: "JoinQuery", + args: params{ + query: "SELECT COUNT(*) FROM transfers JOIN filter_addresses_table ON transfers.tx_from_address = filter_addresses_table.address", + expectedResCount: 99990, + }, + }, + { + name: "LiteralQuery", + args: params{ + query: fmt.Sprintf("SELECT COUNT(*) FROM transfers WHERE tx_from_address IN (%s)", addrValuesStr), + expectedResCount: 99990, + }, + }, + { + name: "ParamQuery", + args: params{ + query: fmt.Sprintf("SELECT COUNT(*) FROM transfers WHERE tx_from_address IN (%s)", addrPlaceholdersStr), + args: refAccounts, + expectedResCount: 99990, + }, + }, + } + + for _, tc := range testCases { + b.Run(tc.name, func(b *testing.B) { + b.ResetTimer() + + for i := 0; i < b.N; i++ { + b.ResetTimer() + + for i := 0; i < b.N; i++ { + res, err := deps.db.Query(tc.args.query, tc.args.args...) + if err != nil { + b.Fatal("failed to query db", err) + } + res.Next() + + var count int + if err := res.Scan(&count); err != nil { + b.Fatal("failed to scan db result", err) + } + if count != tc.args.expectedResCount { + b.Fatalf("unexpected result count: %d, expected: %d", count, tc.args.expectedResCount) + } + + res.Close() + } + } + }) + } +} diff --git a/services/wallet/activity/filter.go b/services/wallet/activity/filter.go index ee5dc0619..77f2c3d53 100644 --- a/services/wallet/activity/filter.go +++ b/services/wallet/activity/filter.go @@ -166,7 +166,7 @@ func GetOldestTimestamp(ctx context.Context, db *sql.DB, addresses []eth.Address transfers.timestamp AS timestamp FROM transfers, filter_conditions WHERE transfers.multi_transaction_id = 0 - AND (filterAllAddresses OR HEX(from_address) IN filter_addresses OR HEX(to_address) IN filter_addresses) + AND (filterAllAddresses OR from_address IN filter_addresses OR to_address IN filter_addresses) UNION ALL @@ -176,7 +176,7 @@ func GetOldestTimestamp(ctx context.Context, db *sql.DB, addresses []eth.Address pending_transactions.timestamp AS timestamp FROM pending_transactions, filter_conditions WHERE pending_transactions.multi_transaction_id = 0 - AND (filterAllAddresses OR HEX(from_address) IN filter_addresses OR HEX(to_address) IN filter_addresses) + AND (filterAllAddresses OR from_address IN filter_addresses OR to_address IN filter_addresses) UNION ALL @@ -185,7 +185,7 @@ func GetOldestTimestamp(ctx context.Context, db *sql.DB, addresses []eth.Address multi_transactions.to_address AS to_address, multi_transactions.timestamp AS timestamp FROM multi_transactions, filter_conditions - WHERE filterAllAddresses OR HEX(from_address) IN filter_addresses OR HEX(to_address) IN filter_addresses + WHERE filterAllAddresses OR from_address IN filter_addresses OR to_address IN filter_addresses ORDER BY timestamp ASC LIMIT 1` diff --git a/services/wallet/activity/filter.sql b/services/wallet/activity/filter.sql index 67c3f75c5..141467f47 100644 --- a/services/wallet/activity/filter.sql +++ b/services/wallet/activity/filter.sql @@ -44,7 +44,7 @@ WITH filter_conditions AS ( ? AS nowTimestamp, ? AS layer2FinalisationDuration, ? AS layer1FinalisationDuration, - '0000000000000000000000000000000000000000' AS zeroAddress + X'0000000000000000000000000000000000000000' AS zeroAddress ), -- This UNION between CTE and TEMP TABLE acts as an optimization. As soon as we drop one or use them interchangeably the performance drops significantly. filter_addresses(address) AS ( @@ -150,7 +150,8 @@ pending_network_ids AS ( pending_transactions.multi_transaction_id ), layer2_networks(network_id) AS ( - VALUES %s + VALUES + %s ) SELECT transfers.hash AS transfer_hash, @@ -178,16 +179,16 @@ SELECT NULL AS mt_from_amount, NULL AS mt_to_amount, CASE - WHEN transfers.status IS 1 THEN - CASE - WHEN transfers.timestamp > 0 AND filter_conditions.nowTimestamp >= transfers.timestamp + - (CASE - WHEN transfers.network_id in layer2_networks THEN layer2FinalisationDuration - ELSE layer1FinalisationDuration - END) - THEN statusFinalized - ELSE statusCompleted - END + WHEN transfers.status IS 1 THEN CASE + WHEN transfers.timestamp > 0 + AND filter_conditions.nowTimestamp >= transfers.timestamp + ( + CASE + WHEN transfers.network_id in layer2_networks THEN layer2FinalisationDuration + ELSE layer1FinalisationDuration + END + ) THEN statusFinalized + ELSE statusCompleted + END ELSE statusFailed END AS agg_status, 1 AS agg_count, @@ -201,10 +202,12 @@ SELECT transfers.type AS type, transfers.contract_address AS contract_address FROM - transfers, - filter_conditions - LEFT JOIN filter_addresses from_join ON HEX(transfers.address) = from_join.address - LEFT JOIN filter_addresses to_join ON HEX(transfers.tx_to_address) = to_join.address + transfers + CROSS JOIN filter_conditions + INNER JOIN filter_to_addresses receiver_join ON filterAllToAddresses != 0 + OR transfers.tx_to_address = receiver_join.address + LEFT JOIN filter_addresses from_join ON transfers.tx_from_address = from_join.address + LEFT JOIN filter_addresses to_join ON transfers.tx_to_address = to_join.address WHERE transfers.loaded == 1 AND transfers.multi_transaction_id = 0 @@ -223,24 +226,22 @@ WHERE filterActivityTypeAll OR ( filterActivityTypeSend - AND tr_type = fromTrType + AND tr_type = fromTrType -- Check NOT ContractDeploymentAT AND NOT ( - tr_type = fromTrType - and transfers.tx_to_address IS NULL + transfers.tx_to_address IS NULL AND transfers.type = 'eth' AND transfers.contract_address IS NOT NULL - AND HEX(transfers.contract_address) != zeroAddress + AND transfers.contract_address != zeroAddress ) ) OR ( filterActivityTypeReceive - AND tr_type = toTrType + AND tr_type = toTrType -- Check NOT MintAT AND NOT ( - tr_type = toTrType - AND transfers.type = 'erc721' + transfers.type = 'erc721' AND ( transfers.tx_from_address IS NULL - OR HEX(transfers.tx_from_address) = zeroAddress + OR transfers.tx_from_address = zeroAddress ) ) ) @@ -250,11 +251,7 @@ WHERE AND transfers.tx_to_address IS NULL AND transfers.type = 'eth' AND transfers.contract_address IS NOT NULL - AND HEX(transfers.contract_address) != zeroAddress - AND ( - filterAllAddresses - OR (HEX(transfers.address) IN filter_addresses) - ) + AND transfers.contract_address != zeroAddress ) OR ( filterActivityTypeMint @@ -262,23 +259,13 @@ WHERE AND transfers.type = 'erc721' AND ( transfers.tx_from_address IS NULL - OR HEX(transfers.tx_from_address) = zeroAddress - ) - AND ( - filterAllAddresses - OR (HEX(transfers.address) IN filter_addresses) + OR transfers.tx_from_address = zeroAddress ) ) ) AND ( - filterAllAddresses - OR (HEX(owner_address) IN filter_addresses) - ) - AND ( - filterAllToAddresses - OR ( - HEX(transfers.tx_to_address) IN filter_to_addresses - ) + filterAllAddresses -- Every account address has an "owned" entry either as to or from + OR (owner_address IN filter_addresses) ) AND ( includeAllTokenTypeAssets @@ -291,7 +278,7 @@ WHERE AND ( ( transfers.network_id, - HEX(transfers.token_address) + transfers.token_address ) IN assets_erc20 ) ) @@ -303,8 +290,8 @@ WHERE AND ( ( transfers.network_id, - HEX(transfers.token_id), - HEX(transfers.token_address) + transfers.token_id, + transfers.token_address ) IN assets_erc721 ) ) @@ -320,7 +307,7 @@ WHERE AND agg_status = statusCompleted ) OR ( - filterStatusFinalized + filterStatusFinalized AND agg_status = statusFinalized ) OR ( @@ -367,10 +354,12 @@ SELECT pending_transactions.type AS type, NULL as contract_address FROM - pending_transactions, - filter_conditions - LEFT JOIN filter_addresses from_join ON HEX(pending_transactions.from_address) = from_join.address - LEFT JOIN filter_addresses to_join ON HEX(pending_transactions.to_address) = to_join.address + pending_transactions + CROSS JOIN filter_conditions + INNER JOIN filter_to_addresses receiver_join ON filterAllToAddresses != 0 + OR pending_transactions.to_address = receiver_join.address + LEFT JOIN filter_addresses from_join ON pending_transactions.from_address = from_join.address + LEFT JOIN filter_addresses to_join ON pending_transactions.to_address = to_join.address WHERE pending_transactions.multi_transaction_id = 0 AND pending_transactions.status = pendingStatus @@ -395,18 +384,7 @@ WHERE ) AND ( filterAllAddresses - OR ( - HEX(pending_transactions.from_address) IN filter_addresses - ) - OR ( - HEX(pending_transactions.to_address) IN filter_addresses - ) - ) - AND ( - filterAllToAddresses - OR ( - HEX(pending_transactions.to_address) IN filter_to_addresses - ) + OR tr_type NOT NULL ) AND ( includeAllTokenTypeAssets @@ -437,15 +415,16 @@ SELECT multi_transactions.from_amount AS mt_from_amount, multi_transactions.to_amount AS mt_to_amount, CASE - WHEN tr_status.min_status = 1 AND COALESCE(pending_status.count, 0) = 0 THEN - CASE - WHEN multi_transactions.timestamp > 0 AND filter_conditions.nowTimestamp >= multi_transactions.timestamp + - (CASE - WHEN multi_transactions.from_network_id in layer2_networks - OR multi_transactions.to_network_id in layer2_networks THEN layer2FinalisationDuration + WHEN tr_status.min_status = 1 + AND COALESCE(pending_status.count, 0) = 0 THEN CASE + WHEN multi_transactions.timestamp > 0 + AND filter_conditions.nowTimestamp >= multi_transactions.timestamp + ( + CASE + WHEN multi_transactions.from_network_id in layer2_networks + OR multi_transactions.to_network_id in layer2_networks THEN layer2FinalisationDuration ELSE layer1FinalisationDuration - END) - THEN statusFinalized + END + ) THEN statusFinalized ELSE statusCompleted END WHEN tr_status.min_status = 0 THEN statusFailed @@ -462,8 +441,10 @@ SELECT NULL AS type, NULL as contract_address FROM - multi_transactions, - filter_conditions + multi_transactions + CROSS JOIN filter_conditions + INNER JOIN filter_to_addresses receiver_join ON filterAllToAddresses != 0 + OR multi_transactions.to_address = receiver_join.address LEFT JOIN tr_status ON multi_transactions.ROWID = tr_status.multi_transaction_id LEFT JOIN pending_status ON multi_transactions.ROWID = pending_status.multi_transaction_id WHERE @@ -487,22 +468,16 @@ WHERE OR ( -- Send multi-transaction types are exclusively for outbound transfers. The receiving end will have a corresponding entry as "owner_address" in the transfers table. mt_type = mTTypeSend - AND HEX(owner_address) IN filter_addresses + AND owner_address IN filter_addresses ) OR ( mt_type != mTTypeSend AND ( - HEX(multi_transactions.from_address) IN filter_addresses - OR HEX(multi_transactions.to_address) IN filter_addresses + multi_transactions.from_address IN filter_addresses + OR multi_transactions.to_address IN filter_addresses ) ) ) - AND ( - filterAllToAddresses - OR ( - HEX(multi_transactions.to_address) IN filter_to_addresses - ) - ) AND ( includeAllTokenTypeAssets OR ( @@ -521,11 +496,11 @@ WHERE AND ( filterAllActivityStatus OR ( - filterStatusCompleted - AND agg_status = statusCompleted + filterStatusCompleted + AND agg_status = statusCompleted ) OR ( - filterStatusFinalized + filterStatusFinalized AND agg_status = statusFinalized ) OR ( diff --git a/services/wallet/transfer/testutils.go b/services/wallet/transfer/testutils.go index 65f50ce14..4473465ed 100644 --- a/services/wallet/transfer/testutils.go +++ b/services/wallet/transfer/testutils.go @@ -68,14 +68,15 @@ func TestTrToToken(t *testing.T, tt *TestTransaction) (token *token.Token, isNat func generateTestTransaction(seed int) TestTransaction { token := SeedToToken(seed) return TestTransaction{ - Hash: eth_common.HexToHash(fmt.Sprintf("0x1%d", seed)), - ChainID: common.ChainID(token.ChainID), - From: eth_common.HexToAddress(fmt.Sprintf("0x2%d", seed)), - Timestamp: int64(seed), - BlkNumber: int64(seed), - Success: true, - Nonce: uint64(seed), - Contract: eth_common.HexToAddress(fmt.Sprintf("0x2%d", seed)), + Hash: eth_common.HexToHash(fmt.Sprintf("0x1%d", seed)), + ChainID: common.ChainID(token.ChainID), + From: eth_common.HexToAddress(fmt.Sprintf("0x2%d", seed)), + Timestamp: int64(seed), + BlkNumber: int64(seed), + Success: true, + Nonce: uint64(seed), + // In practice this is last20Bytes(Keccak256(RLP(From, nonce))) + Contract: eth_common.HexToAddress(fmt.Sprintf("0x4%d", seed)), MultiTransactionID: NoMultiTransactionID, } }