chore(wallet) optimize the filer query

Main changes: - Use tr_type instead of IN clause - Use binary (X'...' syntax) directly into the query instead of converting DB values to HEX - Found to be slightly faster than query parameters in the dedicated benchmark - Didn't see much improvement in filter benchmarks - Tried various combinations of optimizations but without impressive performance results Benchmark results: | Name | Original | tr_type | join | hex | no-db | db_only | last | net_j | |:-----------------------|:-----------|:----------|:---------|:---------|:----------|:----------|---------:|---------:| | RAM_NoFilter-10 | 49580229 | 51253242 | 51112462 | 50915133 | 121217817 | 141691008 | 50908642 | 50239712 | | SSD_NoFilter-10 | 49963604 | 51393588 | 51213038 | 50881483 | 120785679 | 141063467 | 50462767 | 49676867 | | SSD_MovingWindow-10 | 53695712 | 54155292 | 54161733 | 54061325 | 126966633 | 146866017 | 53479929 | 53350475 | | SSD_AllAddr_AllTos-10 | 41382804 | 41195225 | 51684175 | 52107262 | 64348100 | 97608833 | 50523529 | 49968321 | | SSD_OneAddress-10 | 34945275 | 35103850 | 31066429 | 31328762 | 50927300 | 54322971 | 30098529 | 30252546 | | FilterSend_AllAddr-10 | 39546808 | 37566604 | 38389725 | 38260738 | 114820458 | 125588408 | 37127625 | 36864575 | | FilterSend_6Addr-10 | 41221458 | 41111225 | 40848288 | 40135492 | 118629700 | 128200467 | 38942521 | 39012100 | | FilterThreeNetworks-10 | - | - | - | - | - | - | 50058929 | 49854450 | Update status-desktop: #11036
2023-09-14 23:50:51 +02:00 · 2023-09-14 23:50:51 +02:00 · 579f7e4a52
parent 57c7054dd2
commit 579f7e4a52
6 changed files with 387 additions and 257 deletions
--- a/services/wallet/activity/activity.go
+++ b/services/wallet/activity/activity.go
@ -285,7 +285,7 @@ func joinItems[T interface{}](items []T, itemConversion func(T) string) string {

 func joinAddresses(addresses []eth.Address) string {
 	return joinItems(addresses, func(a eth.Address) string {
-		return fmt.Sprintf("'%s'", strings.ToUpper(hex.EncodeToString(a[:])))
+		return fmt.Sprintf("X'%s'", hex.EncodeToString(a[:]))
 	})
 }

@ -337,8 +337,6 @@ type FilterDependencies struct {
 // allAddresses optimization indicates if the passed addresses include all the owners in the wallet DB
 //
 // Adding a no-limit option was never considered or required.
-//
-// TODO: optimization: consider implementing nullable []byte instead of using strings for addresses or insert binary (X'...' syntax) directly into the query
 func getActivityEntries(ctx context.Context, deps FilterDependencies, addresses []eth.Address, allAddresses bool, chainIDs []common.ChainID, filter Filter, offset int, limit int) ([]Entry, error) {
 	if len(addresses) == 0 {
 		return nil, errors.New("no addresses provided")
@ -367,8 +365,7 @@ func getActivityEntries(ctx context.Context, deps FilterDependencies, addresses
 		if sliceChecksCondition(filter.Assets, func(item *Token) bool { return item.TokenType == Erc20 }) {
 			assetsERC20 = joinItems(filter.Assets, func(item Token) string {
 				if item.TokenType == Erc20 {
-					// SQL HEX() (Blob->Hex) conversion returns uppercase digits with no 0x prefix
-					return fmt.Sprintf("%d, '%s'", item.ChainID, strings.ToUpper(item.Address.Hex()[2:]))
+					return fmt.Sprintf("%d, X'%s'", item.ChainID, item.Address.Hex()[2:])
 				}
 				return ""
 			})
@ -379,14 +376,13 @@ func getActivityEntries(ctx context.Context, deps FilterDependencies, addresses
 	assetsERC721 := noEntriesInThreeColumnsTmpTableSQLValues
 	if !includeAllCollectibles && !filter.FilterOutCollectibles {
 		assetsERC721 = joinItems(filter.Collectibles, func(item Token) string {
-			// SQL HEX() (Blob->Hex) conversion returns uppercase digits with no 0x prefix
-			tokenID := strings.ToUpper(item.TokenID.String()[2:])
-			address := strings.ToUpper(item.Address.Hex()[2:])
+			tokenID := item.TokenID.String()[2:]
+			address := item.Address.Hex()[2:]
+			// SQLite mandates that byte length is an even number which hexutil.EncodeBig doesn't guarantee
 			if len(tokenID)%2 == 1 {
-				// Hex length must be divisable by 2, otherwise append '0' at the beginning
 				tokenID = "0" + tokenID
 			}
-			return fmt.Sprintf("%d, '%s', '%s'", item.ChainID, tokenID, address)
+			return fmt.Sprintf("%d, X'%s', X'%s'", item.ChainID, tokenID, address)
 		})
 	}

@ -436,7 +432,7 @@ func getActivityEntries(ctx context.Context, deps FilterDependencies, addresses
 	// The duplicated temporary table UNION with CTE acts as an optimization
 	// As soon as we use filter_addresses CTE or filter_addresses_table temp table
 	// or switch them alternatively for JOIN or IN clauses the performance drops significantly
-	_, err := deps.db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS filter_addresses_table; CREATE TEMP TABLE filter_addresses_table (address VARCHAR PRIMARY KEY); INSERT OR IGNORE INTO filter_addresses_table (address) VALUES %s;\n", involvedAddresses))
+	_, err := deps.db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS filter_addresses_table; CREATE TEMP TABLE filter_addresses_table (address VARCHAR PRIMARY KEY); INSERT INTO filter_addresses_table (address) VALUES %s;\n", involvedAddresses))
 	if err != nil {
 		return nil, err
 	}
--- a/services/wallet/activity/activity_test.go
+++ b/services/wallet/activity/activity_test.go
@ -638,8 +638,6 @@ func TestGetActivityEntriesFilterByType(t *testing.T) {
 	multiTxs[3] = transfer.GenerateTestBridgeMultiTransaction(trs[6], trs[7])
 	multiTxs[4] = transfer.GenerateTestSendMultiTransaction(trs[8]) // trs[9]

-	allAddresses := append(append(append(tdFromAdds, tdToAddrs...), fromAddrs...), toAddrs...)
-
 	var lastMT transfer.MultiTransactionIDType
 	for i := range trs {
 		if i%2 == 0 {
@ -649,7 +647,11 @@ func TestGetActivityEntriesFilterByType(t *testing.T) {
 		transfer.InsertTestTransfer(t, deps.db, trs[i].To, &trs[i])
 	}

-	trsSpecial, _, _ := transfer.GenerateTestTransfers(t, deps.db, 100, 2)
+	trsSpecial, fromSpecial, toSpecial := transfer.GenerateTestTransfers(t, deps.db, 100, 2)
+
+	// Here not to include the modified To and From addresses
+	allAddresses := append(append(append(append(append(tdFromAdds, tdToAddrs...), fromAddrs...), toAddrs...), fromSpecial...), toSpecial...)
+
 	// Insert MintAT
 	trsSpecial[0].From = eth.HexToAddress("0x0")
 	transfer.InsertTestTransferWithOptions(t, deps.db, trsSpecial[0].To, &trsSpecial[0], &transfer.TestTransferOptions{
@ -731,6 +733,7 @@ func TestGetActivityEntriesFilterByType(t *testing.T) {
 	filter.Types = []Type{SendAT}
 	entries, err = getActivityEntries(context.Background(), deps, allAddresses, true, []common.ChainID{}, filter, 0, 15)
 	require.NoError(t, err)
+	// We have 6 but one is not matched because is a receive, having owner the to address
 	require.Equal(t, 5, len(entries))
 }

@ -1294,151 +1297,3 @@ func TestGetMultiTxDetails(t *testing.T) {
 	require.Equal(t, td.multiTx1Tr2.BlkNumber, details.BlockNumber)
 	require.Equal(t, td.multiTx1Tr1.Contract, *details.Contract)
 }
-
-func setupBenchmark(b *testing.B, inMemory bool, resultCount int) (deps FilterDependencies, close func(), accounts []eth.Address) {
-	deps, close = setupTestActivityDBStorageChoice(b, inMemory)
-
-	const transactionCount = 100000
-	const mtSendRatio = 0.2   // 20%
-	const mtSwapRatio = 0.1   // 10%
-	const mtBridgeRatio = 0.1 // 10%
-	const pendingCount = 10
-	const mtSendCount = int(float64(transactionCount) * mtSendRatio)
-	const mtSwapCount = int(float64(transactionCount) * mtSwapRatio)
-	// Bridge requires two transactions
-	const mtBridgeCount = int(float64(transactionCount) * (mtBridgeRatio / 2))
-
-	trs, _, _ := transfer.GenerateTestTransfers(b, deps.db, 0, transactionCount)
-
-	accounts = []eth.Address{trs[0].From, trs[1].From, trs[2].From, trs[3].To, trs[4].To, trs[5].To}
-
-	i := 0
-	multiTxs := make([]transfer.TestMultiTransaction, mtSendCount+mtSwapCount+mtBridgeCount)
-	for ; i < mtSendCount; i++ {
-		multiTxs[i] = transfer.GenerateTestSendMultiTransaction(trs[i])
-		trs[i].From = accounts[i%len(accounts)]
-		multiTxs[i].FromAddress = trs[i].From
-
-		multiTxs[i].MultiTransactionID = transfer.InsertTestMultiTransaction(b, deps.db, &multiTxs[i])
-		trs[i].MultiTransactionID = multiTxs[i].MultiTransactionID
-	}
-
-	for j := 0; j < mtSwapCount; i, j = i+1, j+1 {
-		multiTxs[i] = transfer.GenerateTestSwapMultiTransaction(trs[i], testutils.SntSymbol, int64(i))
-		trs[i].From = accounts[i%len(accounts)]
-		multiTxs[i].FromAddress = trs[i].From
-
-		multiTxs[i].MultiTransactionID = transfer.InsertTestMultiTransaction(b, deps.db, &multiTxs[i])
-		trs[i].MultiTransactionID = multiTxs[i].MultiTransactionID
-	}
-
-	for mtIdx := 0; mtIdx < mtBridgeCount; i, mtIdx = i+2, mtIdx+1 {
-		firstTrIdx := i
-		secondTrIdx := i + 1
-		multiTxs[mtIdx] = transfer.GenerateTestBridgeMultiTransaction(trs[firstTrIdx], trs[secondTrIdx])
-		trs[firstTrIdx].From = accounts[i%len(accounts)]
-		trs[secondTrIdx].To = accounts[(i+3)%len(accounts)]
-		multiTxs[mtIdx].FromAddress = trs[firstTrIdx].From
-		multiTxs[mtIdx].ToAddress = trs[secondTrIdx].To
-		multiTxs[mtIdx].FromAddress = trs[i].From
-
-		multiTxs[mtIdx].MultiTransactionID = transfer.InsertTestMultiTransaction(b, deps.db, &multiTxs[mtIdx])
-		trs[firstTrIdx].MultiTransactionID = multiTxs[mtIdx].MultiTransactionID
-		trs[secondTrIdx].MultiTransactionID = multiTxs[mtIdx].MultiTransactionID
-	}
-
-	for i = 0; i < transactionCount-pendingCount; i++ {
-		trs[i].From = accounts[i%len(accounts)]
-		transfer.InsertTestTransfer(b, deps.db, trs[i].From, &trs[i])
-	}
-
-	for ; i < transactionCount; i++ {
-		trs[i].From = accounts[i%len(accounts)]
-		transfer.InsertTestPendingTransaction(b, deps.db, &trs[i])
-	}
-
-	return
-}
-
-func BenchmarkGetActivityEntries(bArg *testing.B) {
-	type params struct {
-		inMemory               bool
-		resultCount            int
-		generateTestParameters func([]eth.Address) (addresses []eth.Address, allAddresses bool, filter *Filter, startIndex int)
-	}
-	testCases := []struct {
-		name   string
-		params params
-	}{
-		{
-			"RAM_NoFilter",
-			params{
-				true,
-				10,
-				func(addresses []eth.Address) ([]eth.Address, bool, *Filter, int) {
-					return addresses, true, &Filter{}, 0
-				},
-			},
-		},
-		{
-			"SSD_NoFilter",
-			params{
-				false,
-				10,
-				func(addresses []eth.Address) ([]eth.Address, bool, *Filter, int) {
-					return addresses, true, &Filter{}, 0
-				},
-			},
-		},
-		{
-			"SSD_MovingWindow",
-			params{
-				false,
-				10,
-				func(addresses []eth.Address) ([]eth.Address, bool, *Filter, int) {
-					return addresses, true, &Filter{}, 200
-				},
-			},
-		},
-		{
-			"SSD_AllAddresses_AllTos",
-			params{
-				false,
-				10,
-				func(addresses []eth.Address) ([]eth.Address, bool, *Filter, int) {
-					return addresses, true, &Filter{CounterpartyAddresses: addresses[3:]}, 0
-				},
-			},
-		},
-		{
-			"SSD_OneAddress",
-			params{
-				false,
-				10,
-				func(addresses []eth.Address) ([]eth.Address, bool, *Filter, int) {
-					return addresses[0:1], false, &Filter{}, 0
-				},
-			},
-		},
-	}
-
-	deps, closeFn, accounts := setupBenchmark(bArg, true, 10)
-	defer closeFn()
-
-	const resultCount = 10
-	for _, tc := range testCases {
-		addresses, allAddresses, filter, startIndex := tc.params.generateTestParameters(accounts)
-		bArg.Run(tc.name, func(b *testing.B) {
-			// Reset timer after setup
-			b.ResetTimer()
-
-			// Run benchmark
-			for i := 0; i < b.N; i++ {
-				res, err := getActivityEntries(context.Background(), deps, addresses, allAddresses, allNetworksFilter(), *filter, startIndex, resultCount)
-				if err != nil || len(res) != resultCount {
-					b.Error(err)
-				}
-			}
-		})
-	}
-}
--- a/services/wallet/activity/benchmarks_test.go
+++ b/services/wallet/activity/benchmarks_test.go
@ -0,0 +1,303 @@
+package activity
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	eth "github.com/ethereum/go-ethereum/common"
+	"github.com/status-im/status-go/services/wallet/common"
+	"github.com/status-im/status-go/services/wallet/testutils"
+	"github.com/status-im/status-go/services/wallet/transfer"
+)
+
+func setupBenchmark(b *testing.B, accountsCount int, inMemory bool) (deps FilterDependencies, close func(), accounts []eth.Address) {
+	deps, close = setupTestActivityDBStorageChoice(b, inMemory)
+
+	const transactionCount = 100000
+	const mtSendRatio = 0.2   // 20%
+	const mtSwapRatio = 0.1   // 10%
+	const mtBridgeRatio = 0.1 // 10%
+	const pendingCount = 10
+	const mtSendCount = int(float64(transactionCount) * mtSendRatio)
+	const mtSwapCount = int(float64(transactionCount) * mtSwapRatio)
+	// Bridge requires two transactions
+	const mtBridgeCount = int(float64(transactionCount) * (mtBridgeRatio / 2))
+
+	trs, _, _ := transfer.GenerateTestTransfers(b, deps.db, 0, transactionCount)
+
+	accounts = []eth.Address{}
+	for i := 0; i < accountsCount; i++ {
+		if i%2 == 0 {
+			accounts = append(accounts, trs[i].From)
+		} else {
+			accounts = append(accounts, trs[i].To)
+		}
+	}
+
+	i := 0
+	multiTxs := make([]transfer.TestMultiTransaction, mtSendCount+mtSwapCount+mtBridgeCount)
+	for ; i < mtSendCount; i++ {
+		multiTxs[i] = transfer.GenerateTestSendMultiTransaction(trs[i])
+		trs[i].From = accounts[i%len(accounts)]
+		multiTxs[i].FromAddress = trs[i].From
+		// Currently the network ID is not filled in for send transactions
+		multiTxs[i].FromNetworkID = nil
+		multiTxs[i].ToNetworkID = nil
+
+		multiTxs[i].MultiTransactionID = transfer.InsertTestMultiTransaction(b, deps.db, &multiTxs[i])
+		trs[i].MultiTransactionID = multiTxs[i].MultiTransactionID
+	}
+
+	for j := 0; j < mtSwapCount; i, j = i+1, j+1 {
+		multiTxs[i] = transfer.GenerateTestSwapMultiTransaction(trs[i], testutils.SntSymbol, int64(i))
+		trs[i].From = accounts[i%len(accounts)]
+		multiTxs[i].FromAddress = trs[i].From
+
+		multiTxs[i].MultiTransactionID = transfer.InsertTestMultiTransaction(b, deps.db, &multiTxs[i])
+		trs[i].MultiTransactionID = multiTxs[i].MultiTransactionID
+	}
+
+	for mtIdx := 0; mtIdx < mtBridgeCount; i, mtIdx = i+2, mtIdx+1 {
+		firstTrIdx := i
+		secondTrIdx := i + 1
+		multiTxs[mtIdx] = transfer.GenerateTestBridgeMultiTransaction(trs[firstTrIdx], trs[secondTrIdx])
+		trs[firstTrIdx].From = accounts[i%len(accounts)]
+		trs[secondTrIdx].To = accounts[(i+3)%len(accounts)]
+		multiTxs[mtIdx].FromAddress = trs[firstTrIdx].From
+		multiTxs[mtIdx].ToAddress = trs[secondTrIdx].To
+		multiTxs[mtIdx].FromAddress = trs[i].From
+
+		multiTxs[mtIdx].MultiTransactionID = transfer.InsertTestMultiTransaction(b, deps.db, &multiTxs[mtIdx])
+		trs[firstTrIdx].MultiTransactionID = multiTxs[mtIdx].MultiTransactionID
+		trs[secondTrIdx].MultiTransactionID = multiTxs[mtIdx].MultiTransactionID
+	}
+
+	for i = 0; i < transactionCount-pendingCount; i++ {
+		trs[i].From = accounts[i%len(accounts)]
+		transfer.InsertTestTransfer(b, deps.db, trs[i].From, &trs[i])
+	}
+
+	for ; i < transactionCount; i++ {
+		trs[i].From = accounts[i%len(accounts)]
+		transfer.InsertTestPendingTransaction(b, deps.db, &trs[i])
+	}
+
+	return
+}
+
+var allNetEnabled = []common.ChainID(nil)
+
+func BenchmarkGetActivityEntries(bArg *testing.B) {
+	deps, closeFn, accounts := setupBenchmark(bArg, 6, true)
+	defer closeFn()
+
+	type params struct {
+		inMemory bool
+		// resultCount must be nil to expect as many requested
+		resultCount            *int
+		generateTestParameters func() (addresses []eth.Address, allAddresses bool, networks []common.ChainID, filter *Filter, startIndex int)
+	}
+	testCases := []struct {
+		name   string
+		params params
+	}{
+		{
+			"RAM_NoFilter",
+			params{
+				true,
+				nil,
+				func() ([]eth.Address, bool, []common.ChainID, *Filter, int) {
+					return accounts, true, allNetEnabled, &Filter{}, 0
+				},
+			},
+		},
+		{
+			"SSD_NoFilter",
+			params{
+				false,
+				nil,
+				func() ([]eth.Address, bool, []common.ChainID, *Filter, int) {
+					return accounts, true, allNetEnabled, &Filter{}, 0
+				},
+			},
+		},
+		{
+			"SSD_MovingWindow",
+			params{
+				false,
+				nil,
+				func() ([]eth.Address, bool, []common.ChainID, *Filter, int) {
+					return accounts, true, allNetEnabled, &Filter{}, 200
+				},
+			},
+		},
+		{
+			"SSD_AllAddr_AllTos",
+			params{
+				false,
+				nil,
+				func() ([]eth.Address, bool, []common.ChainID, *Filter, int) {
+					return accounts, true, allNetEnabled, &Filter{CounterpartyAddresses: accounts[3:]}, 0
+				},
+			},
+		},
+		{
+			"SSD_OneAddress",
+			params{
+				false,
+				nil,
+				func() ([]eth.Address, bool, []common.ChainID, *Filter, int) {
+					return accounts[0:1], false, allNetEnabled, &Filter{}, 0
+				},
+			},
+		},
+		// All memory from here
+		{
+			"FilterSend_AllAddr",
+			params{
+				true,
+				nil,
+				func() ([]eth.Address, bool, []common.ChainID, *Filter, int) {
+					return accounts, true, allNetEnabled, &Filter{
+						Types: []Type{SendAT},
+					}, 0
+				},
+			},
+		},
+		{
+			"FilterSend_6Addr",
+			params{
+				true,
+				nil,
+				func() ([]eth.Address, bool, []common.ChainID, *Filter, int) {
+					return accounts[len(accounts)-6:], false, allNetEnabled, &Filter{
+						Types: []Type{SendAT},
+					}, 0
+				},
+			},
+		},
+		{
+			"FilterThreeNetworks",
+			params{
+				true,
+				nil,
+				func() ([]eth.Address, bool, []common.ChainID, *Filter, int) {
+					return accounts, true, []common.ChainID{}, &Filter{}, 0
+				},
+			},
+		},
+	}
+
+	const resultCount = 100
+	for _, tc := range testCases {
+		addresses, allAddresses, nets, filter, startIndex := tc.params.generateTestParameters()
+		networks := allNetworksFilter()
+		if len(nets) > 0 {
+			networks = nets
+		}
+
+		bArg.Run(tc.name, func(b *testing.B) {
+			// Reset timer after setup
+			b.ResetTimer()
+
+			// Run benchmark
+			for i := 0; i < b.N; i++ {
+				res, err := getActivityEntries(context.Background(), deps, addresses, allAddresses, networks, *filter, startIndex, resultCount)
+				if err != nil {
+					b.Error(err)
+				} else if tc.params.resultCount != nil {
+					if len(res) != *tc.params.resultCount {
+						b.Error("Got less then expected")
+					}
+				} else if len(res) != resultCount {
+					b.Error("Got less than requested")
+				}
+			}
+		})
+	}
+}
+
+func BenchmarkSQLQuery(b *testing.B) {
+	type params struct {
+		query            string
+		args             []interface{}
+		expectedResCount int
+	}
+
+	deps, closeFn, accounts := setupBenchmark(b, 10000, true)
+	defer closeFn()
+
+	var addrValuesStr, insertAddrValuesStr, addrPlaceholdersStr string
+	var refAccounts []interface{}
+	for _, acc := range accounts {
+		addrValuesStr += fmt.Sprintf("X'%s',", acc.Hex()[2:])
+		insertAddrValuesStr += fmt.Sprintf("(X'%s'),", acc.Hex()[2:])
+		addrPlaceholdersStr += "?,"
+		refAccounts = append(refAccounts, acc)
+	}
+	addrValuesStr = addrValuesStr[:len(addrValuesStr)-1]
+	insertAddrValuesStr = insertAddrValuesStr[:len(insertAddrValuesStr)-1]
+	addrPlaceholdersStr = addrPlaceholdersStr[:len(addrPlaceholdersStr)-1]
+
+	if _, err := deps.db.Exec(fmt.Sprintf("CREATE TEMP TABLE filter_addresses_table (address VARCHAR PRIMARY KEY); INSERT INTO filter_addresses_table (address) VALUES %s;", insertAddrValuesStr)); err != nil {
+		b.Fatal("failed to create temporary table", err)
+	}
+
+	testCases := []struct {
+		name string
+		args params
+		res  testing.BenchmarkResult
+	}{
+		{
+			name: "JoinQuery",
+			args: params{
+				query:            "SELECT COUNT(*) FROM transfers JOIN filter_addresses_table ON transfers.tx_from_address = filter_addresses_table.address",
+				expectedResCount: 99990,
+			},
+		},
+		{
+			name: "LiteralQuery",
+			args: params{
+				query:            fmt.Sprintf("SELECT COUNT(*) FROM transfers WHERE tx_from_address IN (%s)", addrValuesStr),
+				expectedResCount: 99990,
+			},
+		},
+		{
+			name: "ParamQuery",
+			args: params{
+				query:            fmt.Sprintf("SELECT COUNT(*) FROM transfers WHERE tx_from_address IN (%s)", addrPlaceholdersStr),
+				args:             refAccounts,
+				expectedResCount: 99990,
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		b.Run(tc.name, func(b *testing.B) {
+			b.ResetTimer()
+
+			for i := 0; i < b.N; i++ {
+				b.ResetTimer()
+
+				for i := 0; i < b.N; i++ {
+					res, err := deps.db.Query(tc.args.query, tc.args.args...)
+					if err != nil {
+						b.Fatal("failed to query db", err)
+					}
+					res.Next()
+
+					var count int
+					if err := res.Scan(&count); err != nil {
+						b.Fatal("failed to scan db result", err)
+					}
+					if count != tc.args.expectedResCount {
+						b.Fatalf("unexpected result count: %d, expected: %d", count, tc.args.expectedResCount)
+					}
+
+					res.Close()
+				}
+			}
+		})
+	}
+}
--- a/services/wallet/activity/filter.go
+++ b/services/wallet/activity/filter.go
@ -166,7 +166,7 @@ func GetOldestTimestamp(ctx context.Context, db *sql.DB, addresses []eth.Address
 			transfers.timestamp AS timestamp
 		FROM transfers, filter_conditions
 		WHERE transfers.multi_transaction_id = 0
-			AND (filterAllAddresses OR HEX(from_address) IN filter_addresses OR HEX(to_address) IN filter_addresses)
+			AND (filterAllAddresses OR from_address IN filter_addresses OR to_address IN filter_addresses)

 		UNION ALL

@ -176,7 +176,7 @@ func GetOldestTimestamp(ctx context.Context, db *sql.DB, addresses []eth.Address
 			pending_transactions.timestamp AS timestamp
 		FROM pending_transactions, filter_conditions
 		WHERE pending_transactions.multi_transaction_id = 0
-			AND (filterAllAddresses OR HEX(from_address) IN filter_addresses OR HEX(to_address) IN filter_addresses)
+			AND (filterAllAddresses OR from_address IN filter_addresses OR to_address IN filter_addresses)

 		UNION ALL

@ -185,7 +185,7 @@ func GetOldestTimestamp(ctx context.Context, db *sql.DB, addresses []eth.Address
 			multi_transactions.to_address AS to_address,
 			multi_transactions.timestamp AS timestamp
 		FROM multi_transactions, filter_conditions
-		WHERE filterAllAddresses OR HEX(from_address) IN filter_addresses OR HEX(to_address) IN filter_addresses
+		WHERE filterAllAddresses OR from_address IN filter_addresses OR to_address IN filter_addresses
 		ORDER BY timestamp ASC
 		LIMIT 1`

--- a/services/wallet/activity/filter.sql
+++ b/services/wallet/activity/filter.sql
@ -44,7 +44,7 @@ WITH filter_conditions AS (
 		? AS nowTimestamp,
 		? AS layer2FinalisationDuration,
 		? AS layer1FinalisationDuration,
-		'0000000000000000000000000000000000000000' AS zeroAddress
+		X'0000000000000000000000000000000000000000' AS zeroAddress
 ),
 -- This UNION between CTE and TEMP TABLE acts as an optimization. As soon as we drop one or use them interchangeably the performance drops significantly.
 filter_addresses(address) AS (
@ -150,7 +150,8 @@ pending_network_ids AS (
 		pending_transactions.multi_transaction_id
 ),
 layer2_networks(network_id) AS (
-	VALUES %s
+	VALUES
+		%s
 )
 SELECT
 	transfers.hash AS transfer_hash,
@ -178,16 +179,16 @@ SELECT
 	NULL AS mt_from_amount,
 	NULL AS mt_to_amount,
 	CASE
-		WHEN transfers.status IS 1 THEN 
-			CASE
-				WHEN transfers.timestamp > 0 AND filter_conditions.nowTimestamp >= transfers.timestamp + 
-					(CASE
-						WHEN transfers.network_id in layer2_networks THEN layer2FinalisationDuration
-						ELSE layer1FinalisationDuration
-					END) 
-				THEN statusFinalized
-				ELSE statusCompleted
-			END
+		WHEN transfers.status IS 1 THEN CASE
+			WHEN transfers.timestamp > 0
+			AND filter_conditions.nowTimestamp >= transfers.timestamp + (
+				CASE
+					WHEN transfers.network_id in layer2_networks THEN layer2FinalisationDuration
+					ELSE layer1FinalisationDuration
+				END
+			) THEN statusFinalized
+			ELSE statusCompleted
+		END
 		ELSE statusFailed
 	END AS agg_status,
 	1 AS agg_count,
@ -201,10 +202,12 @@ SELECT
 	transfers.type AS type,
 	transfers.contract_address AS contract_address
 FROM
-	transfers,
-	filter_conditions
-	LEFT JOIN filter_addresses from_join ON HEX(transfers.address) = from_join.address
-	LEFT JOIN filter_addresses to_join ON HEX(transfers.tx_to_address) = to_join.address
+	transfers
+	CROSS JOIN filter_conditions
+	INNER JOIN filter_to_addresses receiver_join ON filterAllToAddresses != 0
+	OR transfers.tx_to_address = receiver_join.address
+	LEFT JOIN filter_addresses from_join ON transfers.tx_from_address = from_join.address
+	LEFT JOIN filter_addresses to_join ON transfers.tx_to_address = to_join.address
 WHERE
 	transfers.loaded == 1
 	AND transfers.multi_transaction_id = 0
@ -223,24 +226,22 @@ WHERE
 		filterActivityTypeAll
 		OR (
 			filterActivityTypeSend
-			AND tr_type = fromTrType
+			AND tr_type = fromTrType -- Check NOT ContractDeploymentAT
 			AND NOT (
-				tr_type = fromTrType
-				and transfers.tx_to_address IS NULL
+				transfers.tx_to_address IS NULL
 				AND transfers.type = 'eth'
 				AND transfers.contract_address IS NOT NULL
-				AND HEX(transfers.contract_address) != zeroAddress
+				AND transfers.contract_address != zeroAddress
 			)
 		)
 		OR (
 			filterActivityTypeReceive
-			AND tr_type = toTrType
+			AND tr_type = toTrType -- Check NOT MintAT
 			AND NOT (
-				tr_type = toTrType
-				AND transfers.type = 'erc721'
+				transfers.type = 'erc721'
 				AND (
 					transfers.tx_from_address IS NULL
-					OR HEX(transfers.tx_from_address) = zeroAddress
+					OR transfers.tx_from_address = zeroAddress
 				)
 			)
 		)
@ -250,11 +251,7 @@ WHERE
 			AND transfers.tx_to_address IS NULL
 			AND transfers.type = 'eth'
 			AND transfers.contract_address IS NOT NULL
-			AND HEX(transfers.contract_address) != zeroAddress
-			AND (
-				filterAllAddresses
-				OR (HEX(transfers.address) IN filter_addresses)
-			)
+			AND transfers.contract_address != zeroAddress
 		)
 		OR (
 			filterActivityTypeMint
@ -262,23 +259,13 @@ WHERE
 			AND transfers.type = 'erc721'
 			AND (
 				transfers.tx_from_address IS NULL
-				OR HEX(transfers.tx_from_address) = zeroAddress
-			)
-			AND (
-				filterAllAddresses
-				OR (HEX(transfers.address) IN filter_addresses)
+				OR transfers.tx_from_address = zeroAddress
 			)
 		)
 	)
 	AND (
-		filterAllAddresses
-		OR (HEX(owner_address) IN filter_addresses)
-	)
-	AND (
-		filterAllToAddresses
-		OR (
-			HEX(transfers.tx_to_address) IN filter_to_addresses
-		)
+		filterAllAddresses -- Every account address has an "owned" entry either as to or from
+		OR (owner_address IN filter_addresses)
 	)
 	AND (
 		includeAllTokenTypeAssets
@ -291,7 +278,7 @@ WHERE
 			AND (
 				(
 					transfers.network_id,
-					HEX(transfers.token_address)
+					transfers.token_address
 				) IN assets_erc20
 			)
 		)
@ -303,8 +290,8 @@ WHERE
 			AND (
 				(
 					transfers.network_id,
-					HEX(transfers.token_id),
-					HEX(transfers.token_address)
+					transfers.token_id,
+					transfers.token_address
 				) IN assets_erc721
 			)
 		)
@ -320,7 +307,7 @@ WHERE
 			AND agg_status = statusCompleted
 		)
 		OR (
-			filterStatusFinalized 
+			filterStatusFinalized
 			AND agg_status = statusFinalized
 		)
 		OR (
@ -367,10 +354,12 @@ SELECT
 	pending_transactions.type AS type,
 	NULL as contract_address
 FROM
-	pending_transactions,
-	filter_conditions
-	LEFT JOIN filter_addresses from_join ON HEX(pending_transactions.from_address) = from_join.address
-	LEFT JOIN filter_addresses to_join ON HEX(pending_transactions.to_address) = to_join.address
+	pending_transactions
+	CROSS JOIN filter_conditions
+	INNER JOIN filter_to_addresses receiver_join ON filterAllToAddresses != 0
+	OR pending_transactions.to_address = receiver_join.address
+	LEFT JOIN filter_addresses from_join ON pending_transactions.from_address = from_join.address
+	LEFT JOIN filter_addresses to_join ON pending_transactions.to_address = to_join.address
 WHERE
 	pending_transactions.multi_transaction_id = 0
 	AND pending_transactions.status = pendingStatus
@ -395,18 +384,7 @@ WHERE
 	)
 	AND (
 		filterAllAddresses
-		OR (
-			HEX(pending_transactions.from_address) IN filter_addresses
-		)
-		OR (
-			HEX(pending_transactions.to_address) IN filter_addresses
-		)
-	)
-	AND (
-		filterAllToAddresses
-		OR (
-			HEX(pending_transactions.to_address) IN filter_to_addresses
-		)
+		OR tr_type NOT NULL
 	)
 	AND (
 		includeAllTokenTypeAssets
@ -437,15 +415,16 @@ SELECT
 	multi_transactions.from_amount AS mt_from_amount,
 	multi_transactions.to_amount AS mt_to_amount,
 	CASE
-	WHEN tr_status.min_status = 1 AND COALESCE(pending_status.count, 0) = 0 THEN 
-		CASE
-			WHEN multi_transactions.timestamp > 0 AND filter_conditions.nowTimestamp >= multi_transactions.timestamp + 
-				(CASE
-					WHEN multi_transactions.from_network_id in layer2_networks 
-							OR multi_transactions.to_network_id in layer2_networks THEN layer2FinalisationDuration
+		WHEN tr_status.min_status = 1
+		AND COALESCE(pending_status.count, 0) = 0 THEN CASE
+			WHEN multi_transactions.timestamp > 0
+			AND filter_conditions.nowTimestamp >= multi_transactions.timestamp + (
+				CASE
+					WHEN multi_transactions.from_network_id in layer2_networks
+					OR multi_transactions.to_network_id in layer2_networks THEN layer2FinalisationDuration
 					ELSE layer1FinalisationDuration
-				END)
-			THEN statusFinalized
+				END
+			) THEN statusFinalized
 			ELSE statusCompleted
 		END
 		WHEN tr_status.min_status = 0 THEN statusFailed
@ -462,8 +441,10 @@ SELECT
 	NULL AS type,
 	NULL as contract_address
 FROM
-	multi_transactions,
-	filter_conditions
+	multi_transactions
+	CROSS JOIN filter_conditions
+	INNER JOIN filter_to_addresses receiver_join ON filterAllToAddresses != 0
+	OR multi_transactions.to_address = receiver_join.address
 	LEFT JOIN tr_status ON multi_transactions.ROWID = tr_status.multi_transaction_id
 	LEFT JOIN pending_status ON multi_transactions.ROWID = pending_status.multi_transaction_id
 WHERE
@ -487,22 +468,16 @@ WHERE
 		OR (
 			-- Send multi-transaction types are exclusively for outbound transfers. The receiving end will have a corresponding entry as "owner_address" in the transfers table.
 			mt_type = mTTypeSend
-			AND HEX(owner_address) IN filter_addresses
+			AND owner_address IN filter_addresses
 		)
 		OR (
 			mt_type != mTTypeSend
 			AND (
-				HEX(multi_transactions.from_address) IN filter_addresses
-				OR HEX(multi_transactions.to_address) IN filter_addresses
+				multi_transactions.from_address IN filter_addresses
+				OR multi_transactions.to_address IN filter_addresses
 			)
 		)
 	)
-	AND (
-		filterAllToAddresses
-		OR (
-			HEX(multi_transactions.to_address) IN filter_to_addresses
-		)
-	)
 	AND (
 		includeAllTokenTypeAssets
 		OR (
@ -521,11 +496,11 @@ WHERE
 	AND (
 		filterAllActivityStatus
 		OR (
-			filterStatusCompleted 
-			AND agg_status = statusCompleted 
+			filterStatusCompleted
+			AND agg_status = statusCompleted
 		)
 		OR (
-			filterStatusFinalized 
+			filterStatusFinalized
 			AND agg_status = statusFinalized
 		)
 		OR (
--- a/services/wallet/transfer/testutils.go
+++ b/services/wallet/transfer/testutils.go
@ -68,14 +68,15 @@ func TestTrToToken(t *testing.T, tt *TestTransaction) (token *token.Token, isNat
 func generateTestTransaction(seed int) TestTransaction {
 	token := SeedToToken(seed)
 	return TestTransaction{
-		Hash:               eth_common.HexToHash(fmt.Sprintf("0x1%d", seed)),
-		ChainID:            common.ChainID(token.ChainID),
-		From:               eth_common.HexToAddress(fmt.Sprintf("0x2%d", seed)),
-		Timestamp:          int64(seed),
-		BlkNumber:          int64(seed),
-		Success:            true,
-		Nonce:              uint64(seed),
-		Contract:           eth_common.HexToAddress(fmt.Sprintf("0x2%d", seed)),
+		Hash:      eth_common.HexToHash(fmt.Sprintf("0x1%d", seed)),
+		ChainID:   common.ChainID(token.ChainID),
+		From:      eth_common.HexToAddress(fmt.Sprintf("0x2%d", seed)),
+		Timestamp: int64(seed),
+		BlkNumber: int64(seed),
+		Success:   true,
+		Nonce:     uint64(seed),
+		// In practice this is last20Bytes(Keccak256(RLP(From, nonce)))
+		Contract:           eth_common.HexToAddress(fmt.Sprintf("0x4%d", seed)),
 		MultiTransactionID: NoMultiTransactionID,
 	}
 }