From d29c6c5b6f480da8085de95b813e401204860f12 Mon Sep 17 00:00:00 2001 From: Cuteivist Date: Wed, 20 Sep 2023 10:30:31 +0200 Subject: [PATCH] feat: Calculate finalize status and filter by it (#3969) --- services/wallet/activity/activity.go | 14 +++++- services/wallet/activity/activity_test.go | 49 +++++++++++-------- services/wallet/activity/filter.sql | 57 +++++++++++++++++------ services/wallet/activity/service.go | 4 ++ 4 files changed, 87 insertions(+), 37 deletions(-) diff --git a/services/wallet/activity/activity.go b/services/wallet/activity/activity.go index 729e3647d..b68860c1a 100644 --- a/services/wallet/activity/activity.go +++ b/services/wallet/activity/activity.go @@ -324,6 +324,8 @@ type FilterDependencies struct { tokenSymbol func(token Token) string // use the chainID and symbol to look up token.TokenType and token.Address. Return nil if not found tokenFromSymbol func(chainID *common.ChainID, symbol string) *Token + // use to get current timestamp + currentTimestamp func() int64 } // getActivityEntries queries the transfers, pending_transactions, and multi_transactions tables based on filter parameters and arguments @@ -379,6 +381,11 @@ func getActivityEntries(ctx context.Context, deps FilterDependencies, addresses networks = joinItems(chainIDs, nil) } + layer2Chains := []uint64{common.OptimismMainnet, common.OptimismGoerli, common.ArbitrumMainnet, common.ArbitrumGoerli} + layer2Networks := joinItems(layer2Chains, func(chainID uint64) string { + return fmt.Sprintf("%d", chainID) + }) + startFilterDisabled := !(filter.Period.StartTimestamp > 0) endFilterDisabled := !(filter.Period.EndTimestamp > 0) filterActivityTypeAll := len(filter.Types) == 0 @@ -408,7 +415,7 @@ func getActivityEntries(ctx context.Context, deps FilterDependencies, addresses }) queryString := fmt.Sprintf(queryFormatString, involvedAddresses, toAddresses, assetsTokenCodes, assetsERC20, networks, - joinedMTTypes) + layer2Networks, joinedMTTypes) // The duplicated temporary table UNION with CTE acts as an optimization // As soon as we use filter_addresses CTE or filter_addresses_table temp table @@ -426,10 +433,13 @@ func getActivityEntries(ctx context.Context, deps FilterDependencies, addresses fromTrType, toTrType, allAddresses, filterAllToAddresses, includeAllStatuses, filterStatusCompleted, filterStatusFailed, filterStatusFinalized, filterStatusPending, - FailedAS, CompleteAS, PendingAS, + FailedAS, CompleteAS, FinalizedAS, PendingAS, includeAllTokenTypeAssets, includeAllNetworks, transactions.Pending, + deps.currentTimestamp(), + 648000, // 7.5 days in seconds for layer 2 finalization. 0.5 day is buffer to not create false positive. + 960, // A block on layer 1 is every 12s, finalization require 64 blocks. A buffer of 16 blocks is added to not create false positives. limit, offset) if err != nil { return nil, err diff --git a/services/wallet/activity/activity_test.go b/services/wallet/activity/activity_test.go index 186d96685..c73890222 100644 --- a/services/wallet/activity/activity_test.go +++ b/services/wallet/activity/activity_test.go @@ -5,6 +5,7 @@ import ( "database/sql" "math/big" "testing" + "time" "github.com/status-im/status-go/services/wallet/common" "github.com/status-im/status-go/services/wallet/testutils" @@ -18,6 +19,8 @@ import ( "github.com/stretchr/testify/require" ) +var mockupTime = time.Unix(946724400, 0) // 2000-01-01 12:00:00 + func tokenFromSymbol(chainID *common.ChainID, symbol string) *Token { for i, t := range transfer.TestTokens { if (chainID == nil || t.ChainID == uint64(*chainID)) && t.Symbol == symbol { @@ -70,6 +73,9 @@ func setupTestActivityDBStorageChoice(tb testing.TB, inMemory bool) (deps Filter }, // tokenFromSymbol nil chainID accepts first symbol found tokenFromSymbol: tokenFromSymbol, + currentTimestamp: func() int64 { + return mockupTime.Unix() + }, } return deps, func() { @@ -197,7 +203,7 @@ func TestGetActivityEntriesAll(t *testing.T) { id: td.tr1.MultiTransactionID, timestamp: td.tr1.Timestamp, activityType: ReceiveAT, - activityStatus: CompleteAS, + activityStatus: FinalizedAS, amountOut: (*hexutil.Big)(big.NewInt(0)), amountIn: (*hexutil.Big)(big.NewInt(td.tr1.Value)), tokenOut: nil, @@ -235,7 +241,7 @@ func TestGetActivityEntriesAll(t *testing.T) { id: td.multiTx1ID, timestamp: td.multiTx1.Timestamp, activityType: SendAT, - activityStatus: CompleteAS, + activityStatus: FinalizedAS, amountOut: (*hexutil.Big)(big.NewInt(td.multiTx1.FromAmount)), amountIn: (*hexutil.Big)(big.NewInt(td.multiTx1.ToAmount)), tokenOut: tokenFromSymbol(nil, td.multiTx1.FromToken), @@ -327,7 +333,7 @@ func TestGetActivityEntriesFilterByTime(t *testing.T) { id: 0, timestamp: trs[5].Timestamp, activityType: ReceiveAT, - activityStatus: CompleteAS, + activityStatus: FinalizedAS, amountOut: (*hexutil.Big)(big.NewInt(0)), amountIn: (*hexutil.Big)(big.NewInt(trs[5].Value)), tokenOut: nil, @@ -346,7 +352,7 @@ func TestGetActivityEntriesFilterByTime(t *testing.T) { id: td.multiTx1ID, timestamp: td.multiTx1.Timestamp, activityType: SendAT, - activityStatus: CompleteAS, + activityStatus: FinalizedAS, amountOut: (*hexutil.Big)(big.NewInt(td.multiTx1.FromAmount)), amountIn: (*hexutil.Big)(big.NewInt(td.multiTx1.ToAmount)), tokenOut: tokenFromSymbol(nil, td.multiTx1.FromToken), @@ -373,7 +379,7 @@ func TestGetActivityEntriesFilterByTime(t *testing.T) { id: 0, timestamp: trs[2].Timestamp, activityType: ReceiveAT, - activityStatus: CompleteAS, + activityStatus: FinalizedAS, amountOut: (*hexutil.Big)(big.NewInt(0)), amountIn: (*hexutil.Big)(big.NewInt(trs[2].Value)), tokenOut: nil, @@ -392,7 +398,7 @@ func TestGetActivityEntriesFilterByTime(t *testing.T) { id: td.multiTx1ID, timestamp: td.multiTx1.Timestamp, activityType: SendAT, - activityStatus: CompleteAS, + activityStatus: FinalizedAS, amountOut: (*hexutil.Big)(big.NewInt(td.multiTx1.FromAmount)), amountIn: (*hexutil.Big)(big.NewInt(td.multiTx1.ToAmount)), tokenOut: tokenFromSymbol(nil, td.multiTx1.FromToken), @@ -418,7 +424,7 @@ func TestGetActivityEntriesFilterByTime(t *testing.T) { id: 0, timestamp: trs[2].Timestamp, activityType: ReceiveAT, - activityStatus: CompleteAS, + activityStatus: FinalizedAS, amountOut: (*hexutil.Big)(big.NewInt(0)), amountIn: (*hexutil.Big)(big.NewInt(trs[2].Value)), tokenOut: nil, @@ -437,7 +443,7 @@ func TestGetActivityEntriesFilterByTime(t *testing.T) { id: 0, timestamp: td.tr1.Timestamp, activityType: ReceiveAT, - activityStatus: CompleteAS, + activityStatus: FinalizedAS, amountOut: (*hexutil.Big)(big.NewInt(0)), amountIn: (*hexutil.Big)(big.NewInt(td.tr1.Value)), tokenOut: nil, @@ -483,7 +489,7 @@ func TestGetActivityEntriesCheckOffsetAndLimit(t *testing.T) { id: 0, timestamp: trs[8].Timestamp, activityType: ReceiveAT, - activityStatus: CompleteAS, + activityStatus: FinalizedAS, amountOut: (*hexutil.Big)(big.NewInt(0)), amountIn: (*hexutil.Big)(big.NewInt(trs[8].Value)), tokenOut: nil, @@ -502,7 +508,7 @@ func TestGetActivityEntriesCheckOffsetAndLimit(t *testing.T) { id: 0, timestamp: trs[6].Timestamp, activityType: ReceiveAT, - activityStatus: CompleteAS, + activityStatus: FinalizedAS, amountOut: (*hexutil.Big)(big.NewInt(0)), amountIn: (*hexutil.Big)(big.NewInt(trs[6].Value)), tokenOut: nil, @@ -527,7 +533,7 @@ func TestGetActivityEntriesCheckOffsetAndLimit(t *testing.T) { id: 0, timestamp: trs[6].Timestamp, activityType: ReceiveAT, - activityStatus: CompleteAS, + activityStatus: FinalizedAS, amountOut: (*hexutil.Big)(big.NewInt(0)), amountIn: (*hexutil.Big)(big.NewInt(trs[6].Value)), tokenOut: nil, @@ -546,7 +552,7 @@ func TestGetActivityEntriesCheckOffsetAndLimit(t *testing.T) { id: 0, timestamp: trs[4].Timestamp, activityType: ReceiveAT, - activityStatus: CompleteAS, + activityStatus: FinalizedAS, amountOut: (*hexutil.Big)(big.NewInt(0)), amountIn: (*hexutil.Big)(big.NewInt(trs[4].Value)), tokenOut: nil, @@ -571,7 +577,7 @@ func TestGetActivityEntriesCheckOffsetAndLimit(t *testing.T) { id: 0, timestamp: trs[2].Timestamp, activityType: ReceiveAT, - activityStatus: CompleteAS, + activityStatus: FinalizedAS, amountOut: (*hexutil.Big)(big.NewInt(0)), amountIn: (*hexutil.Big)(big.NewInt(trs[2].Value)), tokenOut: nil, @@ -749,7 +755,7 @@ func TestGetActivityEntriesFilterByAddresses(t *testing.T) { id: 0, timestamp: trs[4].Timestamp, activityType: SendAT, - activityStatus: CompleteAS, + activityStatus: FinalizedAS, amountOut: (*hexutil.Big)(big.NewInt(trs[4].Value)), amountIn: (*hexutil.Big)(big.NewInt(0)), tokenOut: TTrToToken(t, &trs[4].TestTransaction), @@ -768,7 +774,7 @@ func TestGetActivityEntriesFilterByAddresses(t *testing.T) { id: 0, timestamp: trs[1].Timestamp, activityType: SendAT, - activityStatus: CompleteAS, + activityStatus: FinalizedAS, amountOut: (*hexutil.Big)(big.NewInt(trs[1].Value)), amountIn: (*hexutil.Big)(big.NewInt(0)), tokenOut: TTrToToken(t, &trs[1].TestTransaction), @@ -805,9 +811,9 @@ func TestGetActivityEntriesFilterByStatus(t *testing.T) { deps, close := setupTestActivityDB(t) defer close() - // Adds 4 extractable transactions: 1 T, 1 T pending, 1 MT pending, 1 MT with 2xT success + // Adds 4 extractable transactions: 1 T, 1 T pending, 1 MT pending, 1 MT with 2xT finalized td, fromTds, toTds := fillTestData(t, deps.db) - // Add 7 extractable transactions: 1 pending, 1 Tr failed, 1 MT failed, 4 success + // Add 7 extractable transactions: 1 pending, 1 Tr failed, 1 MT failed, 4 finalized trs, fromTrs, toTrs := transfer.GenerateTestTransfers(t, deps.db, td.nextIndex, 7) multiTx := transfer.GenerateTestSendMultiTransaction(trs[6]) failedMTID := transfer.InsertTestMultiTransaction(t, deps.db, &multiTx) @@ -817,6 +823,10 @@ func TestGetActivityEntriesFilterByStatus(t *testing.T) { transfer.InsertTestPendingTransaction(t, deps.db, &trs[i]) } else { trs[i].Success = i != 3 && i != 6 + if trs[i].Success && (i == 2 || i == 5) { + // Finalize status depends on timestamp + trs[i].Timestamp = mockupTime.Unix() - 10 + } transfer.InsertTestTransfer(t, deps.db, trs[i].To, &trs[i]) } } @@ -845,13 +855,12 @@ func TestGetActivityEntriesFilterByStatus(t *testing.T) { filter.Statuses = []Status{CompleteAS} entries, err = getActivityEntries(context.Background(), deps, allAddresses, true, []common.ChainID{}, filter, 0, 15) require.NoError(t, err) - require.Equal(t, 6, len(entries)) + require.Equal(t, 2, len(entries)) - // Finalized is treated as Complete, would need dynamic blockchain status to track the Finalized level filter.Statuses = []Status{FinalizedAS} entries, err = getActivityEntries(context.Background(), deps, allAddresses, true, []common.ChainID{}, filter, 0, 15) require.NoError(t, err) - require.Equal(t, 6, len(entries)) + require.Equal(t, 4, len(entries)) // Combined filter filter.Statuses = []Status{FailedAS, PendingAS} diff --git a/services/wallet/activity/filter.sql b/services/wallet/activity/filter.sql index efaccf072..d6e08557a 100644 --- a/services/wallet/activity/filter.sql +++ b/services/wallet/activity/filter.sql @@ -34,11 +34,15 @@ WITH filter_conditions AS ( ? AS filterStatusFinalized, ? AS filterStatusPending, ? AS statusFailed, - ? AS statusSuccess, + ? AS statusCompleted, + ? AS statusFinalized, ? AS statusPending, ? AS includeAllTokenTypeAssets, ? AS includeAllNetworks, ? AS pendingStatus, + ? AS nowTimestamp, + ? AS layer2FinalisationDuration, + ? AS layer1FinalisationDuration, '0000000000000000000000000000000000000000' AS zeroAddress ), -- This UNION between CTE and TEMP TABLE acts as an optimization. As soon as we drop one or use them interchangeably the performance drops significantly. @@ -139,6 +143,9 @@ pending_network_ids AS ( AND pending_transactions.network_id IN filter_networks GROUP BY pending_transactions.multi_transaction_id +), +layer2_networks(network_id) AS ( + VALUES %s ) SELECT transfers.hash AS transfer_hash, @@ -166,7 +173,16 @@ SELECT NULL AS mt_from_amount, NULL AS mt_to_amount, CASE - WHEN transfers.status IS 1 THEN statusSuccess + WHEN transfers.status IS 1 THEN + CASE + WHEN transfers.timestamp > 0 AND filter_conditions.nowTimestamp >= transfers.timestamp + + (CASE + WHEN transfers.network_id in layer2_networks THEN layer2FinalisationDuration + ELSE layer1FinalisationDuration + END) + THEN statusFinalized + ELSE statusCompleted + END ELSE statusFailed END AS agg_status, 1 AS agg_count, @@ -282,15 +298,16 @@ WHERE AND ( filterAllActivityStatus OR ( - ( - filterStatusCompleted - OR filterStatusFinalized - ) - AND transfers.status = 1 + filterStatusCompleted + AND agg_status = statusCompleted + ) + OR ( + filterStatusFinalized + AND agg_status = statusFinalized ) OR ( filterStatusFailed - AND transfers.status = 0 + AND agg_status = statusFailed ) ) UNION @@ -401,8 +418,17 @@ SELECT multi_transactions.from_amount AS mt_from_amount, multi_transactions.to_amount AS mt_to_amount, CASE - WHEN tr_status.min_status = 1 - AND COALESCE(pending_status.count, 0) = 0 THEN statusSuccess + WHEN tr_status.min_status = 1 AND COALESCE(pending_status.count, 0) = 0 THEN + CASE + WHEN multi_transactions.timestamp > 0 AND filter_conditions.nowTimestamp >= multi_transactions.timestamp + + (CASE + WHEN multi_transactions.from_network_id in layer2_networks + OR multi_transactions.to_network_id in layer2_networks THEN layer2FinalisationDuration + ELSE layer1FinalisationDuration + END) + THEN statusFinalized + ELSE statusCompleted + END WHEN tr_status.min_status = 0 THEN statusFailed ELSE statusPending END AS agg_status, @@ -475,11 +501,12 @@ WHERE AND ( filterAllActivityStatus OR ( - ( - filterStatusCompleted - OR filterStatusFinalized - ) - AND agg_status = statusSuccess + filterStatusCompleted + AND agg_status = statusCompleted + ) + OR ( + filterStatusFinalized + AND agg_status = statusFinalized ) OR ( filterStatusFailed diff --git a/services/wallet/activity/service.go b/services/wallet/activity/service.go index 8e54ba0b7..52cc5561b 100644 --- a/services/wallet/activity/service.go +++ b/services/wallet/activity/service.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "strconv" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/event" @@ -269,6 +270,9 @@ func (s *Service) getDeps() FilterDependencies { Address: t.Address, } }, + currentTimestamp: func() int64 { + return time.Now().Unix() + }, } }