status-go/multiaccounts/accounts/database.go

1173 lines
29 KiB
Go
Raw Normal View History

package accounts
import (
"database/sql"
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
2019-12-11 13:59:37 +00:00
"github.com/status-im/status-go/eth-node/types"
"github.com/status-im/status-go/multiaccounts/common"
Sync Settings (#2478) * Sync Settings * Added valueHandlers and Database singleton Some issues remain, need a way to comparing incoming sql.DB to check if the connection is to a different file or not. Maybe make singleton instance per filename * Added functionality to check the sqlite filename * Refactor of Database.SaveSyncSettings to be used as a handler * Implemented inteface for setting sync protobuf factories * Refactored and completed adhoc send setting sync * Tidying up * Immutability refactor * Refactor settings into dedicated package * Breakout structs * Tidy up * Refactor of bulk settings sync * Bug fixes * Addressing feedback * Fix code dropped during rebase * Fix for db closed * Fix for node config related crashes * Provisional fix for type assertion - issue 2 * Adding robust type assertion checks * Partial fix for null literal db storage and json encoding * Fix for passively handling nil sql.DB, and checking if elem has len and if len is 0 * Added test for preferred name behaviour * Adding saved sync settings to MessengerResponse * Completed granular initial sync and clock from network on save * add Settings to isEmpty * Refactor of protobufs, partially done * Added syncSetting receiver handling, some bug fixes * Fix for sticker packs * Implement inactive flag on sync protobuf factory * Refactor of types and structs * Added SettingField.CanSync functionality * Addressing rebase artifact * Refactor of Setting SELECT queries * Refactor of string return queries * VERSION bump and migration index bump * Deactiveate Sync Settings * Deactiveated preferred_name and send_status_updates Co-authored-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2022-03-23 18:47:00 +00:00
"github.com/status-im/status-go/multiaccounts/settings"
notificationssettings "github.com/status-im/status-go/multiaccounts/settings_notifications"
2022-08-02 12:56:26 +00:00
sociallinkssettings "github.com/status-im/status-go/multiaccounts/settings_social_links"
"github.com/status-im/status-go/nodecfg"
"github.com/status-im/status-go/params"
)
const (
statusChatPath = "m/43'/60'/1581'/0'/0"
statusWalletRootPath = "m/44'/60'/0'/0/"
zeroAddress = "0x0000000000000000000000000000000000000000"
SyncedFromBackup = "backup" // means a account is coming from backed up data
SyncedFromLocalPairing = "local-pairing" // means a account is coming from another device when user is reocovering Status account
)
var (
errDbPassedParameterIsNil = errors.New("accounts: passed parameter is nil")
errDbTransactionIsNil = errors.New("accounts: database transaction is nil")
ErrDbKeypairNotFound = errors.New("accounts: keypair is not found")
ErrDbAccountNotFound = errors.New("accounts: account is not found")
ErrKeypairDifferentAccountsKeyUID = errors.New("cannot store keypair with different accounts' key uid than keypair's key uid")
ErrKeypairWithoutAccounts = errors.New("cannot store keypair without accounts")
)
type Keypair struct {
KeyUID string `json:"key-uid"`
Name string `json:"name"`
Type KeypairType `json:"type"`
DerivedFrom string `json:"derived-from"`
LastUsedDerivationIndex uint64 `json:"last-used-derivation-index,omitempty"`
SyncedFrom string `json:"synced-from,omitempty"` // keeps an info which device this keypair is added from can be one of two values defined in constants or device name (custom)
Clock uint64 `json:"clock,omitempty"`
Accounts []*Account `json:"accounts,omitempty"`
Keycards []*Keycard `json:"keycards,omitempty"`
Removed bool `json:"removed,omitempty"`
}
type Account struct {
Address types.Address `json:"address"`
KeyUID string `json:"key-uid"`
Wallet bool `json:"wallet"`
Chat bool `json:"chat"`
Type AccountType `json:"type,omitempty"`
Path string `json:"path,omitempty"`
PublicKey types.HexBytes `json:"public-key,omitempty"`
Name string `json:"name"`
Emoji string `json:"emoji"`
ColorID common.CustomizationColor `json:"colorId,omitempty"`
Hidden bool `json:"hidden"`
Clock uint64 `json:"clock,omitempty"`
Removed bool `json:"removed,omitempty"`
Operable AccountOperable `json:"operable"` // describes an account's operability (read an explanation at the top of this file)
2023-06-02 07:38:06 +00:00
CreatedAt int64 `json:"createdAt"`
2023-06-20 11:35:22 +00:00
Position int64 `json:"position"`
}
type KeypairType string
type AccountType string
type AccountOperable string
func (a KeypairType) String() string {
return string(a)
}
func (a AccountType) String() string {
return string(a)
}
func (a AccountOperable) String() string {
return string(a)
}
const (
KeypairTypeProfile KeypairType = "profile"
KeypairTypeKey KeypairType = "key"
KeypairTypeSeed KeypairType = "seed"
)
const (
AccountTypeGenerated AccountType = "generated"
AccountTypeKey AccountType = "key"
AccountTypeSeed AccountType = "seed"
AccountTypeWatch AccountType = "watch"
)
const (
AccountNonOperable AccountOperable = "no" // an account is non operable it is not a keycard account and there is no keystore file for it and no keystore file for the address it is derived from
AccountPartiallyOperable AccountOperable = "partially" // an account is partially operable if it is not a keycard account and there is created keystore file for the address it is derived from
AccountFullyOperable AccountOperable = "fully" // an account is fully operable if it is not a keycard account and there is a keystore file for it
)
// IsOwnAccount returns true if this is an account we have the private key for
// NOTE: Wallet flag can't be used as it actually indicates that it's the default
// Wallet
func (a *Account) IsOwnAccount() bool {
2022-07-06 16:12:49 +00:00
return a.Wallet || a.Type == AccountTypeSeed || a.Type == AccountTypeGenerated || a.Type == AccountTypeKey
}
func (a *Account) MarshalJSON() ([]byte, error) {
item := struct {
Address types.Address `json:"address"`
MixedcaseAddress string `json:"mixedcase-address"`
KeyUID string `json:"key-uid"`
Wallet bool `json:"wallet"`
Chat bool `json:"chat"`
Type AccountType `json:"type"`
Path string `json:"path"`
PublicKey types.HexBytes `json:"public-key"`
Name string `json:"name"`
Emoji string `json:"emoji"`
ColorID common.CustomizationColor `json:"colorId"`
Hidden bool `json:"hidden"`
Clock uint64 `json:"clock"`
Removed bool `json:"removed"`
Operable AccountOperable `json:"operable"`
2023-06-02 07:38:06 +00:00
CreatedAt int64 `json:"createdAt"`
2023-06-20 11:35:22 +00:00
Position int64 `json:"position"`
}{
Address: a.Address,
MixedcaseAddress: a.Address.Hex(),
KeyUID: a.KeyUID,
Wallet: a.Wallet,
Chat: a.Chat,
Type: a.Type,
Path: a.Path,
PublicKey: a.PublicKey,
Name: a.Name,
Emoji: a.Emoji,
ColorID: a.ColorID,
Hidden: a.Hidden,
Clock: a.Clock,
Removed: a.Removed,
Operable: a.Operable,
2023-06-02 07:38:06 +00:00
CreatedAt: a.CreatedAt,
2023-06-20 11:35:22 +00:00
Position: a.Position,
}
return json.Marshal(item)
}
func (a *Keypair) MarshalJSON() ([]byte, error) {
item := struct {
KeyUID string `json:"key-uid"`
Name string `json:"name"`
Type KeypairType `json:"type"`
DerivedFrom string `json:"derived-from"`
LastUsedDerivationIndex uint64 `json:"last-used-derivation-index"`
SyncedFrom string `json:"synced-from"`
Clock uint64 `json:"clock"`
Accounts []*Account `json:"accounts"`
Keycards []*Keycard `json:"keycards"`
Removed bool `json:"removed"`
}{
KeyUID: a.KeyUID,
Name: a.Name,
Type: a.Type,
DerivedFrom: a.DerivedFrom,
LastUsedDerivationIndex: a.LastUsedDerivationIndex,
SyncedFrom: a.SyncedFrom,
Clock: a.Clock,
Accounts: a.Accounts,
Keycards: a.Keycards,
Removed: a.Removed,
}
return json.Marshal(item)
}
func (a *Keypair) CopyKeypair() *Keypair {
kp := &Keypair{
Clock: a.Clock,
KeyUID: a.KeyUID,
Name: a.Name,
Type: a.Type,
DerivedFrom: a.DerivedFrom,
LastUsedDerivationIndex: a.LastUsedDerivationIndex,
SyncedFrom: a.SyncedFrom,
Accounts: make([]*Account, len(a.Accounts)),
Keycards: make([]*Keycard, len(a.Keycards)),
Removed: a.Removed,
}
for i, acc := range a.Accounts {
kp.Accounts[i] = &Account{
Address: acc.Address,
KeyUID: acc.KeyUID,
Wallet: acc.Wallet,
Chat: acc.Chat,
Type: acc.Type,
Path: acc.Path,
PublicKey: acc.PublicKey,
Name: acc.Name,
Emoji: acc.Emoji,
ColorID: acc.ColorID,
Hidden: acc.Hidden,
Clock: acc.Clock,
Removed: acc.Removed,
Operable: acc.Operable,
2023-06-20 11:35:22 +00:00
CreatedAt: acc.CreatedAt,
Position: acc.Position,
}
}
for i, kc := range a.Keycards {
kp.Keycards[i] = &Keycard{
KeycardUID: kc.KeycardUID,
KeycardName: kc.KeycardName,
KeycardLocked: kc.KeycardLocked,
AccountsAddresses: kc.AccountsAddresses,
KeyUID: kc.KeyUID,
LastUpdateClock: kc.LastUpdateClock,
}
}
return kp
}
func (a *Keypair) GetChatPublicKey() types.HexBytes {
for _, acc := range a.Accounts {
if acc.Chat {
return acc.PublicKey
}
}
return nil
}
// Database sql wrapper for operations with browser objects.
type Database struct {
Sync Settings (#2478) * Sync Settings * Added valueHandlers and Database singleton Some issues remain, need a way to comparing incoming sql.DB to check if the connection is to a different file or not. Maybe make singleton instance per filename * Added functionality to check the sqlite filename * Refactor of Database.SaveSyncSettings to be used as a handler * Implemented inteface for setting sync protobuf factories * Refactored and completed adhoc send setting sync * Tidying up * Immutability refactor * Refactor settings into dedicated package * Breakout structs * Tidy up * Refactor of bulk settings sync * Bug fixes * Addressing feedback * Fix code dropped during rebase * Fix for db closed * Fix for node config related crashes * Provisional fix for type assertion - issue 2 * Adding robust type assertion checks * Partial fix for null literal db storage and json encoding * Fix for passively handling nil sql.DB, and checking if elem has len and if len is 0 * Added test for preferred name behaviour * Adding saved sync settings to MessengerResponse * Completed granular initial sync and clock from network on save * add Settings to isEmpty * Refactor of protobufs, partially done * Added syncSetting receiver handling, some bug fixes * Fix for sticker packs * Implement inactive flag on sync protobuf factory * Refactor of types and structs * Added SettingField.CanSync functionality * Addressing rebase artifact * Refactor of Setting SELECT queries * Refactor of string return queries * VERSION bump and migration index bump * Deactiveate Sync Settings * Deactiveated preferred_name and send_status_updates Co-authored-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2022-03-23 18:47:00 +00:00
*settings.Database
*notificationssettings.NotificationsSettings
2022-08-02 12:56:26 +00:00
*sociallinkssettings.SocialLinksSettings
*Keycards
db *sql.DB
}
Sync Settings (#2478) * Sync Settings * Added valueHandlers and Database singleton Some issues remain, need a way to comparing incoming sql.DB to check if the connection is to a different file or not. Maybe make singleton instance per filename * Added functionality to check the sqlite filename * Refactor of Database.SaveSyncSettings to be used as a handler * Implemented inteface for setting sync protobuf factories * Refactored and completed adhoc send setting sync * Tidying up * Immutability refactor * Refactor settings into dedicated package * Breakout structs * Tidy up * Refactor of bulk settings sync * Bug fixes * Addressing feedback * Fix code dropped during rebase * Fix for db closed * Fix for node config related crashes * Provisional fix for type assertion - issue 2 * Adding robust type assertion checks * Partial fix for null literal db storage and json encoding * Fix for passively handling nil sql.DB, and checking if elem has len and if len is 0 * Added test for preferred name behaviour * Adding saved sync settings to MessengerResponse * Completed granular initial sync and clock from network on save * add Settings to isEmpty * Refactor of protobufs, partially done * Added syncSetting receiver handling, some bug fixes * Fix for sticker packs * Implement inactive flag on sync protobuf factory * Refactor of types and structs * Added SettingField.CanSync functionality * Addressing rebase artifact * Refactor of Setting SELECT queries * Refactor of string return queries * VERSION bump and migration index bump * Deactiveate Sync Settings * Deactiveated preferred_name and send_status_updates Co-authored-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2022-03-23 18:47:00 +00:00
// NewDB returns a new instance of *Database
func NewDB(db *sql.DB) (*Database, error) {
sDB, err := settings.MakeNewDB(db)
if err != nil {
Sync Settings (#2478) * Sync Settings * Added valueHandlers and Database singleton Some issues remain, need a way to comparing incoming sql.DB to check if the connection is to a different file or not. Maybe make singleton instance per filename * Added functionality to check the sqlite filename * Refactor of Database.SaveSyncSettings to be used as a handler * Implemented inteface for setting sync protobuf factories * Refactored and completed adhoc send setting sync * Tidying up * Immutability refactor * Refactor settings into dedicated package * Breakout structs * Tidy up * Refactor of bulk settings sync * Bug fixes * Addressing feedback * Fix code dropped during rebase * Fix for db closed * Fix for node config related crashes * Provisional fix for type assertion - issue 2 * Adding robust type assertion checks * Partial fix for null literal db storage and json encoding * Fix for passively handling nil sql.DB, and checking if elem has len and if len is 0 * Added test for preferred name behaviour * Adding saved sync settings to MessengerResponse * Completed granular initial sync and clock from network on save * add Settings to isEmpty * Refactor of protobufs, partially done * Added syncSetting receiver handling, some bug fixes * Fix for sticker packs * Implement inactive flag on sync protobuf factory * Refactor of types and structs * Added SettingField.CanSync functionality * Addressing rebase artifact * Refactor of Setting SELECT queries * Refactor of string return queries * VERSION bump and migration index bump * Deactiveate Sync Settings * Deactiveated preferred_name and send_status_updates Co-authored-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2022-03-23 18:47:00 +00:00
return nil, err
}
sn := notificationssettings.NewNotificationsSettings(db)
2022-08-02 12:56:26 +00:00
ssl := sociallinkssettings.NewSocialLinksSettings(db)
kc := NewKeycards(db)
return &Database{sDB, sn, ssl, kc, db}, nil
}
Sync Settings (#2478) * Sync Settings * Added valueHandlers and Database singleton Some issues remain, need a way to comparing incoming sql.DB to check if the connection is to a different file or not. Maybe make singleton instance per filename * Added functionality to check the sqlite filename * Refactor of Database.SaveSyncSettings to be used as a handler * Implemented inteface for setting sync protobuf factories * Refactored and completed adhoc send setting sync * Tidying up * Immutability refactor * Refactor settings into dedicated package * Breakout structs * Tidy up * Refactor of bulk settings sync * Bug fixes * Addressing feedback * Fix code dropped during rebase * Fix for db closed * Fix for node config related crashes * Provisional fix for type assertion - issue 2 * Adding robust type assertion checks * Partial fix for null literal db storage and json encoding * Fix for passively handling nil sql.DB, and checking if elem has len and if len is 0 * Added test for preferred name behaviour * Adding saved sync settings to MessengerResponse * Completed granular initial sync and clock from network on save * add Settings to isEmpty * Refactor of protobufs, partially done * Added syncSetting receiver handling, some bug fixes * Fix for sticker packs * Implement inactive flag on sync protobuf factory * Refactor of types and structs * Added SettingField.CanSync functionality * Addressing rebase artifact * Refactor of Setting SELECT queries * Refactor of string return queries * VERSION bump and migration index bump * Deactiveate Sync Settings * Deactiveated preferred_name and send_status_updates Co-authored-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2022-03-23 18:47:00 +00:00
// DB Gets db sql.DB
func (db *Database) DB() *sql.DB {
Sync Settings (#2478) * Sync Settings * Added valueHandlers and Database singleton Some issues remain, need a way to comparing incoming sql.DB to check if the connection is to a different file or not. Maybe make singleton instance per filename * Added functionality to check the sqlite filename * Refactor of Database.SaveSyncSettings to be used as a handler * Implemented inteface for setting sync protobuf factories * Refactored and completed adhoc send setting sync * Tidying up * Immutability refactor * Refactor settings into dedicated package * Breakout structs * Tidy up * Refactor of bulk settings sync * Bug fixes * Addressing feedback * Fix code dropped during rebase * Fix for db closed * Fix for node config related crashes * Provisional fix for type assertion - issue 2 * Adding robust type assertion checks * Partial fix for null literal db storage and json encoding * Fix for passively handling nil sql.DB, and checking if elem has len and if len is 0 * Added test for preferred name behaviour * Adding saved sync settings to MessengerResponse * Completed granular initial sync and clock from network on save * add Settings to isEmpty * Refactor of protobufs, partially done * Added syncSetting receiver handling, some bug fixes * Fix for sticker packs * Implement inactive flag on sync protobuf factory * Refactor of types and structs * Added SettingField.CanSync functionality * Addressing rebase artifact * Refactor of Setting SELECT queries * Refactor of string return queries * VERSION bump and migration index bump * Deactiveate Sync Settings * Deactiveated preferred_name and send_status_updates Co-authored-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2022-03-23 18:47:00 +00:00
return db.db
}
Sync Settings (#2478) * Sync Settings * Added valueHandlers and Database singleton Some issues remain, need a way to comparing incoming sql.DB to check if the connection is to a different file or not. Maybe make singleton instance per filename * Added functionality to check the sqlite filename * Refactor of Database.SaveSyncSettings to be used as a handler * Implemented inteface for setting sync protobuf factories * Refactored and completed adhoc send setting sync * Tidying up * Immutability refactor * Refactor settings into dedicated package * Breakout structs * Tidy up * Refactor of bulk settings sync * Bug fixes * Addressing feedback * Fix code dropped during rebase * Fix for db closed * Fix for node config related crashes * Provisional fix for type assertion - issue 2 * Adding robust type assertion checks * Partial fix for null literal db storage and json encoding * Fix for passively handling nil sql.DB, and checking if elem has len and if len is 0 * Added test for preferred name behaviour * Adding saved sync settings to MessengerResponse * Completed granular initial sync and clock from network on save * add Settings to isEmpty * Refactor of protobufs, partially done * Added syncSetting receiver handling, some bug fixes * Fix for sticker packs * Implement inactive flag on sync protobuf factory * Refactor of types and structs * Added SettingField.CanSync functionality * Addressing rebase artifact * Refactor of Setting SELECT queries * Refactor of string return queries * VERSION bump and migration index bump * Deactiveate Sync Settings * Deactiveated preferred_name and send_status_updates Co-authored-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2022-03-23 18:47:00 +00:00
// Close closes database.
func (db *Database) Close() error {
Sync Settings (#2478) * Sync Settings * Added valueHandlers and Database singleton Some issues remain, need a way to comparing incoming sql.DB to check if the connection is to a different file or not. Maybe make singleton instance per filename * Added functionality to check the sqlite filename * Refactor of Database.SaveSyncSettings to be used as a handler * Implemented inteface for setting sync protobuf factories * Refactored and completed adhoc send setting sync * Tidying up * Immutability refactor * Refactor settings into dedicated package * Breakout structs * Tidy up * Refactor of bulk settings sync * Bug fixes * Addressing feedback * Fix code dropped during rebase * Fix for db closed * Fix for node config related crashes * Provisional fix for type assertion - issue 2 * Adding robust type assertion checks * Partial fix for null literal db storage and json encoding * Fix for passively handling nil sql.DB, and checking if elem has len and if len is 0 * Added test for preferred name behaviour * Adding saved sync settings to MessengerResponse * Completed granular initial sync and clock from network on save * add Settings to isEmpty * Refactor of protobufs, partially done * Added syncSetting receiver handling, some bug fixes * Fix for sticker packs * Implement inactive flag on sync protobuf factory * Refactor of types and structs * Added SettingField.CanSync functionality * Addressing rebase artifact * Refactor of Setting SELECT queries * Refactor of string return queries * VERSION bump and migration index bump * Deactiveate Sync Settings * Deactiveated preferred_name and send_status_updates Co-authored-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2022-03-23 18:47:00 +00:00
return db.db.Close()
}
func GetAccountTypeForKeypairType(kpType KeypairType) AccountType {
switch kpType {
case KeypairTypeProfile:
return AccountTypeGenerated
case KeypairTypeKey:
return AccountTypeKey
case KeypairTypeSeed:
return AccountTypeSeed
default:
return AccountTypeWatch
}
}
func (db *Database) processKeypairs(rows *sql.Rows) ([]*Keypair, error) {
keypairMap := make(map[string]*Keypair)
var (
kpKeyUID sql.NullString
kpName sql.NullString
kpType sql.NullString
kpDerivedFrom sql.NullString
kpLastUsedDerivationIndex sql.NullInt64
kpSyncedFrom sql.NullString
kpClock sql.NullInt64
)
var (
2023-06-02 07:38:06 +00:00
accAddress sql.NullString
accKeyUID sql.NullString
accPath sql.NullString
accName sql.NullString
accColorID sql.NullString
accEmoji sql.NullString
accWallet sql.NullBool
accChat sql.NullBool
accHidden sql.NullBool
accOperable sql.NullString
accClock sql.NullInt64
accCreatedAt sql.NullTime
2023-06-20 11:35:22 +00:00
accPosition sql.NullInt64
)
for rows.Next() {
kp := &Keypair{}
acc := &Account{}
pubkey := []byte{}
err := rows.Scan(
&kpKeyUID, &kpName, &kpType, &kpDerivedFrom, &kpLastUsedDerivationIndex, &kpSyncedFrom, &kpClock,
&accAddress, &accKeyUID, &pubkey, &accPath, &accName, &accColorID, &accEmoji,
2023-06-20 11:35:22 +00:00
&accWallet, &accChat, &accHidden, &accOperable, &accClock, &accCreatedAt, &accPosition)
if err != nil {
return nil, err
}
// check keypair fields
if kpKeyUID.Valid {
kp.KeyUID = kpKeyUID.String
}
if kpName.Valid {
kp.Name = kpName.String
}
if kpType.Valid {
kp.Type = KeypairType(kpType.String)
}
if kpDerivedFrom.Valid {
kp.DerivedFrom = kpDerivedFrom.String
}
if kpLastUsedDerivationIndex.Valid {
kp.LastUsedDerivationIndex = uint64(kpLastUsedDerivationIndex.Int64)
}
if kpSyncedFrom.Valid {
kp.SyncedFrom = kpSyncedFrom.String
}
if kpClock.Valid {
kp.Clock = uint64(kpClock.Int64)
}
// check keypair accounts fields
if accAddress.Valid {
acc.Address = types.BytesToAddress([]byte(accAddress.String))
}
if accKeyUID.Valid {
acc.KeyUID = accKeyUID.String
}
if accPath.Valid {
acc.Path = accPath.String
}
if accName.Valid {
acc.Name = accName.String
}
if accColorID.Valid {
acc.ColorID = common.CustomizationColor(accColorID.String)
}
if accEmoji.Valid {
acc.Emoji = accEmoji.String
}
if accWallet.Valid {
acc.Wallet = accWallet.Bool
}
if accChat.Valid {
acc.Chat = accChat.Bool
}
if accHidden.Valid {
acc.Hidden = accHidden.Bool
}
if accOperable.Valid {
acc.Operable = AccountOperable(accOperable.String)
}
if accClock.Valid {
acc.Clock = uint64(accClock.Int64)
}
2023-06-02 07:38:06 +00:00
if accCreatedAt.Valid {
acc.CreatedAt = accCreatedAt.Time.UnixMilli()
}
2023-06-20 11:35:22 +00:00
if accPosition.Valid {
acc.Position = accPosition.Int64
}
if lth := len(pubkey); lth > 0 {
acc.PublicKey = make(types.HexBytes, lth)
copy(acc.PublicKey, pubkey)
}
acc.Type = GetAccountTypeForKeypairType(kp.Type)
if _, ok := keypairMap[kp.KeyUID]; !ok {
keypairMap[kp.KeyUID] = kp
}
keypairMap[kp.KeyUID].Accounts = append(keypairMap[kp.KeyUID].Accounts, acc)
}
if err := rows.Err(); err != nil {
return nil, err
}
// Convert map to list
keypairs := make([]*Keypair, 0, len(keypairMap))
for _, keypair := range keypairMap {
keypairs = append(keypairs, keypair)
}
return keypairs, nil
}
// If `keyUID` is passed only keypairs which match the passed `keyUID` will be returned, if `keyUID` is empty, all keypairs will be returned.
func (db *Database) getKeypairs(tx *sql.Tx, keyUID string) ([]*Keypair, error) {
var (
rows *sql.Rows
err error
where string
)
if tx == nil {
tx, err = db.db.Begin()
defer func() {
if err == nil {
err = tx.Commit()
return
}
_ = tx.Rollback()
}()
if err != nil {
return nil, err
}
}
if keyUID != "" {
where = "WHERE k.key_uid = ?"
}
query := fmt.Sprintf( // nolint: gosec
`
SELECT
k.*,
ka.address,
ka.key_uid,
ka.pubkey,
ka.path,
ka.name,
2023-06-02 07:38:06 +00:00
ka.color,
ka.emoji,
ka.wallet,
ka.chat,
ka.hidden,
ka.operable,
2023-06-02 07:38:06 +00:00
ka.clock,
2023-06-20 11:35:22 +00:00
ka.created_at,
ka.position
FROM
keypairs k
LEFT JOIN
keypairs_accounts ka
ON
k.key_uid = ka.key_uid
%s
ORDER BY
2023-06-20 11:35:22 +00:00
ka.position`, where)
stmt, err := tx.Prepare(query)
if err != nil {
return nil, err
}
defer stmt.Close()
if where != "" {
rows, err = stmt.Query(keyUID)
} else {
rows, err = stmt.Query()
}
if err != nil {
return nil, err
}
defer rows.Close()
keypairs, err := db.processKeypairs(rows)
if err != nil {
return nil, err
}
for _, kp := range keypairs {
keycards, err := db.getAllRows(tx, true)
if err != nil {
return nil, err
}
kp.Keycards = keycards
}
return keypairs, nil
}
func (db *Database) getKeypairByKeyUID(tx *sql.Tx, keyUID string) (*Keypair, error) {
keypairs, err := db.getKeypairs(tx, keyUID)
if err != nil && err != sql.ErrNoRows {
return nil, err
}
if len(keypairs) == 0 {
return nil, ErrDbKeypairNotFound
}
return keypairs[0], nil
}
// If `address` is passed only accounts which match the passed `address` will be returned, if `address` is empty, all accounts will be returned.
func (db *Database) getAccounts(tx *sql.Tx, address types.Address) ([]*Account, error) {
var (
rows *sql.Rows
err error
where string
)
if address.String() != zeroAddress {
where = "WHERE ka.address = ?"
}
query := fmt.Sprintf( // nolint: gosec
`
SELECT
k.*,
ka.address,
ka.key_uid,
ka.pubkey,
ka.path,
ka.name,
2023-06-20 11:35:22 +00:00
ka.color,
ka.emoji,
ka.wallet,
ka.chat,
ka.hidden,
ka.operable,
2023-06-02 07:38:06 +00:00
ka.clock,
2023-06-20 11:35:22 +00:00
ka.created_at,
ka.position
FROM
keypairs_accounts ka
LEFT JOIN
keypairs k
ON
ka.key_uid = k.key_uid
%s
ORDER BY
2023-06-20 11:35:22 +00:00
ka.position`, where)
if tx == nil {
if where != "" {
rows, err = db.db.Query(query, address)
} else {
rows, err = db.db.Query(query)
}
if err != nil {
return nil, err
}
} else {
stmt, err := tx.Prepare(query)
if err != nil {
return nil, err
}
defer stmt.Close()
if where != "" {
rows, err = stmt.Query(address)
} else {
rows, err = stmt.Query()
}
if err != nil {
return nil, err
}
}
defer rows.Close()
keypairs, err := db.processKeypairs(rows)
if err != nil {
return nil, err
}
allAccounts := []*Account{}
for _, kp := range keypairs {
allAccounts = append(allAccounts, kp.Accounts...)
}
return allAccounts, nil
}
func (db *Database) getAccountByAddress(tx *sql.Tx, address types.Address) (*Account, error) {
accounts, err := db.getAccounts(tx, address)
if err != nil && err != sql.ErrNoRows {
return nil, err
}
if len(accounts) == 0 {
return nil, ErrDbAccountNotFound
}
return accounts[0], nil
}
func (db *Database) deleteKeypair(tx *sql.Tx, keyUID string) error {
keypairs, err := db.getKeypairs(tx, keyUID)
if err != nil && err != sql.ErrNoRows {
return err
}
if len(keypairs) == 0 {
return ErrDbKeypairNotFound
}
query := `
DELETE
FROM
keypairs
WHERE
key_uid = ?
`
if tx == nil {
_, err := db.db.Exec(query, keyUID)
return err
}
stmt, err := tx.Prepare(query)
if err != nil {
return err
}
defer stmt.Close()
_, err = stmt.Exec(keyUID)
return err
}
func (db *Database) GetKeypairs() ([]*Keypair, error) {
return db.getKeypairs(nil, "")
}
func (db *Database) GetKeypairByKeyUID(keyUID string) (*Keypair, error) {
return db.getKeypairByKeyUID(nil, keyUID)
}
func (db *Database) GetAccounts() ([]*Account, error) {
return db.getAccounts(nil, types.Address{})
}
func (db *Database) GetAccountByAddress(address types.Address) (*Account, error) {
return db.getAccountByAddress(nil, address)
}
func (db *Database) GetWatchOnlyAccounts() (res []*Account, err error) {
accounts, err := db.getAccounts(nil, types.Address{})
if err != nil {
return nil, err
}
for _, acc := range accounts {
if acc.Type == AccountTypeWatch {
res = append(res, acc)
}
}
return
}
func (db *Database) IsAnyAccountPartiallyOrFullyOperableForKeyUID(keyUID string) (bool, error) {
kp, err := db.getKeypairByKeyUID(nil, keyUID)
if err != nil {
return false, err
}
for _, acc := range kp.Accounts {
if acc.Operable != AccountNonOperable {
return true, nil
}
}
return false, nil
}
func (db *Database) DeleteKeypair(keyUID string) error {
return db.deleteKeypair(nil, keyUID)
}
func (db *Database) DeleteAccount(address types.Address, clock uint64) error {
tx, err := db.db.Begin()
defer func() {
if err == nil {
err = tx.Commit()
return
}
_ = tx.Rollback()
}()
if err != nil {
return err
}
acc, err := db.getAccountByAddress(tx, address)
if err != nil {
return err
}
kp, err := db.getKeypairByKeyUID(tx, acc.KeyUID)
if err != nil && err != ErrDbKeypairNotFound {
return err
}
if kp != nil && len(kp.Accounts) == 1 && kp.Accounts[0].Address == address {
return db.deleteKeypair(tx, acc.KeyUID)
}
delete, err := tx.Prepare(`
DELETE
FROM
keypairs_accounts
WHERE
address = ?
`)
if err != nil {
return err
}
defer delete.Close()
_, err = delete.Exec(address)
if err != nil {
return err
}
// Update keypair clock if any but the watch only account was deleted.
if kp != nil {
err = db.updateKeypairClock(tx, acc.KeyUID, clock)
return err
}
return nil
}
func updateKeypairLastUsedIndex(tx *sql.Tx, keyUID string, index uint64, clock uint64, updateKeypairClock bool) error {
if tx == nil {
return errDbTransactionIsNil
}
var (
err error
setClock string
)
if updateKeypairClock {
setClock = ", clock = ?"
}
query := fmt.Sprintf( // nolint: gosec
`
UPDATE
keypairs
SET
last_used_derivation_index = ?
%s
WHERE
key_uid = ?`, setClock)
if setClock != "" {
_, err = tx.Exec(query, index, clock, keyUID)
} else {
_, err = tx.Exec(query, index, keyUID)
}
return err
}
func (db *Database) updateKeypairClock(tx *sql.Tx, keyUID string, clock uint64) error {
if tx == nil {
return errDbTransactionIsNil
}
_, err := tx.Exec(`
UPDATE
keypairs
SET
clock = ?
WHERE
key_uid = ?`,
clock, keyUID)
return err
}
func (db *Database) saveOrUpdateAccounts(tx *sql.Tx, accounts []*Account, updateKeypairClock bool) (err error) {
if tx == nil {
return errDbTransactionIsNil
}
for _, acc := range accounts {
var relatedKeypair *Keypair
// only watch only accounts have an empty `KeyUID` field
var keyUID *string
if acc.KeyUID != "" {
relatedKeypair, err = db.getKeypairByKeyUID(tx, acc.KeyUID)
2022-05-18 10:42:51 +00:00
if err != nil {
if err == sql.ErrNoRows {
// all accounts, except watch only accounts, must have a row in `keypairs` table with the same key uid
continue
}
return err
2022-05-18 10:42:51 +00:00
}
keyUID = &acc.KeyUID
2022-05-18 10:42:51 +00:00
}
2023-06-20 11:35:22 +00:00
var exists bool
err = tx.QueryRow("SELECT EXISTS (SELECT 1 FROM keypairs_accounts WHERE address = ?)", acc.Address).Scan(&exists)
if err != nil {
return err
}
_, err = tx.Exec(`
INSERT OR IGNORE INTO
keypairs_accounts (address, key_uid, pubkey, path, wallet, chat, created_at, updated_at)
VALUES
(?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'));
UPDATE
keypairs_accounts
SET
name = ?,
2023-06-20 11:35:22 +00:00
color = ?,
emoji = ?,
hidden = ?,
operable = ?,
clock = ?,
position = ?,
updated_at = datetime('now')
WHERE
address = ?;
`,
acc.Address, keyUID, acc.PublicKey, acc.Path, acc.Wallet, acc.Chat,
acc.Name, acc.ColorID, acc.Emoji, acc.Hidden, acc.Operable, acc.Clock, acc.Position, acc.Address)
2023-06-20 11:35:22 +00:00
if err != nil {
return err
}
// Update keypair clock if any but the watch only account has changed.
if relatedKeypair != nil && updateKeypairClock {
err = db.updateKeypairClock(tx, acc.KeyUID, acc.Clock)
if err != nil {
return err
}
}
if strings.HasPrefix(acc.Path, statusWalletRootPath) {
accIndex, err := strconv.ParseUint(acc.Path[len(statusWalletRootPath):], 0, 64)
if err != nil {
return err
}
accountsContainPath := func(accounts []*Account, path string) bool {
for _, acc := range accounts {
if acc.Path == path {
return true
}
}
return false
}
expectedNewKeypairIndex := uint64(0)
if relatedKeypair != nil {
expectedNewKeypairIndex = relatedKeypair.LastUsedDerivationIndex
for {
expectedNewKeypairIndex++
if !accountsContainPath(relatedKeypair.Accounts, statusWalletRootPath+strconv.FormatUint(expectedNewKeypairIndex, 10)) {
break
}
}
}
if accIndex == expectedNewKeypairIndex {
err = updateKeypairLastUsedIndex(tx, acc.KeyUID, accIndex, acc.Clock, updateKeypairClock)
if err != nil {
return err
}
}
}
}
return nil
}
func (db *Database) SaveOrUpdateAccounts(accounts []*Account, updateKeypairClock bool) error {
if len(accounts) == 0 {
return errors.New("no provided accounts to save/update")
}
tx, err := db.db.Begin()
if err != nil {
return err
}
defer func() {
if err == nil {
err = tx.Commit()
return
}
_ = tx.Rollback()
}()
err = db.saveOrUpdateAccounts(tx, accounts, updateKeypairClock)
2023-06-20 11:35:22 +00:00
return err
}
func (db *Database) SaveOrUpdateKeypair(keypair *Keypair) error {
if keypair == nil {
return errDbPassedParameterIsNil
}
tx, err := db.db.Begin()
if err != nil {
return err
}
defer func() {
if err == nil {
err = tx.Commit()
return
}
_ = tx.Rollback()
}()
// If keypair is being saved, not updated, then it must be at least one account and all accounts must have the same key uid.
dbKeypair, err := db.getKeypairByKeyUID(tx, keypair.KeyUID)
if err != nil && err != ErrDbKeypairNotFound {
return err
}
if dbKeypair == nil {
if len(keypair.Accounts) == 0 {
return ErrKeypairWithoutAccounts
}
for _, acc := range keypair.Accounts {
if acc.KeyUID == "" || acc.KeyUID != keypair.KeyUID {
return ErrKeypairDifferentAccountsKeyUID
}
}
}
_, err = tx.Exec(`
INSERT OR IGNORE INTO
keypairs (key_uid, type, derived_from)
VALUES
(?, ?, ?);
UPDATE
keypairs
SET
name = ?,
last_used_derivation_index = ?,
synced_from = ?,
clock = ?
WHERE
key_uid = ?;
`, keypair.KeyUID, keypair.Type, keypair.DerivedFrom,
keypair.Name, keypair.LastUsedDerivationIndex, keypair.SyncedFrom, keypair.Clock, keypair.KeyUID)
if err != nil {
return err
}
return db.saveOrUpdateAccounts(tx, keypair.Accounts, false)
}
func (db *Database) UpdateKeypairName(keyUID string, name string, clock uint64, updateChatAccountName bool) error {
tx, err := db.db.Begin()
if err != nil {
return err
}
defer func() {
if err == nil {
err = tx.Commit()
return
}
_ = tx.Rollback()
}()
_, err = db.getKeypairByKeyUID(tx, keyUID)
if err != nil {
return err
}
_, err = tx.Exec(`
UPDATE
keypairs
SET
name = ?,
clock = ?
WHERE
key_uid = ?;
`, name, clock, keyUID)
if err != nil {
return err
}
if updateChatAccountName {
_, err = tx.Exec(`
UPDATE
keypairs_accounts
SET
name = ?,
clock = ?
WHERE
key_uid = ?
AND
path = ?;
`, name, clock, keyUID, statusChatPath)
return err
}
return nil
}
2019-12-11 13:59:37 +00:00
func (db *Database) GetWalletAddress() (rst types.Address, err error) {
err = db.db.QueryRow("SELECT address FROM keypairs_accounts WHERE wallet = 1").Scan(&rst)
return
}
status-im/status-react#9203 Faster tx fetching with less request *** How it worked before this PR on multiaccount creation: - On multiacc creation we scanned chain for eth and erc20 transfers. For each address of a new empty multiaccount this scan required 1. two `eth_getBalance` requests to find out that there is no any balance change between zero and the last block, for eth transfers 2. and `chain-size/100000` (currently ~100) `eth_getLogs` requests, for erc20 transfers - For some reason we scanned an address of the chat account as well, and also accounts were not deduplicated. So even for an empty multiacc we scanned chain twice for each chat and main wallet addresses, in result app had to execute about 400 requests. - As mentioned above, `eth_getBalance` requests were used to check if there were any eth transfers, and that caused empty history in case if user already used all available eth (so that both zero and latest blocks show 0 eth for an address). There might have been transactions but we wouldn't fetch/show them. - There was no upper limit for the number of rpc requests during the scan, so it could require indefinite number of requests; the scanning algorithm was written so that we persisted the whole history of transactions or tried to scan form the beginning again in case of failure, giving up only after 10 minutes of failures. In result addresses with sufficient number of transactions would never be fully scanned and during these 10 minutes app could use gigabytes of internet data. - Failures were caused by `eth_getBlockByNumber`/`eth_getBlockByHash` requests. These requests return significantly bigger responses than `eth_getBalance`/`eth_transactionsCount` and it is likely that execution of thousands of them in parallel caused failures for accounts with hundreds of transactions. Even for an account with 12k we could successfully determine blocks with transaction in a few minutes using `eth_getBalance` requests, but `eth_getBlock...` couldn't be processed for this acc. - There was no caching for for `eth_getBalance` requests, and this caused in average 3-4 times more such requests than is needed. *** How it works now on multiaccount creation: - On multiacc creation we scan chain for last ~30 eth transactions and then check erc20 in the range where these eth transactions were found. For an empty address in multiacc this means: 1. two `eth_getBalance` transactions to determine that there was no balance change between zero and the last block; two `eth_transactionsCount` requests to determine there are no outgoing transactions for this address; total 4 requests for eth transfers 2. 20 `eth_getLogs` for erc20 transfers. This number can be lowered, but that's not a big deal - Deduplication of addresses is added and also we don't scan chat account, so a new multiacc requires ~25 (we also request latest block number and probably execute a few other calls) request to determine that multiacc is empty (comparing to ~400 before) - In case if address contains transactions we: 1. determine the range which contains 20-25 outgoing eth/erc20 transactions. This usually requires up to 10 `eth_transactionCount` requests 2. then we scan chain for eth transfers using `eth_getBalance` and `eth_transactionCount` (for double checking zero balances) 3. we make sure that we do not scan db for more than 30 blocks with transfers. That's important for accounts with mostly incoming transactions, because the range found on the first step might contain any number of incoming transfers, but only 20-25 outgoing transactions 4. when we found ~30 blocks in a given range, we update initial range `from` block using the oldest found block 5. and now we scan db for erc20transfers using `eth_getLogs` `oldest-found-eth-block`-`latest-block`, we make not more than 20 calls 6. when all blocks which contain incoming/outgoing transfers for a given address are found, we save these blocks to db and mark that transfers from these blocks are still to be fetched 7. Then we select latest ~30 (the number can be adjusted) blocks from these which were found and fetch transfers, this requires 3-4 requests per transfer. 8. we persist scanned range so that we know were to start next time 9. we dispatch an event which tells client that transactions are found 10. client fetches latest 20 transfers - when user presses "fetch more" button we check if app's db contains next 20 transfers, if not we scan chain again and return transfers after small fixes
2019-12-18 11:01:46 +00:00
func (db *Database) GetWalletAddresses() (rst []types.Address, err error) {
rows, err := db.db.Query("SELECT address FROM keypairs_accounts WHERE chat = 0 ORDER BY created_at")
status-im/status-react#9203 Faster tx fetching with less request *** How it worked before this PR on multiaccount creation: - On multiacc creation we scanned chain for eth and erc20 transfers. For each address of a new empty multiaccount this scan required 1. two `eth_getBalance` requests to find out that there is no any balance change between zero and the last block, for eth transfers 2. and `chain-size/100000` (currently ~100) `eth_getLogs` requests, for erc20 transfers - For some reason we scanned an address of the chat account as well, and also accounts were not deduplicated. So even for an empty multiacc we scanned chain twice for each chat and main wallet addresses, in result app had to execute about 400 requests. - As mentioned above, `eth_getBalance` requests were used to check if there were any eth transfers, and that caused empty history in case if user already used all available eth (so that both zero and latest blocks show 0 eth for an address). There might have been transactions but we wouldn't fetch/show them. - There was no upper limit for the number of rpc requests during the scan, so it could require indefinite number of requests; the scanning algorithm was written so that we persisted the whole history of transactions or tried to scan form the beginning again in case of failure, giving up only after 10 minutes of failures. In result addresses with sufficient number of transactions would never be fully scanned and during these 10 minutes app could use gigabytes of internet data. - Failures were caused by `eth_getBlockByNumber`/`eth_getBlockByHash` requests. These requests return significantly bigger responses than `eth_getBalance`/`eth_transactionsCount` and it is likely that execution of thousands of them in parallel caused failures for accounts with hundreds of transactions. Even for an account with 12k we could successfully determine blocks with transaction in a few minutes using `eth_getBalance` requests, but `eth_getBlock...` couldn't be processed for this acc. - There was no caching for for `eth_getBalance` requests, and this caused in average 3-4 times more such requests than is needed. *** How it works now on multiaccount creation: - On multiacc creation we scan chain for last ~30 eth transactions and then check erc20 in the range where these eth transactions were found. For an empty address in multiacc this means: 1. two `eth_getBalance` transactions to determine that there was no balance change between zero and the last block; two `eth_transactionsCount` requests to determine there are no outgoing transactions for this address; total 4 requests for eth transfers 2. 20 `eth_getLogs` for erc20 transfers. This number can be lowered, but that's not a big deal - Deduplication of addresses is added and also we don't scan chat account, so a new multiacc requires ~25 (we also request latest block number and probably execute a few other calls) request to determine that multiacc is empty (comparing to ~400 before) - In case if address contains transactions we: 1. determine the range which contains 20-25 outgoing eth/erc20 transactions. This usually requires up to 10 `eth_transactionCount` requests 2. then we scan chain for eth transfers using `eth_getBalance` and `eth_transactionCount` (for double checking zero balances) 3. we make sure that we do not scan db for more than 30 blocks with transfers. That's important for accounts with mostly incoming transactions, because the range found on the first step might contain any number of incoming transfers, but only 20-25 outgoing transactions 4. when we found ~30 blocks in a given range, we update initial range `from` block using the oldest found block 5. and now we scan db for erc20transfers using `eth_getLogs` `oldest-found-eth-block`-`latest-block`, we make not more than 20 calls 6. when all blocks which contain incoming/outgoing transfers for a given address are found, we save these blocks to db and mark that transfers from these blocks are still to be fetched 7. Then we select latest ~30 (the number can be adjusted) blocks from these which were found and fetch transfers, this requires 3-4 requests per transfer. 8. we persist scanned range so that we know were to start next time 9. we dispatch an event which tells client that transactions are found 10. client fetches latest 20 transfers - when user presses "fetch more" button we check if app's db contains next 20 transfers, if not we scan chain again and return transfers after small fixes
2019-12-18 11:01:46 +00:00
if err != nil {
return nil, err
}
defer rows.Close()
status-im/status-react#9203 Faster tx fetching with less request *** How it worked before this PR on multiaccount creation: - On multiacc creation we scanned chain for eth and erc20 transfers. For each address of a new empty multiaccount this scan required 1. two `eth_getBalance` requests to find out that there is no any balance change between zero and the last block, for eth transfers 2. and `chain-size/100000` (currently ~100) `eth_getLogs` requests, for erc20 transfers - For some reason we scanned an address of the chat account as well, and also accounts were not deduplicated. So even for an empty multiacc we scanned chain twice for each chat and main wallet addresses, in result app had to execute about 400 requests. - As mentioned above, `eth_getBalance` requests were used to check if there were any eth transfers, and that caused empty history in case if user already used all available eth (so that both zero and latest blocks show 0 eth for an address). There might have been transactions but we wouldn't fetch/show them. - There was no upper limit for the number of rpc requests during the scan, so it could require indefinite number of requests; the scanning algorithm was written so that we persisted the whole history of transactions or tried to scan form the beginning again in case of failure, giving up only after 10 minutes of failures. In result addresses with sufficient number of transactions would never be fully scanned and during these 10 minutes app could use gigabytes of internet data. - Failures were caused by `eth_getBlockByNumber`/`eth_getBlockByHash` requests. These requests return significantly bigger responses than `eth_getBalance`/`eth_transactionsCount` and it is likely that execution of thousands of them in parallel caused failures for accounts with hundreds of transactions. Even for an account with 12k we could successfully determine blocks with transaction in a few minutes using `eth_getBalance` requests, but `eth_getBlock...` couldn't be processed for this acc. - There was no caching for for `eth_getBalance` requests, and this caused in average 3-4 times more such requests than is needed. *** How it works now on multiaccount creation: - On multiacc creation we scan chain for last ~30 eth transactions and then check erc20 in the range where these eth transactions were found. For an empty address in multiacc this means: 1. two `eth_getBalance` transactions to determine that there was no balance change between zero and the last block; two `eth_transactionsCount` requests to determine there are no outgoing transactions for this address; total 4 requests for eth transfers 2. 20 `eth_getLogs` for erc20 transfers. This number can be lowered, but that's not a big deal - Deduplication of addresses is added and also we don't scan chat account, so a new multiacc requires ~25 (we also request latest block number and probably execute a few other calls) request to determine that multiacc is empty (comparing to ~400 before) - In case if address contains transactions we: 1. determine the range which contains 20-25 outgoing eth/erc20 transactions. This usually requires up to 10 `eth_transactionCount` requests 2. then we scan chain for eth transfers using `eth_getBalance` and `eth_transactionCount` (for double checking zero balances) 3. we make sure that we do not scan db for more than 30 blocks with transfers. That's important for accounts with mostly incoming transactions, because the range found on the first step might contain any number of incoming transfers, but only 20-25 outgoing transactions 4. when we found ~30 blocks in a given range, we update initial range `from` block using the oldest found block 5. and now we scan db for erc20transfers using `eth_getLogs` `oldest-found-eth-block`-`latest-block`, we make not more than 20 calls 6. when all blocks which contain incoming/outgoing transfers for a given address are found, we save these blocks to db and mark that transfers from these blocks are still to be fetched 7. Then we select latest ~30 (the number can be adjusted) blocks from these which were found and fetch transfers, this requires 3-4 requests per transfer. 8. we persist scanned range so that we know were to start next time 9. we dispatch an event which tells client that transactions are found 10. client fetches latest 20 transfers - when user presses "fetch more" button we check if app's db contains next 20 transfers, if not we scan chain again and return transfers after small fixes
2019-12-18 11:01:46 +00:00
for rows.Next() {
addr := types.Address{}
err = rows.Scan(&addr)
if err != nil {
return nil, err
}
rst = append(rst, addr)
}
if err := rows.Err(); err != nil {
return nil, err
}
status-im/status-react#9203 Faster tx fetching with less request *** How it worked before this PR on multiaccount creation: - On multiacc creation we scanned chain for eth and erc20 transfers. For each address of a new empty multiaccount this scan required 1. two `eth_getBalance` requests to find out that there is no any balance change between zero and the last block, for eth transfers 2. and `chain-size/100000` (currently ~100) `eth_getLogs` requests, for erc20 transfers - For some reason we scanned an address of the chat account as well, and also accounts were not deduplicated. So even for an empty multiacc we scanned chain twice for each chat and main wallet addresses, in result app had to execute about 400 requests. - As mentioned above, `eth_getBalance` requests were used to check if there were any eth transfers, and that caused empty history in case if user already used all available eth (so that both zero and latest blocks show 0 eth for an address). There might have been transactions but we wouldn't fetch/show them. - There was no upper limit for the number of rpc requests during the scan, so it could require indefinite number of requests; the scanning algorithm was written so that we persisted the whole history of transactions or tried to scan form the beginning again in case of failure, giving up only after 10 minutes of failures. In result addresses with sufficient number of transactions would never be fully scanned and during these 10 minutes app could use gigabytes of internet data. - Failures were caused by `eth_getBlockByNumber`/`eth_getBlockByHash` requests. These requests return significantly bigger responses than `eth_getBalance`/`eth_transactionsCount` and it is likely that execution of thousands of them in parallel caused failures for accounts with hundreds of transactions. Even for an account with 12k we could successfully determine blocks with transaction in a few minutes using `eth_getBalance` requests, but `eth_getBlock...` couldn't be processed for this acc. - There was no caching for for `eth_getBalance` requests, and this caused in average 3-4 times more such requests than is needed. *** How it works now on multiaccount creation: - On multiacc creation we scan chain for last ~30 eth transactions and then check erc20 in the range where these eth transactions were found. For an empty address in multiacc this means: 1. two `eth_getBalance` transactions to determine that there was no balance change between zero and the last block; two `eth_transactionsCount` requests to determine there are no outgoing transactions for this address; total 4 requests for eth transfers 2. 20 `eth_getLogs` for erc20 transfers. This number can be lowered, but that's not a big deal - Deduplication of addresses is added and also we don't scan chat account, so a new multiacc requires ~25 (we also request latest block number and probably execute a few other calls) request to determine that multiacc is empty (comparing to ~400 before) - In case if address contains transactions we: 1. determine the range which contains 20-25 outgoing eth/erc20 transactions. This usually requires up to 10 `eth_transactionCount` requests 2. then we scan chain for eth transfers using `eth_getBalance` and `eth_transactionCount` (for double checking zero balances) 3. we make sure that we do not scan db for more than 30 blocks with transfers. That's important for accounts with mostly incoming transactions, because the range found on the first step might contain any number of incoming transfers, but only 20-25 outgoing transactions 4. when we found ~30 blocks in a given range, we update initial range `from` block using the oldest found block 5. and now we scan db for erc20transfers using `eth_getLogs` `oldest-found-eth-block`-`latest-block`, we make not more than 20 calls 6. when all blocks which contain incoming/outgoing transfers for a given address are found, we save these blocks to db and mark that transfers from these blocks are still to be fetched 7. Then we select latest ~30 (the number can be adjusted) blocks from these which were found and fetch transfers, this requires 3-4 requests per transfer. 8. we persist scanned range so that we know were to start next time 9. we dispatch an event which tells client that transactions are found 10. client fetches latest 20 transfers - when user presses "fetch more" button we check if app's db contains next 20 transfers, if not we scan chain again and return transfers after small fixes
2019-12-18 11:01:46 +00:00
return rst, nil
}
2019-12-11 13:59:37 +00:00
func (db *Database) GetChatAddress() (rst types.Address, err error) {
err = db.db.QueryRow("SELECT address FROM keypairs_accounts WHERE chat = 1").Scan(&rst)
return
}
2019-12-11 13:59:37 +00:00
func (db *Database) GetAddresses() (rst []types.Address, err error) {
rows, err := db.db.Query("SELECT address FROM keypairs_accounts ORDER BY created_at")
if err != nil {
return nil, err
}
defer rows.Close()
for rows.Next() {
2019-12-11 13:59:37 +00:00
addr := types.Address{}
err = rows.Scan(&addr)
if err != nil {
return nil, err
}
rst = append(rst, addr)
}
if err := rows.Err(); err != nil {
return nil, err
}
return rst, nil
}
// AddressExists returns true if given address is stored in database.
2019-12-11 13:59:37 +00:00
func (db *Database) AddressExists(address types.Address) (exists bool, err error) {
err = db.db.QueryRow("SELECT EXISTS (SELECT 1 FROM keypairs_accounts WHERE address = ?)", address).Scan(&exists)
return exists, err
}
2022-07-06 16:12:49 +00:00
// GetPath returns true if account with given address was recently key and doesn't have a key yet
func (db *Database) GetPath(address types.Address) (path string, err error) {
err = db.db.QueryRow("SELECT path FROM keypairs_accounts WHERE address = ?", address).Scan(&path)
2022-07-06 16:12:49 +00:00
return path, err
}
func (db *Database) GetNodeConfig() (*params.NodeConfig, error) {
2022-04-22 08:11:40 +00:00
return nodecfg.GetNodeConfigFromDB(db.db)
}
// This function should not update the clock, cause it marks accounts locally.
func (db *Database) SetAccountOperability(address types.Address, operable AccountOperable) (err error) {
tx, err := db.db.Begin()
defer func() {
if err == nil {
err = tx.Commit()
return
}
_ = tx.Rollback()
}()
if err != nil {
return err
}
_, err = db.getAccountByAddress(tx, address)
if err != nil {
return err
}
_, err = tx.Exec(`UPDATE keypairs_accounts SET operable = ? WHERE address = ?`, operable, address)
return err
}
2023-06-20 11:35:22 +00:00
func (db *Database) GetPositionForNextNewAccount() (int64, error) {
var pos sql.NullInt64
err := db.db.QueryRow("SELECT MAX(position) FROM keypairs_accounts").Scan(&pos)
2023-06-20 11:35:22 +00:00
if err != nil {
return 0, err
2023-06-20 11:35:22 +00:00
}
if pos.Valid {
return pos.Int64 + 1, nil
2023-06-20 11:35:22 +00:00
}
return 0, nil
}
// The UpdateAccountPosition function rearranges accounts based on the new position assigned to a given address.
// Please note that this function does not ensure a sequential order
func (db *Database) UpdateAccountPosition(address types.Address, newPosition int64, clock uint64) (err error) {
var (
maxPosition int64
minPosition int64
currentPosition int64
)
2023-06-20 11:35:22 +00:00
tx, err := db.db.Begin()
defer func() {
if err == nil {
err = tx.Commit()
return
}
_ = tx.Rollback()
}()
err = tx.QueryRow("SELECT MAX(position), MIN(position) FROM keypairs_accounts").Scan(&maxPosition, &minPosition)
2023-06-20 11:35:22 +00:00
if err != nil {
return err
}
acc, err := db.getAccountByAddress(tx, address)
if err != nil {
return err
}
currentPosition = acc.Position
2023-06-20 11:35:22 +00:00
if newPosition == maxPosition {
newPosition++
} else if newPosition == minPosition {
newPosition--
} else if newPosition > currentPosition {
_, err = tx.Exec("UPDATE keypairs_accounts SET position = position + 1 WHERE position > ?", newPosition)
newPosition++
} else if newPosition < currentPosition {
newPosition--
_, err = tx.Exec("UPDATE keypairs_accounts SET position = position - 1 WHERE position <= ?", newPosition)
}
if err != nil {
return err
}
_, err = tx.Exec("UPDATE keypairs_accounts SET position = ? WHERE address = ?", newPosition, address)
if err != nil {
return err
}
// Update keypair clock if any but the watch only account was deleted.
if acc.KeyUID != "" {
err = db.updateKeypairClock(tx, acc.KeyUID, clock)
return err
}
2023-06-20 11:35:22 +00:00
return nil
}