feat(wallet): move wallet-related tables to a dedicated db.

The only place where appDB is used in wallet is activity,
which refers to `keycards_accounts` table. So a temporary
table `keycards_accounts` is created in wallet db and updated
before each activity query.
This commit is contained in:
Ivan Belyakov 2023-08-11 13:25:14 +02:00 committed by IvanBelyakoff
parent 3a41a2550a
commit d106b449b6
16 changed files with 427 additions and 54 deletions

View File

@ -274,7 +274,7 @@ install-xtools: ##@install Install Miscellaneous Go Tools
GO111MODULE=on go install golang.org/x/tools/go/packages/...@v0.1.5 GO111MODULE=on go install golang.org/x/tools/go/packages/...@v0.1.5
generate: ##@other Regenerate assets and other auto-generated stuff generate: ##@other Regenerate assets and other auto-generated stuff
go generate ./static ./static/mailserver_db_migrations ./t ./multiaccounts/... ./appdatabase/... ./protocol/... go generate ./static ./static/mailserver_db_migrations ./t ./multiaccounts/... ./appdatabase/... ./protocol/... ./walletdatabase/...
prepare-release: clean-release prepare-release: clean-release
mkdir -p $(RELEASE_DIR) mkdir -p $(RELEASE_DIR)
@ -397,6 +397,10 @@ migration:
migration-check: migration-check:
bash _assets/scripts/migration_check.sh bash _assets/scripts/migration_check.sh
migration-wallet: DEFAULT_WALLET_MIGRATION_PATH := walletdatabase/migrations/sql
migration-wallet:
touch $(DEFAULT_WALLET_MIGRATION_PATH)/$(shell date +%s)_$(D).up.sql
install-git-hooks: install-git-hooks:
@ln -s -f $(shell pwd)/_assets/hooks/pre-rebase .git/hooks @ln -s -f $(shell pwd)/_assets/hooks/pre-rebase .git/hooks
@ln -s -f $(shell pwd)/_assets/hooks/pre-merge-commit .git/hooks @ln -s -f $(shell pwd)/_assets/hooks/pre-merge-commit .git/hooks

View File

@ -13,6 +13,7 @@ import (
"time" "time"
"github.com/status-im/status-go/images" "github.com/status-im/status-go/images"
"github.com/status-im/status-go/walletdatabase"
"github.com/imdario/mergo" "github.com/imdario/mergo"
@ -76,6 +77,7 @@ type GethStatusBackend struct {
// rootDataDir is the same for all networks. // rootDataDir is the same for all networks.
rootDataDir string rootDataDir string
appDB *sql.DB appDB *sql.DB
walletDB *sql.DB
config *params.NodeConfig config *params.NodeConfig
statusNode *node.StatusNode statusNode *node.StatusNode
@ -328,6 +330,20 @@ func (b *GethStatusBackend) runDBFileMigrations(account multiaccounts.Account, p
return v4Path, nil return v4Path, nil
} }
func (b *GethStatusBackend) ensureDBsOpened(account multiaccounts.Account, password string) (err error) {
// After wallet DB initial migration, the tables moved to wallet DB are removed from appDB
// so better migrate wallet DB first to avoid removal if wallet DB migration fails
if err = b.ensureWalletDBOpened(account, password); err != nil {
return err
}
if err = b.ensureAppDBOpened(account, password); err != nil {
return err
}
return nil
}
func (b *GethStatusBackend) ensureAppDBOpened(account multiaccounts.Account, password string) (err error) { func (b *GethStatusBackend) ensureAppDBOpened(account multiaccounts.Account, password string) (err error) {
b.mu.Lock() b.mu.Lock()
defer b.mu.Unlock() defer b.mu.Unlock()
@ -352,6 +368,26 @@ func (b *GethStatusBackend) ensureAppDBOpened(account multiaccounts.Account, pas
return nil return nil
} }
func (b *GethStatusBackend) ensureWalletDBOpened(account multiaccounts.Account, password string) (err error) {
b.mu.Lock()
defer b.mu.Unlock()
if b.walletDB != nil {
return nil
}
if len(b.rootDataDir) == 0 {
return errors.New("root datadir wasn't provided")
}
dbWalletPath := filepath.Join(b.rootDataDir, fmt.Sprintf("%s-wallet.db", account.KeyUID))
b.walletDB, err = walletdatabase.InitializeDB(dbWalletPath, password, account.KDFIterations)
if err != nil {
b.log.Error("failed to initialize wallet db", "err", err)
return err
}
b.statusNode.SetWalletDB(b.walletDB)
return nil
}
func (b *GethStatusBackend) setupLogSettings() error { func (b *GethStatusBackend) setupLogSettings() error {
logSettings := logutils.LogSettings{ logSettings := logutils.LogSettings{
Enabled: b.config.LogEnabled, Enabled: b.config.LogEnabled,
@ -382,7 +418,7 @@ func (b *GethStatusBackend) startNodeWithKey(acc multiaccounts.Account, password
acc.KDFIterations = kdfIterations acc.KDFIterations = kdfIterations
} }
err := b.ensureAppDBOpened(acc, password) err := b.ensureDBsOpened(acc, password)
if err != nil { if err != nil {
return err return err
} }
@ -528,7 +564,7 @@ func (b *GethStatusBackend) loginAccount(request *requests.Login) error {
acc.KDFIterations = sqlite.ReducedKDFIterationsNumber acc.KDFIterations = sqlite.ReducedKDFIterationsNumber
} }
err := b.ensureAppDBOpened(acc, password) err := b.ensureDBsOpened(acc, password)
if err != nil { if err != nil {
return err return err
} }
@ -600,7 +636,7 @@ func (b *GethStatusBackend) loginAccount(request *requests.Login) error {
} }
func (b *GethStatusBackend) startNodeWithAccount(acc multiaccounts.Account, password string, inputNodeCfg *params.NodeConfig) error { func (b *GethStatusBackend) startNodeWithAccount(acc multiaccounts.Account, password string, inputNodeCfg *params.NodeConfig) error {
err := b.ensureAppDBOpened(acc, password) err := b.ensureDBsOpened(acc, password)
if err != nil { if err != nil {
return err return err
} }
@ -683,7 +719,7 @@ func (b *GethStatusBackend) GetSettings() (*settings.Settings, error) {
} }
func (b *GethStatusBackend) MigrateKeyStoreDir(acc multiaccounts.Account, password, oldDir, newDir string) error { func (b *GethStatusBackend) MigrateKeyStoreDir(acc multiaccounts.Account, password, oldDir, newDir string) error {
err := b.ensureAppDBOpened(acc, password) err := b.ensureDBsOpened(acc, password)
if err != nil { if err != nil {
return err return err
} }
@ -877,7 +913,7 @@ func (b *GethStatusBackend) ConvertToKeycardAccount(account multiaccounts.Accoun
return err return err
} }
err = b.ensureAppDBOpened(account, password) err = b.ensureDBsOpened(account, password)
if err != nil { if err != nil {
return err return err
} }
@ -1155,7 +1191,7 @@ func (b *GethStatusBackend) ConvertToRegularAccount(mnemonic string, currPasswor
return err return err
} }
err = b.ensureAppDBOpened(multiaccounts.Account{KeyUID: accountInfo.KeyUID, KDFIterations: kdfIterations}, newPassword) err = b.ensureDBsOpened(multiaccounts.Account{KeyUID: accountInfo.KeyUID, KDFIterations: kdfIterations}, newPassword)
if err != nil { if err != nil {
return err return err
} }
@ -1230,7 +1266,7 @@ func (b *GethStatusBackend) VerifyDatabasePassword(keyUID string, password strin
return err return err
} }
err = b.ensureAppDBOpened(multiaccounts.Account{KeyUID: keyUID, KDFIterations: kdfIterations}, password) err = b.ensureDBsOpened(multiaccounts.Account{KeyUID: keyUID, KDFIterations: kdfIterations}, password)
if err != nil { if err != nil {
return err return err
} }
@ -1297,7 +1333,7 @@ func (b *GethStatusBackend) SaveAccountAndStartNodeWithKey(account multiaccounts
if err != nil { if err != nil {
return err return err
} }
err = b.ensureAppDBOpened(account, password) err = b.ensureDBsOpened(account, password)
if err != nil { if err != nil {
return err return err
} }
@ -1328,7 +1364,7 @@ func (b *GethStatusBackend) StartNodeWithAccountAndInitialConfig(
if err != nil { if err != nil {
return err return err
} }
err = b.ensureAppDBOpened(account, password) err = b.ensureDBsOpened(account, password)
if err != nil { if err != nil {
return err return err
} }
@ -1908,6 +1944,7 @@ func (b *GethStatusBackend) Logout() error {
b.AccountManager().Logout() b.AccountManager().Logout()
b.appDB = nil b.appDB = nil
b.walletDB = nil
b.account = nil b.account = nil
if b.statusNode != nil { if b.statusNode != nil {
@ -2003,7 +2040,7 @@ func (b *GethStatusBackend) injectAccountsIntoWakuService(w types.WakuKeyManager
} }
if st != nil { if st != nil {
if err := st.InitProtocol(b.statusNode.GethNode().Config().Name, identity, b.appDB, b.statusNode.HTTPServer(), b.multiaccountsDB, acc, b.accountManager, b.statusNode.RPCClient(), b.statusNode.WalletService(), b.statusNode.CollectiblesService(), logutils.ZapLogger()); err != nil { if err := st.InitProtocol(b.statusNode.GethNode().Config().Name, identity, b.appDB, b.walletDB, b.statusNode.HTTPServer(), b.multiaccountsDB, acc, b.accountManager, b.statusNode.RPCClient(), b.statusNode.WalletService(), b.statusNode.CollectiblesService(), logutils.ZapLogger()); err != nil {
return err return err
} }
// Set initial connection state // Set initial connection state

View File

@ -60,7 +60,7 @@ func setupWalletTest(t *testing.T, password string) (backend *GethStatusBackend,
KeyUID: masterAccInfo.KeyUID, KeyUID: masterAccInfo.KeyUID,
} }
err = backend.ensureAppDBOpened(account, password) err = backend.ensureDBsOpened(account, password)
require.NoError(t, err) require.NoError(t, err)
walletRootAddress := masterAccInfo.Derived[pathWalletRoot].Address walletRootAddress := masterAccInfo.Derived[pathWalletRoot].Address

View File

@ -26,6 +26,13 @@ var customSteps = []sqlite.PostStep{
{Version: 1687193315, CustomMigration: migrateWalletTransferFromToAddresses, RollBackVersion: 1686825075}, {Version: 1687193315, CustomMigration: migrateWalletTransferFromToAddresses, RollBackVersion: 1686825075},
} }
type DbInitializer struct {
}
func (a DbInitializer) Initialize(path, password string, kdfIterationsNumber int) (*sql.DB, error) {
return InitializeDB(path, password, kdfIterationsNumber)
}
func doMigration(db *sql.DB) error { func doMigration(db *sql.DB) error {
lastMigration, migrationTableExists, err := sqlite.GetLastMigrationVersion(db) lastMigration, migrationTableExists, err := sqlite.GetLastMigrationVersion(db)
if err != nil { if err != nil {

View File

@ -0,0 +1,18 @@
DROP TABLE blocks;
DROP TABLE blocks_ranges;
DROP TABLE blocks_ranges_sequential;
DROP TABLE pending_transactions;
DROP TABLE saved_addresses;
-- token_balances is the only table that was placed by mistake in nodeconfig
-- migrations and in tests it is missing if nodeconfig migration is not used.
DROP TABLE IF EXISTS token_balances;
DROP TABLE tokens;
DROP TABLE visible_tokens;
DROP TABLE currency_format_cache;
DROP TABLE multi_transactions;
DROP TABLE balance_history;
DROP TABLE price_cache;
DROP TABLE collectibles_ownership_cache;
DROP TABLE transfers;
-- All indices are automatically removed

View File

@ -0,0 +1,7 @@
package dbsetup
import "database/sql"
type DatabaseInitializer interface {
Initialize(path, password string, kdfIterationsNumber int) (*sql.DB, error)
}

View File

@ -77,6 +77,7 @@ type StatusNode struct {
appDB *sql.DB appDB *sql.DB
multiaccountsDB *multiaccounts.Database multiaccountsDB *multiaccounts.Database
walletDB *sql.DB
config *params.NodeConfig // Status node configuration config *params.NodeConfig // Status node configuration
gethNode *node.Node // reference to Geth P2P stack/node gethNode *node.Node // reference to Geth P2P stack/node
@ -695,3 +696,7 @@ func (n *StatusNode) SetAppDB(db *sql.DB) {
func (n *StatusNode) SetMultiaccountsDB(db *multiaccounts.Database) { func (n *StatusNode) SetMultiaccountsDB(db *multiaccounts.Database) {
n.multiaccountsDB = db n.multiaccountsDB = db
} }
func (n *StatusNode) SetWalletDB(db *sql.DB) {
n.walletDB = db
}

View File

@ -500,7 +500,7 @@ func (b *StatusNode) CollectiblesService() *collectibles.Service {
func (b *StatusNode) walletService(accountsDB *accounts.Database, accountsFeed *event.Feed) *wallet.Service { func (b *StatusNode) walletService(accountsDB *accounts.Database, accountsFeed *event.Feed) *wallet.Service {
if b.walletSrvc == nil { if b.walletSrvc == nil {
b.walletSrvc = wallet.NewService( b.walletSrvc = wallet.NewService(
b.appDB, accountsDB, b.rpcClient, accountsFeed, b.gethAccountManager, b.transactor, b.config, b.walletDB, accountsDB, b.rpcClient, accountsFeed, b.gethAccountManager, b.transactor, b.config,
b.ensService(b.timeSourceNow()), b.ensService(b.timeSourceNow()),
b.stickersService(accountsDB), b.stickersService(accountsDB),
b.rpcFiltersSrvc, b.rpcFiltersSrvc,

View File

@ -28,7 +28,6 @@ import (
"github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p"
"github.com/status-im/status-go/account" "github.com/status-im/status-go/account"
"github.com/status-im/status-go/appdatabase"
"github.com/status-im/status-go/appmetrics" "github.com/status-im/status-go/appmetrics"
"github.com/status-im/status-go/connection" "github.com/status-im/status-go/connection"
"github.com/status-im/status-go/contracts" "github.com/status-im/status-go/contracts"
@ -200,12 +199,6 @@ type mailserverCycle struct {
availabilitySubscriptions []chan struct{} availabilitySubscriptions []chan struct{}
} }
type dbConfig struct {
dbPath string
dbKey string
dbKDFIterations int
}
type EnvelopeEventsInterceptor struct { type EnvelopeEventsInterceptor struct {
EnvelopeEventsHandler transport.EnvelopeEventsHandler EnvelopeEventsHandler transport.EnvelopeEventsHandler
Messenger *Messenger Messenger *Messenger
@ -285,21 +278,12 @@ func NewMessenger(
} }
// Configure the database. // Configure the database.
database := c.db if c.appDb == nil {
if c.db == nil && c.dbConfig == (dbConfig{}) {
return nil, errors.New("database instance or database path needs to be provided") return nil, errors.New("database instance or database path needs to be provided")
} }
if c.db == nil { database := c.appDb
logger.Info("opening a database", zap.String("dbPath", c.dbConfig.dbPath), zap.Int("KDFIterations", c.dbConfig.dbKDFIterations))
var err error
database, err = appdatabase.InitializeDB(c.dbConfig.dbPath, c.dbConfig.dbKey, c.dbConfig.dbKDFIterations)
if err != nil {
return nil, errors.Wrap(err, "failed to initialize database from the db config")
}
}
// Apply any post database creation changes to the database // Apply any post database creation changes to the database
c.db = database
for _, opt := range c.afterDbCreatedHooks { for _, opt := range c.afterDbCreatedHooks {
if err := opt(&c); err != nil { if err := opt(&c); err != nil {
return nil, err return nil, err
@ -466,7 +450,7 @@ func NewMessenger(
mailservers := mailserversDB.NewDB(database) mailservers := mailserversDB.NewDB(database)
savedAddressesManager := wallet.NewSavedAddressesManager(c.db) savedAddressesManager := wallet.NewSavedAddressesManager(c.walletDb)
myPublicKeyString := types.EncodeHex(crypto.FromECDSAPub(&identity.PublicKey)) myPublicKeyString := types.EncodeHex(crypto.FromECDSAPub(&identity.PublicKey))
myContact, err := buildContact(myPublicKeyString, &identity.PublicKey) myContact, err := buildContact(myPublicKeyString, &identity.PublicKey)

View File

@ -76,10 +76,8 @@ type config struct {
featureFlags common.FeatureFlags featureFlags common.FeatureFlags
// A path to a database or a database instance is required. appDb *sql.DB
// The database instance has a higher priority. walletDb *sql.DB
dbConfig dbConfig
db *sql.DB
afterDbCreatedHooks []Option afterDbCreatedHooks []Option
multiAccount *multiaccounts.Database multiAccount *multiaccounts.Database
mailserversDatabase *mailservers.Database mailserversDatabase *mailservers.Database
@ -131,13 +129,6 @@ func WithCustomLogger(logger *zap.Logger) Option {
} }
} }
func WithDatabaseConfig(dbPath string, dbKey string, dbKDFIterations int) Option {
return func(c *config) error {
c.dbConfig = dbConfig{dbPath: dbPath, dbKey: dbKey, dbKDFIterations: dbKDFIterations}
return nil
}
}
func WithVerifyTransactionClient(client EthClient) Option { func WithVerifyTransactionClient(client EthClient) Option {
return func(c *config) error { return func(c *config) error {
c.verifyTransactionClient = client c.verifyTransactionClient = client
@ -147,7 +138,14 @@ func WithVerifyTransactionClient(client EthClient) Option {
func WithDatabase(db *sql.DB) Option { func WithDatabase(db *sql.DB) Option {
return func(c *config) error { return func(c *config) error {
c.db = db c.appDb = db
return nil
}
}
func WithWalletDatabase(db *sql.DB) Option {
return func(c *config) error {
c.walletDb = db
return nil return nil
} }
} }
@ -155,7 +153,7 @@ func WithDatabase(db *sql.DB) Option {
func WithToplevelDatabaseMigrations() Option { func WithToplevelDatabaseMigrations() Option {
return func(c *config) error { return func(c *config) error {
c.afterDbCreatedHooks = append(c.afterDbCreatedHooks, func(c *config) error { c.afterDbCreatedHooks = append(c.afterDbCreatedHooks, func(c *config) error {
return migrations.Migrate(c.db, nil) return migrations.Migrate(c.appDb, nil)
}) })
return nil return nil
} }
@ -173,7 +171,7 @@ func WithAppSettings(s settings.Settings, nc params.NodeConfig) Option {
s.Networks = networks s.Networks = networks
} }
sDB, err := accounts.NewDB(c.db) sDB, err := accounts.NewDB(c.appDb)
if err != nil { if err != nil {
return err return err
} }
@ -209,7 +207,7 @@ func WithBrowserDatabase(bd *browsers.Database) Option {
c.browserDatabase = bd c.browserDatabase = bd
if c.browserDatabase == nil { if c.browserDatabase == nil {
c.afterDbCreatedHooks = append(c.afterDbCreatedHooks, func(c *config) error { c.afterDbCreatedHooks = append(c.afterDbCreatedHooks, func(c *config) error {
c.browserDatabase = browsers.NewDB(c.db) c.browserDatabase = browsers.NewDB(c.appDb)
return nil return nil
}) })
} }

View File

@ -116,7 +116,7 @@ func (s *Service) GetPeer(rawURL string) (*enode.Node, error) {
return enode.ParseV4(rawURL) return enode.ParseV4(rawURL)
} }
func (s *Service) InitProtocol(nodeName string, identity *ecdsa.PrivateKey, db *sql.DB, httpServer *server.MediaServer, multiAccountDb *multiaccounts.Database, acc *multiaccounts.Account, accountManager *account.GethManager, rpcClient *rpc.Client, walletService *wallet.Service, collectiblesService *collectibles.Service, logger *zap.Logger) error { func (s *Service) InitProtocol(nodeName string, identity *ecdsa.PrivateKey, appDb, walletDb *sql.DB, httpServer *server.MediaServer, multiAccountDb *multiaccounts.Database, acc *multiaccounts.Account, accountManager *account.GethManager, rpcClient *rpc.Client, walletService *wallet.Service, collectiblesService *collectibles.Service, logger *zap.Logger) error {
var err error var err error
if !s.config.ShhextConfig.PFSEnabled { if !s.config.ShhextConfig.PFSEnabled {
return nil return nil
@ -148,14 +148,14 @@ func (s *Service) InitProtocol(nodeName string, identity *ecdsa.PrivateKey, db *
EnvelopeEventsHandler: EnvelopeSignalHandler{}, EnvelopeEventsHandler: EnvelopeSignalHandler{},
Logger: logger, Logger: logger,
} }
s.accountsDB, err = accounts.NewDB(db) s.accountsDB, err = accounts.NewDB(appDb)
if err != nil { if err != nil {
return err return err
} }
s.multiAccountsDB = multiAccountDb s.multiAccountsDB = multiAccountDb
s.account = acc s.account = acc
options, err := buildMessengerOptions(s.config, identity, db, httpServer, s.rpcClient, s.multiAccountsDB, acc, envelopesMonitorConfig, s.accountsDB, walletService, collectiblesService, logger, &MessengerSignalsHandler{}) options, err := buildMessengerOptions(s.config, identity, appDb, walletDb, httpServer, s.rpcClient, s.multiAccountsDB, acc, envelopesMonitorConfig, s.accountsDB, walletService, collectiblesService, logger, &MessengerSignalsHandler{})
if err != nil { if err != nil {
return err return err
} }
@ -404,7 +404,8 @@ func (s *Service) Stop() error {
func buildMessengerOptions( func buildMessengerOptions(
config params.NodeConfig, config params.NodeConfig,
identity *ecdsa.PrivateKey, identity *ecdsa.PrivateKey,
db *sql.DB, appDb *sql.DB,
walletDb *sql.DB,
httpServer *server.MediaServer, httpServer *server.MediaServer,
rpcClient *rpc.Client, rpcClient *rpc.Client,
multiAccounts *multiaccounts.Database, multiAccounts *multiaccounts.Database,
@ -419,11 +420,12 @@ func buildMessengerOptions(
options := []protocol.Option{ options := []protocol.Option{
protocol.WithCustomLogger(logger), protocol.WithCustomLogger(logger),
protocol.WithPushNotifications(), protocol.WithPushNotifications(),
protocol.WithDatabase(db), protocol.WithDatabase(appDb),
protocol.WithWalletDatabase(walletDb),
protocol.WithMultiAccounts(multiAccounts), protocol.WithMultiAccounts(multiAccounts),
protocol.WithMailserversDatabase(mailserversDB.NewDB(db)), protocol.WithMailserversDatabase(mailserversDB.NewDB(appDb)),
protocol.WithAccount(account), protocol.WithAccount(account),
protocol.WithBrowserDatabase(browsers.NewDB(db)), protocol.WithBrowserDatabase(browsers.NewDB(appDb)),
protocol.WithEnvelopesMonitorConfig(envelopesMonitorConfig), protocol.WithEnvelopesMonitorConfig(envelopesMonitorConfig),
protocol.WithSignalsHandler(messengerSignalsHandler), protocol.WithSignalsHandler(messengerSignalsHandler),
protocol.WithENSVerificationConfig(publishMessengerResponse, config.ShhextConfig.VerifyENSURL, config.ShhextConfig.VerifyENSContractAddress), protocol.WithENSVerificationConfig(publishMessengerResponse, config.ShhextConfig.VerifyENSURL, config.ShhextConfig.VerifyENSContractAddress),

View File

@ -15,6 +15,7 @@ import (
"github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log"
"github.com/status-im/status-go/multiaccounts/accounts"
"github.com/status-im/status-go/services/wallet/common" "github.com/status-im/status-go/services/wallet/common"
"github.com/status-im/status-go/services/wallet/transfer" "github.com/status-im/status-go/services/wallet/transfer"
@ -30,6 +31,8 @@ const (
PendingTransactionPT PendingTransactionPT
) )
const keypairAccountsTable = "keypairs_accounts"
var ( var (
ZeroAddress = eth.Address{} ZeroAddress = eth.Address{}
) )
@ -575,6 +578,7 @@ const (
type FilterDependencies struct { type FilterDependencies struct {
db *sql.DB db *sql.DB
accountsDb *accounts.Database
tokenSymbol func(token Token) string tokenSymbol func(token Token) string
tokenFromSymbol func(chainID *common.ChainID, symbol string) *Token tokenFromSymbol func(chainID *common.ChainID, symbol string) *Token
} }
@ -655,6 +659,13 @@ func getActivityEntries(ctx context.Context, deps FilterDependencies, addresses
return strconv.Itoa(int(t)) return strconv.Itoa(int(t))
}) })
// Since the filter query needs addresses which are in a different database, we need to update the
// keypairs_accounts table in the current database with the latest addresses from the accounts database
err := updateKeypairsAccountsTable(deps.accountsDb, deps.db)
if err != nil {
return nil, err
}
queryString := fmt.Sprintf(queryFormatString, involvedAddresses, toAddresses, assetsTokenCodes, assetsERC20, networks, queryString := fmt.Sprintf(queryFormatString, involvedAddresses, toAddresses, assetsTokenCodes, assetsERC20, networks,
joinedMTTypes) joinedMTTypes)
@ -907,3 +918,41 @@ func contractTypeFromDBType(dbType string) (transferType *TransferType) {
} }
return transferType return transferType
} }
func updateKeypairsAccountsTable(accountsDb *accounts.Database, db *sql.DB) error {
_, err := db.Exec(fmt.Sprintf("CREATE TEMP TABLE IF NOT EXISTS %s (address VARCHAR PRIMARY KEY)",
keypairAccountsTable))
if err != nil {
log.Error("failed to create 'keypairs_accounts' table", "err", err)
return err
}
addresses, err := accountsDb.GetWalletAddresses()
if err != nil {
log.Error("failed to get wallet addresses", "err", err)
return err
}
tx, err := db.Begin()
if err != nil {
return err
}
defer func() {
if err == nil {
err = tx.Commit()
return
}
_ = tx.Rollback()
}()
for _, address := range addresses {
_, err = tx.Exec(fmt.Sprintf("INSERT OR IGNORE INTO %s (address) VALUES (?)", keypairAccountsTable), address)
if err != nil {
log.Error("failed to insert wallet addresses", "err", err)
return err
}
}
return nil
}

View File

@ -0,0 +1,38 @@
package walletdatabase
import (
"database/sql"
"github.com/status-im/status-go/sqlite"
"github.com/status-im/status-go/walletdatabase/migrations"
)
type DbInitializer struct {
}
func (a DbInitializer) Initialize(path, password string, kdfIterationsNumber int) (*sql.DB, error) {
return InitializeDB(path, password, kdfIterationsNumber)
}
var walletCustomSteps = []sqlite.PostStep{
}
func doMigration(db *sql.DB) error {
// Run all the new migrations
return migrations.Migrate(db, walletCustomSteps)
}
// InitializeDB creates db file at a given path and applies migrations.
func InitializeDB(path, password string, kdfIterationsNumber int) (*sql.DB, error) {
db, err := sqlite.OpenDB(path, password, kdfIterationsNumber)
if err != nil {
return nil, err
}
err = doMigration(db)
if err != nil {
return nil, err
}
return db, nil
}

View File

@ -0,0 +1,30 @@
package migrations
import (
"database/sql"
bindata "github.com/status-im/migrate/v4/source/go_bindata"
"github.com/status-im/status-go/sqlite"
)
// Migrate applies migrations.
// see Migrate in vendor/status-go/sqlite/migrate.go
func Migrate(db *sql.DB, customSteps []sqlite.PostStep) error {
return sqlite.Migrate(db, bindata.Resource(
AssetNames(),
func(name string) ([]byte, error) {
return Asset(name)
},
), customSteps, nil)
}
// MigrateTo is used for testing purposes
func MigrateTo(db *sql.DB, customSteps []sqlite.PostStep, untilVersion uint) error {
return sqlite.Migrate(db, bindata.Resource(
AssetNames(),
func(name string) ([]byte, error) {
return Asset(name)
},
), customSteps, &untilVersion)
}

View File

@ -0,0 +1,191 @@
CREATE TABLE blocks (
network_id UNSIGNED BIGINT NOT NULL,
address VARCHAR NOT NULL,
blk_number BIGINT NOT NULL,
blk_hash BIGINT NOT NULL,
loaded BOOL DEFAULT FALSE,
CONSTRAINT unique_mapping_for_account_to_block_per_network UNIQUE (address,blk_hash,network_id)
);
CREATE TABLE blocks_ranges (
network_id UNSIGNED BIGINT NOT NULL,
address VARCHAR NOT NULL,
blk_from BIGINT NOT NULL,
blk_to BIGINT NOT NULL,
balance BLOB,
nonce INTEGER);
CREATE TABLE blocks_ranges_sequential (
network_id UNSIGNED BIGINT NOT NULL,
address VARCHAR NOT NULL,
blk_start BIGINT,
blk_first BIGINT NOT NULL,
blk_last BIGINT NOT NULL,
PRIMARY KEY (network_id, address)
) WITHOUT ROWID;
CREATE TABLE pending_transactions (
network_id UNSIGNED BIGINT NOT NULL,
hash VARCHAR NOT NULL,
timestamp UNSIGNED BIGINT NOT NULL,
from_address VARCHAR NOT NULL,
to_address VARCHAR,
symbol VARCHAR,
gas_price BLOB,
gas_limit BLOB,
value BLOB,
data TEXT,
type VARCHAR,
additional_data TEXT,
multi_transaction_id INT,
PRIMARY KEY (network_id, hash)
) WITHOUT ROWID;
CREATE TABLE "saved_addresses" (
address VARCHAR NOT NULL,
name TEXT NOT NULL,
favourite BOOLEAN NOT NULL DEFAULT FALSE,
removed BOOLEAN NOT NULL DEFAULT FALSE,
update_clock INT NOT NULL DEFAULT 0,
chain_short_names VARCHAR DEFAULT "",
ens_name VARCHAR DEFAULT "",
is_test BOOLEAN DEFAULT FALSE,
created_at INT DEFAULT 0,
PRIMARY KEY (address, ens_name, is_test)
) WITHOUT ROWID;
CREATE TABLE token_balances (
user_address VARCHAR NOT NULL,
token_name VARCHAR NOT NULL,
token_symbol VARCHAR NOT NULL,
token_address VARCHAR NOT NULL,
token_color VARCHAR NOT NULL DEFAULT "",
token_decimals INT NOT NULL,
token_description VARCHAR NOT NULL DEFAULT "",
token_url VARCHAR NOT NULL DEFAULT "",
balance VARCHAR NOT NULL,
chain_id INT NOT NULL,
PRIMARY KEY (user_address, chain_id, token_symbol) ON CONFLICT REPLACE
);
CREATE TABLE tokens (
address VARCHAR NOT NULL,
network_id UNSIGNED BIGINT NOT NULL,
name TEXT NOT NULL,
symbol VARCHAR NOT NULL,
decimals UNSIGNED INT,
color VARCHAR,
PRIMARY KEY (address, network_id)
) WITHOUT ROWID;
CREATE TABLE visible_tokens (
chain_id UNSIGNED INT,
address VARCHAR NOT NULL
);
CREATE TABLE currency_format_cache (
symbol VARCHAR NOT NULL,
display_decimals INT NOT NULL,
strip_trailing_zeroes BOOLEAN NOT NULL
);
CREATE TABLE multi_transactions (
from_address VARCHAR NOT NULL,
from_asset VARCHAR NOT NULL,
from_amount VARCHAR NOT NULL,
to_address VARCHAR NOT NULL,
to_asset VARCHAR NOT NULL,
type VARCHAR NOT NULL,
timestamp UNSIGNED BIGINT NOT NULL,
to_amount VARCHAR,
from_network_id UNSIGNED BIGINT,
to_network_id UNSIGNED BIGINT,
cross_tx_id VARCHAR DEFAULT "",
from_tx_hash BLOB,
to_tx_hash BLOB
);
CREATE TABLE balance_history (
chain_id UNSIGNED BIGINT NOT NULL,
address VARCHAR NOT NULL,
currency VARCHAR NOT NULL,
block BIGINT NOT NULL,
timestamp INT NOT NULL,
bitset INT NOT NULL,
balance BLOB
);
CREATE TABLE price_cache (
token VARCHAR NOT NULL,
currency VARCHAR NOT NULL,
price REAL NOT NULL
);
CREATE TABLE IF NOT EXISTS collectibles_ownership_cache (
chain_id UNSIGNED BIGINT NOT NULL,
contract_address VARCHAR NOT NULL,
token_id BLOB NOT NULL,
owner_address VARCHAR NOT NULL
);
CREATE TABLE transfers (
network_id UNSIGNED BIGINT NOT NULL,
hash VARCHAR NOT NULL,
address VARCHAR NOT NULL,
blk_hash VARCHAR NOT NULL,
tx BLOB,
sender VARCHAR,
receipt BLOB,
log BLOB,
type VARCHAR NOT NULL,
blk_number BIGINT NOT NULL,
timestamp UNSIGNED BIGINT NOT NULL,
loaded BOOL DEFAULT 1,
multi_transaction_id INT,
base_gas_fee TEXT NOT NULL DEFAULT "",
status INT,
receipt_type INT,
tx_hash BLOB,
log_index INT,
block_hash BLOB,
cumulative_gas_used INT,
contract_address TEXT,
gas_used INT,
tx_index INT,
tx_type INT,
protected BOOLEAN,
gas_limit UNSIGNED INT,
gas_price_clamped64 INT,
gas_tip_cap_clamped64 INT,
gas_fee_cap_clamped64 INT,
amount_padded128hex CHAR(32),
account_nonce INT,
size INT,
token_address BLOB,
token_id BLOB,
tx_from_address BLOB,
tx_to_address BLOB,
FOREIGN KEY(network_id,address,blk_hash) REFERENCES blocks(network_id,address,blk_hash) ON DELETE CASCADE,
CONSTRAINT unique_transfer_per_address_per_network UNIQUE (hash,address,network_id)
);
CREATE INDEX balance_history_filter_entries ON balance_history (chain_id, address, currency, block, timestamp, bitset);
CREATE INDEX idx_transfers_blk_loaded ON transfers(blk_number, loaded);
CREATE UNIQUE INDEX price_cache_identify_entry ON price_cache (token, currency);
CREATE UNIQUE INDEX balance_history_identify_entry ON balance_history (chain_id, address, currency, block);
CREATE UNIQUE INDEX currency_format_cache_identify_entry ON currency_format_cache (symbol);
CREATE INDEX idx_transfers_filter
ON transfers (multi_transaction_id, loaded, timestamp, status, network_id, tx_from_address, tx_to_address, token_address, token_id, type);
CREATE INDEX idx_pending_transactions
ON pending_transactions (multi_transaction_id, from_address, to_address, network_id, timestamp, symbol);
CREATE INDEX idx_multi_transactions
ON multi_transactions (from_address, to_address, type, from_asset, timestamp, to_asset, from_amount, to_amount);
CREATE INDEX IF NOT EXISTS collectibles_ownership_filter_entries ON collectibles_ownership_cache (chain_id, owner_address);

View File

@ -0,0 +1,3 @@
package sql
//go:generate go-bindata -pkg migrations -o ../bindata.go ./