feat(wallet): move wallet-related tables to a dedicated db.

The only place where appDB is used in wallet is activity,
which refers to `keycards_accounts` table. So a temporary
table `keycards_accounts` is created in wallet db and updated
before each activity query.

Makefile

@@ -274,7 +274,7 @@ install-xtools: ##@install Install Miscellaneous Go Tools
 	GO111MODULE=on go install golang.org/x/tools/go/packages/...@v0.1.5
 
 generate: ##@other Regenerate assets and other auto-generated stuff
-	go generate ./static ./static/mailserver_db_migrations ./t ./multiaccounts/... ./appdatabase/... ./protocol/...
+	go generate ./static ./static/mailserver_db_migrations ./t ./multiaccounts/... ./appdatabase/... ./protocol/... ./walletdatabase/...
 
 prepare-release: clean-release
 	mkdir -p $(RELEASE_DIR)
@@ -397,6 +397,10 @@ migration:
 migration-check:
 	bash _assets/scripts/migration_check.sh
 
+migration-wallet: DEFAULT_WALLET_MIGRATION_PATH := walletdatabase/migrations/sql
+migration-wallet:
+	touch $(DEFAULT_WALLET_MIGRATION_PATH)/$(shell date +%s)_$(D).up.sql
+
 install-git-hooks:
 	@ln -s -f $(shell pwd)/_assets/hooks/pre-rebase .git/hooks
 	@ln -s -f $(shell pwd)/_assets/hooks/pre-merge-commit .git/hooks

api/geth_backend.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/status-im/status-go/images"
+	"github.com/status-im/status-go/walletdatabase"
 
 	"github.com/imdario/mergo"
 
@@ -76,6 +77,7 @@ type GethStatusBackend struct {
 	// rootDataDir is the same for all networks.
 	rootDataDir string
 	appDB       *sql.DB
+	walletDB    *sql.DB
 	config      *params.NodeConfig
 
 	statusNode           *node.StatusNode
@@ -328,6 +330,20 @@ func (b *GethStatusBackend) runDBFileMigrations(account multiaccounts.Account, p
 	return v4Path, nil
 }
 
+func (b *GethStatusBackend) ensureDBsOpened(account multiaccounts.Account, password string) (err error) {
+	// After wallet DB initial migration, the tables moved to wallet DB are removed from appDB
+	// so better migrate wallet DB first to avoid removal if wallet DB migration fails
+	if err = b.ensureWalletDBOpened(account, password); err != nil {
+		return err
+	}
+
+	if err = b.ensureAppDBOpened(account, password); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (b *GethStatusBackend) ensureAppDBOpened(account multiaccounts.Account, password string) (err error) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
@@ -352,6 +368,26 @@ func (b *GethStatusBackend) ensureAppDBOpened(account multiaccounts.Account, pas
 	return nil
 }
 
+func (b *GethStatusBackend) ensureWalletDBOpened(account multiaccounts.Account, password string) (err error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.walletDB != nil {
+		return nil
+	}
+	if len(b.rootDataDir) == 0 {
+		return errors.New("root datadir wasn't provided")
+	}
+
+	dbWalletPath := filepath.Join(b.rootDataDir, fmt.Sprintf("%s-wallet.db", account.KeyUID))
+	b.walletDB, err = walletdatabase.InitializeDB(dbWalletPath, password, account.KDFIterations)
+	if err != nil {
+		b.log.Error("failed to initialize wallet db", "err", err)
+		return err
+	}
+	b.statusNode.SetWalletDB(b.walletDB)
+	return nil
+}
+
 func (b *GethStatusBackend) setupLogSettings() error {
 	logSettings := logutils.LogSettings{
 		Enabled:         b.config.LogEnabled,
@@ -382,7 +418,7 @@ func (b *GethStatusBackend) startNodeWithKey(acc multiaccounts.Account, password
 		acc.KDFIterations = kdfIterations
 	}
 
-	err := b.ensureAppDBOpened(acc, password)
+	err := b.ensureDBsOpened(acc, password)
 	if err != nil {
 		return err
 	}
@@ -528,7 +564,7 @@ func (b *GethStatusBackend) loginAccount(request *requests.Login) error {
 		acc.KDFIterations = sqlite.ReducedKDFIterationsNumber
 	}
 
-	err := b.ensureAppDBOpened(acc, password)
+	err := b.ensureDBsOpened(acc, password)
 	if err != nil {
 		return err
 	}
@@ -600,7 +636,7 @@ func (b *GethStatusBackend) loginAccount(request *requests.Login) error {
 }
 
 func (b *GethStatusBackend) startNodeWithAccount(acc multiaccounts.Account, password string, inputNodeCfg *params.NodeConfig) error {
-	err := b.ensureAppDBOpened(acc, password)
+	err := b.ensureDBsOpened(acc, password)
 	if err != nil {
 		return err
 	}
@@ -683,7 +719,7 @@ func (b *GethStatusBackend) GetSettings() (*settings.Settings, error) {
 }
 
 func (b *GethStatusBackend) MigrateKeyStoreDir(acc multiaccounts.Account, password, oldDir, newDir string) error {
-	err := b.ensureAppDBOpened(acc, password)
+	err := b.ensureDBsOpened(acc, password)
 	if err != nil {
 		return err
 	}
@@ -877,7 +913,7 @@ func (b *GethStatusBackend) ConvertToKeycardAccount(account multiaccounts.Accoun
 		return err
 	}
 
-	err = b.ensureAppDBOpened(account, password)
+	err = b.ensureDBsOpened(account, password)
 	if err != nil {
 		return err
 	}
@@ -1155,7 +1191,7 @@ func (b *GethStatusBackend) ConvertToRegularAccount(mnemonic string, currPasswor
 		return err
 	}
 
-	err = b.ensureAppDBOpened(multiaccounts.Account{KeyUID: accountInfo.KeyUID, KDFIterations: kdfIterations}, newPassword)
+	err = b.ensureDBsOpened(multiaccounts.Account{KeyUID: accountInfo.KeyUID, KDFIterations: kdfIterations}, newPassword)
 	if err != nil {
 		return err
 	}
@@ -1230,7 +1266,7 @@ func (b *GethStatusBackend) VerifyDatabasePassword(keyUID string, password strin
 		return err
 	}
 
-	err = b.ensureAppDBOpened(multiaccounts.Account{KeyUID: keyUID, KDFIterations: kdfIterations}, password)
+	err = b.ensureDBsOpened(multiaccounts.Account{KeyUID: keyUID, KDFIterations: kdfIterations}, password)
 	if err != nil {
 		return err
 	}
@@ -1297,7 +1333,7 @@ func (b *GethStatusBackend) SaveAccountAndStartNodeWithKey(account multiaccounts
 	if err != nil {
 		return err
 	}
-	err = b.ensureAppDBOpened(account, password)
+	err = b.ensureDBsOpened(account, password)
 	if err != nil {
 		return err
 	}
@@ -1328,7 +1364,7 @@ func (b *GethStatusBackend) StartNodeWithAccountAndInitialConfig(
 	if err != nil {
 		return err
 	}
-	err = b.ensureAppDBOpened(account, password)
+	err = b.ensureDBsOpened(account, password)
 	if err != nil {
 		return err
 	}
@@ -1908,6 +1944,7 @@ func (b *GethStatusBackend) Logout() error {
 
 	b.AccountManager().Logout()
 	b.appDB = nil
+	b.walletDB = nil
 	b.account = nil
 
 	if b.statusNode != nil {
@@ -2003,7 +2040,7 @@ func (b *GethStatusBackend) injectAccountsIntoWakuService(w types.WakuKeyManager
 	}
 
 	if st != nil {
-		if err := st.InitProtocol(b.statusNode.GethNode().Config().Name, identity, b.appDB, b.statusNode.HTTPServer(), b.multiaccountsDB, acc, b.accountManager, b.statusNode.RPCClient(), b.statusNode.WalletService(), b.statusNode.CollectiblesService(), logutils.ZapLogger()); err != nil {
+		if err := st.InitProtocol(b.statusNode.GethNode().Config().Name, identity, b.appDB, b.walletDB, b.statusNode.HTTPServer(), b.multiaccountsDB, acc, b.accountManager, b.statusNode.RPCClient(), b.statusNode.WalletService(), b.statusNode.CollectiblesService(), logutils.ZapLogger()); err != nil {
 			return err
 		}
 		// Set initial connection state

api/test_helpers.go

@@ -60,7 +60,7 @@ func setupWalletTest(t *testing.T, password string) (backend *GethStatusBackend,
 		KeyUID:         masterAccInfo.KeyUID,
 	}
 
-	err = backend.ensureAppDBOpened(account, password)
+	err = backend.ensureDBsOpened(account, password)
 	require.NoError(t, err)
 
 	walletRootAddress := masterAccInfo.Derived[pathWalletRoot].Address

appdatabase/database.go

@@ -26,6 +26,13 @@ var customSteps = []sqlite.PostStep{
 	{Version: 1687193315, CustomMigration: migrateWalletTransferFromToAddresses, RollBackVersion: 1686825075},
 }
 
+type DbInitializer struct {
+}
+
+func (a DbInitializer) Initialize(path, password string, kdfIterationsNumber int) (*sql.DB, error) {
+	return InitializeDB(path, password, kdfIterationsNumber)
+}
+
 func doMigration(db *sql.DB) error {
 	lastMigration, migrationTableExists, err := sqlite.GetLastMigrationVersion(db)
 	if err != nil {

appdatabase/migrations/sql/1691753758_move_wallet_tables_to_wallet_db.up.sql

@@ -0,0 +1,18 @@
+DROP TABLE blocks;
+DROP TABLE blocks_ranges;
+DROP TABLE blocks_ranges_sequential;
+DROP TABLE pending_transactions;
+DROP TABLE saved_addresses;
+-- token_balances is the only table that was placed by mistake in nodeconfig
+-- migrations and in tests it is missing if nodeconfig migration is not used.
+DROP TABLE IF EXISTS token_balances;
+DROP TABLE tokens;
+DROP TABLE visible_tokens;
+DROP TABLE currency_format_cache;
+DROP TABLE multi_transactions;
+DROP TABLE balance_history;
+DROP TABLE price_cache;
+DROP TABLE collectibles_ownership_cache;
+DROP TABLE transfers;
+
+-- All indices are automatically removed

common/dbsetup/db_setup.go

@@ -0,0 +1,7 @@
+package dbsetup
+
+import "database/sql"
+
+type DatabaseInitializer interface {
+	Initialize(path, password string, kdfIterationsNumber int) (*sql.DB, error)
+}

node/get_status_node.go

@@ -77,6 +77,7 @@ type StatusNode struct {
 
 	appDB           *sql.DB
 	multiaccountsDB *multiaccounts.Database
+	walletDB        *sql.DB
 
 	config    *params.NodeConfig // Status node configuration
 	gethNode  *node.Node         // reference to Geth P2P stack/node
@@ -695,3 +696,7 @@ func (n *StatusNode) SetAppDB(db *sql.DB) {
 func (n *StatusNode) SetMultiaccountsDB(db *multiaccounts.Database) {
 	n.multiaccountsDB = db
 }
+
+func (n *StatusNode) SetWalletDB(db *sql.DB) {
+	n.walletDB = db
+}

node/status_node_services.go

@@ -500,7 +500,7 @@ func (b *StatusNode) CollectiblesService() *collectibles.Service {
 func (b *StatusNode) walletService(accountsDB *accounts.Database, accountsFeed *event.Feed) *wallet.Service {
 	if b.walletSrvc == nil {
 		b.walletSrvc = wallet.NewService(
-			b.appDB, accountsDB, b.rpcClient, accountsFeed, b.gethAccountManager, b.transactor, b.config,
+			b.walletDB, accountsDB, b.rpcClient, accountsFeed, b.gethAccountManager, b.transactor, b.config,
 			b.ensService(b.timeSourceNow()),
 			b.stickersService(accountsDB),
 			b.rpcFiltersSrvc,

protocol/messenger.go

@@ -28,7 +28,6 @@ import (
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/status-im/status-go/account"
-	"github.com/status-im/status-go/appdatabase"
 	"github.com/status-im/status-go/appmetrics"
 	"github.com/status-im/status-go/connection"
 	"github.com/status-im/status-go/contracts"
@@ -200,12 +199,6 @@ type mailserverCycle struct {
 	availabilitySubscriptions []chan struct{}
 }
 
-type dbConfig struct {
-	dbPath          string
-	dbKey           string
-	dbKDFIterations int
-}
-
 type EnvelopeEventsInterceptor struct {
 	EnvelopeEventsHandler transport.EnvelopeEventsHandler
 	Messenger             *Messenger
@@ -285,21 +278,12 @@ func NewMessenger(
 	}
 
 	// Configure the database.
-	database := c.db
+	if c.appDb == nil {
-	if c.db == nil && c.dbConfig == (dbConfig{}) {
 		return nil, errors.New("database instance or database path needs to be provided")
 	}
-	if c.db == nil {
+	database := c.appDb
-		logger.Info("opening a database", zap.String("dbPath", c.dbConfig.dbPath), zap.Int("KDFIterations", c.dbConfig.dbKDFIterations))
-		var err error
-		database, err = appdatabase.InitializeDB(c.dbConfig.dbPath, c.dbConfig.dbKey, c.dbConfig.dbKDFIterations)
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to initialize database from the db config")
-		}
-	}
 
 	// Apply any post database creation changes to the database
-	c.db = database
 	for _, opt := range c.afterDbCreatedHooks {
 		if err := opt(&c); err != nil {
 			return nil, err
@@ -466,7 +450,7 @@ func NewMessenger(
 
 	mailservers := mailserversDB.NewDB(database)
 
-	savedAddressesManager := wallet.NewSavedAddressesManager(c.db)
+	savedAddressesManager := wallet.NewSavedAddressesManager(c.walletDb)
 
 	myPublicKeyString := types.EncodeHex(crypto.FromECDSAPub(&identity.PublicKey))
 	myContact, err := buildContact(myPublicKeyString, &identity.PublicKey)

protocol/messenger_config.go

@@ -76,10 +76,8 @@ type config struct {
 
 	featureFlags common.FeatureFlags
 
-	// A path to a database or a database instance is required.
+	appDb               *sql.DB
-	// The database instance has a higher priority.
+	walletDb            *sql.DB
-	dbConfig            dbConfig
-	db                  *sql.DB
 	afterDbCreatedHooks []Option
 	multiAccount        *multiaccounts.Database
 	mailserversDatabase *mailservers.Database
@@ -131,13 +129,6 @@ func WithCustomLogger(logger *zap.Logger) Option {
 	}
 }
 
-func WithDatabaseConfig(dbPath string, dbKey string, dbKDFIterations int) Option {
-	return func(c *config) error {
-		c.dbConfig = dbConfig{dbPath: dbPath, dbKey: dbKey, dbKDFIterations: dbKDFIterations}
-		return nil
-	}
-}
-
 func WithVerifyTransactionClient(client EthClient) Option {
 	return func(c *config) error {
 		c.verifyTransactionClient = client
@@ -147,7 +138,14 @@ func WithVerifyTransactionClient(client EthClient) Option {
 
 func WithDatabase(db *sql.DB) Option {
 	return func(c *config) error {
-		c.db = db
+		c.appDb = db
+		return nil
+	}
+}
+
+func WithWalletDatabase(db *sql.DB) Option {
+	return func(c *config) error {
+		c.walletDb = db
 		return nil
 	}
 }
@@ -155,7 +153,7 @@ func WithDatabase(db *sql.DB) Option {
 func WithToplevelDatabaseMigrations() Option {
 	return func(c *config) error {
 		c.afterDbCreatedHooks = append(c.afterDbCreatedHooks, func(c *config) error {
-			return migrations.Migrate(c.db, nil)
+			return migrations.Migrate(c.appDb, nil)
 		})
 		return nil
 	}
@@ -173,7 +171,7 @@ func WithAppSettings(s settings.Settings, nc params.NodeConfig) Option {
 				s.Networks = networks
 			}
 
-			sDB, err := accounts.NewDB(c.db)
+			sDB, err := accounts.NewDB(c.appDb)
 			if err != nil {
 				return err
 			}
@@ -209,7 +207,7 @@ func WithBrowserDatabase(bd *browsers.Database) Option {
 		c.browserDatabase = bd
 		if c.browserDatabase == nil {
 			c.afterDbCreatedHooks = append(c.afterDbCreatedHooks, func(c *config) error {
-				c.browserDatabase = browsers.NewDB(c.db)
+				c.browserDatabase = browsers.NewDB(c.appDb)
 				return nil
 			})
 		}

services/ext/service.go

@@ -116,7 +116,7 @@ func (s *Service) GetPeer(rawURL string) (*enode.Node, error) {
 	return enode.ParseV4(rawURL)
 }
 
-func (s *Service) InitProtocol(nodeName string, identity *ecdsa.PrivateKey, db *sql.DB, httpServer *server.MediaServer, multiAccountDb *multiaccounts.Database, acc *multiaccounts.Account, accountManager *account.GethManager, rpcClient *rpc.Client, walletService *wallet.Service, collectiblesService *collectibles.Service, logger *zap.Logger) error {
+func (s *Service) InitProtocol(nodeName string, identity *ecdsa.PrivateKey, appDb, walletDb *sql.DB, httpServer *server.MediaServer, multiAccountDb *multiaccounts.Database, acc *multiaccounts.Account, accountManager *account.GethManager, rpcClient *rpc.Client, walletService *wallet.Service, collectiblesService *collectibles.Service, logger *zap.Logger) error {
 	var err error
 	if !s.config.ShhextConfig.PFSEnabled {
 		return nil
@@ -148,14 +148,14 @@ func (s *Service) InitProtocol(nodeName string, identity *ecdsa.PrivateKey, db *
 		EnvelopeEventsHandler: EnvelopeSignalHandler{},
 		Logger:                logger,
 	}
-	s.accountsDB, err = accounts.NewDB(db)
+	s.accountsDB, err = accounts.NewDB(appDb)
 	if err != nil {
 		return err
 	}
 	s.multiAccountsDB = multiAccountDb
 	s.account = acc
 
-	options, err := buildMessengerOptions(s.config, identity, db, httpServer, s.rpcClient, s.multiAccountsDB, acc, envelopesMonitorConfig, s.accountsDB, walletService, collectiblesService, logger, &MessengerSignalsHandler{})
+	options, err := buildMessengerOptions(s.config, identity, appDb, walletDb, httpServer, s.rpcClient, s.multiAccountsDB, acc, envelopesMonitorConfig, s.accountsDB, walletService, collectiblesService, logger, &MessengerSignalsHandler{})
 	if err != nil {
 		return err
 	}
@@ -404,7 +404,8 @@ func (s *Service) Stop() error {
 func buildMessengerOptions(
 	config params.NodeConfig,
 	identity *ecdsa.PrivateKey,
-	db *sql.DB,
+	appDb *sql.DB,
+	walletDb *sql.DB,
 	httpServer *server.MediaServer,
 	rpcClient *rpc.Client,
 	multiAccounts *multiaccounts.Database,
@@ -419,11 +420,12 @@ func buildMessengerOptions(
 	options := []protocol.Option{
 		protocol.WithCustomLogger(logger),
 		protocol.WithPushNotifications(),
-		protocol.WithDatabase(db),
+		protocol.WithDatabase(appDb),
+		protocol.WithWalletDatabase(walletDb),
 		protocol.WithMultiAccounts(multiAccounts),
-		protocol.WithMailserversDatabase(mailserversDB.NewDB(db)),
+		protocol.WithMailserversDatabase(mailserversDB.NewDB(appDb)),
 		protocol.WithAccount(account),
-		protocol.WithBrowserDatabase(browsers.NewDB(db)),
+		protocol.WithBrowserDatabase(browsers.NewDB(appDb)),
 		protocol.WithEnvelopesMonitorConfig(envelopesMonitorConfig),
 		protocol.WithSignalsHandler(messengerSignalsHandler),
 		protocol.WithENSVerificationConfig(publishMessengerResponse, config.ShhextConfig.VerifyENSURL, config.ShhextConfig.VerifyENSContractAddress),

services/wallet/activity/activity.go

@@ -15,6 +15,7 @@ import (
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/log"
 
+	"github.com/status-im/status-go/multiaccounts/accounts"
 	"github.com/status-im/status-go/services/wallet/common"
 	"github.com/status-im/status-go/services/wallet/transfer"
 
@@ -30,6 +31,8 @@ const (
 	PendingTransactionPT
 )
 
+const keypairAccountsTable = "keypairs_accounts"
+
 var (
 	ZeroAddress = eth.Address{}
 )
@@ -575,6 +578,7 @@ const (
 
 type FilterDependencies struct {
 	db              *sql.DB
+	accountsDb      *accounts.Database
 	tokenSymbol     func(token Token) string
 	tokenFromSymbol func(chainID *common.ChainID, symbol string) *Token
 }
@@ -655,6 +659,13 @@ func getActivityEntries(ctx context.Context, deps FilterDependencies, addresses
 		return strconv.Itoa(int(t))
 	})
 
+	// Since the filter query needs addresses which are in a different database, we need to update the
+	// keypairs_accounts table in the current database with the latest addresses from the accounts database
+	err := updateKeypairsAccountsTable(deps.accountsDb, deps.db)
+	if err != nil {
+		return nil, err
+	}
+
 	queryString := fmt.Sprintf(queryFormatString, involvedAddresses, toAddresses, assetsTokenCodes, assetsERC20, networks,
 		joinedMTTypes)
 
@@ -907,3 +918,41 @@ func contractTypeFromDBType(dbType string) (transferType *TransferType) {
 	}
 	return transferType
 }
+
+func updateKeypairsAccountsTable(accountsDb *accounts.Database, db *sql.DB) error {
+	_, err := db.Exec(fmt.Sprintf("CREATE TEMP TABLE IF NOT EXISTS %s (address VARCHAR PRIMARY KEY)",
+		keypairAccountsTable))
+	if err != nil {
+		log.Error("failed to create 'keypairs_accounts' table", "err", err)
+		return err
+	}
+
+	addresses, err := accountsDb.GetWalletAddresses()
+	if err != nil {
+		log.Error("failed to get wallet addresses", "err", err)
+		return err
+	}
+
+	tx, err := db.Begin()
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if err == nil {
+			err = tx.Commit()
+			return
+		}
+		_ = tx.Rollback()
+	}()
+
+	for _, address := range addresses {
+		_, err = tx.Exec(fmt.Sprintf("INSERT OR IGNORE INTO %s (address) VALUES (?)", keypairAccountsTable), address)
+		if err != nil {
+			log.Error("failed to insert wallet addresses", "err", err)
+			return err
+		}
+	}
+
+	return nil
+}

walletdatabase/database.go

@@ -0,0 +1,38 @@
+package walletdatabase
+
+import (
+	"database/sql"
+
+	"github.com/status-im/status-go/sqlite"
+	"github.com/status-im/status-go/walletdatabase/migrations"
+)
+
+type DbInitializer struct {
+}
+
+func (a DbInitializer) Initialize(path, password string, kdfIterationsNumber int) (*sql.DB, error) {
+	return InitializeDB(path, password, kdfIterationsNumber)
+}
+
+var walletCustomSteps = []sqlite.PostStep{
+}
+
+func doMigration(db *sql.DB) error {
+	// Run all the new migrations
+	return migrations.Migrate(db, walletCustomSteps)
+}
+
+// InitializeDB creates db file at a given path and applies migrations.
+func InitializeDB(path, password string, kdfIterationsNumber int) (*sql.DB, error) {
+	db, err := sqlite.OpenDB(path, password, kdfIterationsNumber)
+	if err != nil {
+		return nil, err
+	}
+
+	err = doMigration(db)
+	if err != nil {
+		return nil, err
+	}
+
+	return db, nil
+}

walletdatabase/migrations/migrate.go

@@ -0,0 +1,30 @@
+package migrations
+
+import (
+	"database/sql"
+
+	bindata "github.com/status-im/migrate/v4/source/go_bindata"
+
+	"github.com/status-im/status-go/sqlite"
+)
+
+// Migrate applies migrations.
+// see Migrate in vendor/status-go/sqlite/migrate.go
+func Migrate(db *sql.DB, customSteps []sqlite.PostStep) error {
+	return sqlite.Migrate(db, bindata.Resource(
+		AssetNames(),
+		func(name string) ([]byte, error) {
+			return Asset(name)
+		},
+	), customSteps, nil)
+}
+
+// MigrateTo is used for testing purposes
+func MigrateTo(db *sql.DB, customSteps []sqlite.PostStep, untilVersion uint) error {
+	return sqlite.Migrate(db, bindata.Resource(
+		AssetNames(),
+		func(name string) ([]byte, error) {
+			return Asset(name)
+		},
+	), customSteps, &untilVersion)
+}

walletdatabase/migrations/sql/1691753758_initial.up.sql

@@ -0,0 +1,191 @@
+CREATE TABLE blocks (
+    network_id UNSIGNED BIGINT NOT NULL,
+    address VARCHAR NOT NULL,
+    blk_number BIGINT NOT NULL,
+    blk_hash BIGINT NOT NULL,
+    loaded BOOL DEFAULT FALSE,
+    CONSTRAINT unique_mapping_for_account_to_block_per_network UNIQUE (address,blk_hash,network_id)
+);
+
+CREATE TABLE blocks_ranges (
+    network_id UNSIGNED BIGINT NOT NULL,
+    address VARCHAR NOT NULL,
+    blk_from BIGINT NOT NULL,
+    blk_to BIGINT NOT NULL,
+    balance BLOB,
+    nonce INTEGER);
+
+CREATE TABLE blocks_ranges_sequential (
+    network_id UNSIGNED BIGINT NOT NULL,
+    address VARCHAR NOT NULL,
+    blk_start BIGINT,
+    blk_first BIGINT NOT NULL,
+    blk_last BIGINT NOT NULL,
+    PRIMARY KEY (network_id, address)
+) WITHOUT ROWID;
+
+CREATE TABLE pending_transactions (
+    network_id UNSIGNED BIGINT NOT NULL,
+    hash VARCHAR NOT NULL,
+    timestamp UNSIGNED BIGINT NOT NULL,
+    from_address VARCHAR NOT NULL,
+    to_address VARCHAR,
+    symbol VARCHAR,
+    gas_price BLOB,
+    gas_limit BLOB,
+    value BLOB,
+    data TEXT,
+    type VARCHAR,
+    additional_data TEXT,
+    multi_transaction_id INT,
+    PRIMARY KEY (network_id, hash)
+) WITHOUT ROWID;
+
+CREATE TABLE "saved_addresses" (
+    address VARCHAR NOT NULL,
+    name TEXT NOT NULL,
+    favourite BOOLEAN NOT NULL DEFAULT FALSE,
+    removed BOOLEAN NOT NULL DEFAULT FALSE,
+    update_clock INT NOT NULL DEFAULT 0,
+    chain_short_names VARCHAR DEFAULT "",
+    ens_name VARCHAR DEFAULT "",
+    is_test BOOLEAN DEFAULT FALSE,
+    created_at INT DEFAULT 0,
+    PRIMARY KEY (address, ens_name, is_test)
+) WITHOUT ROWID;
+
+CREATE TABLE token_balances (
+    user_address VARCHAR NOT NULL,
+    token_name VARCHAR NOT NULL,
+    token_symbol VARCHAR NOT NULL,
+    token_address VARCHAR NOT NULL,
+    token_color VARCHAR NOT NULL DEFAULT "",
+    token_decimals INT NOT NULL,
+    token_description VARCHAR NOT NULL DEFAULT "",
+    token_url VARCHAR NOT NULL DEFAULT "",
+    balance VARCHAR NOT NULL,
+    chain_id INT NOT NULL,
+    PRIMARY KEY (user_address, chain_id, token_symbol) ON CONFLICT REPLACE
+);
+
+CREATE TABLE tokens (
+    address VARCHAR NOT NULL,
+    network_id UNSIGNED BIGINT NOT NULL,
+    name TEXT NOT NULL,
+    symbol VARCHAR NOT NULL,
+    decimals UNSIGNED INT,
+    color VARCHAR,
+    PRIMARY KEY (address, network_id)
+) WITHOUT ROWID;
+
+CREATE TABLE visible_tokens (
+    chain_id UNSIGNED INT,
+    address VARCHAR NOT NULL
+);
+
+CREATE TABLE currency_format_cache (
+    symbol VARCHAR NOT NULL,
+    display_decimals INT NOT NULL,
+    strip_trailing_zeroes BOOLEAN NOT NULL
+);
+
+CREATE TABLE multi_transactions (
+    from_address VARCHAR NOT NULL,
+    from_asset VARCHAR NOT NULL,
+    from_amount VARCHAR NOT NULL,
+    to_address VARCHAR NOT NULL,
+    to_asset VARCHAR NOT NULL,
+    type VARCHAR NOT NULL,
+    timestamp UNSIGNED BIGINT NOT NULL,
+    to_amount VARCHAR,
+    from_network_id UNSIGNED BIGINT,
+    to_network_id UNSIGNED BIGINT,
+    cross_tx_id VARCHAR DEFAULT "",
+    from_tx_hash BLOB,
+    to_tx_hash BLOB
+);
+
+CREATE TABLE balance_history (
+    chain_id UNSIGNED BIGINT NOT NULL,
+    address VARCHAR NOT NULL,
+    currency VARCHAR NOT NULL,
+    block BIGINT NOT NULL,
+    timestamp INT NOT NULL,
+    bitset INT NOT NULL,
+    balance BLOB
+);
+
+CREATE TABLE price_cache (
+    token VARCHAR NOT NULL,
+    currency VARCHAR NOT NULL,
+    price REAL NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS collectibles_ownership_cache (
+    chain_id UNSIGNED BIGINT NOT NULL,
+    contract_address VARCHAR NOT NULL,
+    token_id BLOB NOT NULL,
+    owner_address VARCHAR NOT NULL
+);
+
+CREATE TABLE transfers (
+    network_id UNSIGNED BIGINT NOT NULL,
+    hash VARCHAR NOT NULL,
+    address VARCHAR NOT NULL,
+    blk_hash VARCHAR NOT NULL,
+    tx BLOB,
+    sender VARCHAR,
+    receipt BLOB,
+    log BLOB,
+    type VARCHAR NOT NULL,
+    blk_number BIGINT NOT NULL,
+    timestamp UNSIGNED BIGINT NOT NULL,
+    loaded BOOL DEFAULT 1,
+    multi_transaction_id INT,
+    base_gas_fee TEXT NOT NULL DEFAULT "",
+    status INT,
+    receipt_type INT,
+    tx_hash BLOB,
+    log_index INT,
+    block_hash BLOB,
+    cumulative_gas_used INT,
+    contract_address TEXT,
+    gas_used INT,
+    tx_index INT,
+    tx_type INT,
+    protected BOOLEAN,
+    gas_limit UNSIGNED INT,
+    gas_price_clamped64 INT,
+    gas_tip_cap_clamped64 INT,
+    gas_fee_cap_clamped64 INT,
+    amount_padded128hex CHAR(32),
+    account_nonce INT,
+    size INT,
+    token_address BLOB,
+    token_id BLOB,
+    tx_from_address BLOB,
+    tx_to_address BLOB,
+    FOREIGN KEY(network_id,address,blk_hash) REFERENCES blocks(network_id,address,blk_hash) ON DELETE CASCADE,
+    CONSTRAINT unique_transfer_per_address_per_network UNIQUE (hash,address,network_id)
+);
+
+CREATE INDEX balance_history_filter_entries ON balance_history (chain_id, address, currency, block, timestamp, bitset);
+
+CREATE INDEX idx_transfers_blk_loaded ON transfers(blk_number, loaded);
+
+CREATE UNIQUE INDEX price_cache_identify_entry ON price_cache (token, currency);
+
+CREATE UNIQUE INDEX balance_history_identify_entry ON balance_history (chain_id, address, currency, block);
+
+CREATE UNIQUE INDEX currency_format_cache_identify_entry ON currency_format_cache (symbol);
+
+CREATE INDEX idx_transfers_filter
+ON transfers (multi_transaction_id, loaded, timestamp, status, network_id, tx_from_address, tx_to_address, token_address, token_id, type);
+
+CREATE INDEX idx_pending_transactions
+ON pending_transactions (multi_transaction_id, from_address, to_address, network_id, timestamp, symbol);
+
+CREATE INDEX idx_multi_transactions
+ON multi_transactions (from_address, to_address, type, from_asset, timestamp, to_asset, from_amount, to_amount);
+
+CREATE INDEX IF NOT EXISTS collectibles_ownership_filter_entries ON collectibles_ownership_cache (chain_id, owner_address);

walletdatabase/migrations/sql/doc.go

@@ -0,0 +1,3 @@
+package sql
+
+//go:generate go-bindata -pkg migrations -o ../bindata.go ./