feat(migration): sqlite migration improvements.

Some functions split to be more cohesive, custom PostSteps are
stored as pointers to allow their retrieval and parameters passing
in runtime if needed (some extra work is dropped and TBD
when needed)

appdatabase/database.go

@@ -20,7 +20,7 @@ import (
 
 const nodeCfgMigrationDate = 1640111208
 
-var customSteps = []sqlite.PostStep{
+var customSteps = []*sqlite.PostStep{
 	{Version: 1674136690, CustomMigration: migrateEnsUsernames},
 	{Version: 1686048341, CustomMigration: migrateWalletJSONBlobs, RollBackVersion: 1686041510},
 	{Version: 1687193315, CustomMigration: migrateWalletTransferFromToAddresses, RollBackVersion: 1686825075},

appdatabase/database_test.go

@@ -182,7 +182,7 @@ func TestMigrateWalletJsonBlobs(t *testing.T) {
 	err = insertTestTransaction(9, uniswapV3TxTestData, uniswapV3ReceiptTestData, uniswapV3LogTestData, false)
 	require.NoError(t, err)
 
-	failMigrationSteps := []sqlite.PostStep{
+	failMigrationSteps := []*sqlite.PostStep{
 		{
 			Version: customSteps[1].Version,
 			CustomMigration: func(sqlTx *sql.Tx) error {

appdatabase/migrations/migrate.go

@@ -10,7 +10,7 @@ import (
 
 // Migrate applies migrations.
 // see Migrate in vendor/status-go/sqlite/migrate.go
-func Migrate(db *sql.DB, customSteps []sqlite.PostStep) error {
+func Migrate(db *sql.DB, customSteps []*sqlite.PostStep) error {
 	return sqlite.Migrate(db, bindata.Resource(
 		AssetNames(),
 		func(name string) ([]byte, error) {
@@ -20,7 +20,7 @@ func Migrate(db *sql.DB, customSteps []sqlite.PostStep) error {
 }
 
 // MigrateTo is used for testing purposes
-func MigrateTo(db *sql.DB, customSteps []sqlite.PostStep, untilVersion uint) error {
+func MigrateTo(db *sql.DB, customSteps []*sqlite.PostStep, untilVersion uint) error {
 	return sqlite.Migrate(db, bindata.Resource(
 		AssetNames(),
 		func(name string) ([]byte, error) {

multiaccounts/migrations/migrate.go

@@ -10,7 +10,7 @@ import (
 
 // Migrate applies migrations.
 // see Migrate in vendor/status-go/sqlite/migrate.go
-func Migrate(db *sql.DB, customSteps []sqlite.PostStep) error {
+func Migrate(db *sql.DB, customSteps []*sqlite.PostStep) error {
 	return sqlite.Migrate(db, bindata.Resource(
 		AssetNames(),
 		func(name string) ([]byte, error) {

sqlite/migrate.go

@@ -10,9 +10,11 @@ import (
 	bindata "github.com/status-im/migrate/v4/source/go_bindata"
 )
 
+type CustomMigrationFunc func(tx *sql.Tx) error
+
 type PostStep struct {
 	Version         uint
-	CustomMigration func(tx *sql.Tx) error
+	CustomMigration CustomMigrationFunc
 	RollBackVersion uint
 }
 
@@ -36,7 +38,7 @@ var migrationTable = "status_go_" + sqlcipher.DefaultMigrationsTable
 //
 // untilVersion, for testing purposes optional parameter, can be used to limit the migration to a specific version.
 // Pass nil to migrate to the latest available version.
-func Migrate(db *sql.DB, resources *bindata.AssetSource, customSteps []PostStep, untilVersion *uint) error {
+func Migrate(db *sql.DB, resources *bindata.AssetSource, customSteps []*PostStep, untilVersion *uint) error {
 	source, err := bindata.WithInstance(resources)
 	if err != nil {
 		return fmt.Errorf("failed to create bindata migration source: %w", err)
@@ -82,7 +84,7 @@ func Migrate(db *sql.DB, resources *bindata.AssetSource, customSteps []PostStep,
 
 // runCustomMigrations performs source migrations from current to each custom steps, then runs custom migration callback
 // until it executes all custom migrations or an error occurs and it tries to rollback to RollBackVersion if > 0.
-func runCustomMigrations(m *migrate.Migrate, db *sql.DB, customSteps []PostStep, customIndex int, untilVersion *uint) error {
+func runCustomMigrations(m *migrate.Migrate, db *sql.DB, customSteps []*PostStep, customIndex int, untilVersion *uint) error {
 	for customIndex < len(customSteps) && (untilVersion == nil || customSteps[customIndex].Version <= *untilVersion) {
 		customStep := customSteps[customIndex]
 
@@ -99,7 +101,8 @@ func runCustomMigrations(m *migrate.Migrate, db *sql.DB, customSteps []PostStep,
 	return nil
 }
 
-func runCustomMigrationStep(db *sql.DB, customStep PostStep, m *migrate.Migrate) error {
+func runCustomMigrationStep(db *sql.DB, customStep *PostStep, m *migrate.Migrate) error {
+
 	sqlTx, err := db.Begin()
 	if err != nil {
 		return fmt.Errorf("failed to begin transaction: %w", err)
@@ -116,7 +119,7 @@ func runCustomMigrationStep(db *sql.DB, customStep PostStep, m *migrate.Migrate)
 	return nil
 }
 
-func rollbackCustomMigration(m *migrate.Migrate, customStep PostStep, customErr error) error {
+func rollbackCustomMigration(m *migrate.Migrate, customStep *PostStep, customErr error) error {
 	if customStep.RollBackVersion > 0 {
 		err := m.Migrate(customStep.RollBackVersion)
 		newV, _, _ := m.Version()
@@ -135,7 +138,8 @@ func runRemainingMigrations(m *migrate.Migrate, untilVersion *uint) error {
 		}
 	} else {
 		if err := m.Up(); err != nil && err != migrate.ErrNoChange {
-			return fmt.Errorf("failed to migrate up: %w", err)
+			ver, _, _ := m.Version()
+			return fmt.Errorf("failed to migrate up: %w, current version: %d", err, ver)
 		}
 	}
 	return nil

sqlite/sqlite.go

@@ -27,6 +27,7 @@ const (
 	InMemoryPath     = ":memory:"
 	V4CipherPageSize = 8192
 	V3CipherPageSize = 1024
+	sqlMainDatabase  = "main"
 )
 
 // DecryptDB completely removes the encryption from the db
@@ -58,7 +59,22 @@ func encryptDB(db *sql.DB, encryptedPath string, key string, kdfIterationsNumber
 		defer onEnd()
 	}
 
-	_, err := db.Exec(`ATTACH DATABASE '` + encryptedPath + `' AS encrypted KEY '` + key + `'`)
+	attachedDbName := "encrypted"
+	err := attachDatabaseWithDefaultSettings(db, encryptedPath, attachedDbName, key, kdfIterationsNumber)
+	if err != nil {
+		return err
+	}
+
+	_, err = db.Exec(fmt.Sprintf(`SELECT sqlcipher_export('%s')`, attachedDbName))
+	if err != nil {
+		return err
+	}
+	_, err = db.Exec(fmt.Sprintf(`DETACH DATABASE %s`, attachedDbName))
+	return err
+}
+
+func attachDatabaseWithDefaultSettings(db *sql.DB, attachedDbPath string, attachedDbName string, key string, kdfIterationsNumber int) error {
+	_, err := db.Exec(fmt.Sprintf(`ATTACH DATABASE '%s' AS %s KEY '%s'`, attachedDbPath, attachedDbName, key))
 	if err != nil {
 		return err
 	}
@@ -67,31 +83,39 @@ func encryptDB(db *sql.DB, encryptedPath string, key string, kdfIterationsNumber
 		kdfIterationsNumber = sqlite.ReducedKDFIterationsNumber
 	}
 
-	_, err = db.Exec(fmt.Sprintf("PRAGMA encrypted.kdf_iter = '%d'", kdfIterationsNumber))
+	if _, err := db.Exec(fmt.Sprintf(`PRAGMA %s.busy_timeout = 60000`, attachedDbName)); err != nil {
+		return errors.New("failed to set `busy_timeout` pragma on attached db")
+	}
+
+	return setDatabaseCipherSettings(db, kdfIterationsNumber, attachedDbName)
+}
+
+func setDatabaseCipherSettings(db *sql.DB, kdfIterationsNumber int, dbNameOpt ...string) error {
+	dbName := sqlMainDatabase
+	if len(dbNameOpt) > 0 {
+		dbName = dbNameOpt[0]
+	}
+
+	_, err := db.Exec(fmt.Sprintf("PRAGMA %s.kdf_iter = '%d'", dbName, kdfIterationsNumber))
 	if err != nil {
 		return err
 	}
 
-	if _, err := db.Exec(fmt.Sprintf("PRAGMA encrypted.cipher_page_size = %d", V4CipherPageSize)); err != nil {
+	if _, err := db.Exec(fmt.Sprintf("PRAGMA %s.cipher_page_size = %d", dbName, V4CipherPageSize)); err != nil {
 		fmt.Println("failed to set cipher_page_size pragma")
 		return err
 	}
-	if _, err := db.Exec("PRAGMA encrypted.cipher_hmac_algorithm = HMAC_SHA1"); err != nil {
+	if _, err := db.Exec(fmt.Sprintf("PRAGMA %s.cipher_hmac_algorithm = HMAC_SHA1", dbName)); err != nil {
 		fmt.Println("failed to set cipher_hmac_algorithm pragma")
 		return err
 	}
 
-	if _, err := db.Exec("PRAGMA encrypted.cipher_kdf_algorithm = PBKDF2_HMAC_SHA1"); err != nil {
+	if _, err := db.Exec(fmt.Sprintf("PRAGMA %s.cipher_kdf_algorithm = PBKDF2_HMAC_SHA1", dbName)); err != nil {
 		fmt.Println("failed to set cipher_kdf_algorithm pragma")
 		return err
 	}
 
-	_, err = db.Exec(`SELECT sqlcipher_export('encrypted')`)
-	if err != nil {
-		return err
-	}
-	_, err = db.Exec(`DETACH DATABASE encrypted`)
-	return err
+	return nil
 }
 
 // EncryptDB takes a plaintext database and adds encryption
@@ -140,7 +164,7 @@ func buildSqlcipherDSN(path string) (string, error) {
 	return path + queryOperator + "_txlock=immediate", nil
 }
 
-func openDB(path string, key string, kdfIterationsNumber int, chiperPageSize int) (*sql.DB, error) {
+func openDB(path string, key string, kdfIterationsNumber int, cipherPageSize int) (*sql.DB, error) {
 	driverName := fmt.Sprintf("sqlcipher_with_extensions-%d", len(sql.Drivers()))
 	sql.Register(driverName, &sqlcipher.SQLiteDriver{
 		ConnectHook: func(conn *sqlcipher.SQLiteConn) error {
@@ -156,7 +180,7 @@ func openDB(path string, key string, kdfIterationsNumber int, chiperPageSize int
 				kdfIterationsNumber = sqlite.ReducedKDFIterationsNumber
 			}
 
-			if _, err := conn.Exec(fmt.Sprintf("PRAGMA cipher_page_size = %d", chiperPageSize), nil); err != nil {
+			if _, err := conn.Exec(fmt.Sprintf("PRAGMA cipher_page_size = %d", cipherPageSize), nil); err != nil {
 				fmt.Println("failed to set cipher_page_size pragma")
 				return err
 			}
@@ -189,7 +213,6 @@ func openDB(path string, key string, kdfIterationsNumber int, chiperPageSize int
 	})
 
 	dsn, err := buildSqlcipherDSN(path)
-
 	if err != nil {
 		return nil, err
 	}
@@ -223,7 +246,7 @@ func openDB(path string, key string, kdfIterationsNumber int, chiperPageSize int
 	return db, nil
 }
 
-// OpenDB opens not-encrypted database.
+// OpenDB opens encrypted database.
 func OpenDB(path string, key string, kdfIterationsNumber int) (*sql.DB, error) {
 	return openDB(path, key, kdfIterationsNumber, V4CipherPageSize)
 }

walletdatabase/database.go

@@ -14,8 +14,7 @@ func (a DbInitializer) Initialize(path, password string, kdfIterationsNumber int
 	return InitializeDB(path, password, kdfIterationsNumber)
 }
 
-var walletCustomSteps = []sqlite.PostStep{
-}
+var walletCustomSteps = []*sqlite.PostStep{}
 
 func doMigration(db *sql.DB) error {
 	// Run all the new migrations

walletdatabase/migrations/migrate.go

@@ -10,7 +10,7 @@ import (
 
 // Migrate applies migrations.
 // see Migrate in vendor/status-go/sqlite/migrate.go
-func Migrate(db *sql.DB, customSteps []sqlite.PostStep) error {
+func Migrate(db *sql.DB, customSteps []*sqlite.PostStep) error {
 	return sqlite.Migrate(db, bindata.Resource(
 		AssetNames(),
 		func(name string) ([]byte, error) {
@@ -20,7 +20,7 @@ func Migrate(db *sql.DB, customSteps []sqlite.PostStep) error {
 }
 
 // MigrateTo is used for testing purposes
-func MigrateTo(db *sql.DB, customSteps []sqlite.PostStep, untilVersion uint) error {
+func MigrateTo(db *sql.DB, customSteps []*sqlite.PostStep, untilVersion uint) error {
 	return sqlite.Migrate(db, bindata.Resource(
 		AssetNames(),
 		func(name string) ([]byte, error) {