From aa3d33a58f46e49aef28dfe007278c323acab404 Mon Sep 17 00:00:00 2001 From: Ivan Belyakov Date: Tue, 8 Aug 2023 08:55:13 +0200 Subject: [PATCH] feat(migration): sqlite migration improvements. Some functions split to be more cohesive, custom PostSteps are stored as pointers to allow their retrieval and parameters passing in runtime if needed (some extra work is dropped and TBD when needed) --- appdatabase/database.go | 2 +- appdatabase/database_test.go | 2 +- appdatabase/migrations/migrate.go | 4 +-- multiaccounts/migrations/migrate.go | 2 +- sqlite/migrate.go | 16 +++++---- sqlite/sqlite.go | 53 ++++++++++++++++++++-------- walletdatabase/database.go | 3 +- walletdatabase/migrations/migrate.go | 4 +-- 8 files changed, 56 insertions(+), 30 deletions(-) diff --git a/appdatabase/database.go b/appdatabase/database.go index 7d2d82e6d..80cd38116 100644 --- a/appdatabase/database.go +++ b/appdatabase/database.go @@ -20,7 +20,7 @@ import ( const nodeCfgMigrationDate = 1640111208 -var customSteps = []sqlite.PostStep{ +var customSteps = []*sqlite.PostStep{ {Version: 1674136690, CustomMigration: migrateEnsUsernames}, {Version: 1686048341, CustomMigration: migrateWalletJSONBlobs, RollBackVersion: 1686041510}, {Version: 1687193315, CustomMigration: migrateWalletTransferFromToAddresses, RollBackVersion: 1686825075}, diff --git a/appdatabase/database_test.go b/appdatabase/database_test.go index 07b334127..7fbb55fd2 100644 --- a/appdatabase/database_test.go +++ b/appdatabase/database_test.go @@ -182,7 +182,7 @@ func TestMigrateWalletJsonBlobs(t *testing.T) { err = insertTestTransaction(9, uniswapV3TxTestData, uniswapV3ReceiptTestData, uniswapV3LogTestData, false) require.NoError(t, err) - failMigrationSteps := []sqlite.PostStep{ + failMigrationSteps := []*sqlite.PostStep{ { Version: customSteps[1].Version, CustomMigration: func(sqlTx *sql.Tx) error { diff --git a/appdatabase/migrations/migrate.go b/appdatabase/migrations/migrate.go index bb6b130b5..53c4dd655 100644 --- a/appdatabase/migrations/migrate.go +++ b/appdatabase/migrations/migrate.go @@ -10,7 +10,7 @@ import ( // Migrate applies migrations. // see Migrate in vendor/status-go/sqlite/migrate.go -func Migrate(db *sql.DB, customSteps []sqlite.PostStep) error { +func Migrate(db *sql.DB, customSteps []*sqlite.PostStep) error { return sqlite.Migrate(db, bindata.Resource( AssetNames(), func(name string) ([]byte, error) { @@ -20,7 +20,7 @@ func Migrate(db *sql.DB, customSteps []sqlite.PostStep) error { } // MigrateTo is used for testing purposes -func MigrateTo(db *sql.DB, customSteps []sqlite.PostStep, untilVersion uint) error { +func MigrateTo(db *sql.DB, customSteps []*sqlite.PostStep, untilVersion uint) error { return sqlite.Migrate(db, bindata.Resource( AssetNames(), func(name string) ([]byte, error) { diff --git a/multiaccounts/migrations/migrate.go b/multiaccounts/migrations/migrate.go index 9e0352815..ee5132cd3 100644 --- a/multiaccounts/migrations/migrate.go +++ b/multiaccounts/migrations/migrate.go @@ -10,7 +10,7 @@ import ( // Migrate applies migrations. // see Migrate in vendor/status-go/sqlite/migrate.go -func Migrate(db *sql.DB, customSteps []sqlite.PostStep) error { +func Migrate(db *sql.DB, customSteps []*sqlite.PostStep) error { return sqlite.Migrate(db, bindata.Resource( AssetNames(), func(name string) ([]byte, error) { diff --git a/sqlite/migrate.go b/sqlite/migrate.go index 2fba2b49d..3c268f6d9 100644 --- a/sqlite/migrate.go +++ b/sqlite/migrate.go @@ -10,9 +10,11 @@ import ( bindata "github.com/status-im/migrate/v4/source/go_bindata" ) +type CustomMigrationFunc func(tx *sql.Tx) error + type PostStep struct { Version uint - CustomMigration func(tx *sql.Tx) error + CustomMigration CustomMigrationFunc RollBackVersion uint } @@ -36,7 +38,7 @@ var migrationTable = "status_go_" + sqlcipher.DefaultMigrationsTable // // untilVersion, for testing purposes optional parameter, can be used to limit the migration to a specific version. // Pass nil to migrate to the latest available version. -func Migrate(db *sql.DB, resources *bindata.AssetSource, customSteps []PostStep, untilVersion *uint) error { +func Migrate(db *sql.DB, resources *bindata.AssetSource, customSteps []*PostStep, untilVersion *uint) error { source, err := bindata.WithInstance(resources) if err != nil { return fmt.Errorf("failed to create bindata migration source: %w", err) @@ -82,7 +84,7 @@ func Migrate(db *sql.DB, resources *bindata.AssetSource, customSteps []PostStep, // runCustomMigrations performs source migrations from current to each custom steps, then runs custom migration callback // until it executes all custom migrations or an error occurs and it tries to rollback to RollBackVersion if > 0. -func runCustomMigrations(m *migrate.Migrate, db *sql.DB, customSteps []PostStep, customIndex int, untilVersion *uint) error { +func runCustomMigrations(m *migrate.Migrate, db *sql.DB, customSteps []*PostStep, customIndex int, untilVersion *uint) error { for customIndex < len(customSteps) && (untilVersion == nil || customSteps[customIndex].Version <= *untilVersion) { customStep := customSteps[customIndex] @@ -99,7 +101,8 @@ func runCustomMigrations(m *migrate.Migrate, db *sql.DB, customSteps []PostStep, return nil } -func runCustomMigrationStep(db *sql.DB, customStep PostStep, m *migrate.Migrate) error { +func runCustomMigrationStep(db *sql.DB, customStep *PostStep, m *migrate.Migrate) error { + sqlTx, err := db.Begin() if err != nil { return fmt.Errorf("failed to begin transaction: %w", err) @@ -116,7 +119,7 @@ func runCustomMigrationStep(db *sql.DB, customStep PostStep, m *migrate.Migrate) return nil } -func rollbackCustomMigration(m *migrate.Migrate, customStep PostStep, customErr error) error { +func rollbackCustomMigration(m *migrate.Migrate, customStep *PostStep, customErr error) error { if customStep.RollBackVersion > 0 { err := m.Migrate(customStep.RollBackVersion) newV, _, _ := m.Version() @@ -135,7 +138,8 @@ func runRemainingMigrations(m *migrate.Migrate, untilVersion *uint) error { } } else { if err := m.Up(); err != nil && err != migrate.ErrNoChange { - return fmt.Errorf("failed to migrate up: %w", err) + ver, _, _ := m.Version() + return fmt.Errorf("failed to migrate up: %w, current version: %d", err, ver) } } return nil diff --git a/sqlite/sqlite.go b/sqlite/sqlite.go index 91252cd8b..b7ba5ac37 100644 --- a/sqlite/sqlite.go +++ b/sqlite/sqlite.go @@ -27,6 +27,7 @@ const ( InMemoryPath = ":memory:" V4CipherPageSize = 8192 V3CipherPageSize = 1024 + sqlMainDatabase = "main" ) // DecryptDB completely removes the encryption from the db @@ -58,7 +59,22 @@ func encryptDB(db *sql.DB, encryptedPath string, key string, kdfIterationsNumber defer onEnd() } - _, err := db.Exec(`ATTACH DATABASE '` + encryptedPath + `' AS encrypted KEY '` + key + `'`) + attachedDbName := "encrypted" + err := attachDatabaseWithDefaultSettings(db, encryptedPath, attachedDbName, key, kdfIterationsNumber) + if err != nil { + return err + } + + _, err = db.Exec(fmt.Sprintf(`SELECT sqlcipher_export('%s')`, attachedDbName)) + if err != nil { + return err + } + _, err = db.Exec(fmt.Sprintf(`DETACH DATABASE %s`, attachedDbName)) + return err +} + +func attachDatabaseWithDefaultSettings(db *sql.DB, attachedDbPath string, attachedDbName string, key string, kdfIterationsNumber int) error { + _, err := db.Exec(fmt.Sprintf(`ATTACH DATABASE '%s' AS %s KEY '%s'`, attachedDbPath, attachedDbName, key)) if err != nil { return err } @@ -67,31 +83,39 @@ func encryptDB(db *sql.DB, encryptedPath string, key string, kdfIterationsNumber kdfIterationsNumber = sqlite.ReducedKDFIterationsNumber } - _, err = db.Exec(fmt.Sprintf("PRAGMA encrypted.kdf_iter = '%d'", kdfIterationsNumber)) + if _, err := db.Exec(fmt.Sprintf(`PRAGMA %s.busy_timeout = 60000`, attachedDbName)); err != nil { + return errors.New("failed to set `busy_timeout` pragma on attached db") + } + + return setDatabaseCipherSettings(db, kdfIterationsNumber, attachedDbName) +} + +func setDatabaseCipherSettings(db *sql.DB, kdfIterationsNumber int, dbNameOpt ...string) error { + dbName := sqlMainDatabase + if len(dbNameOpt) > 0 { + dbName = dbNameOpt[0] + } + + _, err := db.Exec(fmt.Sprintf("PRAGMA %s.kdf_iter = '%d'", dbName, kdfIterationsNumber)) if err != nil { return err } - if _, err := db.Exec(fmt.Sprintf("PRAGMA encrypted.cipher_page_size = %d", V4CipherPageSize)); err != nil { + if _, err := db.Exec(fmt.Sprintf("PRAGMA %s.cipher_page_size = %d", dbName, V4CipherPageSize)); err != nil { fmt.Println("failed to set cipher_page_size pragma") return err } - if _, err := db.Exec("PRAGMA encrypted.cipher_hmac_algorithm = HMAC_SHA1"); err != nil { + if _, err := db.Exec(fmt.Sprintf("PRAGMA %s.cipher_hmac_algorithm = HMAC_SHA1", dbName)); err != nil { fmt.Println("failed to set cipher_hmac_algorithm pragma") return err } - if _, err := db.Exec("PRAGMA encrypted.cipher_kdf_algorithm = PBKDF2_HMAC_SHA1"); err != nil { + if _, err := db.Exec(fmt.Sprintf("PRAGMA %s.cipher_kdf_algorithm = PBKDF2_HMAC_SHA1", dbName)); err != nil { fmt.Println("failed to set cipher_kdf_algorithm pragma") return err } - _, err = db.Exec(`SELECT sqlcipher_export('encrypted')`) - if err != nil { - return err - } - _, err = db.Exec(`DETACH DATABASE encrypted`) - return err + return nil } // EncryptDB takes a plaintext database and adds encryption @@ -140,7 +164,7 @@ func buildSqlcipherDSN(path string) (string, error) { return path + queryOperator + "_txlock=immediate", nil } -func openDB(path string, key string, kdfIterationsNumber int, chiperPageSize int) (*sql.DB, error) { +func openDB(path string, key string, kdfIterationsNumber int, cipherPageSize int) (*sql.DB, error) { driverName := fmt.Sprintf("sqlcipher_with_extensions-%d", len(sql.Drivers())) sql.Register(driverName, &sqlcipher.SQLiteDriver{ ConnectHook: func(conn *sqlcipher.SQLiteConn) error { @@ -156,7 +180,7 @@ func openDB(path string, key string, kdfIterationsNumber int, chiperPageSize int kdfIterationsNumber = sqlite.ReducedKDFIterationsNumber } - if _, err := conn.Exec(fmt.Sprintf("PRAGMA cipher_page_size = %d", chiperPageSize), nil); err != nil { + if _, err := conn.Exec(fmt.Sprintf("PRAGMA cipher_page_size = %d", cipherPageSize), nil); err != nil { fmt.Println("failed to set cipher_page_size pragma") return err } @@ -189,7 +213,6 @@ func openDB(path string, key string, kdfIterationsNumber int, chiperPageSize int }) dsn, err := buildSqlcipherDSN(path) - if err != nil { return nil, err } @@ -223,7 +246,7 @@ func openDB(path string, key string, kdfIterationsNumber int, chiperPageSize int return db, nil } -// OpenDB opens not-encrypted database. +// OpenDB opens encrypted database. func OpenDB(path string, key string, kdfIterationsNumber int) (*sql.DB, error) { return openDB(path, key, kdfIterationsNumber, V4CipherPageSize) } diff --git a/walletdatabase/database.go b/walletdatabase/database.go index e180951a5..5f178652c 100644 --- a/walletdatabase/database.go +++ b/walletdatabase/database.go @@ -14,8 +14,7 @@ func (a DbInitializer) Initialize(path, password string, kdfIterationsNumber int return InitializeDB(path, password, kdfIterationsNumber) } -var walletCustomSteps = []sqlite.PostStep{ -} +var walletCustomSteps = []*sqlite.PostStep{} func doMigration(db *sql.DB) error { // Run all the new migrations diff --git a/walletdatabase/migrations/migrate.go b/walletdatabase/migrations/migrate.go index bb6b130b5..53c4dd655 100644 --- a/walletdatabase/migrations/migrate.go +++ b/walletdatabase/migrations/migrate.go @@ -10,7 +10,7 @@ import ( // Migrate applies migrations. // see Migrate in vendor/status-go/sqlite/migrate.go -func Migrate(db *sql.DB, customSteps []sqlite.PostStep) error { +func Migrate(db *sql.DB, customSteps []*sqlite.PostStep) error { return sqlite.Migrate(db, bindata.Resource( AssetNames(), func(name string) ([]byte, error) { @@ -20,7 +20,7 @@ func Migrate(db *sql.DB, customSteps []sqlite.PostStep) error { } // MigrateTo is used for testing purposes -func MigrateTo(db *sql.DB, customSteps []sqlite.PostStep, untilVersion uint) error { +func MigrateTo(db *sql.DB, customSteps []*sqlite.PostStep, untilVersion uint) error { return sqlite.Migrate(db, bindata.Resource( AssetNames(), func(name string) ([]byte, error) {