From 43b2c3b7ce4b1269d23cecd596e7a42fc9ecf87a Mon Sep 17 00:00:00 2001 From: Alex Jbanca <47811206+alexjba@users.noreply.github.com> Date: Tue, 13 Jun 2023 18:20:21 +0300 Subject: [PATCH] perf(sqlCipher): Increase cipher page size to 8192 (#3591) * perf(sqlCipher): Increase cipher page size to 8192 Increasing the cipher page size to 8192 requires DB re-encryption. The process is as follows: //Login to v3 DB PRAGMA key = 'key'; PRAGMA cipher_page_size = 1024"; // old Page size PRAGMA cipher_hmac_algorithm = HMAC_SHA1"; PRAGMA cipher_kdf_algorithm = PBKDF2_HMAC_SHA1"; PRAGMA kdf_iter = kdfIterationsNumber"; //Create V4 DB with increased page size ATTACH DATABASE 'newdb.db' AS newdb KEY 'key'; PRAGMA newdb.cipher_page_size = 8192; // new Page size PRAGMA newdb.cipher_hmac_algorithm = HMAC_SHA1"; // same as in v3 PRAGMA newdb.cipher_kdf_algorithm = PBKDF2_HMAC_SHA1"; // same as in v3 PRAGMA newdb.kdf_iter = kdfIterationsNumber"; // same as in v3 SELECT sqlcipher_export('newdb'); DETACH DATABASE newdb; //Login to V4 DB ... Worth noting: The DB migration will happen on the first successful login. The new DB version will have a different name to be able to distinguish between different DB versions.Versions naming mirrors sqlcipher major version (naming conventions used by sqlcipher), meaning that we're migrating from V3 to V4 DB (even if we're not fully aligned with V4 standards). The DB is not migrated to the v4 standard `SHA512` due to performance reasons. Our custom `SHA1` implementation is fully optimised for perfomance. * perf(sqlCipher): Fixing failing tests Update the new DB file format in Delete account, Change password and Decrypt database flows * perf(SQLCipher): Increase page size - send events to notify when the DB re-encryption starts/ends --- api/geth_backend.go | 87 ++++++++++++++++++++++------------------- appdatabase/database.go | 4 ++ signal/events_db.go | 18 +++++++++ sqlite/sqlite.go | 66 +++++++++++++++++++++++-------- 4 files changed, 117 insertions(+), 58 deletions(-) create mode 100644 signal/events_db.go diff --git a/api/geth_backend.go b/api/geth_backend.go index 728ceb61e..2dcabb7d4 100644 --- a/api/geth_backend.go +++ b/api/geth_backend.go @@ -248,6 +248,9 @@ func (b *GethStatusBackend) DeleteMultiaccount(keyUID string, keyStoreDir string filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db", keyUID)), filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db-shm", keyUID)), filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db-wal", keyUID)), + filepath.Join(b.rootDataDir, fmt.Sprintf("%s-v4.db", keyUID)), + filepath.Join(b.rootDataDir, fmt.Sprintf("%s-v4.db-shm", keyUID)), + filepath.Join(b.rootDataDir, fmt.Sprintf("%s-v4.db-wal", keyUID)), } for _, path := range dbFiles { if _, err := os.Stat(path); err == nil { @@ -289,6 +292,39 @@ func (b *GethStatusBackend) DeleteImportedKey(address, password, keyStoreDir str return err } +func (b *GethStatusBackend) runDBFileMigrations(account multiaccounts.Account, password string) (string, error) { + // Migrate file path to fix issue https://github.com/status-im/status-go/issues/2027 + unsupportedPath := filepath.Join(b.rootDataDir, fmt.Sprintf("app-%x.sql", account.KeyUID)) + v3Path := filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db", account.KeyUID)) + v4Path := filepath.Join(b.rootDataDir, fmt.Sprintf("%s-v4.db", account.KeyUID)) + + _, err := os.Stat(unsupportedPath) + if err == nil { + err := os.Rename(unsupportedPath, v3Path) + if err != nil { + return "", err + } + + // rename journals as well, but ignore errors + _ = os.Rename(unsupportedPath+"-shm", v3Path+"-shm") + _ = os.Rename(unsupportedPath+"-wal", v3Path+"-wal") + } + + if _, err = os.Stat(v3Path); err == nil { + if err := appdatabase.MigrateV3ToV4(v3Path, v4Path, password, account.KDFIterations); err != nil { + _ = os.Remove(v4Path) + _ = os.Remove(v4Path + "-shm") + _ = os.Remove(v4Path + "-wal") + return "", errors.New("Failed to migrate v3 db to v4: " + err.Error()) + } + _ = os.Remove(v3Path) + _ = os.Remove(v3Path + "-shm") + _ = os.Remove(v3Path + "-wal") + } + + return v4Path, nil +} + func (b *GethStatusBackend) ensureAppDBOpened(account multiaccounts.Account, password string) (err error) { b.mu.Lock() defer b.mu.Unlock() @@ -299,23 +335,12 @@ func (b *GethStatusBackend) ensureAppDBOpened(account multiaccounts.Account, pas return errors.New("root datadir wasn't provided") } - // Migrate file path to fix issue https://github.com/status-im/status-go/issues/2027 - oldPath := filepath.Join(b.rootDataDir, fmt.Sprintf("app-%x.sql", account.KeyUID)) - newPath := filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db", account.KeyUID)) - - _, err = os.Stat(oldPath) - if err == nil { - err := os.Rename(oldPath, newPath) - if err != nil { - return err - } - - // rename journals as well, but ignore errors - _ = os.Rename(oldPath+"-shm", newPath+"-shm") - _ = os.Rename(oldPath+"-wal", newPath+"-wal") + dbFilePath, err := b.runDBFileMigrations(account, password) + if err != nil { + return errors.New("Failed to migrate db file: " + err.Error()) } - b.appDB, err = appdatabase.InitializeDB(newPath, password, account.KDFIterations) + b.appDB, err = appdatabase.InitializeDB(dbFilePath, password, account.KDFIterations) if err != nil { b.log.Error("failed to initialize db", "err", err) return err @@ -691,23 +716,12 @@ func (b *GethStatusBackend) ExportUnencryptedDatabase(acc multiaccounts.Account, return errors.New("root datadir wasn't provided") } - // Migrate file path to fix issue https://github.com/status-im/status-go/issues/2027 - oldPath := filepath.Join(b.rootDataDir, fmt.Sprintf("app-%x.sql", acc.KeyUID)) - newPath := filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db", acc.KeyUID)) - - _, err := os.Stat(oldPath) - if err == nil { - err := os.Rename(oldPath, newPath) - if err != nil { - return err - } - - // rename journals as well, but ignore errors - _ = os.Rename(oldPath+"-shm", newPath+"-shm") - _ = os.Rename(oldPath+"-wal", newPath+"-wal") + dbPath, err := b.runDBFileMigrations(acc, password) + if err != nil { + return err } - err = appdatabase.DecryptDatabase(newPath, directory, password, acc.KDFIterations) + err = appdatabase.DecryptDatabase(dbPath, directory, password, acc.KDFIterations) if err != nil { b.log.Error("failed to initialize db", "err", err) return err @@ -725,7 +739,7 @@ func (b *GethStatusBackend) ImportUnencryptedDatabase(acc multiaccounts.Account, return errors.New("root datadir wasn't provided") } - path := filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db", acc.KeyUID)) + path := filepath.Join(b.rootDataDir, fmt.Sprintf("%s-v4.db", acc.KeyUID)) err := appdatabase.EncryptDatabase(databasePath, path, password, acc.KDFIterations) if err != nil { @@ -736,7 +750,7 @@ func (b *GethStatusBackend) ImportUnencryptedDatabase(acc multiaccounts.Account, } func (b *GethStatusBackend) ChangeDatabasePassword(keyUID string, password string, newPassword string) error { - dbPath := filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db", keyUID)) + dbPath := filepath.Join(b.rootDataDir, fmt.Sprintf("%s-v4.db", keyUID)) config := b.StatusNode().Config() keyDir := "" if config == nil { @@ -1130,15 +1144,6 @@ func (b *GethStatusBackend) VerifyDatabasePassword(keyUID string, password strin return err } - accountsDB, err := accounts.NewDB(b.appDB) - if err != nil { - return err - } - _, err = accountsDB.GetWalletAddress() - if err != nil { - return err - } - err = b.closeAppDB() if err != nil { return err diff --git a/appdatabase/database.go b/appdatabase/database.go index b39ad63e9..0ebd7fb42 100644 --- a/appdatabase/database.go +++ b/appdatabase/database.go @@ -200,6 +200,10 @@ func migrateEnsUsernames(sqlTx *sql.Tx) error { return nil } +func MigrateV3ToV4(v3Path string, v4Path string, password string, kdfIterationsNumber int) error { + return sqlite.MigrateV3ToV4(v3Path, v4Path, password, kdfIterationsNumber) +} + const ( batchSize = 1000 ) diff --git a/signal/events_db.go b/signal/events_db.go new file mode 100644 index 000000000..5cac7b30d --- /dev/null +++ b/signal/events_db.go @@ -0,0 +1,18 @@ +package signal + +const ( + // ReEncryptionStarted is sent when db reencryption was started. + ReEncryptionStarted = "db.reEncryption.started" + // ReEncryptionFinished is sent when db reencryption was finished. + ReEncryptionFinished = "db.reEncryption.finished" +) + +// Send db.reencryption.started signal. +func SendReEncryptionStarted() { + send(ReEncryptionStarted, nil) +} + +// Send db.reencryption.finished signal. +func SendReEncryptionFinished() { + send(ReEncryptionFinished, nil) +} diff --git a/sqlite/sqlite.go b/sqlite/sqlite.go index 4b63fd786..fd5be8a7e 100644 --- a/sqlite/sqlite.go +++ b/sqlite/sqlite.go @@ -13,6 +13,7 @@ import ( sqlcipher "github.com/mutecomm/go-sqlcipher/v4" // We require go sqlcipher that overrides default implementation "github.com/status-im/status-go/protocol/sqlite" + "github.com/status-im/status-go/signal" ) const ( @@ -23,14 +24,16 @@ const ( ReducedKDFIterationsNumber = 3200 // WALMode for sqlite. - WALMode = "wal" - InMemoryPath = ":memory:" + WALMode = "wal" + InMemoryPath = ":memory:" + V4CipherPageSize = 8192 + V3CipherPageSize = 1024 ) // DecryptDB completely removes the encryption from the db func DecryptDB(oldPath string, newPath string, key string, kdfIterationsNumber int) error { - db, err := openDB(oldPath, key, kdfIterationsNumber) + db, err := openDB(oldPath, key, kdfIterationsNumber, V4CipherPageSize) if err != nil { return err } @@ -48,16 +51,11 @@ func DecryptDB(oldPath string, newPath string, key string, kdfIterationsNumber i return err } -// EncryptDB takes a plaintext database and adds encryption -func EncryptDB(unencryptedPath string, encryptedPath string, key string, kdfIterationsNumber int) error { - _ = os.Remove(encryptedPath) +func encryptDB(db *sql.DB, encryptedPath string, key string, kdfIterationsNumber int) error { + signal.SendReEncryptionStarted() + defer signal.SendReEncryptionFinished() - db, err := OpenUnecryptedDB(unencryptedPath) - if err != nil { - return err - } - - _, err = db.Exec(`ATTACH DATABASE '` + encryptedPath + `' AS encrypted KEY '` + key + `'`) + _, err := db.Exec(`ATTACH DATABASE '` + encryptedPath + `' AS encrypted KEY '` + key + `'`) if err != nil { return err } @@ -71,7 +69,7 @@ func EncryptDB(unencryptedPath string, encryptedPath string, key string, kdfIter return err } - if _, err := db.Exec("PRAGMA encrypted.cipher_page_size = 1024"); err != nil { + if _, err := db.Exec(fmt.Sprintf("PRAGMA encrypted.cipher_page_size = %d", V4CipherPageSize)); err != nil { fmt.Println("failed to set cipher_page_size pragma") return err } @@ -93,6 +91,17 @@ func EncryptDB(unencryptedPath string, encryptedPath string, key string, kdfIter return err } +// EncryptDB takes a plaintext database and adds encryption +func EncryptDB(unencryptedPath string, encryptedPath string, key string, kdfIterationsNumber int) error { + _ = os.Remove(encryptedPath) + + db, err := OpenUnecryptedDB(unencryptedPath) + if err != nil { + return err + } + return encryptDB(db, encryptedPath, key, kdfIterationsNumber) +} + func buildSqlcipherDSN(path string) (string, error) { if path == InMemoryPath { return InMemoryPath, nil @@ -118,7 +127,7 @@ func buildSqlcipherDSN(path string) (string, error) { return path + queryOperator + "_txlock=immediate", nil } -func openDB(path string, key string, kdfIterationsNumber int) (*sql.DB, error) { +func openDB(path string, key string, kdfIterationsNumber int, chiperPageSize int) (*sql.DB, error) { driverName := fmt.Sprintf("sqlcipher_with_extensions-%d", len(sql.Drivers())) sql.Register(driverName, &sqlcipher.SQLiteDriver{ ConnectHook: func(conn *sqlcipher.SQLiteConn) error { @@ -134,7 +143,7 @@ func openDB(path string, key string, kdfIterationsNumber int) (*sql.DB, error) { kdfIterationsNumber = sqlite.ReducedKDFIterationsNumber } - if _, err := conn.Exec("PRAGMA cipher_page_size = 1024", nil); err != nil { + if _, err := conn.Exec(fmt.Sprintf("PRAGMA cipher_page_size = %d", chiperPageSize), nil); err != nil { fmt.Println("failed to set cipher_page_size pragma") return err } @@ -192,12 +201,18 @@ func openDB(path string, key string, kdfIterationsNumber int) (*sql.DB, error) { db.SetMaxIdleConns(nproc) } + // Dummy select to check if the key is correct. Will return last error from initialization + if _, err := db.Exec("SELECT 'Key check'"); err != nil { + db.Close() + return nil, err + } + return db, nil } // OpenDB opens not-encrypted database. func OpenDB(path string, key string, kdfIterationsNumber int) (*sql.DB, error) { - return openDB(path, key, kdfIterationsNumber) + return openDB(path, key, kdfIterationsNumber, V4CipherPageSize) } // OpenUnecryptedDB opens database with setting PRAGMA key. @@ -229,11 +244,14 @@ func OpenUnecryptedDB(path string) (*sql.DB, error) { } func ChangeEncryptionKey(path string, key string, kdfIterationsNumber int, newKey string) error { + signal.SendReEncryptionStarted() + defer signal.SendReEncryptionFinished() + if kdfIterationsNumber <= 0 { kdfIterationsNumber = sqlite.ReducedKDFIterationsNumber } - db, err := openDB(path, key, kdfIterationsNumber) + db, err := openDB(path, key, kdfIterationsNumber, V4CipherPageSize) if err != nil { return err @@ -246,3 +264,17 @@ func ChangeEncryptionKey(path string, key string, kdfIterationsNumber int, newKe return nil } + +// MigrateV3ToV4 migrates database from v3 to v4 format with encryption. +func MigrateV3ToV4(v3Path string, v4Path string, key string, kdfIterationsNumber int) error { + + db, err := openDB(v3Path, key, kdfIterationsNumber, V3CipherPageSize) + + if err != nil { + fmt.Println("failed to open db", err) + return err + } + defer db.Close() + + return encryptDB(db, v4Path, key, kdfIterationsNumber) +}