perf(sqlCipher): Increase cipher page size to 8192 (#3591)

* perf(sqlCipher): Increase cipher page size to 8192

Increasing the cipher page size to 8192 requires DB re-encryption. The process is as follows:
//Login to v3 DB
PRAGMA key = 'key';
PRAGMA cipher_page_size = 1024"; // old Page size
PRAGMA cipher_hmac_algorithm = HMAC_SHA1";
PRAGMA cipher_kdf_algorithm = PBKDF2_HMAC_SHA1";
PRAGMA kdf_iter = kdfIterationsNumber";

//Create V4 DB with increased page size
ATTACH DATABASE 'newdb.db' AS newdb KEY 'key';
PRAGMA newdb.cipher_page_size = 8192; // new Page size
PRAGMA newdb.cipher_hmac_algorithm = HMAC_SHA1"; // same as in v3
PRAGMA newdb.cipher_kdf_algorithm = PBKDF2_HMAC_SHA1"; // same as in v3
PRAGMA newdb.kdf_iter = kdfIterationsNumber"; // same as in v3
SELECT sqlcipher_export('newdb');
DETACH DATABASE newdb;

//Login to V4 DB
...

Worth noting:
The DB migration will happen on the first successful login.
The new DB version will have a different name to be able to distinguish between different DB versions.Versions naming mirrors sqlcipher major version (naming conventions used by sqlcipher), meaning that we're migrating from V3 to V4 DB (even if we're not fully aligned with V4 standards). The DB is not migrated to the v4 standard `SHA512` due to performance reasons. Our custom `SHA1` implementation is fully optimised for perfomance.

* perf(sqlCipher): Fixing failing tests

Update the new DB file format in Delete account, Change password and Decrypt database flows

* perf(SQLCipher): Increase page size - send events to notify when the DB re-encryption starts/ends
This commit is contained in:
Alex Jbanca 2023-06-13 18:20:21 +03:00 committed by GitHub
parent 3f231f53e3
commit 43b2c3b7ce
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 117 additions and 58 deletions

View File

@ -248,6 +248,9 @@ func (b *GethStatusBackend) DeleteMultiaccount(keyUID string, keyStoreDir string
filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db", keyUID)), filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db", keyUID)),
filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db-shm", keyUID)), filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db-shm", keyUID)),
filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db-wal", keyUID)), filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db-wal", keyUID)),
filepath.Join(b.rootDataDir, fmt.Sprintf("%s-v4.db", keyUID)),
filepath.Join(b.rootDataDir, fmt.Sprintf("%s-v4.db-shm", keyUID)),
filepath.Join(b.rootDataDir, fmt.Sprintf("%s-v4.db-wal", keyUID)),
} }
for _, path := range dbFiles { for _, path := range dbFiles {
if _, err := os.Stat(path); err == nil { if _, err := os.Stat(path); err == nil {
@ -289,6 +292,39 @@ func (b *GethStatusBackend) DeleteImportedKey(address, password, keyStoreDir str
return err return err
} }
func (b *GethStatusBackend) runDBFileMigrations(account multiaccounts.Account, password string) (string, error) {
// Migrate file path to fix issue https://github.com/status-im/status-go/issues/2027
unsupportedPath := filepath.Join(b.rootDataDir, fmt.Sprintf("app-%x.sql", account.KeyUID))
v3Path := filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db", account.KeyUID))
v4Path := filepath.Join(b.rootDataDir, fmt.Sprintf("%s-v4.db", account.KeyUID))
_, err := os.Stat(unsupportedPath)
if err == nil {
err := os.Rename(unsupportedPath, v3Path)
if err != nil {
return "", err
}
// rename journals as well, but ignore errors
_ = os.Rename(unsupportedPath+"-shm", v3Path+"-shm")
_ = os.Rename(unsupportedPath+"-wal", v3Path+"-wal")
}
if _, err = os.Stat(v3Path); err == nil {
if err := appdatabase.MigrateV3ToV4(v3Path, v4Path, password, account.KDFIterations); err != nil {
_ = os.Remove(v4Path)
_ = os.Remove(v4Path + "-shm")
_ = os.Remove(v4Path + "-wal")
return "", errors.New("Failed to migrate v3 db to v4: " + err.Error())
}
_ = os.Remove(v3Path)
_ = os.Remove(v3Path + "-shm")
_ = os.Remove(v3Path + "-wal")
}
return v4Path, nil
}
func (b *GethStatusBackend) ensureAppDBOpened(account multiaccounts.Account, password string) (err error) { func (b *GethStatusBackend) ensureAppDBOpened(account multiaccounts.Account, password string) (err error) {
b.mu.Lock() b.mu.Lock()
defer b.mu.Unlock() defer b.mu.Unlock()
@ -299,23 +335,12 @@ func (b *GethStatusBackend) ensureAppDBOpened(account multiaccounts.Account, pas
return errors.New("root datadir wasn't provided") return errors.New("root datadir wasn't provided")
} }
// Migrate file path to fix issue https://github.com/status-im/status-go/issues/2027 dbFilePath, err := b.runDBFileMigrations(account, password)
oldPath := filepath.Join(b.rootDataDir, fmt.Sprintf("app-%x.sql", account.KeyUID)) if err != nil {
newPath := filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db", account.KeyUID)) return errors.New("Failed to migrate db file: " + err.Error())
_, err = os.Stat(oldPath)
if err == nil {
err := os.Rename(oldPath, newPath)
if err != nil {
return err
}
// rename journals as well, but ignore errors
_ = os.Rename(oldPath+"-shm", newPath+"-shm")
_ = os.Rename(oldPath+"-wal", newPath+"-wal")
} }
b.appDB, err = appdatabase.InitializeDB(newPath, password, account.KDFIterations) b.appDB, err = appdatabase.InitializeDB(dbFilePath, password, account.KDFIterations)
if err != nil { if err != nil {
b.log.Error("failed to initialize db", "err", err) b.log.Error("failed to initialize db", "err", err)
return err return err
@ -691,23 +716,12 @@ func (b *GethStatusBackend) ExportUnencryptedDatabase(acc multiaccounts.Account,
return errors.New("root datadir wasn't provided") return errors.New("root datadir wasn't provided")
} }
// Migrate file path to fix issue https://github.com/status-im/status-go/issues/2027 dbPath, err := b.runDBFileMigrations(acc, password)
oldPath := filepath.Join(b.rootDataDir, fmt.Sprintf("app-%x.sql", acc.KeyUID)) if err != nil {
newPath := filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db", acc.KeyUID)) return err
_, err := os.Stat(oldPath)
if err == nil {
err := os.Rename(oldPath, newPath)
if err != nil {
return err
}
// rename journals as well, but ignore errors
_ = os.Rename(oldPath+"-shm", newPath+"-shm")
_ = os.Rename(oldPath+"-wal", newPath+"-wal")
} }
err = appdatabase.DecryptDatabase(newPath, directory, password, acc.KDFIterations) err = appdatabase.DecryptDatabase(dbPath, directory, password, acc.KDFIterations)
if err != nil { if err != nil {
b.log.Error("failed to initialize db", "err", err) b.log.Error("failed to initialize db", "err", err)
return err return err
@ -725,7 +739,7 @@ func (b *GethStatusBackend) ImportUnencryptedDatabase(acc multiaccounts.Account,
return errors.New("root datadir wasn't provided") return errors.New("root datadir wasn't provided")
} }
path := filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db", acc.KeyUID)) path := filepath.Join(b.rootDataDir, fmt.Sprintf("%s-v4.db", acc.KeyUID))
err := appdatabase.EncryptDatabase(databasePath, path, password, acc.KDFIterations) err := appdatabase.EncryptDatabase(databasePath, path, password, acc.KDFIterations)
if err != nil { if err != nil {
@ -736,7 +750,7 @@ func (b *GethStatusBackend) ImportUnencryptedDatabase(acc multiaccounts.Account,
} }
func (b *GethStatusBackend) ChangeDatabasePassword(keyUID string, password string, newPassword string) error { func (b *GethStatusBackend) ChangeDatabasePassword(keyUID string, password string, newPassword string) error {
dbPath := filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db", keyUID)) dbPath := filepath.Join(b.rootDataDir, fmt.Sprintf("%s-v4.db", keyUID))
config := b.StatusNode().Config() config := b.StatusNode().Config()
keyDir := "" keyDir := ""
if config == nil { if config == nil {
@ -1130,15 +1144,6 @@ func (b *GethStatusBackend) VerifyDatabasePassword(keyUID string, password strin
return err return err
} }
accountsDB, err := accounts.NewDB(b.appDB)
if err != nil {
return err
}
_, err = accountsDB.GetWalletAddress()
if err != nil {
return err
}
err = b.closeAppDB() err = b.closeAppDB()
if err != nil { if err != nil {
return err return err

View File

@ -200,6 +200,10 @@ func migrateEnsUsernames(sqlTx *sql.Tx) error {
return nil return nil
} }
func MigrateV3ToV4(v3Path string, v4Path string, password string, kdfIterationsNumber int) error {
return sqlite.MigrateV3ToV4(v3Path, v4Path, password, kdfIterationsNumber)
}
const ( const (
batchSize = 1000 batchSize = 1000
) )

18
signal/events_db.go Normal file
View File

@ -0,0 +1,18 @@
package signal
const (
// ReEncryptionStarted is sent when db reencryption was started.
ReEncryptionStarted = "db.reEncryption.started"
// ReEncryptionFinished is sent when db reencryption was finished.
ReEncryptionFinished = "db.reEncryption.finished"
)
// Send db.reencryption.started signal.
func SendReEncryptionStarted() {
send(ReEncryptionStarted, nil)
}
// Send db.reencryption.finished signal.
func SendReEncryptionFinished() {
send(ReEncryptionFinished, nil)
}

View File

@ -13,6 +13,7 @@ import (
sqlcipher "github.com/mutecomm/go-sqlcipher/v4" // We require go sqlcipher that overrides default implementation sqlcipher "github.com/mutecomm/go-sqlcipher/v4" // We require go sqlcipher that overrides default implementation
"github.com/status-im/status-go/protocol/sqlite" "github.com/status-im/status-go/protocol/sqlite"
"github.com/status-im/status-go/signal"
) )
const ( const (
@ -23,14 +24,16 @@ const (
ReducedKDFIterationsNumber = 3200 ReducedKDFIterationsNumber = 3200
// WALMode for sqlite. // WALMode for sqlite.
WALMode = "wal" WALMode = "wal"
InMemoryPath = ":memory:" InMemoryPath = ":memory:"
V4CipherPageSize = 8192
V3CipherPageSize = 1024
) )
// DecryptDB completely removes the encryption from the db // DecryptDB completely removes the encryption from the db
func DecryptDB(oldPath string, newPath string, key string, kdfIterationsNumber int) error { func DecryptDB(oldPath string, newPath string, key string, kdfIterationsNumber int) error {
db, err := openDB(oldPath, key, kdfIterationsNumber) db, err := openDB(oldPath, key, kdfIterationsNumber, V4CipherPageSize)
if err != nil { if err != nil {
return err return err
} }
@ -48,16 +51,11 @@ func DecryptDB(oldPath string, newPath string, key string, kdfIterationsNumber i
return err return err
} }
// EncryptDB takes a plaintext database and adds encryption func encryptDB(db *sql.DB, encryptedPath string, key string, kdfIterationsNumber int) error {
func EncryptDB(unencryptedPath string, encryptedPath string, key string, kdfIterationsNumber int) error { signal.SendReEncryptionStarted()
_ = os.Remove(encryptedPath) defer signal.SendReEncryptionFinished()
db, err := OpenUnecryptedDB(unencryptedPath) _, err := db.Exec(`ATTACH DATABASE '` + encryptedPath + `' AS encrypted KEY '` + key + `'`)
if err != nil {
return err
}
_, err = db.Exec(`ATTACH DATABASE '` + encryptedPath + `' AS encrypted KEY '` + key + `'`)
if err != nil { if err != nil {
return err return err
} }
@ -71,7 +69,7 @@ func EncryptDB(unencryptedPath string, encryptedPath string, key string, kdfIter
return err return err
} }
if _, err := db.Exec("PRAGMA encrypted.cipher_page_size = 1024"); err != nil { if _, err := db.Exec(fmt.Sprintf("PRAGMA encrypted.cipher_page_size = %d", V4CipherPageSize)); err != nil {
fmt.Println("failed to set cipher_page_size pragma") fmt.Println("failed to set cipher_page_size pragma")
return err return err
} }
@ -93,6 +91,17 @@ func EncryptDB(unencryptedPath string, encryptedPath string, key string, kdfIter
return err return err
} }
// EncryptDB takes a plaintext database and adds encryption
func EncryptDB(unencryptedPath string, encryptedPath string, key string, kdfIterationsNumber int) error {
_ = os.Remove(encryptedPath)
db, err := OpenUnecryptedDB(unencryptedPath)
if err != nil {
return err
}
return encryptDB(db, encryptedPath, key, kdfIterationsNumber)
}
func buildSqlcipherDSN(path string) (string, error) { func buildSqlcipherDSN(path string) (string, error) {
if path == InMemoryPath { if path == InMemoryPath {
return InMemoryPath, nil return InMemoryPath, nil
@ -118,7 +127,7 @@ func buildSqlcipherDSN(path string) (string, error) {
return path + queryOperator + "_txlock=immediate", nil return path + queryOperator + "_txlock=immediate", nil
} }
func openDB(path string, key string, kdfIterationsNumber int) (*sql.DB, error) { func openDB(path string, key string, kdfIterationsNumber int, chiperPageSize int) (*sql.DB, error) {
driverName := fmt.Sprintf("sqlcipher_with_extensions-%d", len(sql.Drivers())) driverName := fmt.Sprintf("sqlcipher_with_extensions-%d", len(sql.Drivers()))
sql.Register(driverName, &sqlcipher.SQLiteDriver{ sql.Register(driverName, &sqlcipher.SQLiteDriver{
ConnectHook: func(conn *sqlcipher.SQLiteConn) error { ConnectHook: func(conn *sqlcipher.SQLiteConn) error {
@ -134,7 +143,7 @@ func openDB(path string, key string, kdfIterationsNumber int) (*sql.DB, error) {
kdfIterationsNumber = sqlite.ReducedKDFIterationsNumber kdfIterationsNumber = sqlite.ReducedKDFIterationsNumber
} }
if _, err := conn.Exec("PRAGMA cipher_page_size = 1024", nil); err != nil { if _, err := conn.Exec(fmt.Sprintf("PRAGMA cipher_page_size = %d", chiperPageSize), nil); err != nil {
fmt.Println("failed to set cipher_page_size pragma") fmt.Println("failed to set cipher_page_size pragma")
return err return err
} }
@ -192,12 +201,18 @@ func openDB(path string, key string, kdfIterationsNumber int) (*sql.DB, error) {
db.SetMaxIdleConns(nproc) db.SetMaxIdleConns(nproc)
} }
// Dummy select to check if the key is correct. Will return last error from initialization
if _, err := db.Exec("SELECT 'Key check'"); err != nil {
db.Close()
return nil, err
}
return db, nil return db, nil
} }
// OpenDB opens not-encrypted database. // OpenDB opens not-encrypted database.
func OpenDB(path string, key string, kdfIterationsNumber int) (*sql.DB, error) { func OpenDB(path string, key string, kdfIterationsNumber int) (*sql.DB, error) {
return openDB(path, key, kdfIterationsNumber) return openDB(path, key, kdfIterationsNumber, V4CipherPageSize)
} }
// OpenUnecryptedDB opens database with setting PRAGMA key. // OpenUnecryptedDB opens database with setting PRAGMA key.
@ -229,11 +244,14 @@ func OpenUnecryptedDB(path string) (*sql.DB, error) {
} }
func ChangeEncryptionKey(path string, key string, kdfIterationsNumber int, newKey string) error { func ChangeEncryptionKey(path string, key string, kdfIterationsNumber int, newKey string) error {
signal.SendReEncryptionStarted()
defer signal.SendReEncryptionFinished()
if kdfIterationsNumber <= 0 { if kdfIterationsNumber <= 0 {
kdfIterationsNumber = sqlite.ReducedKDFIterationsNumber kdfIterationsNumber = sqlite.ReducedKDFIterationsNumber
} }
db, err := openDB(path, key, kdfIterationsNumber) db, err := openDB(path, key, kdfIterationsNumber, V4CipherPageSize)
if err != nil { if err != nil {
return err return err
@ -246,3 +264,17 @@ func ChangeEncryptionKey(path string, key string, kdfIterationsNumber int, newKe
return nil return nil
} }
// MigrateV3ToV4 migrates database from v3 to v4 format with encryption.
func MigrateV3ToV4(v3Path string, v4Path string, key string, kdfIterationsNumber int) error {
db, err := openDB(v3Path, key, kdfIterationsNumber, V3CipherPageSize)
if err != nil {
fmt.Println("failed to open db", err)
return err
}
defer db.Close()
return encryptDB(db, v4Path, key, kdfIterationsNumber)
}