status-im
/
status-go
mirror of https://github.com/status-im/status-go.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

perf(sqlCipher): Increase cipher page size to 8192 (#3591) 

* perf(sqlCipher): Increase cipher page size to 8192

Increasing the cipher page size to 8192 requires DB re-encryption. The process is as follows:
//Login to v3 DB
PRAGMA key = 'key';
PRAGMA cipher_page_size = 1024"; // old Page size
PRAGMA cipher_hmac_algorithm = HMAC_SHA1";
PRAGMA cipher_kdf_algorithm = PBKDF2_HMAC_SHA1";
PRAGMA kdf_iter = kdfIterationsNumber";

//Create V4 DB with increased page size
ATTACH DATABASE 'newdb.db' AS newdb KEY 'key';
PRAGMA newdb.cipher_page_size = 8192; // new Page size
PRAGMA newdb.cipher_hmac_algorithm = HMAC_SHA1"; // same as in v3
PRAGMA newdb.cipher_kdf_algorithm = PBKDF2_HMAC_SHA1"; // same as in v3
PRAGMA newdb.kdf_iter = kdfIterationsNumber"; // same as in v3
SELECT sqlcipher_export('newdb');
DETACH DATABASE newdb;

//Login to V4 DB
...

Worth noting:
The DB migration will happen on the first successful login.
The new DB version will have a different name to be able to distinguish between different DB versions.Versions naming mirrors sqlcipher major version (naming conventions used by sqlcipher), meaning that we're migrating from V3 to V4 DB (even if we're not fully aligned with V4 standards). The DB is not migrated to the v4 standard `SHA512` due to performance reasons. Our custom `SHA1` implementation is fully optimised for perfomance.

* perf(sqlCipher): Fixing failing tests

Update the new DB file format in Delete account, Change password and Decrypt database flows

* perf(SQLCipher): Increase page size - send events to notify when the DB re-encryption starts/ends
This commit is contained in:
Alex Jbanca 2023-06-13 18:20:21 +03:00 committed by GitHub
parent 3f231f53e3
commit 43b2c3b7ce
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 117 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
api/geth_backend.go
View File

@ -248,6 +248,9 @@ func (b *GethStatusBackend) DeleteMultiaccount(keyUID string, keyStoreDir string
filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db", keyUID)),
filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db-shm", keyUID)),
filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db-wal", keyUID)),
filepath.Join(b.rootDataDir, fmt.Sprintf("%s-v4.db", keyUID)),
filepath.Join(b.rootDataDir, fmt.Sprintf("%s-v4.db-shm", keyUID)),
filepath.Join(b.rootDataDir, fmt.Sprintf("%s-v4.db-wal", keyUID)),
}
for _, path := range dbFiles {
if _, err := os.Stat(path); err == nil {
@ -289,6 +292,39 @@ func (b *GethStatusBackend) DeleteImportedKey(address, password, keyStoreDir str
return err
}
func (b *GethStatusBackend) runDBFileMigrations(account multiaccounts.Account, password string) (string, error) {
// Migrate file path to fix issue https://github.com/status-im/status-go/issues/2027
unsupportedPath := filepath.Join(b.rootDataDir, fmt.Sprintf("app-%x.sql", account.KeyUID))
v3Path := filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db", account.KeyUID))
v4Path := filepath.Join(b.rootDataDir, fmt.Sprintf("%s-v4.db", account.KeyUID))
_, err := os.Stat(unsupportedPath)
if err == nil {
err := os.Rename(unsupportedPath, v3Path)
if err != nil {
return "", err
}
// rename journals as well, but ignore errors
_ = os.Rename(unsupportedPath+"-shm", v3Path+"-shm")
_ = os.Rename(unsupportedPath+"-wal", v3Path+"-wal")
}
if _, err = os.Stat(v3Path); err == nil {
if err := appdatabase.MigrateV3ToV4(v3Path, v4Path, password, account.KDFIterations); err != nil {
_ = os.Remove(v4Path)
_ = os.Remove(v4Path + "-shm")
_ = os.Remove(v4Path + "-wal")
return "", errors.New("Failed to migrate v3 db to v4: " + err.Error())
}
_ = os.Remove(v3Path)
_ = os.Remove(v3Path + "-shm")
_ = os.Remove(v3Path + "-wal")
}
return v4Path, nil
}
func (b *GethStatusBackend) ensureAppDBOpened(account multiaccounts.Account, password string) (err error) {
b.mu.Lock()
defer b.mu.Unlock()
@ -299,23 +335,12 @@ func (b *GethStatusBackend) ensureAppDBOpened(account multiaccounts.Account, pas
return errors.New("root datadir wasn't provided")
}
// Migrate file path to fix issue https://github.com/status-im/status-go/issues/2027
oldPath := filepath.Join(b.rootDataDir, fmt.Sprintf("app-%x.sql", account.KeyUID))
newPath := filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db", account.KeyUID))
_, err = os.Stat(oldPath)
if err == nil {
err := os.Rename(oldPath, newPath)
if err != nil {
return err
}
// rename journals as well, but ignore errors
_ = os.Rename(oldPath+"-shm", newPath+"-shm")
_ = os.Rename(oldPath+"-wal", newPath+"-wal")
dbFilePath, err := b.runDBFileMigrations(account, password)
if err != nil {
return errors.New("Failed to migrate db file: " + err.Error())
}
b.appDB, err = appdatabase.InitializeDB(newPath, password, account.KDFIterations)
b.appDB, err = appdatabase.InitializeDB(dbFilePath, password, account.KDFIterations)
if err != nil {
b.log.Error("failed to initialize db", "err", err)
return err
@ -691,23 +716,12 @@ func (b *GethStatusBackend) ExportUnencryptedDatabase(acc multiaccounts.Account,
return errors.New("root datadir wasn't provided")
}
// Migrate file path to fix issue https://github.com/status-im/status-go/issues/2027
oldPath := filepath.Join(b.rootDataDir, fmt.Sprintf("app-%x.sql", acc.KeyUID))
newPath := filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db", acc.KeyUID))
_, err := os.Stat(oldPath)
if err == nil {
err := os.Rename(oldPath, newPath)
if err != nil {
return err
}
// rename journals as well, but ignore errors
_ = os.Rename(oldPath+"-shm", newPath+"-shm")
_ = os.Rename(oldPath+"-wal", newPath+"-wal")
dbPath, err := b.runDBFileMigrations(acc, password)
if err != nil {
return err
}
err = appdatabase.DecryptDatabase(newPath, directory, password, acc.KDFIterations)
err = appdatabase.DecryptDatabase(dbPath, directory, password, acc.KDFIterations)
if err != nil {
b.log.Error("failed to initialize db", "err", err)
return err
@ -725,7 +739,7 @@ func (b *GethStatusBackend) ImportUnencryptedDatabase(acc multiaccounts.Account,
return errors.New("root datadir wasn't provided")
}
path := filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db", acc.KeyUID))
path := filepath.Join(b.rootDataDir, fmt.Sprintf("%s-v4.db", acc.KeyUID))
err := appdatabase.EncryptDatabase(databasePath, path, password, acc.KDFIterations)
if err != nil {
@ -736,7 +750,7 @@ func (b *GethStatusBackend) ImportUnencryptedDatabase(acc multiaccounts.Account,
}
func (b *GethStatusBackend) ChangeDatabasePassword(keyUID string, password string, newPassword string) error {
dbPath := filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db", keyUID))
dbPath := filepath.Join(b.rootDataDir, fmt.Sprintf("%s-v4.db", keyUID))
config := b.StatusNode().Config()
keyDir := ""
if config == nil {
@ -1130,15 +1144,6 @@ func (b *GethStatusBackend) VerifyDatabasePassword(keyUID string, password strin
return err
}
accountsDB, err := accounts.NewDB(b.appDB)
if err != nil {
return err
}
_, err = accountsDB.GetWalletAddress()
if err != nil {
return err
}
err = b.closeAppDB()
if err != nil {
return err

4
appdatabase/database.go
View File

@ -200,6 +200,10 @@ func migrateEnsUsernames(sqlTx *sql.Tx) error {
return nil
}
func MigrateV3ToV4(v3Path string, v4Path string, password string, kdfIterationsNumber int) error {
return sqlite.MigrateV3ToV4(v3Path, v4Path, password, kdfIterationsNumber)
}
const (
batchSize = 1000
)

18
signal/events_db.go Normal file
View File

@ -0,0 +1,18 @@
package signal
const (
// ReEncryptionStarted is sent when db reencryption was started.
ReEncryptionStarted = "db.reEncryption.started"
// ReEncryptionFinished is sent when db reencryption was finished.
ReEncryptionFinished = "db.reEncryption.finished"
)
// Send db.reencryption.started signal.
func SendReEncryptionStarted() {
send(ReEncryptionStarted, nil)
}
// Send db.reencryption.finished signal.
func SendReEncryptionFinished() {
send(ReEncryptionFinished, nil)
}

66
sqlite/sqlite.go
View File

@ -13,6 +13,7 @@ import (
sqlcipher "github.com/mutecomm/go-sqlcipher/v4" // We require go sqlcipher that overrides default implementation
"github.com/status-im/status-go/protocol/sqlite"
"github.com/status-im/status-go/signal"
)
const (
@ -23,14 +24,16 @@ const (
ReducedKDFIterationsNumber = 3200
// WALMode for sqlite.
WALMode = "wal"
InMemoryPath = ":memory:"
WALMode = "wal"
InMemoryPath = ":memory:"
V4CipherPageSize = 8192
V3CipherPageSize = 1024
)
// DecryptDB completely removes the encryption from the db
func DecryptDB(oldPath string, newPath string, key string, kdfIterationsNumber int) error {
db, err := openDB(oldPath, key, kdfIterationsNumber)
db, err := openDB(oldPath, key, kdfIterationsNumber, V4CipherPageSize)
if err != nil {
return err
}
@ -48,16 +51,11 @@ func DecryptDB(oldPath string, newPath string, key string, kdfIterationsNumber i
return err
}
// EncryptDB takes a plaintext database and adds encryption
func EncryptDB(unencryptedPath string, encryptedPath string, key string, kdfIterationsNumber int) error {
_ = os.Remove(encryptedPath)
func encryptDB(db *sql.DB, encryptedPath string, key string, kdfIterationsNumber int) error {
signal.SendReEncryptionStarted()
defer signal.SendReEncryptionFinished()
db, err := OpenUnecryptedDB(unencryptedPath)
if err != nil {
return err
}
_, err = db.Exec(`ATTACH DATABASE '` + encryptedPath + `' AS encrypted KEY '` + key + `'`)
_, err := db.Exec(`ATTACH DATABASE '` + encryptedPath + `' AS encrypted KEY '` + key + `'`)
if err != nil {
return err
}
@ -71,7 +69,7 @@ func EncryptDB(unencryptedPath string, encryptedPath string, key string, kdfIter
return err
}
if _, err := db.Exec("PRAGMA encrypted.cipher_page_size = 1024"); err != nil {
if _, err := db.Exec(fmt.Sprintf("PRAGMA encrypted.cipher_page_size = %d", V4CipherPageSize)); err != nil {
fmt.Println("failed to set cipher_page_size pragma")
return err
}
@ -93,6 +91,17 @@ func EncryptDB(unencryptedPath string, encryptedPath string, key string, kdfIter
return err
}
// EncryptDB takes a plaintext database and adds encryption
func EncryptDB(unencryptedPath string, encryptedPath string, key string, kdfIterationsNumber int) error {
_ = os.Remove(encryptedPath)
db, err := OpenUnecryptedDB(unencryptedPath)
if err != nil {
return err
}
return encryptDB(db, encryptedPath, key, kdfIterationsNumber)
}
func buildSqlcipherDSN(path string) (string, error) {
if path == InMemoryPath {
return InMemoryPath, nil
@ -118,7 +127,7 @@ func buildSqlcipherDSN(path string) (string, error) {
return path + queryOperator + "_txlock=immediate", nil
}
func openDB(path string, key string, kdfIterationsNumber int) (*sql.DB, error) {
func openDB(path string, key string, kdfIterationsNumber int, chiperPageSize int) (*sql.DB, error) {
driverName := fmt.Sprintf("sqlcipher_with_extensions-%d", len(sql.Drivers()))
sql.Register(driverName, &sqlcipher.SQLiteDriver{
ConnectHook: func(conn *sqlcipher.SQLiteConn) error {
@ -134,7 +143,7 @@ func openDB(path string, key string, kdfIterationsNumber int) (*sql.DB, error) {
kdfIterationsNumber = sqlite.ReducedKDFIterationsNumber
}
if _, err := conn.Exec("PRAGMA cipher_page_size = 1024", nil); err != nil {
if _, err := conn.Exec(fmt.Sprintf("PRAGMA cipher_page_size = %d", chiperPageSize), nil); err != nil {
fmt.Println("failed to set cipher_page_size pragma")
return err
}
@ -192,12 +201,18 @@ func openDB(path string, key string, kdfIterationsNumber int) (*sql.DB, error) {
db.SetMaxIdleConns(nproc)
}
// Dummy select to check if the key is correct. Will return last error from initialization
if _, err := db.Exec("SELECT 'Key check'"); err != nil {
db.Close()
return nil, err
}
return db, nil
}
// OpenDB opens not-encrypted database.
func OpenDB(path string, key string, kdfIterationsNumber int) (*sql.DB, error) {
return openDB(path, key, kdfIterationsNumber)
return openDB(path, key, kdfIterationsNumber, V4CipherPageSize)
}
// OpenUnecryptedDB opens database with setting PRAGMA key.
@ -229,11 +244,14 @@ func OpenUnecryptedDB(path string) (*sql.DB, error) {
}
func ChangeEncryptionKey(path string, key string, kdfIterationsNumber int, newKey string) error {
signal.SendReEncryptionStarted()
defer signal.SendReEncryptionFinished()
if kdfIterationsNumber <= 0 {
kdfIterationsNumber = sqlite.ReducedKDFIterationsNumber
}
db, err := openDB(path, key, kdfIterationsNumber)
db, err := openDB(path, key, kdfIterationsNumber, V4CipherPageSize)
if err != nil {
return err
@ -246,3 +264,17 @@ func ChangeEncryptionKey(path string, key string, kdfIterationsNumber int, newKe
return nil
}
// MigrateV3ToV4 migrates database from v3 to v4 format with encryption.
func MigrateV3ToV4(v3Path string, v4Path string, key string, kdfIterationsNumber int) error {
db, err := openDB(v3Path, key, kdfIterationsNumber, V3CipherPageSize)
if err != nil {
fmt.Println("failed to open db", err)
return err
}
defer db.Close()
return encryptDB(db, v4Path, key, kdfIterationsNumber)
}