perf(sqlCipher): Increase cipher page size to 8192 (#3591)

* perf(sqlCipher): Increase cipher page size to 8192

Increasing the cipher page size to 8192 requires DB re-encryption. The process is as follows:
//Login to v3 DB
PRAGMA key = 'key';
PRAGMA cipher_page_size = 1024"; // old Page size
PRAGMA cipher_hmac_algorithm = HMAC_SHA1";
PRAGMA cipher_kdf_algorithm = PBKDF2_HMAC_SHA1";
PRAGMA kdf_iter = kdfIterationsNumber";

//Create V4 DB with increased page size
ATTACH DATABASE 'newdb.db' AS newdb KEY 'key';
PRAGMA newdb.cipher_page_size = 8192; // new Page size
PRAGMA newdb.cipher_hmac_algorithm = HMAC_SHA1"; // same as in v3
PRAGMA newdb.cipher_kdf_algorithm = PBKDF2_HMAC_SHA1"; // same as in v3
PRAGMA newdb.kdf_iter = kdfIterationsNumber"; // same as in v3
SELECT sqlcipher_export('newdb');
DETACH DATABASE newdb;

//Login to V4 DB
...

Worth noting:
The DB migration will happen on the first successful login.
The new DB version will have a different name to be able to distinguish between different DB versions.Versions naming mirrors sqlcipher major version (naming conventions used by sqlcipher), meaning that we're migrating from V3 to V4 DB (even if we're not fully aligned with V4 standards). The DB is not migrated to the v4 standard `SHA512` due to performance reasons. Our custom `SHA1` implementation is fully optimised for perfomance.

* perf(sqlCipher): Fixing failing tests

Update the new DB file format in Delete account, Change password and Decrypt database flows

* perf(SQLCipher): Increase page size - send events to notify when the DB re-encryption starts/ends

api/geth_backend.go

@@ -248,6 +248,9 @@ func (b *GethStatusBackend) DeleteMultiaccount(keyUID string, keyStoreDir string
 		filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db", keyUID)),
 		filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db-shm", keyUID)),
 		filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db-wal", keyUID)),
+		filepath.Join(b.rootDataDir, fmt.Sprintf("%s-v4.db", keyUID)),
+		filepath.Join(b.rootDataDir, fmt.Sprintf("%s-v4.db-shm", keyUID)),
+		filepath.Join(b.rootDataDir, fmt.Sprintf("%s-v4.db-wal", keyUID)),
 	}
 	for _, path := range dbFiles {
 		if _, err := os.Stat(path); err == nil {
@@ -289,6 +292,39 @@ func (b *GethStatusBackend) DeleteImportedKey(address, password, keyStoreDir str
 	return err
 }
 
+func (b *GethStatusBackend) runDBFileMigrations(account multiaccounts.Account, password string) (string, error) {
+	// Migrate file path to fix issue https://github.com/status-im/status-go/issues/2027
+	unsupportedPath := filepath.Join(b.rootDataDir, fmt.Sprintf("app-%x.sql", account.KeyUID))
+	v3Path := filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db", account.KeyUID))
+	v4Path := filepath.Join(b.rootDataDir, fmt.Sprintf("%s-v4.db", account.KeyUID))
+
+	_, err := os.Stat(unsupportedPath)
+	if err == nil {
+		err := os.Rename(unsupportedPath, v3Path)
+		if err != nil {
+			return "", err
+		}
+
+		// rename journals as well, but ignore errors
+		_ = os.Rename(unsupportedPath+"-shm", v3Path+"-shm")
+		_ = os.Rename(unsupportedPath+"-wal", v3Path+"-wal")
+	}
+
+	if _, err = os.Stat(v3Path); err == nil {
+		if err := appdatabase.MigrateV3ToV4(v3Path, v4Path, password, account.KDFIterations); err != nil {
+			_ = os.Remove(v4Path)
+			_ = os.Remove(v4Path + "-shm")
+			_ = os.Remove(v4Path + "-wal")
+			return "", errors.New("Failed to migrate v3 db to v4: " + err.Error())
+		}
+		_ = os.Remove(v3Path)
+		_ = os.Remove(v3Path + "-shm")
+		_ = os.Remove(v3Path + "-wal")
+	}
+
+	return v4Path, nil
+}
+
 func (b *GethStatusBackend) ensureAppDBOpened(account multiaccounts.Account, password string) (err error) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
@@ -299,23 +335,12 @@ func (b *GethStatusBackend) ensureAppDBOpened(account multiaccounts.Account, pas
 		return errors.New("root datadir wasn't provided")
 	}
 
-	// Migrate file path to fix issue https://github.com/status-im/status-go/issues/2027
+	dbFilePath, err := b.runDBFileMigrations(account, password)
-	oldPath := filepath.Join(b.rootDataDir, fmt.Sprintf("app-%x.sql", account.KeyUID))
+	if err != nil {
-	newPath := filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db", account.KeyUID))
+		return errors.New("Failed to migrate db file: " + err.Error())
-
-	_, err = os.Stat(oldPath)
-	if err == nil {
-		err := os.Rename(oldPath, newPath)
-		if err != nil {
-			return err
-		}
-
-		// rename journals as well, but ignore errors
-		_ = os.Rename(oldPath+"-shm", newPath+"-shm")
-		_ = os.Rename(oldPath+"-wal", newPath+"-wal")
 	}
 
-	b.appDB, err = appdatabase.InitializeDB(newPath, password, account.KDFIterations)
+	b.appDB, err = appdatabase.InitializeDB(dbFilePath, password, account.KDFIterations)
 	if err != nil {
 		b.log.Error("failed to initialize db", "err", err)
 		return err
@@ -691,23 +716,12 @@ func (b *GethStatusBackend) ExportUnencryptedDatabase(acc multiaccounts.Account,
 		return errors.New("root datadir wasn't provided")
 	}
 
-	// Migrate file path to fix issue https://github.com/status-im/status-go/issues/2027
+	dbPath, err := b.runDBFileMigrations(acc, password)
-	oldPath := filepath.Join(b.rootDataDir, fmt.Sprintf("app-%x.sql", acc.KeyUID))
+	if err != nil {
-	newPath := filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db", acc.KeyUID))
+		return err
-
-	_, err := os.Stat(oldPath)
-	if err == nil {
-		err := os.Rename(oldPath, newPath)
-		if err != nil {
-			return err
-		}
-
-		// rename journals as well, but ignore errors
-		_ = os.Rename(oldPath+"-shm", newPath+"-shm")
-		_ = os.Rename(oldPath+"-wal", newPath+"-wal")
 	}
 
-	err = appdatabase.DecryptDatabase(newPath, directory, password, acc.KDFIterations)
+	err = appdatabase.DecryptDatabase(dbPath, directory, password, acc.KDFIterations)
 	if err != nil {
 		b.log.Error("failed to initialize db", "err", err)
 		return err
@@ -725,7 +739,7 @@ func (b *GethStatusBackend) ImportUnencryptedDatabase(acc multiaccounts.Account,
 		return errors.New("root datadir wasn't provided")
 	}
 
-	path := filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db", acc.KeyUID))
+	path := filepath.Join(b.rootDataDir, fmt.Sprintf("%s-v4.db", acc.KeyUID))
 
 	err := appdatabase.EncryptDatabase(databasePath, path, password, acc.KDFIterations)
 	if err != nil {
@@ -736,7 +750,7 @@ func (b *GethStatusBackend) ImportUnencryptedDatabase(acc multiaccounts.Account,
 }
 
 func (b *GethStatusBackend) ChangeDatabasePassword(keyUID string, password string, newPassword string) error {
-	dbPath := filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db", keyUID))
+	dbPath := filepath.Join(b.rootDataDir, fmt.Sprintf("%s-v4.db", keyUID))
 	config := b.StatusNode().Config()
 	keyDir := ""
 	if config == nil {
@@ -1130,15 +1144,6 @@ func (b *GethStatusBackend) VerifyDatabasePassword(keyUID string, password strin
 		return err
 	}
 
-	accountsDB, err := accounts.NewDB(b.appDB)
-	if err != nil {
-		return err
-	}
-	_, err = accountsDB.GetWalletAddress()
-	if err != nil {
-		return err
-	}
-
 	err = b.closeAppDB()
 	if err != nil {
 		return err

appdatabase/database.go

@@ -200,6 +200,10 @@ func migrateEnsUsernames(sqlTx *sql.Tx) error {
 	return nil
 }
 
+func MigrateV3ToV4(v3Path string, v4Path string, password string, kdfIterationsNumber int) error {
+	return sqlite.MigrateV3ToV4(v3Path, v4Path, password, kdfIterationsNumber)
+}
+
 const (
 	batchSize = 1000
 )

signal/events_db.go

@@ -0,0 +1,18 @@
+package signal
+
+const (
+	// ReEncryptionStarted is sent when db reencryption was started.
+	ReEncryptionStarted = "db.reEncryption.started"
+	// ReEncryptionFinished is sent when db reencryption was finished.
+	ReEncryptionFinished = "db.reEncryption.finished"
+)
+
+// Send db.reencryption.started signal.
+func SendReEncryptionStarted() {
+	send(ReEncryptionStarted, nil)
+}
+
+// Send db.reencryption.finished signal.
+func SendReEncryptionFinished() {
+	send(ReEncryptionFinished, nil)
+}

sqlite/sqlite.go

@@ -13,6 +13,7 @@ import (
 	sqlcipher "github.com/mutecomm/go-sqlcipher/v4" // We require go sqlcipher that overrides default implementation
 
 	"github.com/status-im/status-go/protocol/sqlite"
+	"github.com/status-im/status-go/signal"
 )
 
 const (
@@ -23,14 +24,16 @@ const (
 	ReducedKDFIterationsNumber = 3200
 
 	// WALMode for sqlite.
-	WALMode      = "wal"
+	WALMode          = "wal"
-	InMemoryPath = ":memory:"
+	InMemoryPath     = ":memory:"
+	V4CipherPageSize = 8192
+	V3CipherPageSize = 1024
 )
 
 // DecryptDB completely removes the encryption from the db
 func DecryptDB(oldPath string, newPath string, key string, kdfIterationsNumber int) error {
 
-	db, err := openDB(oldPath, key, kdfIterationsNumber)
+	db, err := openDB(oldPath, key, kdfIterationsNumber, V4CipherPageSize)
 	if err != nil {
 		return err
 	}
@@ -48,16 +51,11 @@ func DecryptDB(oldPath string, newPath string, key string, kdfIterationsNumber i
 	return err
 }
 
-// EncryptDB takes a plaintext database and adds encryption
+func encryptDB(db *sql.DB, encryptedPath string, key string, kdfIterationsNumber int) error {
-func EncryptDB(unencryptedPath string, encryptedPath string, key string, kdfIterationsNumber int) error {
+	signal.SendReEncryptionStarted()
-	_ = os.Remove(encryptedPath)
+	defer signal.SendReEncryptionFinished()
 
-	db, err := OpenUnecryptedDB(unencryptedPath)
+	_, err := db.Exec(`ATTACH DATABASE '` + encryptedPath + `' AS encrypted KEY '` + key + `'`)
-	if err != nil {
-		return err
-	}
-
-	_, err = db.Exec(`ATTACH DATABASE '` + encryptedPath + `' AS encrypted KEY '` + key + `'`)
 	if err != nil {
 		return err
 	}
@@ -71,7 +69,7 @@ func EncryptDB(unencryptedPath string, encryptedPath string, key string, kdfIter
 		return err
 	}
 
-	if _, err := db.Exec("PRAGMA encrypted.cipher_page_size = 1024"); err != nil {
+	if _, err := db.Exec(fmt.Sprintf("PRAGMA encrypted.cipher_page_size = %d", V4CipherPageSize)); err != nil {
 		fmt.Println("failed to set cipher_page_size pragma")
 		return err
 	}
@@ -93,6 +91,17 @@ func EncryptDB(unencryptedPath string, encryptedPath string, key string, kdfIter
 	return err
 }
 
+// EncryptDB takes a plaintext database and adds encryption
+func EncryptDB(unencryptedPath string, encryptedPath string, key string, kdfIterationsNumber int) error {
+	_ = os.Remove(encryptedPath)
+
+	db, err := OpenUnecryptedDB(unencryptedPath)
+	if err != nil {
+		return err
+	}
+	return encryptDB(db, encryptedPath, key, kdfIterationsNumber)
+}
+
 func buildSqlcipherDSN(path string) (string, error) {
 	if path == InMemoryPath {
 		return InMemoryPath, nil
@@ -118,7 +127,7 @@ func buildSqlcipherDSN(path string) (string, error) {
 	return path + queryOperator + "_txlock=immediate", nil
 }
 
-func openDB(path string, key string, kdfIterationsNumber int) (*sql.DB, error) {
+func openDB(path string, key string, kdfIterationsNumber int, chiperPageSize int) (*sql.DB, error) {
 	driverName := fmt.Sprintf("sqlcipher_with_extensions-%d", len(sql.Drivers()))
 	sql.Register(driverName, &sqlcipher.SQLiteDriver{
 		ConnectHook: func(conn *sqlcipher.SQLiteConn) error {
@@ -134,7 +143,7 @@ func openDB(path string, key string, kdfIterationsNumber int) (*sql.DB, error) {
 				kdfIterationsNumber = sqlite.ReducedKDFIterationsNumber
 			}
 
-			if _, err := conn.Exec("PRAGMA cipher_page_size = 1024", nil); err != nil {
+			if _, err := conn.Exec(fmt.Sprintf("PRAGMA cipher_page_size = %d", chiperPageSize), nil); err != nil {
 				fmt.Println("failed to set cipher_page_size pragma")
 				return err
 			}
@@ -192,12 +201,18 @@ func openDB(path string, key string, kdfIterationsNumber int) (*sql.DB, error) {
 		db.SetMaxIdleConns(nproc)
 	}
 
+	// Dummy select to check if the key is correct. Will return last error from initialization
+	if _, err := db.Exec("SELECT 'Key check'"); err != nil {
+		db.Close()
+		return nil, err
+	}
+
 	return db, nil
 }
 
 // OpenDB opens not-encrypted database.
 func OpenDB(path string, key string, kdfIterationsNumber int) (*sql.DB, error) {
-	return openDB(path, key, kdfIterationsNumber)
+	return openDB(path, key, kdfIterationsNumber, V4CipherPageSize)
 }
 
 // OpenUnecryptedDB opens database with setting PRAGMA key.
@@ -229,11 +244,14 @@ func OpenUnecryptedDB(path string) (*sql.DB, error) {
 }
 
 func ChangeEncryptionKey(path string, key string, kdfIterationsNumber int, newKey string) error {
+	signal.SendReEncryptionStarted()
+	defer signal.SendReEncryptionFinished()
+
 	if kdfIterationsNumber <= 0 {
 		kdfIterationsNumber = sqlite.ReducedKDFIterationsNumber
 	}
 
-	db, err := openDB(path, key, kdfIterationsNumber)
+	db, err := openDB(path, key, kdfIterationsNumber, V4CipherPageSize)
 
 	if err != nil {
 		return err
@@ -246,3 +264,17 @@ func ChangeEncryptionKey(path string, key string, kdfIterationsNumber int, newKe
 
 	return nil
 }
+
+// MigrateV3ToV4 migrates database from v3 to v4 format with encryption.
+func MigrateV3ToV4(v3Path string, v4Path string, key string, kdfIterationsNumber int) error {
+
+	db, err := openDB(v3Path, key, kdfIterationsNumber, V3CipherPageSize)
+
+	if err != nil {
+		fmt.Println("failed to open db", err)
+		return err
+	}
+	defer db.Close()
+
+	return encryptDB(db, v4Path, key, kdfIterationsNumber)
+}