From b6302e30a24ea3cf5ce26a9a4a04c4aa6d3c893d Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Mon, 7 May 2012 11:05:22 -0400 Subject: [PATCH 1/7] disable key initialization on the no-encryption speed test --- tool/crypto-speedtest.tcl | 2 -- 1 file changed, 2 deletions(-) diff --git a/tool/crypto-speedtest.tcl b/tool/crypto-speedtest.tcl index ca6e0b4..fa923a0 100755 --- a/tool/crypto-speedtest.tcl +++ b/tool/crypto-speedtest.tcl @@ -95,8 +95,6 @@ catch {exec /bin/sh -c {rm -f perftest*.db}} set fd [open perftest0.sql w] puts $fd { -PRAGMA key='xyzzy'; -PRAGMA cipher_use_hmac=OFF; } close $fd From 8a734f2219fecc07cf7c29d7b5f86bd3cc90c361 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Thu, 17 May 2012 17:29:01 -0400 Subject: [PATCH 2/7] expand rekey tests --- test/crypto.test | 107 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/test/crypto.test b/test/crypto.test index cefe533..33f82a0 100644 --- a/test/crypto.test +++ b/test/crypto.test @@ -201,6 +201,113 @@ do_test rekey-as-first-operation { db close file delete -force test.db +# create a new database, insert some data +# then rekey it with the same password +do_test rekey-same-passkey { + sqlite_orig db test.db + + execsql { + PRAGMA key = 'test123'; + CREATE TABLE t1(a,b); + BEGIN; + } + + for {set i 1} {$i<=1000} {incr i} { + set r [expr {int(rand()*500000)}] + execsql "INSERT INTO t1 VALUES($i,'value $r');" + } + + execsql { + COMMIT; + SELECT count(*) FROM t1; + PRAGMA rekey = 'test123'; + SELECT count(*) FROM t1; + } +} {1000 1000} +db close +file delete -force test.db + +# create a new database, insert some data +# then rekey it. Make sure it is immediately +# readable. Then close it and make sure it can be +# read back +do_test rekey-and-query-1 { + sqlite_orig db test.db + + execsql { + PRAGMA key = 'test123'; + CREATE TABLE t1(a,b); + BEGIN; + } + + for {set i 1} {$i<=1000} {incr i} { + set r [expr {int(rand()*500000)}] + execsql "INSERT INTO t1 VALUES($i,'value $r');" + } + + execsql { + COMMIT; + SELECT count(*) FROM t1; + PRAGMA rekey = 'test321'; + SELECT count(*) FROM t1; + } +} {1000 1000} + +db close + +do_test rekey-and-query-2 { + sqlite_orig db test.db + execsql { + PRAGMA key = 'test321'; + SELECT count(*) FROM t1; + } +} {1000} +db close +file delete -force test.db + +# create a new database, insert some data +# delete about 50% of the data +# write some new data +# delete another 50% +# then rekey it. Make sure it is immediately +# readable. Then close it and make sure it can be +# read back +do_test rekey-delete-and-query-1 { + sqlite_orig db test.db + + execsql { + PRAGMA key = 'test123'; + CREATE TABLE t1(a,b); + CREATE INDEX ta_a ON t1(a); + BEGIN; + } + + for {set i 1} {$i<10000} {incr i} { + set r [expr {int(rand()*500000)}] + execsql "INSERT INTO t1 VALUES($i,'value $r');" + } + + execsql { + COMMIT; + DELETE FROM t1 WHERE a > 5000; + BEGIN; + } + + for {set i 10000} {$i<12500} {incr i} { + set r [expr {int(rand()*500000)}] + execsql "INSERT INTO t1 VALUES($i,'value $r');" + } + + execsql { + PRAGMA rekey = 'test321'; + SELECT count(*) FROM t1; + } +} {7500} + +db close +file delete -force test.db + + # attach an encrypted database # where both database have the same # key From 1a17b573ebe7f26657ab68f7333bfc01f7372f94 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Thu, 17 May 2012 17:29:27 -0400 Subject: [PATCH 3/7] fix context comparisons and log rekey errors --- src/crypto.c | 9 ++++++--- src/crypto_impl.c | 2 ++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/crypto.c b/src/crypto.c index 8928fb9..ed1411f 100644 --- a/src/crypto.c +++ b/src/crypto.c @@ -313,11 +313,14 @@ int sqlite3_rekey(sqlite3 *db, const void *pKey, int nKey) { rc = sqlite3PagerGet(pPager, pgno, &page); if(rc == SQLITE_OK) { /* write page see pager_incr_changecounter for example */ rc = sqlite3PagerWrite(page); - //printf("sqlite3PagerWrite(%d)\n", pgno); if(rc == SQLITE_OK) { sqlite3PagerUnref(page); - } - } + } else { + CODEC_TRACE(("sqlite3_rekey: error %d occurred writing page %d\n", rc, pgno)); + } + } else { + CODEC_TRACE(("sqlite3_rekey: error %d occurred getting page %d\n", rc, pgno)); + } } } diff --git a/src/crypto_impl.c b/src/crypto_impl.c index 07b21d9..9e86be8 100644 --- a/src/crypto_impl.c +++ b/src/crypto_impl.c @@ -208,6 +208,8 @@ int sqlcipher_cipher_ctx_cmp(cipher_ctx *c1, cipher_ctx *c2) { && c1->fast_kdf_iter == c2->fast_kdf_iter && c1->key_sz == c2->key_sz && c1->pass_sz == c2->pass_sz + && c1->use_hmac == c2->use_hmac + && c1->hmac_sz == c2->hmac_sz && ( c1->pass == c2->pass || !sqlcipher_memcmp((const unsigned char*)c1->pass, From 0e8cdf4b25e6747824656f3ed897e9d922d0aeb7 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Thu, 17 May 2012 19:07:57 -0400 Subject: [PATCH 4/7] test for rekey after a large delete --- test/crypto.test | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/test/crypto.test b/test/crypto.test index 33f82a0..ba65505 100644 --- a/test/crypto.test +++ b/test/crypto.test @@ -282,31 +282,38 @@ do_test rekey-delete-and-query-1 { BEGIN; } - for {set i 1} {$i<10000} {incr i} { - set r [expr {int(rand()*500000)}] - execsql "INSERT INTO t1 VALUES($i,'value $r');" + for {set i 1} {$i<1000} {incr i} { + set r [expr {int(rand()*32767)}] + set r1 [expr {int(rand()*32767)}] + execsql "INSERT INTO t1 VALUES($r,$r1);" } + set r [expr {int(rand()*32767)}] + set r1 [expr {int(rand()*32767)}] + execsql "UPDATE t1 SET b = $r WHERE a < $r1;" + + set r [expr {int(rand()*32767)}] + + execsql "DELETE FROM t1 WHERE a < $r;" execsql { COMMIT; - DELETE FROM t1 WHERE a > 5000; - BEGIN; - } - - for {set i 10000} {$i<12500} {incr i} { - set r [expr {int(rand()*500000)}] - execsql "INSERT INTO t1 VALUES($i,'value $r');" + SELECT (count(*) > 0) FROM t1; } - - execsql { - PRAGMA rekey = 'test321'; - SELECT count(*) FROM t1; - } -} {7500} +} {1} db close -file delete -force test.db +file copy -force test.db test-rekey.db +do_test rekey-delete-and-query-1 { + sqlite_orig db test.db + execsql { + PRAGMA key = 'test123'; + PRAGMA rekey = 'test321'; + SELECT count(*) > 1 FROM t1; + } +} {} +db close +file delete -force test.db # attach an encrypted database # where both database have the same From c2f7cd7926d3751b02e9162b720e0d6d6285ef51 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Fri, 18 May 2012 02:22:40 -0400 Subject: [PATCH 5/7] enable secure delete on all encrypted databases --- src/crypto.c | 5 +++++ test/crypto.test | 6 ++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/crypto.c b/src/crypto.c index ed1411f..7b11367 100644 --- a/src/crypto.c +++ b/src/crypto.c @@ -239,6 +239,11 @@ int sqlite3CodecAttach(sqlite3* db, int nDb, const void *zKey, int nKey) { codec_set_btree_to_codec_pagesize(db, pDb, ctx); + /* force secure delete. This has the benefit of wiping internal data when deleted + and also ensures that all pages are written to disk (i.e. not skipped by + sqlite3PagerDontWrite optimizations) */ + sqlite3BtreeSecureDelete(pDb->pBt, 1); + /* if fd is null, then this is an in-memory database and we dont' want to overwrite the AutoVacuum settings if not null, then set to the default */ diff --git a/test/crypto.test b/test/crypto.test index ba65505..6b313fc 100644 --- a/test/crypto.test +++ b/test/crypto.test @@ -271,7 +271,9 @@ file delete -force test.db # delete another 50% # then rekey it. Make sure it is immediately # readable. Then close it and make sure it can be -# read back +# read back. This test will ensure that Secure Delete +# is enabled and all pages are being written and are not +# being optimized out by sqlite3PagerDontWrite do_test rekey-delete-and-query-1 { sqlite_orig db test.db @@ -311,7 +313,7 @@ do_test rekey-delete-and-query-1 { PRAGMA rekey = 'test321'; SELECT count(*) > 1 FROM t1; } -} {} +} {1} db close file delete -force test.db From 07321926ba511a0605cc6965d2a8a1bd3096bcb5 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Fri, 18 May 2012 02:38:25 -0400 Subject: [PATCH 6/7] round out rekey tests for WAL --- test/crypto.test | 68 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 65 insertions(+), 3 deletions(-) diff --git a/test/crypto.test b/test/crypto.test index 6b313fc..32ca7b6 100644 --- a/test/crypto.test +++ b/test/crypto.test @@ -302,11 +302,9 @@ do_test rekey-delete-and-query-1 { SELECT (count(*) > 0) FROM t1; } } {1} - db close -file copy -force test.db test-rekey.db -do_test rekey-delete-and-query-1 { +do_test rekey-delete-and-query-2 { sqlite_orig db test.db execsql { PRAGMA key = 'test123'; @@ -315,6 +313,70 @@ do_test rekey-delete-and-query-1 { } } {1} db close + +do_test rekey-delete-and-query-3 { + sqlite_orig db test.db + execsql { + PRAGMA key = 'test321'; + SELECT count(*) > 1 FROM t1; + } +} {1} +db close +file delete -force test.db + + +# same as previous test, but use WAL +do_test rekey-delete-and-query-wal-1 { + sqlite_orig db test.db + + execsql { + PRAGMA key = 'test123'; + PRAGMA journal_mode = WAL; + CREATE TABLE t1(a,b); + CREATE INDEX ta_a ON t1(a); + BEGIN; + } + + for {set i 1} {$i<1000} {incr i} { + set r [expr {int(rand()*32767)}] + set r1 [expr {int(rand()*32767)}] + execsql "INSERT INTO t1 VALUES($r,$r1);" + } + set r [expr {int(rand()*32767)}] + set r1 [expr {int(rand()*32767)}] + execsql "UPDATE t1 SET b = $r WHERE a < $r1;" + + set r [expr {int(rand()*32767)}] + + execsql "DELETE FROM t1 WHERE a < $r;" + + execsql { + COMMIT; + SELECT (count(*) > 0) FROM t1; + } +} {1} +db close + +do_test rekey-delete-and-query-wal-2 { + sqlite_orig db test.db + execsql { + PRAGMA key = 'test123'; + PRAGMA journal_mode = WAL; + PRAGMA rekey = 'test321'; + SELECT count(*) > 1 FROM t1; + } +} {wal 1} +db close + +do_test rekey-delete-and-query-wal-3 { + sqlite_orig db test.db + execsql { + PRAGMA key = 'test321'; + PRAGMA journal_mode = WAL; + SELECT count(*) > 1 FROM t1; + } +} {wal 1} +db close file delete -force test.db # attach an encrypted database From ac8385cb1465a4c564cd6fdf2f78ebf93cf9a110 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Fri, 18 May 2012 10:45:41 -0400 Subject: [PATCH 7/7] basic tests for autovacuum --- test/crypto.test | 72 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/test/crypto.test b/test/crypto.test index 32ca7b6..eb43a16 100644 --- a/test/crypto.test +++ b/test/crypto.test @@ -310,8 +310,9 @@ do_test rekey-delete-and-query-2 { PRAGMA key = 'test123'; PRAGMA rekey = 'test321'; SELECT count(*) > 1 FROM t1; + PRAGMA integrity_check; } -} {1} +} {1 ok} db close do_test rekey-delete-and-query-3 { @@ -364,8 +365,9 @@ do_test rekey-delete-and-query-wal-2 { PRAGMA journal_mode = WAL; PRAGMA rekey = 'test321'; SELECT count(*) > 1 FROM t1; + PRAGMA integrity_check; } -} {wal 1} +} {wal 1 ok} db close do_test rekey-delete-and-query-wal-3 { @@ -1395,4 +1397,70 @@ do_test verify-pragma-cipher-version { db close file delete -force test.db +# create a new database, insert some data +# and delete some data with +# auto_vacuum on +do_test auto-vacuum-full { + sqlite_orig db test.db + + execsql { + PRAGMA key = 'test123'; + PRAGMA auto_vacuum = FULL; + CREATE TABLE t1(a,b); + BEGIN; + } + + for {set i 1} {$i<10000} {incr i} { + set r [expr {int(rand()*32767)}] + set r1 [expr {int(rand()*32767)}] + execsql "INSERT INTO t1 VALUES($r,$r1);" + } + set r [expr {int(rand()*32767)}] + execsql "DELETE FROM t1 WHERE a < $r;" + + execsql { + COMMIT; + PRAGMA integrity_check; + PRAGMA freelist_count; + SELECT (count(*) > 0) FROM t1; + } +} {ok 0 1} +db close +file delete -force test.db + +# create a new database, insert some data +# and delete some data with +# auto_vacuum incremental +do_test auto-vacuum-incremental { + sqlite_orig db test.db + + execsql { + PRAGMA key = 'test123'; + PRAGMA auto_vacuum = INCREMENTAL; + CREATE TABLE t1(a,b); + BEGIN; + } + + for {set i 1} {$i<10000} {incr i} { + set r [expr {int(rand()*32767)}] + set r1 [expr {int(rand()*32767)}] + execsql "INSERT INTO t1 VALUES($r,$r1);" + } + set r [expr {int(rand()*32767)}] + execsql "DELETE FROM t1 WHERE a < $r;" + + execsql { + COMMIT; + PRAGMA incremental_vacuum; + PRAGMA freelist_count; + PRAGMA integrity_check; + SELECT (count(*) > 0) FROM t1; + } +} {0 ok 1} +db close +file delete -force test.db + + + + finish_test