From 3703f3638ee5c83360a2f58e1f5e5d6ebab46520 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Fri, 24 May 2013 16:17:15 -0400 Subject: [PATCH 01/12] rework attach to require explicit key --- src/crypto.c | 17 ++++------------- test/crypto.test | 33 +++++++++++++++++++++++++++------ 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/src/crypto.c b/src/crypto.c index 50213b5..9dc53fb 100644 --- a/src/crypto.c +++ b/src/crypto.c @@ -314,8 +314,9 @@ int sqlite3CodecAttach(sqlite3* db, int nDb, const void *zKey, int nKey) { sqlite3BtreeSetAutoVacuum(pDb->pBt, SQLITE_DEFAULT_AUTOVACUUM); } sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; } - return SQLITE_OK; + return SQLITE_ERROR; } void sqlite3_activate_see(const char* in) { @@ -409,18 +410,8 @@ int sqlite3_rekey(sqlite3 *db, const void *pKey, int nKey) { void sqlite3CodecGetKey(sqlite3* db, int nDb, void **zKey, int *nKey) { struct Db *pDb = &db->aDb[nDb]; CODEC_TRACE(("sqlite3CodecGetKey: entered db=%p, nDb=%d\n", db, nDb)); - - if( pDb->pBt ) { - codec_ctx *ctx; - sqlite3pager_get_codec(pDb->pBt->pBt->pPager, (void **) &ctx); - - if(ctx) { /* if the codec has an attached codec_context user the raw key data */ - sqlcipher_codec_get_pass(ctx, zKey, nKey); - } else { - *zKey = NULL; - *nKey = 0; - } - } + *zKey = NULL; + *nKey = 0; } diff --git a/test/crypto.test b/test/crypto.test index e8bbe85..055b2cd 100644 --- a/test/crypto.test +++ b/test/crypto.test @@ -381,9 +381,30 @@ do_test rekey-delete-and-query-wal-3 { db close file delete -force test.db +# attach an encrypted database +# without specifying key, verify it fails +setup test.db "'testkey'" +do_test attach-database-with-default-key { + sqlite_orig db2 test2.db + + execsql { + PRAGMA key = 'testkey'; + CREATE TABLE t2(a,b); + INSERT INTO t2 VALUES ('test1', 'test2'); + } db2 + + catchsql { + ATTACH 'test.db' AS db; + } db2 + +} {1 {unable to open database: test.db}} +db2 close +file delete -force test.db +file delete -force test2.db + # attach an encrypted database # where both database have the same -# key +# key explicitly setup test.db "'testkey'" do_test attach-database-with-same-key { sqlite_orig db2 test2.db @@ -396,7 +417,7 @@ do_test attach-database-with-same-key { execsql { SELECT count(*) FROM t2; - ATTACH 'test.db' AS db; + ATTACH 'test.db' AS db KEY 'testkey'; SELECT count(*) FROM db.t1; } db2 @@ -581,7 +602,7 @@ file delete -force test.db # create an unencrypted database, attach a new encrypted volume # copy data between, verify the encypted database is good afterwards -do_test unencryped-attach { +do_test unencrypted-attach { sqlite_orig db test.db execsql { @@ -1335,7 +1356,7 @@ do_test default-use-hmac-attach { PRAGMA cipher_default_use_hmac = OFF; PRAGMA key = 'testkey'; SELECT count(*) FROM t1; - ATTACH 'sqlcipher-1.1.8-testkey.db' AS db2; + ATTACH 'sqlcipher-1.1.8-testkey.db' AS db2 KEY 'testkey'; SELECT count(*) from db2.t1; PRAGMA cipher_default_use_hmac = ON; } @@ -1352,7 +1373,7 @@ do_test attach-1.1.8-database-from-2.0-fails { catchsql { PRAGMA key = 'testkey'; CREATE table t1(a,b); - ATTACH 'sqlcipher-1.1.8-testkey.db' AS db2; + ATTACH 'sqlcipher-1.1.8-testkey.db' AS db2 KEY 'testkey'; } } {1 {file is encrypted or is not a database}} db close @@ -1376,7 +1397,7 @@ do_test change-default-use-hmac-attach { PRAGMA key = 'testkey'; SELECT count(*) FROM t1; PRAGMA cipher_default_use_hmac = OFF; - ATTACH 'sqlcipher-1.1.8-testkey.db' AS db2; + ATTACH 'sqlcipher-1.1.8-testkey.db' AS db2 KEY 'testkey'; SELECT count(*) from db2.t1; PRAGMA cipher_default_use_hmac = ON; } From a2bb641e485661395c2c31a78c7e061ea373f27d Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Thu, 27 Jun 2013 19:07:58 -0400 Subject: [PATCH 02/12] 64k iterations plus pragma cipher_default_kdf_iter function --- sqlcipher-2.3-testkey.db | Bin 0 -> 2048 bytes src/crypto.c | 9 ++++++ src/crypto.h | 5 +++- src/crypto_impl.c | 12 +++++++- test/crypto.test | 58 +++++++++++++++++++++++++++++++++++---- 5 files changed, 77 insertions(+), 7 deletions(-) create mode 100644 sqlcipher-2.3-testkey.db diff --git a/sqlcipher-2.3-testkey.db b/sqlcipher-2.3-testkey.db new file mode 100644 index 0000000000000000000000000000000000000000..da27623373d27c6cf722ffb2eae8e2fb84f171ff GIT binary patch literal 2048 zcmV+b2>NOk^TF^{L~6~or%gh^OHlH zZ^~hcKOYfQfHjnIAI{gOi0q&=$`OeS##B^lWtu3;V6BlMTCW#_yrEIWiaazWMKMlC zHltH}hygk7VkMmROU_hwD3`bAwTYNj>5JvOFA&=(BB?gU!jO^Rpl^+j8jj*DEFXu> zRsmB{2H+hl5vkR+3eo9%2yr^^U~Mt*1%3bky%g9i-*zOnjZ=<`Qd{RU{8zZ5(`Ffu zo!^2iGdAW)RfOaJb@IA1HL?+@GqzPkH59th0*+;JJN=Ve)q4PZ3abxPM=8zXLz2;Q za0>6=c`+;89LS*ol%AG2(X&+#THCt$+h!MalRpUT*s~F+#kp{8FobE;7KcBSH)hHZ z_HDFJ7#^a4qssud6iLFj?v+1L4%ma*!^0bickPX~eH#Sn$OU!WSY&$HGA=z@{25*T zV@+qVzBd8A94Gdl8g*UwEV9Ff`Kyi!U7LSJ%72bt+otG`=D5gqk{_=o?$3u;BFSuE zxB;e-$BxgJBNC|fyoYh`PL671kvH-6BX?Vf2Y5!Y!QNw(VNr*%Obd^)9osmRGvm(T4|rB+#iI0w=$0e$*kTM11+=$I{>*P<*zW=X$H zRm4Fb{ZUHr#)i|0Kv%N&HlXR>xe|Ji1UB|H3b;O`ehq%VaI*ky299C@D6U>hQx+GG zI*Vl7`M+tRlTZQDB*J!Q_#v1w@3mwE z9w$>_v~#@>P-gEzg>DDUxpx^(fO7SKRYv30PlTiJB)>%)ZlX;_sHrI4^{}qB%5b0* zeKZTbOVg8E%E&&%9@{X9j`v37b0phjV9248{r<7tWdmTVh8j_@obWk?U=*r2k?ezs z>=w(#PdxyhpE8i#?ej=L^A1!)y>mrwD__CZELdjb~5*s zAuIC)3+&L%Crm6EGk9MU682s^1-9U7ZB*UN9y`O4n5%3iL+meDSv!mfqZ0uTsRn+o zFCGGR*2pNAVn?%)x70#37cFpZ$plzn&2OwZOo`kCcu3JFG^ETk zLp(eyyzNfKj_`T~NNvyB>FG)A)%#{hxj7S&sV_JvPeyPmZMk`I)AH)Wqr2EXyypr6 zeKSey5kCyxADA4xAMW+e@tV>`KCc4peBZ)a#Natwd;)9vFzzc-Gh$h z0z0Z&s$%b9_nuMdmJU%lWB428V8Jix75R7aaD$Mvf0K?waV9t*MuA7XbzUlv;Y~b+ zaxP-2uv`YGRg7wP3ln|b%PA~i%VN&4*3TAY*YRLtBYg%iU!kPzm9eQe5w!>FKVIt*PW)xraKN{_P)JOC*bV{%eHN^_*l7Q<`d|A;BYs@hQkigo<6f0THX2~jJ z>Hm^g^OESg=n@3xMiDPRJR5z5*9xxuSbaIDp#$Kr#@~|+XGRm9yS}{Li*Q5j1K&@? z-kXeot;1Kh)|S0k=-ZB}gvj|NtWV`D-}YDOTbulket2Y>yh#QDyAYnji3ZKw&zXe{ z$D^T?{`W(zP-gnZMDs~0Awj*z3(&&z(ijk6A*-jzS0xOT`^g`-<)w;QgEcdUnat1I zI|EM{!+tQ&tjK2sxc9#UYMcP71fxn2=^!M)%D^(Gr&X!i)v145)Dg z7CA-&ws|scttJQ1oB<)tE0554OPDt4eauZr{UlloG3#^ABj|go=|*ZcO6QsY%d_#N zN8$bIrKs$YC=9&i=F!!Npay1=m^fS&gkFicp1A12JDxzvvM%;jc=Bw{JI-3p3)pm( zJBSAwLI!e3Z~IS2{)v=*3?O0EA^2DKzuRJI^KpcTkT1D4_NkN0-cD-cp|kJKkk1_oONZWU}#emj=)U z_D-2`>~>omJVRfH%yfTdi~W0IN_@n~*iuv3flags */ @@ -173,6 +173,9 @@ int sqlcipher_codec_ctx_set_pagesize(codec_ctx *, int); int sqlcipher_codec_ctx_get_pagesize(codec_ctx *); int sqlcipher_codec_ctx_get_reservesize(codec_ctx *); +void sqlcipher_set_default_kdf_iter(int iter); +int sqlcipher_get_default_kdf_iter(); + int sqlcipher_codec_ctx_set_kdf_iter(codec_ctx *, int, int); int sqlcipher_codec_ctx_get_kdf_iter(codec_ctx *ctx, int); diff --git a/src/crypto_impl.c b/src/crypto_impl.c index 589696d..76bd9be 100644 --- a/src/crypto_impl.c +++ b/src/crypto_impl.c @@ -66,6 +66,7 @@ typedef struct { static unsigned int default_flags = DEFAULT_CIPHER_FLAGS; static unsigned char hmac_salt_mask = HMAC_SALT_MASK; +static int default_kdf_iter = PBKDF2_ITER; static sqlcipher_provider *default_provider = NULL; @@ -386,6 +387,15 @@ const char* sqlcipher_codec_ctx_get_cipher(codec_ctx *ctx, int for_ctx) { return c_ctx->provider->get_cipher(c_ctx->provider_ctx); } +/* set the global default KDF iteration */ +void sqlcipher_set_default_kdf_iter(int iter) { + default_kdf_iter = iter; +} + +int sqlcipher_get_default_kdf_iter() { + return default_kdf_iter; +} + int sqlcipher_codec_ctx_set_kdf_iter(codec_ctx *ctx, int kdf_iter, int for_ctx) { cipher_ctx *c_ctx = for_ctx ? ctx->write_ctx : ctx->read_ctx; int rc; @@ -572,7 +582,7 @@ int sqlcipher_codec_ctx_init(codec_ctx **iCtx, Db *pDb, Pager *pPager, sqlite3_f } if((rc = sqlcipher_codec_ctx_set_cipher(ctx, CIPHER, 0)) != SQLITE_OK) return rc; - if((rc = sqlcipher_codec_ctx_set_kdf_iter(ctx, PBKDF2_ITER, 0)) != SQLITE_OK) return rc; + if((rc = sqlcipher_codec_ctx_set_kdf_iter(ctx, default_kdf_iter, 0)) != SQLITE_OK) return rc; if((rc = sqlcipher_codec_ctx_set_fast_kdf_iter(ctx, FAST_PBKDF2_ITER, 0)) != SQLITE_OK) return rc; if((rc = sqlcipher_codec_ctx_set_pass(ctx, zKey, nKey, 0)) != SQLITE_OK) return rc; diff --git a/test/crypto.test b/test/crypto.test index 90439c6..94a72f2 100644 --- a/test/crypto.test +++ b/test/crypto.test @@ -879,6 +879,7 @@ do_test open-1.1.8-database { execsql { PRAGMA key = 'testkey'; PRAGMA cipher_use_hmac = OFF; + PRAGMA kdf_iter = 4000; SELECT count(*) FROM t1; SELECT * FROM t1; } @@ -893,6 +894,7 @@ do_test attach-and-copy-1.1.8 { execsql { PRAGMA key = 'testkey'; PRAGMA cipher_use_hmac = OFF; + PRAGMA kdf_iter = 4000; ATTACH DATABASE 'test.db' AS db2 KEY 'testkey-hmac'; CREATE TABLE db2.t1(a,b); INSERT INTO db2.t1 SELECT * FROM main.t1; @@ -1350,20 +1352,22 @@ do_test cipher-options-before-keys { db close file delete -force test.db -# open a 1.1.8 database (no HMAC), then +# open a 1.1.8 database (no HMAC, 4K iter), then # try to open another 1.1.8 database. The # attached database should have the same hmac # setting as the original -do_test default-use-hmac-attach { +do_test default-hmac-kdf-attach { file copy -force sqlcipher-1.1.8-testkey.db test.db sqlite_orig db test.db execsql { PRAGMA cipher_default_use_hmac = OFF; + PRAGMA cipher_default_kdf_iter = 4000; PRAGMA key = 'testkey'; SELECT count(*) FROM t1; ATTACH 'sqlcipher-1.1.8-testkey.db' AS db2; SELECT count(*) from db2.t1; PRAGMA cipher_default_use_hmac = ON; + PRAGMA cipher_default_kdf_iter = 64000; } } {4 4} db close @@ -1384,12 +1388,12 @@ do_test attach-1.1.8-database-from-2.0-fails { db close file delete -force test.db -# open a 2.0 database (with HMAC), then +# open a 2.0 database (with HMAC, 4k iter), then # set the default hmac setting to OFF. # try to a 1.1.8 database. this should # succeed now that hmac is off by default # before the attach -do_test change-default-use-hmac-attach { +do_test change-default-hmac-kdf-attach { sqlite_orig db test.db execsql { PRAGMA key = 'testkey'; @@ -1402,9 +1406,11 @@ do_test change-default-use-hmac-attach { PRAGMA key = 'testkey'; SELECT count(*) FROM t1; PRAGMA cipher_default_use_hmac = OFF; + PRAGMA cipher_default_kdf_iter = 4000; ATTACH 'sqlcipher-1.1.8-testkey.db' AS db2; SELECT count(*) from db2.t1; PRAGMA cipher_default_use_hmac = ON; + PRAGMA cipher_default_kdf_iter = 64000; } } {1 4} db close @@ -1580,12 +1586,25 @@ do_test multipage-schema-autovacuum-shortread-wal { db close file delete -force test.db +# open a 2.3 database with little endian hmac page numbers (default) +# verify it can be opened +do_test open-2.3-le-database { + sqlite_orig db sqlcipher-2.3-testkey.db + execsql { + PRAGMA key = 'testkey'; + SELECT count(*) FROM t1; + SELECT * FROM t1; + } +} {4 1 1 one one 1 2 one two} +db close + # open a 2.0 database with little endian hmac page numbers (default) # verify it can be opened do_test open-2.0-le-database { sqlite_orig db sqlcipher-2.0-le-testkey.db execsql { PRAGMA key = 'testkey'; + PRAGMA kdf_iter = 4000; SELECT count(*) FROM t1; SELECT * FROM t1; } @@ -1599,6 +1618,7 @@ do_test open-2.0-be-database { execsql { PRAGMA key = 'testkey'; PRAGMA cipher_hmac_pgno = be; + PRAGMA kdf_iter = 4000; SELECT count(*) FROM t1; SELECT * FROM t1; } @@ -1615,6 +1635,7 @@ do_test be-to-le-migration { execsql { PRAGMA key = 'testkey'; PRAGMA cipher_hmac_pgno = be; + PRAGMA kdf_iter = 4000; ATTACH DATABASE 'test.db' AS db2 KEY 'testkey'; CREATE TABLE db2.t1(a,b); INSERT INTO db2.t1 SELECT * FROM main.t1; @@ -1684,6 +1705,31 @@ do_test verify-pragma-cipher-default-use-hmac-off { db close file delete -force test.db +# verify the pragma default_cipher_kdf_iter +# is set to 64000 by default +do_test verify-pragma-cipher-default-kdf-iter-default { + sqlite_orig db test.db + execsql { + PRAGMA cipher_default_kdf_iter; + } +} {64000} +db close +file delete -force test.db + + +# verify the pragma default_cipher_kdf_ter +# reports changes +do_test verify-pragma-cipher-default-use-hmac-off { + sqlite_orig db test.db + execsql { + PRAGMA cipher_default_kdf_iter = 1000; + PRAGMA cipher_default_kdf_iter; + PRAGMA cipher_default_kdf_iter = 64000; + } +} {1000} +db close +file delete -force test.db + # verify the pragma kdf_iter # reports the default value do_test verify-pragma-kdf-iter-reports-default { @@ -1692,7 +1738,7 @@ do_test verify-pragma-kdf-iter-reports-default { PRAGMA key = 'test'; PRAGMA kdf_iter; } -} {4000} +} {64000} db close file delete -force test.db @@ -1843,6 +1889,7 @@ do_test open-2.0-beta-database { sqlite_orig db sqlcipher-2.0-beta-testkey.db execsql { PRAGMA key = 'testkey'; + PRAGMA kdf_iter = 4000; PRAGMA fast_kdf_iter = 4000; PRAGMA cipher_hmac_salt_mask = "x'00'"; SELECT count(*) FROM t1; @@ -1861,6 +1908,7 @@ do_test 2.0-beta-to-2.0-migration { execsql { PRAGMA key = 'testkey'; PRAGMA cipher_hmac_salt_mask = "x'00'"; + PRAGMA kdf_iter = 4000; PRAGMA fast_kdf_iter = 4000; SELECT count(*) FROM sqlite_master; From 6cd1fdf60b5c41be513d2a70240e54a6886f63dc Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Fri, 19 Jul 2013 12:50:14 -0400 Subject: [PATCH 03/12] finalize new attach behavior - key can be specified in hex as key plus salt together - source passphrase is cleared after derivation - key specification containing derived key and salt stored on cipher_ctx for use in attach on an encrypted main database when no key is provided with the attach statement --- src/crypto.c | 15 +++++-- src/crypto.h | 9 +++- src/crypto_impl.c | 103 +++++++++++++++++++++++++++++++++++----------- test/crypto.test | 50 ++++++++++++++++++---- 4 files changed, 142 insertions(+), 35 deletions(-) diff --git a/src/crypto.c b/src/crypto.c index 847e262..ca24521 100644 --- a/src/crypto.c +++ b/src/crypto.c @@ -317,9 +317,8 @@ int sqlite3CodecAttach(sqlite3* db, int nDb, const void *zKey, int nKey) { sqlite3BtreeSetAutoVacuum(pDb->pBt, SQLITE_DEFAULT_AUTOVACUUM); } sqlite3_mutex_leave(db->mutex); - return SQLITE_OK; } - return SQLITE_ERROR; + return SQLITE_OK; } void sqlite3_activate_see(const char* in) { @@ -413,8 +412,16 @@ int sqlite3_rekey(sqlite3 *db, const void *pKey, int nKey) { void sqlite3CodecGetKey(sqlite3* db, int nDb, void **zKey, int *nKey) { struct Db *pDb = &db->aDb[nDb]; CODEC_TRACE(("sqlite3CodecGetKey: entered db=%p, nDb=%d\n", db, nDb)); - *zKey = NULL; - *nKey = 0; + if( pDb->pBt ) { + codec_ctx *ctx; + sqlite3pager_get_codec(pDb->pBt->pBt->pPager, (void **) &ctx); + if(ctx) { /* if the codec has an attached codec_context user the raw key data */ + sqlcipher_codec_get_keyspec(ctx, zKey, nKey); + } else { + *zKey = NULL; + *nKey = 0; + } + } } #ifndef OMIT_EXPORT diff --git a/src/crypto.h b/src/crypto.h index 39924f7..aa77670 100644 --- a/src/crypto.h +++ b/src/crypto.h @@ -149,6 +149,13 @@ static void cipher_hex2bin(const char *hex, int sz, unsigned char *out){ } } +static void cipher_bin2hex(const unsigned char* in, int sz, char *out) { + int i; + for(i=0; i < sz; i++) { + sprintf(out + (i*2), "%02x ", in[i]); + } +} + /* extensions defined in crypto_impl.c */ typedef struct codec_ctx codec_ctx; @@ -167,7 +174,7 @@ int sqlcipher_page_cipher(codec_ctx *, int, Pgno, int, int, unsigned char *, uns void sqlcipher_codec_ctx_set_error(codec_ctx *, int); int sqlcipher_codec_ctx_set_pass(codec_ctx *, const void *, int, int); -void sqlcipher_codec_get_pass(codec_ctx *, void **zKey, int *nKey); +void sqlcipher_codec_get_keyspec(codec_ctx *, void **zKey, int *nKey); int sqlcipher_codec_ctx_set_pagesize(codec_ctx *, int); int sqlcipher_codec_ctx_get_pagesize(codec_ctx *); diff --git a/src/crypto_impl.c b/src/crypto_impl.c index 589696d..5510073 100644 --- a/src/crypto_impl.c +++ b/src/crypto_impl.c @@ -56,10 +56,12 @@ typedef struct { int pass_sz; int reserve_sz; int hmac_sz; + int keyspec_sz; unsigned int flags; unsigned char *key; unsigned char *hmac_key; char *pass; + char *keyspec; sqlcipher_provider *provider; void *provider_ctx; } cipher_ctx; @@ -260,6 +262,7 @@ static void sqlcipher_cipher_ctx_free(cipher_ctx **iCtx) { sqlcipher_free(ctx->key, ctx->key_sz); sqlcipher_free(ctx->hmac_key, ctx->key_sz); sqlcipher_free(ctx->pass, ctx->pass_sz); + sqlcipher_free(ctx->keyspec, ctx->keyspec_sz); sqlcipher_free(ctx, sizeof(cipher_ctx)); } @@ -307,8 +310,9 @@ static int sqlcipher_cipher_ctx_copy(cipher_ctx *target, cipher_ctx *source) { CODEC_TRACE(("sqlcipher_cipher_ctx_copy: entered target=%p, source=%p\n", target, source)); sqlcipher_free(target->pass, target->pass_sz); + sqlcipher_free(target->keyspec, target->keyspec_sz); memcpy(target, source, sizeof(cipher_ctx)); - + target->key = key; //restore pointer to previously allocated key data memcpy(target->key, source->key, CIPHER_MAX_KEY_SZ); @@ -321,31 +325,67 @@ static int sqlcipher_cipher_ctx_copy(cipher_ctx *target, cipher_ctx *source) { target->provider_ctx = provider_ctx; // restore pointer to previouly allocated provider context; target->provider->ctx_copy(target->provider_ctx, source->provider_ctx); - target->pass = sqlcipher_malloc(source->pass_sz); - if(target->pass == NULL) return SQLITE_NOMEM; - memcpy(target->pass, source->pass, source->pass_sz); + if(source->pass && source->pass_sz) { + target->pass = sqlcipher_malloc(source->pass_sz); + if(target->pass == NULL) return SQLITE_NOMEM; + memcpy(target->pass, source->pass, source->pass_sz); + } + if(source->keyspec && source->keyspec_sz) { + target->keyspec = sqlcipher_malloc(source->keyspec_sz); + if(target->keyspec == NULL) return SQLITE_NOMEM; + memcpy(target->keyspec, source->keyspec, source->keyspec_sz); + } + return SQLITE_OK; +} + +/** + * Set the keyspec for the cipher_ctx + * + * returns SQLITE_OK if assignment was successfull + * returns SQLITE_NOMEM if an error occured allocating memory + */ +static int sqlcipher_cipher_ctx_set_keyspec(cipher_ctx *ctx, const unsigned char *key, int key_sz, const unsigned char *salt, int salt_sz) { + + /* free, zero existing pointers and size */ + sqlcipher_free(ctx->keyspec, ctx->keyspec_sz); + ctx->keyspec = NULL; + ctx->keyspec_sz = 0; + + /* establic a hex-formated key specification, containing the raw encryption key and + the salt used to generate it */ + ctx->keyspec_sz = ((key_sz + salt_sz) * 2) + 3; + ctx->keyspec = sqlcipher_malloc(ctx->keyspec_sz); + if(ctx->keyspec == NULL) return SQLITE_NOMEM; + + ctx->keyspec[0] = 'x'; + ctx->keyspec[1] = '\''; + ctx->keyspec[ctx->keyspec_sz - 1] = '\''; + cipher_bin2hex(key, key_sz, ctx->keyspec + 2); + cipher_bin2hex(salt, salt_sz, ctx->keyspec + (key_sz * 2) + 2); return SQLITE_OK; } - /** - * Set the raw password / key data for a cipher context + * Set the passphrase for the cipher_ctx * * returns SQLITE_OK if assignment was successfull * returns SQLITE_NOMEM if an error occured allocating memory - * returns SQLITE_ERROR if the key couldn't be set because the pass was null or size was zero */ static int sqlcipher_cipher_ctx_set_pass(cipher_ctx *ctx, const void *zKey, int nKey) { + + /* free, zero existing pointers and size */ sqlcipher_free(ctx->pass, ctx->pass_sz); - ctx->pass_sz = nKey; - if(zKey && nKey) { + ctx->pass = NULL; + ctx->pass_sz = 0; + + if(zKey && nKey) { /* if new password is provided, copy it */ + ctx->pass_sz = nKey; ctx->pass = sqlcipher_malloc(nKey); if(ctx->pass == NULL) return SQLITE_NOMEM; memcpy(ctx->pass, zKey, nKey); - return SQLITE_OK; - } - return SQLITE_ERROR; + } + return SQLITE_OK; } int sqlcipher_codec_ctx_set_pass(codec_ctx *ctx, const void *zKey, int nKey, int for_ctx) { @@ -508,9 +548,9 @@ void* sqlcipher_codec_ctx_get_kdf_salt(codec_ctx *ctx) { return ctx->kdf_salt; } -void sqlcipher_codec_get_pass(codec_ctx *ctx, void **zKey, int *nKey) { - *zKey = ctx->read_ctx->pass; - *nKey = ctx->read_ctx->pass_sz; +void sqlcipher_codec_get_keyspec(codec_ctx *ctx, void **zKey, int *nKey) { + *zKey = ctx->read_ctx->keyspec; + *nKey = ctx->read_ctx->keyspec_sz; } int sqlcipher_codec_ctx_set_pagesize(codec_ctx *ctx, int size) { @@ -721,7 +761,11 @@ int sqlcipher_page_cipher(codec_ctx *ctx, int for_ctx, Pgno pgno, int mode, int * Derive an encryption key for a cipher contex key based on the raw password. * * If the raw key data is formated as x'hex' and there are exactly enough hex chars to fill - * the key space (i.e 64 hex chars for a 256 bit key) then the key data will be used directly. + * the key (i.e 64 hex chars for a 256 bit key) then the key data will be used directly. + + * Else, if the raw key data is formated as x'hex' and there are exactly enough hex chars to fill + * the key and the salt (i.e 92 hex chars for a 256 bit key and 16 byte salt) then it will be unpacked + * as the key followed by the salt. * * Otherwise, a key data will be derived using PBKDF2 * @@ -729,7 +773,8 @@ int sqlcipher_page_cipher(codec_ctx *ctx, int for_ctx, Pgno pgno, int mode, int * returns SQLITE_ERROR if the key could't be derived (for instance if pass is NULL or pass_sz is 0) */ static int sqlcipher_cipher_ctx_key_derive(codec_ctx *ctx, cipher_ctx *c_ctx) { - CODEC_TRACE(("codec_key_derive: entered c_ctx->pass=%s, c_ctx->pass_sz=%d \ + int rc; + CODEC_TRACE(("cipher_ctx_key_derive: entered c_ctx->pass=%s, c_ctx->pass_sz=%d \ ctx->kdf_salt=%p ctx->kdf_salt_sz=%d c_ctx->kdf_iter=%d \ ctx->hmac_kdf_salt=%p, c_ctx->fast_kdf_iter=%d c_ctx->key_sz=%d\n", c_ctx->pass, c_ctx->pass_sz, ctx->kdf_salt, ctx->kdf_salt_sz, c_ctx->kdf_iter, @@ -737,19 +782,26 @@ static int sqlcipher_cipher_ctx_key_derive(codec_ctx *ctx, cipher_ctx *c_ctx) { if(c_ctx->pass && c_ctx->pass_sz) { // if pass is not null - if (c_ctx->pass_sz == ((c_ctx->key_sz*2)+3) && sqlite3StrNICmp(c_ctx->pass ,"x'", 2) == 0) { + if (c_ctx->pass_sz == ((c_ctx->key_sz * 2) + 3) && sqlite3StrNICmp(c_ctx->pass ,"x'", 2) == 0) { int n = c_ctx->pass_sz - 3; /* adjust for leading x' and tailing ' */ const char *z = c_ctx->pass + 2; /* adjust lead offset of x' */ - CODEC_TRACE(("codec_key_derive: using raw key from hex\n")); + CODEC_TRACE(("cipher_ctx_key_derive: using raw key from hex\n")); cipher_hex2bin(z, n, c_ctx->key); + } else if (c_ctx->pass_sz == (((c_ctx->key_sz + ctx->kdf_salt_sz) * 2) + 3) && sqlite3StrNICmp(c_ctx->pass ,"x'", 2) == 0) { + const char *z = c_ctx->pass + 2; /* adjust lead offset of x' */ + CODEC_TRACE(("cipher_ctx_key_derive: using raw key from hex\n")); + cipher_hex2bin(z, (c_ctx->key_sz * 2), c_ctx->key); + cipher_hex2bin(z + (c_ctx->key_sz * 2), (ctx->kdf_salt_sz * 2), ctx->kdf_salt); } else { - CODEC_TRACE(("codec_key_derive: deriving key using full PBKDF2 with %d iterations\n", c_ctx->kdf_iter)); + CODEC_TRACE(("cipher_ctx_key_derive: deriving key using full PBKDF2 with %d iterations\n", c_ctx->kdf_iter)); c_ctx->provider->kdf(c_ctx->provider_ctx, (const char*) c_ctx->pass, c_ctx->pass_sz, ctx->kdf_salt, ctx->kdf_salt_sz, c_ctx->kdf_iter, c_ctx->key_sz, c_ctx->key); - } + /* set the context "keyspec" containing the hex-formatted key and salt to be used when attaching databases */ + if((rc = sqlcipher_cipher_ctx_set_keyspec(c_ctx, c_ctx->key, c_ctx->key_sz, ctx->kdf_salt, ctx->kdf_salt_sz)) != SQLITE_OK) return rc; + /* if this context is setup to use hmac checks, generate a seperate and different key for HMAC. In this case, we use the output of the previous KDF as the input to this KDF run. This ensures a distinct but predictable HMAC key. */ @@ -766,7 +818,7 @@ static int sqlcipher_cipher_ctx_key_derive(codec_ctx *ctx, cipher_ctx *c_ctx) { ctx->hmac_kdf_salt[i] ^= hmac_salt_mask; } - CODEC_TRACE(("codec_key_derive: deriving hmac key from encryption key using PBKDF2 with %d iterations\n", + CODEC_TRACE(("cipher_ctx_key_derive: deriving hmac key from encryption key using PBKDF2 with %d iterations\n", c_ctx->fast_kdf_iter)); @@ -789,12 +841,17 @@ int sqlcipher_codec_key_derive(codec_ctx *ctx) { if(ctx->write_ctx->derive_key) { if(sqlcipher_cipher_ctx_cmp(ctx->write_ctx, ctx->read_ctx) == 0) { - // the relevant parameters are the same, just copy read key + /* the relevant parameters are the same, just copy read key */ if(sqlcipher_cipher_ctx_copy(ctx->write_ctx, ctx->read_ctx) != SQLITE_OK) return SQLITE_ERROR; } else { if(sqlcipher_cipher_ctx_key_derive(ctx, ctx->write_ctx) != SQLITE_OK) return SQLITE_ERROR; } } + + /* TODO: wipe and free passphrase after key derivation */ + sqlcipher_cipher_ctx_set_pass(ctx->read_ctx, NULL, 0); + sqlcipher_cipher_ctx_set_pass(ctx->write_ctx, NULL, 0); + return SQLITE_OK; } diff --git a/test/crypto.test b/test/crypto.test index a78ec2c..0238cb7 100644 --- a/test/crypto.test +++ b/test/crypto.test @@ -4,8 +4,6 @@ # http://zetetic.net # # Copyright (c) 2009, ZETETIC LLC -# All rights reserved. -# # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # * Redistributions of source code must retain the above copyright @@ -409,6 +407,8 @@ file delete -force test.db # attach an encrypted database # without specifying key, verify it fails +# even if the source passwords are the same +# because the kdf salts are different setup test.db "'testkey'" do_test attach-database-with-default-key { sqlite_orig db2 test2.db @@ -423,7 +423,7 @@ do_test attach-database-with-default-key { ATTACH 'test.db' AS db; } db2 -} {1 {unable to open database: test.db}} +} {1 {file is encrypted or is not a database}} db2 close file delete -force test.db file delete -force test2.db @@ -662,7 +662,7 @@ file delete -force test2.db # create an unencrypted database, attach a new encrypted volume # using a raw key copy data between, verify the encypted # database is good afterwards -do_test unencryped-attach-raw-key { +do_test unencrypted-attach-raw-key { sqlite_orig db test.db execsql { @@ -693,9 +693,45 @@ db2 close file delete -force test.db file delete -force test2.db +# create an encrypted database, attach an default-key encrypted volume +# copy data between, verify the second database +do_test encrypted-attach-default-key { + sqlite_orig db test.db + + execsql { + PRAGMA key='testkey'; + CREATE TABLE t1(a,b); + BEGIN; + } + + for {set i 1} {$i<=1000} {incr i} { + set r [expr {int(rand()*500000)}] + execsql "INSERT INTO t1 VALUES($i,$r);" + } + + execsql { + COMMIT; + ATTACH DATABASE 'test2.db' AS test; + CREATE TABLE test.t1(a,b); + INSERT INTO test.t1 SELECT * FROM t1; + DETACH DATABASE test; + } + + sqlite_orig db2 test2.db + + execsql { + PRAGMA key='testkey'; + SELECT count(*) FROM t1; + } db2 +} {1000} +db close +db2 close +file delete -force test.db +file delete -force test2.db + # create an encrypted database, attach an unencrypted volume # copy data between, verify the unencypted database is good afterwards -do_test encryped-attach-unencrypted { +do_test encrypted-attach-unencrypted { sqlite_orig db test.db execsql { @@ -704,7 +740,7 @@ do_test encryped-attach-unencrypted { sqlite_orig db2 test2.db execsql { - PRAGMA key='testkey'; + PRAGMA key = 'testkey'; CREATE TABLE t1(a,b); BEGIN; } db2 @@ -732,7 +768,7 @@ file delete -force test2.db # create an unencrypted database, attach an unencrypted volume # copy data between, verify the unencypted database is good afterwards -do_test unencryped-attach-unencrypted { +do_test unencrypted-attach-unencrypted { sqlite_orig db test.db execsql { From 4bc6ba08f84179ececdd04d8a7193ea5571403d2 Mon Sep 17 00:00:00 2001 From: Nick Parker Date: Wed, 24 Jul 2013 17:07:42 -0500 Subject: [PATCH 04/12] Initial work on a PRAGMA cipher_migrate Swapping of the Btree meta data is not working properly now. --- src/crypto.c | 7 +++++ src/crypto.h | 1 + src/crypto_impl.c | 74 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+) diff --git a/src/crypto.c b/src/crypto.c index ca24521..32b2184 100644 --- a/src/crypto.c +++ b/src/crypto.c @@ -89,6 +89,13 @@ int codec_pragma(sqlite3* db, int iDb, Parse *pParse, const char *zLeft, const c CODEC_TRACE(("codec_pragma: entered db=%p iDb=%d pParse=%p zLeft=%s zRight=%s ctx=%p\n", db, iDb, pParse, zLeft, zRight, ctx)); + if( sqlite3StrICmp(zLeft, "cipher_migrate")==0 && !zRight ){ + if(ctx){ + char *migrate_status = sqlite3_mprintf("%d", sqlcipher_codec_ctx_migrate(ctx)); + codec_vdbe_return_static_string(pParse, "sqlcipher_migrate", migrate_status); + sqlite3_free(migrate_status); + } + } else if( sqlite3StrICmp(zLeft, "cipher_provider")==0 && !zRight ){ if(ctx) { codec_vdbe_return_static_string(pParse, "cipher_provider", sqlcipher_codec_get_cipher_provider(ctx)); diff --git a/src/crypto.h b/src/crypto.h index aa77670..c4849bc 100644 --- a/src/crypto.h +++ b/src/crypto.h @@ -209,6 +209,7 @@ int sqlcipher_codec_ctx_unset_flag(codec_ctx *ctx, unsigned int flag); int sqlcipher_codec_ctx_get_flag(codec_ctx *ctx, unsigned int flag, int for_ctx); const char* sqlcipher_codec_get_cipher_provider(codec_ctx *ctx); +int sqlcipher_codec_ctx_migrate(codec_ctx *ctx); #endif #endif /* END SQLCIPHER */ diff --git a/src/crypto_impl.c b/src/crypto_impl.c index 5510073..c4c9f1d 100644 --- a/src/crypto_impl.c +++ b/src/crypto_impl.c @@ -867,5 +867,79 @@ const char* sqlcipher_codec_get_cipher_provider(codec_ctx *ctx) { return ctx->read_ctx->provider->get_provider_name(ctx->read_ctx); } +int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { + u32 meta; + int rc = 0; + int command_idx = 0; + sqlite3 *db = ctx->pBt->db; + const char *db_filename = sqlite3_db_filename(db, "main"); + const char *migrated_db_filename = sqlite3_mprintf("%s-migrated", db_filename); + const char *key = ctx->read_ctx->pass; + int db_idx = db->nDb; + CODEC_TRACE(("current database count:%d\n", db_idx)); + if(db_filename){ + char *attach_command = sqlite3_mprintf("ATTACH DATABASE '%s-migrated' as migrate KEY '%s';", + db_filename, key); + const char *commands[] = { + "PRAGMA kdf_iter = '4000';", + attach_command, + "SELECT sqlcipher_export('migrate');", + }; + for(command_idx = 0; command_idx < ArraySize(commands); command_idx++){ + const char *command = commands[command_idx]; + rc = sqlite3_exec(db, command, NULL, NULL, NULL); + if(rc != SQLITE_OK){ + break; + } + } + sqlite3_free(attach_command); + if(rc == SQLITE_OK){ + + static const unsigned char aCopy[] = { + BTREE_SCHEMA_VERSION, 1, /* Add one to the old schema cookie */ + BTREE_DEFAULT_CACHE_SIZE, 0, /* Preserve the default page cache size */ + BTREE_TEXT_ENCODING, 0, /* Preserve the text encoding */ + BTREE_USER_VERSION, 0, /* Preserve the user version */ + BTREE_APPLICATION_ID, 0, /* Preserve the application id */ + }; + + CODEC_TRACE(("current database count:%d\n", db->nDb)); + Btree *pDest = db->aDb[0].pBt; + Btree *pSrc = db->aDb[db->nDb-1].pBt; + + CODEC_TRACE(("pSrc is '%p'\n", (void*)pSrc)); + CODEC_TRACE(("pDest is '%p'\n", (void*)pDest)); + + //rc = sqlite3_exec(db, "BEGIN;", NULL, NULL, NULL); + rc = sqlite3BtreeBeginTrans(pSrc, 2); + rc = sqlite3BtreeBeginTrans(pDest, 2); + + assert( 1==sqlite3BtreeIsInTrans(pDest) ); + assert( 1==sqlite3BtreeIsInTrans(pSrc) ); + + CODEC_TRACE(("before metadata copy\n")); + int i = 0; + for(i=0; i Date: Thu, 25 Jul 2013 16:22:58 -0500 Subject: [PATCH 05/12] More work on cipher_migrate, still unusable --- src/crypto_impl.c | 74 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 53 insertions(+), 21 deletions(-) diff --git a/src/crypto_impl.c b/src/crypto_impl.c index c4c9f1d..2be4446 100644 --- a/src/crypto_impl.c +++ b/src/crypto_impl.c @@ -871,12 +871,23 @@ int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { u32 meta; int rc = 0; int command_idx = 0; + int saved_flags; + int saved_nChange; + int saved_nTotalChange; + void (*saved_xTrace)(void*,const char*); + Db *pDb = 0; sqlite3 *db = ctx->pBt->db; const char *db_filename = sqlite3_db_filename(db, "main"); const char *migrated_db_filename = sqlite3_mprintf("%s-migrated", db_filename); const char *key = ctx->read_ctx->pass; - int db_idx = db->nDb; - CODEC_TRACE(("current database count:%d\n", db_idx)); + static const unsigned char aCopy[] = { + BTREE_SCHEMA_VERSION, 1, /* Add one to the old schema cookie */ + BTREE_DEFAULT_CACHE_SIZE, 0, /* Preserve the default page cache size */ + BTREE_TEXT_ENCODING, 0, /* Preserve the text encoding */ + BTREE_USER_VERSION, 0, /* Preserve the user version */ + BTREE_APPLICATION_ID, 0, /* Preserve the application id */ + }; + if(db_filename){ char *attach_command = sqlite3_mprintf("ATTACH DATABASE '%s-migrated' as migrate KEY '%s';", db_filename, key); @@ -893,31 +904,39 @@ int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { } } sqlite3_free(attach_command); + if(rc == SQLITE_OK){ + if( !db->autoCommit ){ + CODEC_TRACE(("cannot migrate from within a transaction")); + goto handle_error; + } + if( db->activeVdbeCnt>1 ){ + CODEC_TRACE(("cannot migrate - SQL statements in progress")); + goto handle_error; + } + + /* Save the current value of the database flags so that it can be + ** restored before returning. Then set the writable-schema flag, and + ** disable CHECK and foreign key constraints. */ + saved_flags = db->flags; + saved_nChange = db->nChange; + saved_nTotalChange = db->nTotalChange; + saved_xTrace = db->xTrace; + db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks | SQLITE_PreferBuiltin; + db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder); + db->xTrace = 0; - static const unsigned char aCopy[] = { - BTREE_SCHEMA_VERSION, 1, /* Add one to the old schema cookie */ - BTREE_DEFAULT_CACHE_SIZE, 0, /* Preserve the default page cache size */ - BTREE_TEXT_ENCODING, 0, /* Preserve the text encoding */ - BTREE_USER_VERSION, 0, /* Preserve the user version */ - BTREE_APPLICATION_ID, 0, /* Preserve the application id */ - }; - - CODEC_TRACE(("current database count:%d\n", db->nDb)); Btree *pDest = db->aDb[0].pBt; - Btree *pSrc = db->aDb[db->nDb-1].pBt; + pDb = &(db->aDb[db->nDb-1]); + Btree *pSrc = pDb->pBt; - CODEC_TRACE(("pSrc is '%p'\n", (void*)pSrc)); - CODEC_TRACE(("pDest is '%p'\n", (void*)pDest)); - - //rc = sqlite3_exec(db, "BEGIN;", NULL, NULL, NULL); + rc = sqlite3_exec(db, "BEGIN;", NULL, NULL, NULL); rc = sqlite3BtreeBeginTrans(pSrc, 2); - rc = sqlite3BtreeBeginTrans(pDest, 2); + //rc = sqlite3BtreeBeginTrans(pDest, 2); assert( 1==sqlite3BtreeIsInTrans(pDest) ); assert( 1==sqlite3BtreeIsInTrans(pSrc) ); - CODEC_TRACE(("before metadata copy\n")); int i = 0; for(i=0; iflags = saved_flags; + db->nChange = saved_nChange; + db->nTotalChange = saved_nTotalChange; + db->xTrace = saved_xTrace; + sqlite3BtreeSetPageSize(pDest, -1, -1, 1); + db->autoCommit = 1; + if( pDb ){ + sqlite3BtreeClose(pDb->pBt); + pDb->pBt = 0; + pDb->pSchema = 0; + } + sqlite3ResetAllSchemasOfConnection(db); remove(migrated_db_filename); + sqlite3_free(migrated_db_filename); } - sqlite3_free(migrated_db_filename); } goto exit; handle_error: - CODEC_TRACE(("An error occurred\n")); + CODEC_TRACE(("an error occurred\n")); + rc = SQLITE_ERROR; exit: return rc; From fb14d957ca8d41251421ceefab851c91e6889746 Mon Sep 17 00:00:00 2001 From: Nick Parker Date: Fri, 26 Jul 2013 09:20:18 -0500 Subject: [PATCH 06/12] Swap btree for write, reset password on codec - working. --- src/crypto_impl.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/crypto_impl.c b/src/crypto_impl.c index 2be4446..77ff645 100644 --- a/src/crypto_impl.c +++ b/src/crypto_impl.c @@ -871,6 +871,7 @@ int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { u32 meta; int rc = 0; int command_idx = 0; + int password_sz; int saved_flags; int saved_nChange; int saved_nTotalChange; @@ -879,7 +880,7 @@ int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { sqlite3 *db = ctx->pBt->db; const char *db_filename = sqlite3_db_filename(db, "main"); const char *migrated_db_filename = sqlite3_mprintf("%s-migrated", db_filename); - const char *key = ctx->read_ctx->pass; + char *key = ctx->read_ctx->pass; static const unsigned char aCopy[] = { BTREE_SCHEMA_VERSION, 1, /* Add one to the old schema cookie */ BTREE_DEFAULT_CACHE_SIZE, 0, /* Preserve the default page cache size */ @@ -932,10 +933,13 @@ int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { rc = sqlite3_exec(db, "BEGIN;", NULL, NULL, NULL); rc = sqlite3BtreeBeginTrans(pSrc, 2); - //rc = sqlite3BtreeBeginTrans(pDest, 2); + rc = sqlite3BtreeBeginTrans(pDest, 2); assert( 1==sqlite3BtreeIsInTrans(pDest) ); assert( 1==sqlite3BtreeIsInTrans(pSrc) ); + + sqlite3CodecGetKey(db, db->nDb - 1, &key, &password_sz); + sqlcipher_codec_ctx_set_pass(ctx, key, password_sz, 2); int i = 0; for(i=0; i Date: Fri, 26 Jul 2013 09:29:36 -0500 Subject: [PATCH 07/12] Clean up compiler warnings --- src/crypto_impl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/crypto_impl.c b/src/crypto_impl.c index 77ff645..ba4ff48 100644 --- a/src/crypto_impl.c +++ b/src/crypto_impl.c @@ -879,7 +879,7 @@ int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { Db *pDb = 0; sqlite3 *db = ctx->pBt->db; const char *db_filename = sqlite3_db_filename(db, "main"); - const char *migrated_db_filename = sqlite3_mprintf("%s-migrated", db_filename); + char *migrated_db_filename = sqlite3_mprintf("%s-migrated", db_filename); char *key = ctx->read_ctx->pass; static const unsigned char aCopy[] = { BTREE_SCHEMA_VERSION, 1, /* Add one to the old schema cookie */ @@ -938,7 +938,7 @@ int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { assert( 1==sqlite3BtreeIsInTrans(pDest) ); assert( 1==sqlite3BtreeIsInTrans(pSrc) ); - sqlite3CodecGetKey(db, db->nDb - 1, &key, &password_sz); + sqlite3CodecGetKey(db, db->nDb - 1, (void**)&key, &password_sz); sqlcipher_codec_ctx_set_pass(ctx, key, password_sz, 2); int i = 0; From 9b415f640161838d7e89f1e170dc32a041148d40 Mon Sep 17 00:00:00 2001 From: Nick Parker Date: Fri, 26 Jul 2013 16:50:36 -0500 Subject: [PATCH 08/12] Work to configure the proper env. for the migration --- src/crypto_impl.c | 85 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/src/crypto_impl.c b/src/crypto_impl.c index ba4ff48..e2df79f 100644 --- a/src/crypto_impl.c +++ b/src/crypto_impl.c @@ -880,7 +880,14 @@ int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { sqlite3 *db = ctx->pBt->db; const char *db_filename = sqlite3_db_filename(db, "main"); char *migrated_db_filename = sqlite3_mprintf("%s-migrated", db_filename); + char *query_sqlite_master = "SELECT count(*) from sqlite_master;"; + char *pragma_hmac_off = "PRAGMA cipher_use_hmac = OFF;"; + char *pragma_4k_kdf_iter = "PRAGMA kdf_iter = 4000;"; char *key = ctx->read_ctx->pass; + int upgrade_1x_format = 0; + int upgrade_4k_format = 0; + sqlite3 *test; + char *err = 0; static const unsigned char aCopy[] = { BTREE_SCHEMA_VERSION, 1, /* Add one to the old schema cookie */ BTREE_DEFAULT_CACHE_SIZE, 0, /* Preserve the default page cache size */ @@ -890,15 +897,49 @@ int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { }; if(db_filename){ + char *attach_command = sqlite3_mprintf("ATTACH DATABASE '%s-migrated' as migrate KEY '%s';", db_filename, key); + + int rc = sqlcipher_check_connection(db_filename, key, ""); + if(rc == SQLITE_OK){ + // no upgrade required + goto exit; + } + + // check for 1x format + rc = sqlcipher_check_connection(db_filename, key, pragma_hmac_off); + if(rc == SQLITE_OK) { + upgrade_1x_format = 1; + } + + // check for 4k format + rc = sqlcipher_check_connection(db_filename, key, pragma_4k_kdf_iter); + if(rc == SQLITE_OK) { + upgrade_4k_format = 1; + } + + // check both 1x and 4k together + char *pragma_1x_and_4k = sqlite3_mprintf("%s%s", pragma_hmac_off, + pragma_4k_kdf_iter); + rc = sqlcipher_check_connection(db_filename, key, pragma_1x_and_4k); + sqlite3_free(pragma_1x_and_4k); + if(rc == SQLITE_OK) { + upgrade_1x_format = 1; + upgrade_4k_format = 1; + } + const char *commands[] = { - "PRAGMA kdf_iter = '4000';", + upgrade_4k_format == 1 ? pragma_4k_kdf_iter : "", + upgrade_1x_format == 1 ? pragma_hmac_off : "", attach_command, "SELECT sqlcipher_export('migrate');", }; for(command_idx = 0; command_idx < ArraySize(commands); command_idx++){ const char *command = commands[command_idx]; + if(strcmp(command, "") == 0){ + continue; + } rc = sqlite3_exec(db, command, NULL, NULL, NULL); if(rc != SQLITE_OK){ break; @@ -978,5 +1019,47 @@ int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { return rc; } +int sqlcipher_check_connection(char *filename, char *key, char *sql) { + int rc; + sqlite3 *db; + char *errMsg; + sqlite3_stmt *statement; + int status = SQLITE_ERROR; + char *query_sqlite_master = "SELECT count(*) FROM sqlite_master;"; + + rc = sqlite3_open(filename, &db); + if(rc != SQLITE_OK){ + goto cleanup; + } + rc = sqlite3_key(db, key, (int)strlen(key)); + if(rc != SQLITE_OK){ + goto cleanup; + } + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + if(rc != SQLITE_OK){ + goto cleanup; + } + rc = sqlite3_prepare(db, query_sqlite_master, -1, &statement, NULL); + if(rc != SQLITE_OK){ + goto cleanup; + } + if(sqlite3_step(statement) == SQLITE_ROW){ + status = SQLITE_OK; + } + goto cleanup; + +cleanup: + if(statement){ + sqlite3_finalize(statement); + } + if(db){ + sqlite3_close(db); + } + + exit: + return status; + +} + #endif /* END SQLCIPHER */ From f844a5dfc3f99623a2b9e65c7e0a5853fbe160d1 Mon Sep 17 00:00:00 2001 From: Nick Parker Date: Mon, 29 Jul 2013 11:04:29 -0500 Subject: [PATCH 09/12] Attach new codec before btree copy --- src/crypto_impl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/crypto_impl.c b/src/crypto_impl.c index e2df79f..fed1398 100644 --- a/src/crypto_impl.c +++ b/src/crypto_impl.c @@ -979,8 +979,9 @@ int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { assert( 1==sqlite3BtreeIsInTrans(pDest) ); assert( 1==sqlite3BtreeIsInTrans(pSrc) ); + sqlite3CodecGetKey(db, db->nDb - 1, (void**)&key, &password_sz); - sqlcipher_codec_ctx_set_pass(ctx, key, password_sz, 2); + sqlite3CodecAttach(db, 0, key, password_sz); int i = 0; for(i=0; inChange = saved_nChange; db->nTotalChange = saved_nTotalChange; db->xTrace = saved_xTrace; - sqlite3BtreeSetPageSize(pDest, -1, -1, 1); db->autoCommit = 1; if( pDb ){ sqlite3BtreeClose(pDb->pBt); From dce806aae5c8e5138699df1382aee3b84956827b Mon Sep 17 00:00:00 2001 From: Nick Parker Date: Mon, 12 Aug 2013 15:09:11 -0500 Subject: [PATCH 10/12] Add tests for cipher_migrate --- test/crypto.test | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/test/crypto.test b/test/crypto.test index 0238cb7..822e4c7 100644 --- a/test/crypto.test +++ b/test/crypto.test @@ -1959,4 +1959,45 @@ if_built_with_commoncrypto verify-default-cipher { db close file delete -force test.db +do_test migrate-1.1.8-database-to-3x-format { + file delete -force test.db-migrated + file delete -force test.db + file copy -force sqlcipher-1.1.8-testkey.db test.db + sqlite_orig db test.db + execsql { + PRAGMA key = 'testkey'; + PRAGMA cipher_migrate; + } + db close + + sqlite_orig db test.db + execsql { + PRAGMA key = 'testkey'; + SELECT count(*) FROM sqlite_master; + } +} {1} +db close +file delete -force test.db +file delete -force test.db-migrated + +do_test migrate-2-0-le-database-to-3x-format { + file delete -force test.db + file copy -force sqlcipher-2.0-le-testkey.db test.db + sqlite_orig db test.db + execsql { + PRAGMA key = 'testkey'; + PRAGMA cipher_migrate; + } + db close + + sqlite_orig db test.db + execsql { + PRAGMA key = 'testkey'; + SELECT count(*) FROM sqlite_master; + } +} {1} +db close +file delete -force test.db +file delete -force test.db-migrated + finish_test From c3f6cf1e02bbb341bed5fbbcfb736c6b05e1ce54 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Tue, 13 Aug 2013 12:54:16 -0400 Subject: [PATCH 11/12] null termination of key value --- src/crypto_impl.c | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/crypto_impl.c b/src/crypto_impl.c index fed1398..7efb6d9 100644 --- a/src/crypto_impl.c +++ b/src/crypto_impl.c @@ -883,7 +883,8 @@ int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { char *query_sqlite_master = "SELECT count(*) from sqlite_master;"; char *pragma_hmac_off = "PRAGMA cipher_use_hmac = OFF;"; char *pragma_4k_kdf_iter = "PRAGMA kdf_iter = 4000;"; - char *key = ctx->read_ctx->pass; + char *key; + int key_sz; int upgrade_1x_format = 0; int upgrade_4k_format = 0; sqlite3 *test; @@ -896,6 +897,11 @@ int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { BTREE_APPLICATION_ID, 0, /* Preserve the application id */ }; + key_sz = ctx->read_ctx->pass_sz + 1; + key = sqlcipher_malloc(key_sz); + memset(key, 0, key_sz); + memcpy(key, ctx->read_ctx->pass, ctx->read_ctx->pass_sz); + if(db_filename){ char *attach_command = sqlite3_mprintf("ATTACH DATABASE '%s-migrated' as migrate KEY '%s';", @@ -908,18 +914,18 @@ int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { } // check for 1x format - rc = sqlcipher_check_connection(db_filename, key, pragma_hmac_off); - if(rc == SQLITE_OK) { - upgrade_1x_format = 1; - } + //rc = sqlcipher_check_connection(db_filename, key, pragma_hmac_off); + //if(rc == SQLITE_OK) { + // upgrade_1x_format = 1; + //} - // check for 4k format + // Version 2 - check for 4k with hmac format rc = sqlcipher_check_connection(db_filename, key, pragma_4k_kdf_iter); if(rc == SQLITE_OK) { upgrade_4k_format = 1; } - // check both 1x and 4k together + // Version 1 - check both no hmac and 4k together char *pragma_1x_and_4k = sqlite3_mprintf("%s%s", pragma_hmac_off, pragma_4k_kdf_iter); rc = sqlcipher_check_connection(db_filename, key, pragma_1x_and_4k); @@ -929,6 +935,8 @@ int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { upgrade_4k_format = 1; } + if(upgrade_1x_format == 0 || upgrade_4k_format == 0) goto handle_error; + const char *commands[] = { upgrade_4k_format == 1 ? pragma_4k_kdf_iter : "", upgrade_1x_format == 1 ? pragma_hmac_off : "", @@ -946,6 +954,7 @@ int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { } } sqlite3_free(attach_command); + sqlcipher_free(key, key_sz); if(rc == SQLITE_OK){ if( !db->autoCommit ){ From 6f4a0f2d8d452ebfb6ed62135d45d5ef84fd67ed Mon Sep 17 00:00:00 2001 From: Nick Parker Date: Tue, 13 Aug 2013 14:53:08 -0500 Subject: [PATCH 12/12] Adjustments to cipher_migrate - And format flags when checking for exit condition - Pass key size to sqlcipher_check_connection - Trace version upgrade logic --- src/crypto_impl.c | 36 ++++++++++++++++++------------------ test/crypto.test | 5 ----- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/src/crypto_impl.c b/src/crypto_impl.c index 7efb6d9..7e5d437 100644 --- a/src/crypto_impl.c +++ b/src/crypto_impl.c @@ -896,7 +896,6 @@ int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { BTREE_USER_VERSION, 0, /* Preserve the user version */ BTREE_APPLICATION_ID, 0, /* Preserve the application id */ }; - key_sz = ctx->read_ctx->pass_sz + 1; key = sqlcipher_malloc(key_sz); memset(key, 0, key_sz); @@ -907,35 +906,34 @@ int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { char *attach_command = sqlite3_mprintf("ATTACH DATABASE '%s-migrated' as migrate KEY '%s';", db_filename, key); - int rc = sqlcipher_check_connection(db_filename, key, ""); + int rc = sqlcipher_check_connection(db_filename, key, key_sz, ""); if(rc == SQLITE_OK){ - // no upgrade required + CODEC_TRACE(("No upgrade required - exiting\n")); goto exit; } - // check for 1x format - //rc = sqlcipher_check_connection(db_filename, key, pragma_hmac_off); - //if(rc == SQLITE_OK) { - // upgrade_1x_format = 1; - //} - // Version 2 - check for 4k with hmac format - rc = sqlcipher_check_connection(db_filename, key, pragma_4k_kdf_iter); + rc = sqlcipher_check_connection(db_filename, key, key_sz, pragma_4k_kdf_iter); if(rc == SQLITE_OK) { + CODEC_TRACE(("Version 2 format found\n")); upgrade_4k_format = 1; } // Version 1 - check both no hmac and 4k together char *pragma_1x_and_4k = sqlite3_mprintf("%s%s", pragma_hmac_off, pragma_4k_kdf_iter); - rc = sqlcipher_check_connection(db_filename, key, pragma_1x_and_4k); + rc = sqlcipher_check_connection(db_filename, key, key_sz, pragma_1x_and_4k); sqlite3_free(pragma_1x_and_4k); if(rc == SQLITE_OK) { + CODEC_TRACE(("Version 1 format found\n")); upgrade_1x_format = 1; upgrade_4k_format = 1; } - if(upgrade_1x_format == 0 || upgrade_4k_format == 0) goto handle_error; + if(upgrade_1x_format == 0 && upgrade_4k_format == 0) { + CODEC_TRACE(("Upgrade format not determined\n")); + goto handle_error; + } const char *commands[] = { upgrade_4k_format == 1 ? pragma_4k_kdf_iter : "", @@ -1016,31 +1014,33 @@ int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { sqlite3ResetAllSchemasOfConnection(db); remove(migrated_db_filename); sqlite3_free(migrated_db_filename); + } else { + CODEC_TRACE(("*** migration failure** \n\n")); } + } goto exit; handle_error: - CODEC_TRACE(("an error occurred\n")); + CODEC_TRACE(("An error occurred attempting to migrate the database\n")); rc = SQLITE_ERROR; exit: return rc; } -int sqlcipher_check_connection(char *filename, char *key, char *sql) { +int sqlcipher_check_connection(char *filename, char *key, int key_sz, char *sql) { int rc; sqlite3 *db; char *errMsg; sqlite3_stmt *statement; - int status = SQLITE_ERROR; char *query_sqlite_master = "SELECT count(*) FROM sqlite_master;"; rc = sqlite3_open(filename, &db); if(rc != SQLITE_OK){ goto cleanup; } - rc = sqlite3_key(db, key, (int)strlen(key)); + rc = sqlite3_key(db, key, key_sz); if(rc != SQLITE_OK){ goto cleanup; } @@ -1053,7 +1053,7 @@ int sqlcipher_check_connection(char *filename, char *key, char *sql) { goto cleanup; } if(sqlite3_step(statement) == SQLITE_ROW){ - status = SQLITE_OK; + rc = SQLITE_OK; } goto cleanup; @@ -1066,7 +1066,7 @@ cleanup: } exit: - return status; + return rc; } diff --git a/test/crypto.test b/test/crypto.test index 822e4c7..3f32f55 100644 --- a/test/crypto.test +++ b/test/crypto.test @@ -1960,8 +1960,6 @@ db close file delete -force test.db do_test migrate-1.1.8-database-to-3x-format { - file delete -force test.db-migrated - file delete -force test.db file copy -force sqlcipher-1.1.8-testkey.db test.db sqlite_orig db test.db execsql { @@ -1978,10 +1976,8 @@ do_test migrate-1.1.8-database-to-3x-format { } {1} db close file delete -force test.db -file delete -force test.db-migrated do_test migrate-2-0-le-database-to-3x-format { - file delete -force test.db file copy -force sqlcipher-2.0-le-testkey.db test.db sqlite_orig db test.db execsql { @@ -1998,6 +1994,5 @@ do_test migrate-2-0-le-database-to-3x-format { } {1} db close file delete -force test.db -file delete -force test.db-migrated finish_test